diff --git a/compute/core/src/contracts.ts b/compute/core/src/contracts.ts index 0abe295..bb0a8d7 100644 --- a/compute/core/src/contracts.ts +++ b/compute/core/src/contracts.ts @@ -61,6 +61,15 @@ export interface WorkerJobTiming { computeMs?: number; } +export type PdfLayoutProgressPhase = 'infer' | 'merge'; + +export interface PdfLayoutProgress { + totalPages: number; + pagesParsed: number; + currentPage?: number; + phase: PdfLayoutProgressPhase; +} + export interface WorkerJobStatusResponse { status: WorkerJobState; result?: Result; @@ -96,4 +105,5 @@ export interface WorkerOperationState { result?: Result; error?: WorkerJobErrorShape; timing?: WorkerJobTiming; + progress?: PdfLayoutProgress; } diff --git a/compute/core/src/local-runtime.ts b/compute/core/src/local-runtime.ts index 5a794b8..15fe92e 100644 --- a/compute/core/src/local-runtime.ts +++ b/compute/core/src/local-runtime.ts @@ -25,10 +25,16 @@ export async function runWhisperAlignmentFromAudioBuffer(input: { export async function runPdfLayoutFromPdfBuffer(input: { documentId: string; pdfBytes: ArrayBuffer; + onPageParsed?: (input: { + pageNumber: number; + totalPages: number; + pageMs: number; + }) => void | Promise; }) { const parsed = await parsePdf({ documentId: input.documentId, pdfBytes: input.pdfBytes, + onPageParsed: input.onPageParsed, }); return { parsed }; } diff --git a/compute/core/src/pdf-layout/parsePdf.ts b/compute/core/src/pdf-layout/parsePdf.ts index 74c9dfe..0207b8f 100644 --- a/compute/core/src/pdf-layout/parsePdf.ts +++ b/compute/core/src/pdf-layout/parsePdf.ts @@ -11,6 +11,11 @@ import { renderPage } from './renderPage'; interface ParsePdfInput { documentId: string; pdfBytes: ArrayBuffer; + onPageParsed?: (input: { + pageNumber: number; + totalPages: number; + pageMs: number; + }) => void | Promise; } const LAYOUT_RENDER_SCALE = 1.5; @@ -80,6 +85,7 @@ export async function parsePdf(input: ParsePdfInput): Promise let sawText = false; for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) { + const pageStartedAt = Date.now(); const page = await pdf.getPage(pageNumber); const viewport = page.getViewport({ scale: 1.0 }); const textContent = await page.getTextContent(); @@ -143,6 +149,14 @@ export async function parsePdf(input: ParsePdfInput): Promise height: viewport.height, blocks, }); + + if (input.onPageParsed) { + await input.onPageParsed({ + pageNumber, + totalPages: pdf.numPages, + pageMs: Date.now() - pageStartedAt, + }); + } } if (!sawText) { diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts index 6712192..36c818b 100644 --- a/compute/worker/src/server.ts +++ b/compute/worker/src/server.ts @@ -37,6 +37,7 @@ import { type WorkerOperationKind, type WorkerOperationRequest, type WorkerOperationState, + type PdfLayoutProgress, } from '@openreader/compute-core/contracts'; import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; @@ -75,6 +76,7 @@ interface StoredJobState { result?: Result; error?: WorkerJobErrorShape; timing?: WorkerJobTiming; + progress?: PdfLayoutProgress; } interface OpIndexEntry { @@ -772,6 +774,7 @@ async function main(): Promise { const runLayout = async ( payload: PdfLayoutJobRequest, queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, ): Promise => { const parsed = layoutSchema.parse(payload); @@ -779,17 +782,38 @@ async function main(): Promise { const pdfBytes = await readObjectByKey(parsed.documentObjectKey); const s3FetchMs = Date.now() - s3FetchStartedAt; + let lastTotalPages = 0; + let lastPagesParsed = 0; const computeStartedAt = Date.now(); const result = await withTimeout( runPdfLayoutFromPdfBuffer({ documentId: parsed.documentId, pdfBytes, + onPageParsed: async ({ pageNumber, totalPages }) => { + lastTotalPages = totalPages; + lastPagesParsed = pageNumber; + if (!hooks?.onProgress) return; + await hooks.onProgress({ + totalPages, + pagesParsed: pageNumber, + currentPage: pageNumber, + phase: 'infer', + }); + }, }), pdfTimeoutMs, 'pdf layout job', ); const computeMs = Date.now() - computeStartedAt; + if (hooks?.onProgress && lastTotalPages > 0) { + await hooks.onProgress({ + totalPages: lastTotalPages, + pagesParsed: lastPagesParsed, + currentPage: lastPagesParsed || undefined, + phase: 'merge', + }); + } const parsedObjectKey = await putParsedObject(parsed.documentId, parsed.namespace, result.parsed); return { parsedObjectKey, @@ -804,11 +828,16 @@ async function main(): Promise { async function processMessage(input: { msg: JsMsg; codec: JsonCodec>; - run: (payload: TPayload, queueWaitMs: number) => Promise; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; workerLabel: string; }): Promise { let decoded: QueuedJob | null = null; let heartbeat: NodeJS.Timeout | null = null; + let latestProgress: PdfLayoutProgress | undefined; try { decoded = input.codec.decode(input.msg.data); const startedAt = Date.now(); @@ -824,6 +853,7 @@ async function main(): Promise { startedAt, updatedAt: startedAt, ...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }; await putOpState(runningState); @@ -837,11 +867,11 @@ async function main(): Promise { startedAt, updatedAt: startedAt, ...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }); - heartbeat = setInterval(() => { - const now = Date.now(); - const heartbeatState: WorkerOperationState = { + const persistRunningState = async (updatedAt: number): Promise => { + const runningOpState: WorkerOperationState = { opId: decoded!.opId, opKey: decoded!.opKey, kind: decoded!.kind, @@ -849,18 +879,13 @@ async function main(): Promise { status: 'running', queuedAt: decoded!.queuedAt, startedAt, - updatedAt: now, + updatedAt, ...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }; - void putOpState(heartbeatState).catch((stateError) => { - app.log.error({ - worker: input.workerLabel, - opId: decoded?.opId, - jobId: decoded?.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist running heartbeat op state'); - }); - void putJobState({ + + await putOpState(runningOpState); + await putJobState({ jobId: decoded!.jobId, opId: decoded!.opId, opKey: decoded!.opKey, @@ -868,19 +893,30 @@ async function main(): Promise { status: 'running', timestamp: decoded!.queuedAt, startedAt, - updatedAt: now, + updatedAt, ...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}), - }).catch((stateError) => { + ...(latestProgress ? { progress: latestProgress } : {}), + }); + }; + + heartbeat = setInterval(() => { + const now = Date.now(); + void persistRunningState(now).catch((stateError) => { app.log.error({ worker: input.workerLabel, opId: decoded?.opId, jobId: decoded?.jobId, error: toErrorMessage(stateError), - }, 'failed to persist running heartbeat job state'); + }, 'failed to persist running heartbeat state'); }); }, RUNNING_HEARTBEAT_MS); - const result = await input.run(decoded.payload, queueWaitMs ?? 0); + const result = await input.run(decoded.payload, queueWaitMs ?? 0, { + onProgress: async (progress) => { + latestProgress = progress; + await persistRunningState(Date.now()); + }, + }); const resultTiming = result && typeof result === 'object' && 'timing' in result ? (result as { timing?: WorkerJobTiming }).timing : undefined; @@ -897,6 +933,7 @@ async function main(): Promise { updatedAt: now, result: result as WhisperAlignJobResult | PdfLayoutJobResult, ...(resultTiming ? { timing: resultTiming } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }; await putOpState(succeededState); @@ -911,6 +948,7 @@ async function main(): Promise { updatedAt: now, result: result as WhisperAlignJobResult | PdfLayoutJobResult, ...(resultTiming ? { timing: resultTiming } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }); input.msg.ack(); @@ -941,6 +979,7 @@ async function main(): Promise { updatedAt: now, ...(status === 'failed' ? { error: { message } } : {}), ...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }; await putOpState(opState).catch((stateError) => { @@ -962,6 +1001,7 @@ async function main(): Promise { updatedAt: now, ...(status === 'failed' ? { error: { message } } : {}), ...(typeof queueWaitMs === 'number' ? { timing: { queueWaitMs } } : {}), + ...(latestProgress ? { progress: latestProgress } : {}), }).catch((stateError) => { app.log.error({ worker: input.workerLabel, @@ -1001,7 +1041,11 @@ async function main(): Promise { async function createWorkerLoop(input: { consumer: Consumer; codec: JsonCodec>; - run: (payload: TPayload, queueWaitMs: number) => Promise; + run: ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; workerLabel: string; }): Promise { while (!stopping) { diff --git a/drizzle/postgres/0007_pdf_parse_progress.sql b/drizzle/postgres/0007_pdf_parse_progress.sql new file mode 100644 index 0000000..9c2d188 --- /dev/null +++ b/drizzle/postgres/0007_pdf_parse_progress.sql @@ -0,0 +1,2 @@ +ALTER TABLE "documents" ADD COLUMN "parse_state" text;--> statement-breakpoint +ALTER TABLE "documents" DROP COLUMN "parse_status"; \ No newline at end of file diff --git a/drizzle/postgres/meta/0007_snapshot.json b/drizzle/postgres/meta/0007_snapshot.json new file mode 100644 index 0000000..ffc0c25 --- /dev/null +++ b/drizzle/postgres/meta/0007_snapshot.json @@ -0,0 +1,1831 @@ +{ + "id": "68baf959-cd1d-418b-a04e-11397aa05c39", + "prevId": "4d83c2e9-61f5-4a0a-a49d-9b68e7912375", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.admin_providers": { + "name": "admin_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_settings": { + "name": "admin_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value_json": { + "name": "value_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobook_chapters": { + "name": "audiobook_chapters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "name": "audiobook_chapters_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audiobooks": { + "name": "audiobooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "name": "audiobooks_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_previews": { + "name": "document_previews", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_until_ms", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "name": "document_previews_document_id_namespace_variant_pk", + "columns": [ + "document_id", + "namespace", + "variant" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.document_settings": { + "name": "document_settings", + "schema": "", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "name": "document_settings_document_id_user_id_pk", + "columns": [ + "document_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_modified": { + "name": "last_modified", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parse_state": { + "name": "parse_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parsed_json_key": { + "name": "parsed_json_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_modified", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "name": "documents_id_user_id_pk", + "columns": [ + "id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_entries": { + "name": "tts_segment_entries", + "schema": "", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_version": { + "name": "document_version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_reader_rank", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_char_offset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_spine_href", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_location", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_index", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "locator_identity_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "document_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "name": "tts_segment_entries_segment_entry_id_user_id_pk", + "columns": [ + "segment_entry_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tts_segment_variants": { + "name": "tts_segment_variants", + "schema": "", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "settings_json": { + "name": "settings_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "segment_entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "settings_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "name": "tts_segment_variants_segment_id_user_id_pk", + "columns": [ + "segment_id", + "user_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_document_progress": { + "name": "user_document_progress", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "name": "user_document_progress_user_id_document_id_pk", + "columns": [ + "user_id", + "document_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preferences": { + "name": "user_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "data_json": { + "name": "data_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_tts_chars": { + "name": "user_tts_chars", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "(extract(epoch from now()) * 1000)::bigint" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "name": "user_tts_chars_user_id_date_pk", + "columns": [ + "user_id", + "date" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/postgres/meta/_journal.json b/drizzle/postgres/meta/_journal.json index ee17326..eccd0d6 100644 --- a/drizzle/postgres/meta/_journal.json +++ b/drizzle/postgres/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1779102950715, "tag": "0006_preview-defaults-cleanup", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1779378127846, + "tag": "0007_pdf_parse_progress", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/sqlite/0007_pdf_parse_progress.sql b/drizzle/sqlite/0007_pdf_parse_progress.sql new file mode 100644 index 0000000..b9a4631 --- /dev/null +++ b/drizzle/sqlite/0007_pdf_parse_progress.sql @@ -0,0 +1,2 @@ +ALTER TABLE `documents` ADD `parse_state` text;--> statement-breakpoint +ALTER TABLE `documents` DROP COLUMN `parse_status`; \ No newline at end of file diff --git a/drizzle/sqlite/meta/0007_snapshot.json b/drizzle/sqlite/meta/0007_snapshot.json new file mode 100644 index 0000000..37cc0da --- /dev/null +++ b/drizzle/sqlite/meta/0007_snapshot.json @@ -0,0 +1,1693 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "031a62ae-47a7-4476-a577-7b14ff787eba", + "prevId": "b4c531bb-7ddb-40e6-a946-815a6521653e", + "tables": { + "admin_providers": { + "name": "admin_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_type": { + "name": "provider_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_iv": { + "name": "api_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_last4": { + "name": "api_key_last4", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_instructions": { + "name": "default_instructions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "admin_providers_slug_unique": { + "name": "admin_providers_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "admin_settings": { + "name": "admin_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'admin'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobook_chapters": { + "name": "audiobook_chapters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "book_id": { + "name": "book_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chapter_index": { + "name": "chapter_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "audiobook_chapters_user_id_user_id_fk": { + "name": "audiobook_chapters_user_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk": { + "name": "audiobook_chapters_book_id_user_id_audiobooks_id_user_id_fk", + "tableFrom": "audiobook_chapters", + "tableTo": "audiobooks", + "columnsFrom": [ + "book_id", + "user_id" + ], + "columnsTo": [ + "id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobook_chapters_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobook_chapters_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audiobooks": { + "name": "audiobooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cover_path": { + "name": "cover_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "audiobooks_user_id_user_id_fk": { + "name": "audiobooks_user_id_user_id_fk", + "tableFrom": "audiobooks", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "audiobooks_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "audiobooks_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_previews": { + "name": "document_previews", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "source_last_modified_ms": { + "name": "source_last_modified_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'image/jpeg'" + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until_ms": { + "name": "lease_until_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at_ms": { + "name": "created_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at_ms": { + "name": "updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_document_previews_status_lease": { + "name": "idx_document_previews_status_lease", + "columns": [ + "status", + "lease_until_ms" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "document_previews_document_id_namespace_variant_pk": { + "columns": [ + "document_id", + "namespace", + "variant" + ], + "name": "document_previews_document_id_namespace_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_settings": { + "name": "document_settings", + "columns": { + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_document_settings_user_id": { + "name": "idx_document_settings_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_settings_user_id_user_id_fk": { + "name": "document_settings_user_id_user_id_fk", + "tableFrom": "document_settings", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "document_settings_document_id_user_id_pk": { + "columns": [ + "document_id", + "user_id" + ], + "name": "document_settings_document_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parse_state": { + "name": "parse_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parsed_json_key": { + "name": "parsed_json_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_documents_user_id": { + "name": "idx_documents_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_documents_user_id_last_modified": { + "name": "idx_documents_user_id_last_modified", + "columns": [ + "user_id", + "last_modified" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_user_id_user_id_fk": { + "name": "documents_user_id_user_id_fk", + "tableFrom": "documents", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "documents_id_user_id_pk": { + "columns": [ + "id", + "user_id" + ], + "name": "documents_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_entries": { + "name": "tts_segment_entries", + "columns": { + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_version": { + "name": "document_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_index": { + "name": "segment_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_key": { + "name": "segment_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "locator_reader_rank": { + "name": "locator_reader_rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_reader_type": { + "name": "locator_reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_page": { + "name": "locator_page", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_index": { + "name": "locator_spine_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_spine_href": { + "name": "locator_spine_href", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_char_offset": { + "name": "locator_char_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_location": { + "name": "locator_location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locator_identity_key": { + "name": "locator_identity_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_hash": { + "name": "text_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text_length": { + "name": "text_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_entries_manifest_sort": { + "name": "idx_tts_segment_entries_manifest_sort", + "columns": [ + "user_id", + "document_id", + "document_version", + "locator_reader_rank", + "locator_spine_index", + "locator_char_offset", + "locator_spine_href", + "locator_page", + "locator_location", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_manifest_group": { + "name": "idx_tts_segment_entries_manifest_group", + "columns": [ + "user_id", + "document_id", + "document_version", + "segment_index", + "locator_identity_key" + ], + "isUnique": false + }, + "idx_tts_segment_entries_scope": { + "name": "idx_tts_segment_entries_scope", + "columns": [ + "user_id", + "document_id", + "document_version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_entries_user_id_user_id_fk": { + "name": "tts_segment_entries_user_id_user_id_fk", + "tableFrom": "tts_segment_entries", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_entries_segment_entry_id_user_id_pk": { + "columns": [ + "segment_entry_id", + "user_id" + ], + "name": "tts_segment_entries_segment_entry_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tts_segment_variants": { + "name": "tts_segment_variants", + "columns": { + "segment_id": { + "name": "segment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "segment_entry_id": { + "name": "segment_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_hash": { + "name": "settings_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "audio_key": { + "name": "audio_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audio_format": { + "name": "audio_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mp3'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alignment_json": { + "name": "alignment_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_tts_segment_variants_entry": { + "name": "idx_tts_segment_variants_entry", + "columns": [ + "user_id", + "segment_entry_id", + "updated_at" + ], + "isUnique": false + }, + "idx_tts_segment_variants_status": { + "name": "idx_tts_segment_variants_status", + "columns": [ + "user_id", + "status" + ], + "isUnique": false + }, + "idx_tts_segment_variants_unique_settings": { + "name": "idx_tts_segment_variants_unique_settings", + "columns": [ + "user_id", + "segment_entry_id", + "settings_hash" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tts_segment_variants_user_id_user_id_fk": { + "name": "tts_segment_variants_user_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk": { + "name": "tts_segment_variants_segment_entry_id_user_id_tts_segment_entries_segment_entry_id_user_id_fk", + "tableFrom": "tts_segment_variants", + "tableTo": "tts_segment_entries", + "columnsFrom": [ + "segment_entry_id", + "user_id" + ], + "columnsTo": [ + "segment_entry_id", + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tts_segment_variants_segment_id_user_id_pk": { + "columns": [ + "segment_id", + "user_id" + ], + "name": "tts_segment_variants_segment_id_user_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_document_progress": { + "name": "user_document_progress", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reader_type": { + "name": "reader_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_document_progress_user_id_updated_at": { + "name": "idx_user_document_progress_user_id_updated_at", + "columns": [ + "user_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "user_document_progress_user_id_user_id_fk": { + "name": "user_document_progress_user_id_user_id_fk", + "tableFrom": "user_document_progress", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_document_progress_user_id_document_id_pk": { + "columns": [ + "user_id", + "document_id" + ], + "name": "user_document_progress_user_id_document_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_preferences": { + "name": "user_preferences", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data_json": { + "name": "data_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "client_updated_at_ms": { + "name": "client_updated_at_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": {}, + "foreignKeys": { + "user_preferences_user_id_user_id_fk": { + "name": "user_preferences_user_id_user_id_fk", + "tableFrom": "user_preferences", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_tts_chars": { + "name": "user_tts_chars", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "date": { + "name": "date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "idx_user_tts_chars_date": { + "name": "idx_user_tts_chars_date", + "columns": [ + "date" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "user_tts_chars_user_id_date_pk": { + "columns": [ + "user_id", + "date" + ], + "name": "user_tts_chars_user_id_date_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "is_anonymous": { + "name": "is_anonymous", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/sqlite/meta/_journal.json b/drizzle/sqlite/meta/_journal.json index 32df1e6..8db8d88 100644 --- a/drizzle/sqlite/meta/_journal.json +++ b/drizzle/sqlite/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1779102950385, "tag": "0006_preview-defaults-cleanup", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1779378125905, + "tag": "0007_pdf_parse_progress", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/(app)/pdf/[id]/page.tsx b/src/app/(app)/pdf/[id]/page.tsx index 9646da7..52b6f9c 100644 --- a/src/app/(app)/pdf/[id]/page.tsx +++ b/src/app/(app)/pdf/[id]/page.tsx @@ -42,6 +42,7 @@ export default function PDFViewerPage() { currDocPage, currDocPages, parseStatus, + parseProgress, documentSettings, updateDocumentSettings, parsedOverlayEnabled, @@ -240,30 +241,113 @@ export default function PDFViewerPage() { } const renderPdfStatusLoader = () => { + const totalPages = parseProgress?.totalPages ?? 0; + const pagesParsed = parseProgress?.pagesParsed ?? 0; + const progressPercent = totalPages > 0 + ? Math.max(0, Math.min(100, (pagesParsed / totalPages) * 100)) + : 0; + const hasMeasuredProgress = totalPages > 0; + const isMerging = parseProgress?.phase === 'merge'; + let statusText = 'Loading PDF...'; + let statusSubText = 'Initializing document renderer'; if (!isLoading) { if (parseState === 'pending') { - statusText = 'Preparing PDF layout...'; + statusText = parseProgress + ? `Page ${Math.max(0, parseProgress.pagesParsed)} / ${parseProgress.totalPages} parsed` + : 'Preparing PDF layout...'; + statusSubText = parseProgress?.phase === 'merge' + ? 'Finalizing stitched block structure' + : 'Queueing parser and preparing page extraction'; } else if (parseState === 'running') { - statusText = 'Parsing PDF layout blocks...'; + statusText = parseProgress + ? `Page ${Math.max(0, parseProgress.pagesParsed)} / ${parseProgress.totalPages} parsed` + : 'Parsing PDF layout blocks...'; + statusSubText = parseProgress?.phase === 'merge' + ? 'Merging cross-page sections' + : 'Inferring reading order and text regions'; } else if (parseState === 'failed') { statusText = 'PDF parsing failed. Retry to continue.'; + statusSubText = 'The parser could not build a usable layout map'; } } + const stageOneComplete = !isLoading; + const stageTwoActive = parseState === 'running' && !isMerging; + const stageTwoComplete = (parseState === 'running' && isMerging) || parseState === 'ready'; + const stageThreeActive = parseState === 'running' && isMerging; + const stageThreeComplete = parseState === 'ready'; + return ( -
- -

{statusText}

- {!isLoading && parseState === 'failed' ? ( - - ) : null} +
+
+
+
+
+
+

PDF Layout Parse

+

{statusText}

+

{statusSubText}

+
+
+ + + {parseState === 'failed' ? 'blocked' : (isMerging ? 'merge' : 'infer')} + +
+
+ +
+
+ Progress + + {hasMeasuredProgress ? `${Math.round(progressPercent)}%` : 'Starting'} + +
+
+
+
+
+ {hasMeasuredProgress ? `Page ${pagesParsed}/${totalPages}` : 'Awaiting first page'} + • + {isMerging ? 'Cross-page merge' : 'Page inference'} +
+
+
+ +
+
+ + + Prepare + + + + Infer + + + + Merge + +
+
+ + {!isLoading && parseState === 'failed' ? ( +
+ +
+ ) : null} +
+
); }; diff --git a/src/app/(app)/pdf/[id]/usePdfDocument.ts b/src/app/(app)/pdf/[id]/usePdfDocument.ts index 3476931..4c50459 100644 --- a/src/app/(app)/pdf/[id]/usePdfDocument.ts +++ b/src/app/(app)/pdf/[id]/usePdfDocument.ts @@ -24,6 +24,7 @@ import { getDocumentSettings, getParsedPdfDocument, putDocumentSettings, + subscribeParsedPdfDocumentEvents, } from '@/lib/client/api/documents'; import { createPdfAudiobookSourceAdapter } from '@/lib/client/audiobooks/adapters/pdf'; import { regenerateAudiobookChapter, runAudiobookGeneration } from '@/lib/client/audiobooks/pipeline'; @@ -44,7 +45,7 @@ import { type DocumentSettings, } from '@/types/document-settings'; import { mergeDocumentSettings } from '@/lib/shared/document-settings'; -import type { ParsedPdfDocument, ParsedPdfPage, PdfParseStatus } from '@/types/parsed-pdf'; +import type { ParsedPdfDocument, ParsedPdfPage, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; import type { TTSSentenceAlignment, @@ -68,6 +69,7 @@ export interface PdfDocumentState { pdfDocument: PDFDocumentProxy | undefined; parsedDocument: ParsedPdfDocument | null; parseStatus: PdfParseStatus | null; + parseProgress: PdfParseProgress | null; documentSettings: DocumentSettings; updateDocumentSettings: (settings: DocumentSettings) => Promise; parsedOverlayEnabled: boolean; @@ -143,6 +145,7 @@ export function usePdfDocument(): PdfDocumentState { const [pdfDocument, setPdfDocument] = useState(); const [parsedDocument, setParsedDocument] = useState(null); const [parseStatus, setParseStatus] = useState(null); + const [parseProgress, setParseProgress] = useState(null); const [documentSettings, setDocumentSettings] = useState(DEFAULT_DOCUMENT_SETTINGS); const [parsedOverlayEnabled, setParsedOverlayEnabled] = useState(false); const [isAudioCombining] = useState(false); @@ -164,6 +167,7 @@ export function usePdfDocument(): PdfDocumentState { const docLoadSeqRef = useRef(0); const docLoadAbortRef = useRef(null); const parsePollAbortRef = useRef(null); + const parseSseCloseRef = useRef<(() => void) | null>(null); const fetchParsedDocument = useCallback(async ( documentId: string, @@ -174,11 +178,11 @@ export function usePdfDocument(): PdfDocumentState { // document backfills parse output via the parsed endpoint polling path. const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending'; setParseStatus(effectiveInitialStatus); - - const maxAttempts = 25; + setParseProgress(null); const delayMs = 1200; const retryFailed = effectiveInitialStatus === 'failed'; - for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + let attempt = 0; + while (!signal.aborted) { if (signal.aborted) return; const result = await getParsedPdfDocument(documentId, { signal, @@ -187,13 +191,16 @@ export function usePdfDocument(): PdfDocumentState { if (result.status === 'ready') { setParsedDocument(result.parsed); setParseStatus('ready'); + setParseProgress(null); return; } setParseStatus(result.status); + setParseProgress(result.parseProgress ?? null); if (result.status === 'failed') { setParsedDocument(null); return; } + attempt += 1; await new Promise((resolve) => setTimeout(resolve, delayMs)); } }, []); @@ -211,13 +218,54 @@ export function usePdfDocument(): PdfDocumentState { const startParsedPolling = useCallback((documentId: string, initialStatus: PdfParseStatus | null) => { parsePollAbortRef.current?.abort(); + parseSseCloseRef.current?.(); + parseSseCloseRef.current = null; + setParseProgress(null); const controller = new AbortController(); parsePollAbortRef.current = controller; - void fetchParsedDocument(documentId, initialStatus, controller.signal).finally(() => { + + const closeSse = subscribeParsedPdfDocumentEvents(documentId, { + onSnapshot: (snapshot) => { + if (controller.signal.aborted) return; + setParseStatus(snapshot.parseStatus); + setParseProgress(snapshot.parseProgress); + if (snapshot.parseStatus === 'ready' || snapshot.parseStatus === 'failed') { + if (snapshot.parseStatus === 'failed') { + setParsedDocument(null); + } else { + void fetchParsedDocument(documentId, 'ready', controller.signal); + } + closeSse(); + parseSseCloseRef.current = null; + if (parsePollAbortRef.current === controller) { + parsePollAbortRef.current = null; + } + return; + } + }, + onError: () => { + // Fall back to parsed polling if browser SSE disconnects. + if (controller.signal.aborted) return; + closeSse(); + parseSseCloseRef.current = null; + void fetchParsedDocument(documentId, initialStatus, controller.signal).finally(() => { + if (parsePollAbortRef.current === controller) { + parsePollAbortRef.current = null; + } + }); + }, + }); + parseSseCloseRef.current = closeSse; + + controller.signal.addEventListener('abort', () => { + closeSse(); + if (parseSseCloseRef.current === closeSse) { + parseSseCloseRef.current = null; + } if (parsePollAbortRef.current === controller) { parsePollAbortRef.current = null; } - }); + }, { once: true }); }, [fetchParsedDocument]); useEffect(() => { @@ -392,6 +440,8 @@ export function usePdfDocument(): PdfDocumentState { loadSeqRef.current += 1; parsePollAbortRef.current?.abort(); parsePollAbortRef.current = null; + parseSseCloseRef.current?.(); + parseSseCloseRef.current = null; pageTextCacheRef.current.clear(); setPdfDocument(undefined); setCurrDocPages(undefined); @@ -401,6 +451,7 @@ export function usePdfDocument(): PdfDocumentState { setCurrDocData(undefined); setParsedDocument(null); setParseStatus(null); + setParseProgress(null); setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS); const meta = await getDocumentMetadata(id, { signal: controller.signal }); @@ -465,6 +516,7 @@ export function usePdfDocument(): PdfDocumentState { await forceReparsePdfDocument(currDocId); setParsedDocument(null); setParseStatus('pending'); + setParseProgress(null); startParsedPolling(currDocId, 'pending'); } catch (error) { console.error('Failed to force PDF reparse:', error); @@ -485,6 +537,8 @@ export function usePdfDocument(): PdfDocumentState { docLoadAbortRef.current = null; parsePollAbortRef.current?.abort(); parsePollAbortRef.current = null; + parseSseCloseRef.current?.(); + parseSseCloseRef.current = null; setCurrDocId(undefined); setCurrDocName(undefined); setCurrDocData(undefined); @@ -493,6 +547,7 @@ export function usePdfDocument(): PdfDocumentState { setPdfDocument(undefined); setParsedDocument(null); setParseStatus(null); + setParseProgress(null); setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS); pageTextCacheRef.current.clear(); stop(); @@ -595,6 +650,7 @@ export function usePdfDocument(): PdfDocumentState { currDocText, parsedDocument, parseStatus, + parseProgress, documentSettings, updateDocumentSettings, parsedOverlayEnabled, @@ -621,6 +677,7 @@ export function usePdfDocument(): PdfDocumentState { currDocText, parsedDocument, parseStatus, + parseProgress, documentSettings, updateDocumentSettings, parsedOverlayEnabled, diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts new file mode 100644 index 0000000..76c4955 --- /dev/null +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -0,0 +1,191 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { and, eq, inArray } from 'drizzle-orm'; +import { db } from '@/db'; +import { documents } from '@/db/schema'; +import { requireAuthContext } from '@/lib/server/auth/auth'; +import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob'; +import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; +import { isS3Configured } from '@/lib/server/storage/s3'; +import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; +import { isValidDocumentId } from '@/lib/server/documents/blobstore'; +import { normalizeParseStatus, parseDocumentParseState } from '@/lib/server/documents/parse-state'; + +export const dynamic = 'force-dynamic'; + +const SSE_POLL_INTERVAL_MS = 1200; + +type ParseRow = { + id: string; + userId: string; + parseState: string | null; +}; + +type ParsedSnapshot = { + parseStatus: PdfParseStatus; + parseProgress: PdfParseProgress | null; +}; + +function s3NotConfiguredResponse(): NextResponse { + return NextResponse.json( + { error: 'Documents storage is not configured. Set S3_* environment variables.' }, + { status: 503 }, + ); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function toSnapshot(row: ParseRow): ParsedSnapshot { + const state = parseDocumentParseState(row.parseState); + const parseStatus = normalizeParseStatus(state.status); + return { + parseStatus, + parseProgress: state.progress ?? null, + }; +} + +async function loadPreferredRow(input: { + documentId: string; + storageUserId: string; + allowedUserIds: string[]; +}): Promise { + const rows = (await db + .select({ + id: documents.id, + userId: documents.userId, + parseState: documents.parseState, + }) + .from(documents) + .where(and(eq(documents.id, input.documentId), inArray(documents.userId, input.allowedUserIds)))) as ParseRow[]; + + return rows.find((candidate) => candidate.userId === input.storageUserId) ?? rows[0] ?? null; +} + +export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { + try { + if (!isS3Configured()) return s3NotConfiguredResponse(); + + const authCtxOrRes = await requireAuthContext(req); + if (authCtxOrRes instanceof Response) return authCtxOrRes; + + const params = await ctx.params; + const id = (params.id || '').trim().toLowerCase(); + if (!isValidDocumentId(id)) { + return NextResponse.json({ error: 'Invalid document id' }, { status: 400 }); + } + + const testNamespace = getOpenReaderTestNamespace(req.headers); + const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace); + const storageUserId = authCtxOrRes.userId ?? unclaimedUserId; + const allowedUserIds = authCtxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId]; + + const row = await loadPreferredRow({ + documentId: id, + storageUserId, + allowedUserIds, + }); + + if (!row) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + const initial = toSnapshot(row); + if (initial.parseStatus === 'pending') { + enqueueParsePdfJob({ + documentId: id, + userId: row.userId, + namespace: testNamespace, + }); + } + + const encoder = new TextEncoder(); + + const stream = new ReadableStream({ + start(controller) { + let closed = false; + + const writeSnapshot = (snapshot: ParsedSnapshot): void => { + controller.enqueue(encoder.encode(`event: snapshot\ndata: ${JSON.stringify(snapshot)}\n\n`)); + }; + + const run = async () => { + let current = initial; + writeSnapshot(current); + let signature = JSON.stringify(current); + + while (!closed) { + if (current.parseStatus === 'ready' || current.parseStatus === 'failed') break; + await sleep(SSE_POLL_INTERVAL_MS); + if (closed) break; + + const nextRow = await loadPreferredRow({ + documentId: id, + storageUserId, + allowedUserIds, + }); + if (!nextRow) break; + + const next = toSnapshot(nextRow); + if (next.parseStatus === 'pending') { + enqueueParsePdfJob({ + documentId: id, + userId: nextRow.userId, + namespace: testNamespace, + }); + } + + const nextSignature = JSON.stringify(next); + if (nextSignature !== signature) { + current = next; + signature = nextSignature; + writeSnapshot(current); + } + } + }; + + void run() + .catch((error) => { + if (!closed) { + controller.enqueue(encoder.encode(`event: error\ndata: ${JSON.stringify({ error: String(error) })}\n\n`)); + } + }) + .finally(() => { + if (!closed) { + closed = true; + try { + controller.close(); + } catch { + // no-op + } + } + }); + + req.signal.addEventListener('abort', () => { + if (closed) return; + closed = true; + try { + controller.close(); + } catch { + // no-op + } + }, { once: true }); + }, + cancel() { + return; + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream; charset=utf-8', + 'Cache-Control': 'no-cache, no-transform', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', + }, + }); + } catch (error) { + console.error('Error streaming parsed PDF progress:', error); + return NextResponse.json({ error: 'Failed to stream parsed PDF progress' }, { status: 500 }); + } +} diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 23580ae..4fe7ea5 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -10,6 +10,11 @@ import { isValidDocumentId, } from '@/lib/server/documents/blobstore'; import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob'; +import { + normalizeParseStatus, + parseDocumentParseState, + stringifyDocumentParseState, +} from '@/lib/server/documents/parse-state'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; import type { ParsedPdfDocument } from '@/types/parsed-pdf'; @@ -28,15 +33,6 @@ function hasAnyParsedBlocks(doc: ParsedPdfDocument | null): boolean { return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0); } -function normalizeParseStatus( - status: string | null, -): 'pending' | 'running' | 'ready' | 'failed' { - if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') { - return status; - } - return 'pending'; -} - export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { try { if (!isS3Configured()) return s3NotConfiguredResponse(); @@ -60,14 +56,14 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string .select({ id: documents.id, userId: documents.userId, - parseStatus: documents.parseStatus, + parseState: documents.parseState, parsedJsonKey: documents.parsedJsonKey, }) .from(documents) .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ id: string; userId: string; - parseStatus: string | null; + parseState: string | null; parsedJsonKey: string | null; }>; @@ -76,31 +72,26 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - const effectiveStatus = normalizeParseStatus(row.parseStatus); - if (row.parseStatus !== effectiveStatus) { - await db - .update(documents) - .set({ parseStatus: 'pending' }) - .where(and(eq(documents.id, id), eq(documents.userId, row.userId))); - enqueueParsePdfJob({ - documentId: id, - userId: row.userId, - namespace: testNamespace, - }); - return NextResponse.json({ parseStatus: 'pending' }, { status: 202 }); - } + const state = parseDocumentParseState(row.parseState); + const effectiveStatus = normalizeParseStatus(state.status); if (effectiveStatus === 'failed' && retryFailed) { await db .update(documents) - .set({ parseStatus: 'pending' }) + .set({ + parseState: stringifyDocumentParseState({ + status: 'pending', + progress: null, + updatedAt: Date.now(), + }), + }) .where(and(eq(documents.id, id), eq(documents.userId, row.userId))); enqueueParsePdfJob({ documentId: id, userId: row.userId, namespace: testNamespace, }); - return NextResponse.json({ parseStatus: 'pending' }, { status: 202 }); + return NextResponse.json({ parseStatus: 'pending', parseProgress: null }, { status: 202 }); } if (effectiveStatus !== 'ready') { @@ -111,7 +102,10 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string namespace: testNamespace, }); } - return NextResponse.json({ parseStatus: effectiveStatus }, { status: 202 }); + return NextResponse.json({ + parseStatus: effectiveStatus, + parseProgress: state.progress ?? null, + }, { status: 202 }); } try { @@ -174,14 +168,14 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string .select({ id: documents.id, userId: documents.userId, - parseStatus: documents.parseStatus, + parseState: documents.parseState, parsedJsonKey: documents.parsedJsonKey, }) .from(documents) .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ id: string; userId: string; - parseStatus: string | null; + parseState: string | null; parsedJsonKey: string | null; }>; @@ -190,12 +184,19 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - const effectiveStatus = normalizeParseStatus(row.parseStatus); + const state = parseDocumentParseState(row.parseState); + const effectiveStatus = normalizeParseStatus(state.status); if (effectiveStatus !== 'running') { await db .update(documents) - .set({ parseStatus: 'pending' }) + .set({ + parseState: stringifyDocumentParseState({ + status: 'pending', + progress: null, + updatedAt: Date.now(), + }), + }) .where(and(eq(documents.id, id), eq(documents.userId, row.userId))); } @@ -206,7 +207,10 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string }); return NextResponse.json( - { parseStatus: effectiveStatus === 'running' ? 'running' : 'pending' }, + { + parseStatus: effectiveStatus === 'running' ? 'running' : 'pending', + parseProgress: effectiveStatus === 'running' ? (state.progress ?? null) : null, + }, { status: 202 }, ); } catch (error) { diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 0d6a7b1..05c6aeb 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -11,6 +11,11 @@ import { } from '@/lib/server/documents/previews'; import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob'; import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; +import { + normalizeParseStatus, + parseDocumentParseState, + stringifyDocumentParseState, +} from '@/lib/server/documents/parse-state'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { isS3Configured } from '@/lib/server/storage/s3'; import type { BaseDocument, DocumentType } from '@/types/documents'; @@ -25,16 +30,6 @@ type RegisterDocument = { lastModified: number; }; -function normalizeParseStatus( - status: string | null, -): 'pending' | 'running' | 'ready' | 'failed' | null { - if (status === null) return null; - if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') { - return status; - } - return 'pending'; -} - function s3NotConfiguredResponse(): NextResponse { return NextResponse.json( { error: 'Documents storage is not configured. Set S3_* environment variables.' }, @@ -140,7 +135,9 @@ export async function POST(req: NextRequest) { size: headSize, lastModified: doc.lastModified, filePath: doc.id, - parseStatus: doc.type === 'pdf' ? 'pending' : null, + parseState: doc.type === 'pdf' + ? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }) + : null, parsedJsonKey: null, }) .onConflictDoUpdate({ @@ -151,7 +148,9 @@ export async function POST(req: NextRequest) { size: headSize, lastModified: doc.lastModified, filePath: doc.id, - parseStatus: doc.type === 'pdf' ? 'pending' : null, + parseState: doc.type === 'pdf' + ? stringifyDocumentParseState({ status: 'pending', progress: null, updatedAt: Date.now() }) + : null, parsedJsonKey: null, }, }); @@ -229,7 +228,7 @@ export async function GET(req: NextRequest) { size: number; lastModified: number; filePath: string; - parseStatus: string | null; + parseState: string | null; parsedJsonKey: string | null; }>; @@ -255,7 +254,7 @@ export async function GET(req: NextRequest) { size: Number(doc.size), lastModified: Number(doc.lastModified), type, - parseStatus: normalizeParseStatus(doc.parseStatus), + parseStatus: type === 'pdf' ? normalizeParseStatus(parseDocumentParseState(doc.parseState).status) : null, parsedJsonKey: doc.parsedJsonKey, scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user', }; diff --git a/src/db/schema_postgres.ts b/src/db/schema_postgres.ts index 6de592f..13f3061 100644 --- a/src/db/schema_postgres.ts +++ b/src/db/schema_postgres.ts @@ -12,7 +12,7 @@ export const documents = pgTable('documents', { size: bigint('size', { mode: 'number' }).notNull(), lastModified: bigint('last_modified', { mode: 'number' }).notNull(), filePath: text('file_path').notNull(), - parseStatus: text('parse_status'), + parseState: text('parse_state'), parsedJsonKey: text('parsed_json_key'), createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS), }, (table) => [ diff --git a/src/db/schema_sqlite.ts b/src/db/schema_sqlite.ts index 7bf0ade..2781e7d 100644 --- a/src/db/schema_sqlite.ts +++ b/src/db/schema_sqlite.ts @@ -12,7 +12,7 @@ export const documents = sqliteTable('documents', { size: integer('size').notNull(), lastModified: integer('last_modified').notNull(), filePath: text('file_path').notNull(), - parseStatus: text('parse_status'), + parseState: text('parse_state'), parsedJsonKey: text('parsed_json_key'), createdAt: integer('created_at').default(SQLITE_NOW_MS), }, (table) => [ diff --git a/src/lib/client/api/documents.ts b/src/lib/client/api/documents.ts index 882ad4c..5e73292 100644 --- a/src/lib/client/api/documents.ts +++ b/src/lib/client/api/documents.ts @@ -1,6 +1,6 @@ import { sha256HexFromArrayBuffer } from '@/lib/client/sha256'; import type { BaseDocument, DocumentType } from '@/types/documents'; -import type { ParsedPdfDocument } from '@/types/parsed-pdf'; +import type { ParsedPdfDocument, PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; import type { DocumentSettings } from '@/types/document-settings'; export type UploadSource = { @@ -81,7 +81,10 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort export async function getParsedPdfDocument( id: string, options?: { signal?: AbortSignal; retryFailed?: boolean }, -): Promise<{ status: 'ready'; parsed: ParsedPdfDocument } | { status: 'pending' | 'running' | 'failed' }> { +): Promise< + | { status: 'ready'; parsed: ParsedPdfDocument } + | { status: 'pending' | 'running' | 'failed'; parseProgress?: PdfParseProgress | null } +> { const query = options?.retryFailed ? '?retry=1' : ''; const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, { signal: options?.signal, @@ -89,12 +92,12 @@ export async function getParsedPdfDocument( }); if (res.status === 202) { - const data = (await res.json().catch(() => null)) as { parseStatus?: string } | null; + const data = (await res.json().catch(() => null)) as { parseStatus?: string; parseProgress?: PdfParseProgress | null } | null; const parseStatus = data?.parseStatus; if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed') { - return { status: parseStatus }; + return { status: parseStatus, parseProgress: data?.parseProgress ?? null }; } - return { status: 'pending' }; + return { status: 'pending', parseProgress: data?.parseProgress ?? null }; } if (!res.ok) { @@ -106,6 +109,31 @@ export async function getParsedPdfDocument( return { status: 'ready', parsed }; } +export function subscribeParsedPdfDocumentEvents( + id: string, + handlers: { + onSnapshot: (snapshot: { parseStatus: PdfParseStatus; parseProgress: PdfParseProgress | null }) => void; + onError?: (error: Event) => void; + }, +): () => void { + const source = new EventSource(`/api/documents/${encodeURIComponent(id)}/parsed/events`); + source.addEventListener('snapshot', (event) => { + if (!(event instanceof MessageEvent)) return; + try { + const payload = JSON.parse(event.data) as { parseStatus: PdfParseStatus; parseProgress: PdfParseProgress | null }; + handlers.onSnapshot(payload); + } catch { + // Ignore malformed payloads to avoid breaking active streams. + } + }); + source.addEventListener('error', (event) => { + handlers.onError?.(event); + }); + return () => { + source.close(); + }; +} + export async function forceReparsePdfDocument( id: string, options?: { signal?: AbortSignal }, diff --git a/src/lib/server/compute/local.ts b/src/lib/server/compute/local.ts index 0315682..f4e0922 100644 --- a/src/lib/server/compute/local.ts +++ b/src/lib/server/compute/local.ts @@ -35,6 +35,17 @@ export class LocalComputeBackend implements ComputeBackend { if (!pdfBytes) { throw new Error('Local compute PDF layout requires pdfBytes or (documentId + namespace)'); } - return { parsed: (await runPdfLayoutFromPdfBuffer({ documentId: input.documentId, pdfBytes })).parsed }; + return { + parsed: (await runPdfLayoutFromPdfBuffer({ + documentId: input.documentId, + pdfBytes, + onPageParsed: (page) => input.onProgress?.({ + totalPages: page.totalPages, + pagesParsed: page.pageNumber, + currentPage: page.pageNumber, + phase: 'infer', + }), + })).parsed, + }; } } diff --git a/src/lib/server/compute/types.ts b/src/lib/server/compute/types.ts index aacbb10..9b86c20 100644 --- a/src/lib/server/compute/types.ts +++ b/src/lib/server/compute/types.ts @@ -1,4 +1,9 @@ -import type { TTSAudioBuffer, TTSSentenceAlignment, ParsedPdfDocument } from '@openreader/compute-core/contracts'; +import type { + TTSAudioBuffer, + TTSSentenceAlignment, + ParsedPdfDocument, + PdfLayoutProgress, +} from '@openreader/compute-core/contracts'; export type ComputeMode = 'local' | 'worker'; @@ -19,6 +24,7 @@ export interface PdfLayoutInput { namespace?: string | null; documentObjectKey?: string; pdfBytes?: ArrayBuffer; + onProgress?: (progress: PdfLayoutProgress) => void | Promise; } export type PdfLayoutResult = diff --git a/src/lib/server/compute/worker-contract.ts b/src/lib/server/compute/worker-contract.ts index 19e6bc5..950e62a 100644 --- a/src/lib/server/compute/worker-contract.ts +++ b/src/lib/server/compute/worker-contract.ts @@ -1,6 +1,8 @@ export { type PdfLayoutJobRequest, type PdfLayoutJobResult, + type PdfLayoutProgress, + type PdfLayoutProgressPhase, type PdfLayoutOperationRequest, type WhisperAlignJobRequest, type WhisperAlignJobResult, diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts index 55c9197..733c133 100644 --- a/src/lib/server/compute/worker.ts +++ b/src/lib/server/compute/worker.ts @@ -375,6 +375,11 @@ export class WorkerComputeBackend implements ComputeBackend { opKeyHash, documentId: input.documentId, attempt, + onSnapshot: (snapshot) => { + if (snapshot.progress) { + void input.onProgress?.(snapshot.progress); + } + }, }); if (final.status !== 'succeeded' || !final.result) { @@ -482,7 +487,9 @@ export class WorkerComputeBackend implements ComputeBackend { private async waitForOperation( opId: string, - context: Record = {}, + context: Record & { + onSnapshot?: (snapshot: WorkerOperationState) => void + } = {}, ): Promise> { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), this.waitTimeoutMs); @@ -577,6 +584,7 @@ export class WorkerComputeBackend implements ComputeBackend { eventCount += 1; latest = snapshot; + context.onSnapshot?.(snapshot); if (snapshot.status !== lastStatus) { lastStatus = snapshot.status; logWorker('info', 'sse.wait.status', { diff --git a/src/lib/server/documents/parse-state.ts b/src/lib/server/documents/parse-state.ts new file mode 100644 index 0000000..133a6bf --- /dev/null +++ b/src/lib/server/documents/parse-state.ts @@ -0,0 +1,72 @@ +import type { PdfParseProgress, PdfParseStatus } from '@/types/parsed-pdf'; + +export interface DocumentParseState { + status: PdfParseStatus; + progress?: PdfParseProgress | null; + updatedAt?: number; + error?: string | null; + opId?: string; + jobId?: string; +} + +export function normalizeParseStatus(status: string | null | undefined): PdfParseStatus { + if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') { + return status; + } + return 'pending'; +} + +function normalizeProgress(progress: unknown): PdfParseProgress | null { + if (!progress || typeof progress !== 'object') return null; + const rec = progress as Record; + const totalPages = Number(rec.totalPages ?? 0); + if (!Number.isFinite(totalPages) || totalPages <= 0) return null; + const pagesParsedRaw = Number(rec.pagesParsed ?? 0); + const pagesParsed = Math.max(0, Math.min(totalPages, Number.isFinite(pagesParsedRaw) ? pagesParsedRaw : 0)); + const phase = rec.phase === 'merge' ? 'merge' : 'infer'; + const currentPageRaw = Number(rec.currentPage ?? pagesParsed); + const currentPage = Number.isFinite(currentPageRaw) && currentPageRaw > 0 + ? Math.max(1, Math.min(totalPages, currentPageRaw)) + : undefined; + return { + totalPages, + pagesParsed, + currentPage, + phase, + }; +} + +export function parseDocumentParseState(value: string | null): DocumentParseState { + if (!value) return { status: 'pending', progress: null }; + try { + const parsed = JSON.parse(value) as Record; + const status = normalizeParseStatus(typeof parsed.status === 'string' ? parsed.status : null); + const progress = normalizeProgress(parsed.progress); + const updatedAtRaw = Number(parsed.updatedAt ?? 0); + const updatedAt = Number.isFinite(updatedAtRaw) && updatedAtRaw > 0 ? updatedAtRaw : undefined; + const error = typeof parsed.error === 'string' ? parsed.error : null; + const opId = typeof parsed.opId === 'string' ? parsed.opId : undefined; + const jobId = typeof parsed.jobId === 'string' ? parsed.jobId : undefined; + return { + status, + progress, + ...(typeof updatedAt === 'number' ? { updatedAt } : {}), + ...(error ? { error } : {}), + ...(opId ? { opId } : {}), + ...(jobId ? { jobId } : {}), + }; + } catch { + return { status: 'pending', progress: null }; + } +} + +export function stringifyDocumentParseState(state: DocumentParseState): string { + return JSON.stringify({ + status: normalizeParseStatus(state.status), + progress: state.progress ?? null, + updatedAt: typeof state.updatedAt === 'number' ? state.updatedAt : Date.now(), + ...(state.error ? { error: state.error } : {}), + ...(state.opId ? { opId: state.opId } : {}), + ...(state.jobId ? { jobId: state.jobId } : {}), + }); +} diff --git a/src/lib/server/jobs/parsePdfJob.ts b/src/lib/server/jobs/parsePdfJob.ts index 593d06d..bd4e3d3 100644 --- a/src/lib/server/jobs/parsePdfJob.ts +++ b/src/lib/server/jobs/parsePdfJob.ts @@ -2,8 +2,10 @@ import { and, eq } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; +import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; import { getCompute } from '@/lib/server/compute'; import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache'; +import type { PdfLayoutProgress } from '@openreader/compute-core/contracts'; interface ParsePdfJobInput { documentId: string; @@ -23,16 +25,41 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise { running.add(key); try { + const now = Date.now(); await db .update(documents) - .set({ parseStatus: 'running' }) + .set({ + parseState: stringifyDocumentParseState({ + status: 'running', + progress: null, + updatedAt: now, + }), + }) .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); const compute = await getCompute(); + const writeProgress = async (progress: PdfLayoutProgress): Promise => { + await db + .update(documents) + .set({ + parseState: stringifyDocumentParseState({ + status: 'running', + progress: { + totalPages: progress.totalPages, + pagesParsed: progress.pagesParsed, + currentPage: progress.currentPage, + phase: progress.phase, + }, + updatedAt: Date.now(), + }), + }) + .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); + }; const layout = await compute.parsePdfLayout({ documentId: input.documentId, namespace: input.namespace, documentObjectKey: documentKey(input.documentId, input.namespace), + onProgress: writeProgress, }); let parsedJsonKey = layout.parsedObjectKey ?? null; @@ -56,7 +83,14 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise { await db .update(documents) - .set({ parseStatus: 'ready', parsedJsonKey }) + .set({ + parseState: stringifyDocumentParseState({ + status: 'ready', + progress: null, + updatedAt: Date.now(), + }), + parsedJsonKey, + }) .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -66,7 +100,14 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise { try { await db .update(documents) - .set({ parseStatus }) + .set({ + parseState: stringifyDocumentParseState({ + status: 'failed', + progress: null, + updatedAt: Date.now(), + error: message, + }), + }) .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); } catch (statusError) { console.error('[parsePdfJob] failed to write parse status', { diff --git a/src/types/parsed-pdf.ts b/src/types/parsed-pdf.ts index 2ee67af..a7d5650 100644 --- a/src/types/parsed-pdf.ts +++ b/src/types/parsed-pdf.ts @@ -77,3 +77,11 @@ export interface ParsedPdfDocument { } export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed'; +export type PdfParsePhase = 'infer' | 'merge'; + +export interface PdfParseProgress { + totalPages: number; + pagesParsed: number; + currentPage?: number; + phase: PdfParsePhase; +} diff --git a/tests/unit/transfer-user-documents.spec.ts b/tests/unit/transfer-user-documents.spec.ts index c349897..ca58e28 100644 --- a/tests/unit/transfer-user-documents.spec.ts +++ b/tests/unit/transfer-user-documents.spec.ts @@ -20,7 +20,7 @@ test.describe('transferUserDocuments', () => { size INTEGER NOT NULL, last_modified INTEGER NOT NULL, file_path TEXT NOT NULL, - parse_status TEXT, + parse_state TEXT, parsed_json_key TEXT, created_at INTEGER, PRIMARY KEY (id, user_id)