fix: limit upload to 5 MB / 20 pages max and increase conversion timeout
Prevents PyTorch/Docling pipeline crashes on HF Spaces CPU by: - Reducing max file size from 50 MB to 5 MB - Adding configurable MAX_PAGE_COUNT setting (env var, default unlimited) - Increasing conversion timeout from 600s to 900s - Adding frontend upload validation with explicit error messages - Exposing maxPageCount via /api/health for dynamic UI hints
This commit is contained in:
parent
11eaeb4cd6
commit
09145206d2
14 changed files with 128 additions and 45 deletions
|
|
@ -9,12 +9,15 @@
|
|||
# DOCLING_SERVE_URL=http://localhost:5001
|
||||
# DOCLING_SERVE_API_KEY=
|
||||
|
||||
# Max seconds per conversion (default: 600)
|
||||
# CONVERSION_TIMEOUT=600
|
||||
# Max seconds per conversion (default: 900)
|
||||
# CONVERSION_TIMEOUT=900
|
||||
|
||||
# Max parallel analysis jobs (default: 3)
|
||||
# MAX_CONCURRENT_ANALYSES=3
|
||||
|
||||
# Max pages per PDF (default: 0 = unlimited). Set to 20 for HF Spaces.
|
||||
# MAX_PAGE_COUNT=0
|
||||
|
||||
# Deployment mode: "self-hosted" (default) or "huggingface"
|
||||
# Shows disclaimer banner when set to "huggingface"
|
||||
# DEPLOYMENT_MODE=self-hosted
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ async def upload(file: UploadFile) -> DocumentResponse:
|
|||
|
||||
# Reject early if Content-Length exceeds limit (before reading body)
|
||||
if file.size and file.size > document_service.MAX_FILE_SIZE:
|
||||
raise HTTPException(status_code=413, detail="File too large (max 50 MB)")
|
||||
raise HTTPException(status_code=413, detail="File too large (max 5 MB)")
|
||||
|
||||
# Read in chunks to avoid holding the full upload in a single allocation
|
||||
chunks: list[bytes] = []
|
||||
|
|
@ -43,7 +43,7 @@ async def upload(file: UploadFile) -> DocumentResponse:
|
|||
while chunk := await file.read(_READ_CHUNK_SIZE):
|
||||
total += len(chunk)
|
||||
if total > document_service.MAX_FILE_SIZE:
|
||||
raise HTTPException(status_code=413, detail="File too large (max 50 MB)")
|
||||
raise HTTPException(status_code=413, detail="File too large (max 5 MB)")
|
||||
chunks.append(chunk)
|
||||
content = b"".join(chunks)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,9 @@ class Settings:
|
|||
deployment_mode: str = "self-hosted" # "self-hosted" or "huggingface"
|
||||
docling_serve_url: str = "http://localhost:5001"
|
||||
docling_serve_api_key: str | None = None
|
||||
conversion_timeout: int = 600
|
||||
conversion_timeout: int = 900
|
||||
max_concurrent_analyses: int = 3
|
||||
max_page_count: int = 0 # 0 = unlimited
|
||||
upload_dir: str = "./uploads"
|
||||
db_path: str = "./data/docling_studio.db"
|
||||
cors_origins: list[str] = field(
|
||||
|
|
@ -31,8 +32,9 @@ class Settings:
|
|||
deployment_mode=os.environ.get("DEPLOYMENT_MODE", "self-hosted"),
|
||||
docling_serve_url=os.environ.get("DOCLING_SERVE_URL", "http://localhost:5001"),
|
||||
docling_serve_api_key=os.environ.get("DOCLING_SERVE_API_KEY"),
|
||||
conversion_timeout=int(os.environ.get("CONVERSION_TIMEOUT", "600")),
|
||||
conversion_timeout=int(os.environ.get("CONVERSION_TIMEOUT", "900")),
|
||||
max_concurrent_analyses=int(os.environ.get("MAX_CONCURRENT_ANALYSES", "3")),
|
||||
max_page_count=int(os.environ.get("MAX_PAGE_COUNT", "0")),
|
||||
upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"),
|
||||
db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"),
|
||||
cors_origins=[o.strip() for o in cors_raw.split(",")],
|
||||
|
|
|
|||
|
|
@ -113,10 +113,13 @@ async def health() -> dict[str, str]:
|
|||
logger.warning("Health check: database unreachable", exc_info=True)
|
||||
|
||||
status = "ok" if db_status == "ok" else "degraded"
|
||||
return {
|
||||
result: dict[str, str | int] = {
|
||||
"status": status,
|
||||
"version": settings.app_version,
|
||||
"engine": settings.conversion_engine,
|
||||
"deploymentMode": settings.deployment_mode,
|
||||
"database": db_status,
|
||||
}
|
||||
if settings.max_page_count > 0:
|
||||
result["maxPageCount"] = settings.max_page_count
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ from persistence import analysis_repo, document_repo
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
UPLOAD_DIR = settings.upload_dir
|
||||
MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB
|
||||
MAX_FILE_SIZE = 5 * 1024 * 1024 # 5 MB
|
||||
MAX_PAGE_COUNT = settings.max_page_count # 0 = unlimited
|
||||
|
||||
|
||||
# PDF magic bytes: %PDF
|
||||
|
|
@ -32,7 +33,7 @@ async def upload(filename: str, content_type: str, file_content: bytes) -> Docum
|
|||
Writes the file in fixed-size chunks to keep peak memory usage low.
|
||||
"""
|
||||
if len(file_content) > MAX_FILE_SIZE:
|
||||
raise ValueError("File too large (max 50 MB)")
|
||||
raise ValueError("File too large (max 5 MB)")
|
||||
|
||||
if not file_content[:4].startswith(_PDF_MAGIC):
|
||||
raise ValueError("Invalid file: not a PDF document")
|
||||
|
|
@ -51,6 +52,10 @@ async def upload(filename: str, content_type: str, file_content: bytes) -> Docum
|
|||
# Count PDF pages
|
||||
page_count = _count_pages(file_content)
|
||||
|
||||
if MAX_PAGE_COUNT > 0 and page_count is not None and page_count > MAX_PAGE_COUNT:
|
||||
os.unlink(file_path)
|
||||
raise ValueError(f"Too many pages ({page_count}). Maximum allowed: {MAX_PAGE_COUNT}")
|
||||
|
||||
doc = Document(
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ class TestDocumentEndpoints:
|
|||
|
||||
@patch("services.document_service.upload", new_callable=AsyncMock)
|
||||
def test_upload_too_large(self, mock_upload, client):
|
||||
mock_upload.side_effect = ValueError("File too large (max 50 MB)")
|
||||
mock_upload.side_effect = ValueError("File too large (max 5 MB)")
|
||||
|
||||
resp = client.post(
|
||||
"/api/documents/upload",
|
||||
|
|
|
|||
|
|
@ -24,6 +24,50 @@ class TestUploadValidation:
|
|||
with pytest.raises(ValueError, match="not a PDF"):
|
||||
await document_service.upload("fake.pdf", "application/pdf", content)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_too_many_pages(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(document_service, "MAX_PAGE_COUNT", 20)
|
||||
|
||||
with patch.object(document_service, "_count_pages", return_value=40):
|
||||
content = b"%PDF-1.4 fake pdf content"
|
||||
with pytest.raises(ValueError, match="Too many pages"):
|
||||
await document_service.upload("big.pdf", "application/pdf", content)
|
||||
|
||||
# Verify temp file was cleaned up
|
||||
assert len(os.listdir(tmp_path)) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allows_pdf_under_page_limit(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(document_service, "MAX_PAGE_COUNT", 20)
|
||||
|
||||
mock_insert = AsyncMock()
|
||||
with (
|
||||
patch("persistence.document_repo.insert", mock_insert),
|
||||
patch.object(document_service, "_count_pages", return_value=15),
|
||||
):
|
||||
content = b"%PDF-1.4 fake pdf content"
|
||||
doc = await document_service.upload("ok.pdf", "application/pdf", content)
|
||||
|
||||
assert doc.page_count == 15
|
||||
mock_insert.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unlimited_pages_when_zero(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(document_service, "MAX_PAGE_COUNT", 0)
|
||||
|
||||
mock_insert = AsyncMock()
|
||||
with (
|
||||
patch("persistence.document_repo.insert", mock_insert),
|
||||
patch.object(document_service, "_count_pages", return_value=100),
|
||||
):
|
||||
content = b"%PDF-1.4 fake pdf content"
|
||||
doc = await document_service.upload("big.pdf", "application/pdf", content)
|
||||
|
||||
assert doc.page_count == 100
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accepts_valid_pdf(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ class TestSettingsDefaults:
|
|||
assert s.deployment_mode == "self-hosted"
|
||||
assert s.docling_serve_url == "http://localhost:5001"
|
||||
assert s.docling_serve_api_key is None
|
||||
assert s.conversion_timeout == 600
|
||||
assert s.conversion_timeout == 900
|
||||
assert s.max_page_count == 0
|
||||
assert s.upload_dir == "./uploads"
|
||||
assert s.db_path == "./data/docling_studio.db"
|
||||
assert "http://localhost:3000" in s.cors_origins
|
||||
|
|
@ -35,6 +36,7 @@ class TestSettingsFromEnv:
|
|||
monkeypatch.setenv("DOCLING_SERVE_URL", "http://serve:9000")
|
||||
monkeypatch.setenv("DOCLING_SERVE_API_KEY", "secret-key")
|
||||
monkeypatch.setenv("CONVERSION_TIMEOUT", "120")
|
||||
monkeypatch.setenv("MAX_PAGE_COUNT", "20")
|
||||
monkeypatch.setenv("UPLOAD_DIR", "/data/uploads")
|
||||
monkeypatch.setenv("DB_PATH", "/data/test.db")
|
||||
monkeypatch.setenv("CORS_ORIGINS", "http://a.com, http://b.com")
|
||||
|
|
@ -47,6 +49,7 @@ class TestSettingsFromEnv:
|
|||
assert s.docling_serve_url == "http://serve:9000"
|
||||
assert s.docling_serve_api_key == "secret-key"
|
||||
assert s.conversion_timeout == 120
|
||||
assert s.max_page_count == 20
|
||||
assert s.upload_dir == "/data/uploads"
|
||||
assert s.db_path == "/data/test.db"
|
||||
assert s.cors_origins == ["http://a.com", "http://b.com"]
|
||||
|
|
@ -60,6 +63,7 @@ class TestSettingsFromEnv:
|
|||
"DOCLING_SERVE_URL",
|
||||
"DOCLING_SERVE_API_KEY",
|
||||
"CONVERSION_TIMEOUT",
|
||||
"MAX_PAGE_COUNT",
|
||||
"UPLOAD_DIR",
|
||||
"DB_PATH",
|
||||
"CORS_ORIGINS",
|
||||
|
|
@ -70,7 +74,8 @@ class TestSettingsFromEnv:
|
|||
|
||||
assert s.app_version == "dev"
|
||||
assert s.conversion_engine == "local"
|
||||
assert s.conversion_timeout == 600
|
||||
assert s.conversion_timeout == 900
|
||||
assert s.max_page_count == 0
|
||||
|
||||
def test_cors_origins_split(self, monkeypatch):
|
||||
monkeypatch.setenv("CORS_ORIGINS", "http://a.com,http://b.com,http://c.com")
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ server {
|
|||
proxy_pass http://document-parser:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_read_timeout 600s;
|
||||
proxy_send_timeout 600s;
|
||||
client_max_body_size 50M;
|
||||
proxy_read_timeout 900s;
|
||||
proxy_send_timeout 900s;
|
||||
client_max_body_size 5M;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { ref } from 'vue'
|
|||
import type { Document } from '../../shared/types'
|
||||
import * as api from './api'
|
||||
|
||||
const MAX_FILE_SIZE = 50 * 1024 * 1024 // 50 MB
|
||||
const MAX_FILE_SIZE = 5 * 1024 * 1024 // 5 MB
|
||||
|
||||
export const useDocumentStore = defineStore('document', () => {
|
||||
const documents = ref<Document[]>([])
|
||||
|
|
@ -27,7 +27,7 @@ export const useDocumentStore = defineStore('document', () => {
|
|||
|
||||
async function upload(file: File): Promise<Document> {
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
error.value = 'File too large (max 50 MB)'
|
||||
error.value = 'File too large (max 5 MB)'
|
||||
throw new Error(error.value)
|
||||
}
|
||||
uploading.value = true
|
||||
|
|
|
|||
|
|
@ -23,55 +23,67 @@
|
|||
<path d="M12 16V4m0 0L8 8m4-4l4 4M4 17v2a1 1 0 001 1h14a1 1 0 001-1v-2" />
|
||||
</svg>
|
||||
<span class="upload-text">{{ t('upload.drop') }}</span>
|
||||
<span class="upload-hint">{{ t('upload.maxSize') }}</span>
|
||||
<span class="upload-hint">{{ uploadHint }}</span>
|
||||
<span v-if="store.error" class="upload-error">{{ store.error }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
import { useDocumentStore } from '../store'
|
||||
import { useFeatureFlagStore } from '../../feature-flags/store'
|
||||
import { useI18n } from '../../../shared/i18n'
|
||||
|
||||
const emit = defineEmits<{ uploaded: [docId: string] }>()
|
||||
|
||||
const store = useDocumentStore()
|
||||
const flags = useFeatureFlagStore()
|
||||
const { t } = useI18n()
|
||||
const fileInput = ref<HTMLInputElement | null>(null)
|
||||
const dragging = ref(false)
|
||||
|
||||
const uploadHint = computed(() => {
|
||||
const size = t('upload.maxSize')
|
||||
if (flags.maxPageCount > 0) {
|
||||
return `${size} · ${t('upload.maxPages').replace('{n}', String(flags.maxPageCount))}`
|
||||
}
|
||||
return size
|
||||
})
|
||||
|
||||
function openFilePicker() {
|
||||
fileInput.value?.click()
|
||||
}
|
||||
|
||||
function isPdf(file: File): boolean {
|
||||
return file.type === 'application/pdf' || file.name.toLowerCase().endsWith('.pdf')
|
||||
}
|
||||
|
||||
async function handleFile(file: File) {
|
||||
store.clearError()
|
||||
if (!isPdf(file)) {
|
||||
store.error = t('upload.invalidFormat')
|
||||
return
|
||||
}
|
||||
try {
|
||||
const doc = await store.upload(file)
|
||||
if (doc) emit('uploaded', doc.id)
|
||||
} catch {
|
||||
// error is already set in store.upload
|
||||
}
|
||||
}
|
||||
|
||||
async function onFileSelect(e: Event) {
|
||||
const target = e.target as HTMLInputElement
|
||||
const file = target.files?.[0]
|
||||
if (file && (file.type === 'application/pdf' || file.name.toLowerCase().endsWith('.pdf'))) {
|
||||
try {
|
||||
store.clearError()
|
||||
const doc = await store.upload(file)
|
||||
if (doc) emit('uploaded', doc.id)
|
||||
} catch {
|
||||
// error is already set in store.upload
|
||||
}
|
||||
}
|
||||
if (file) await handleFile(file)
|
||||
target.value = ''
|
||||
}
|
||||
|
||||
async function onDrop(e: DragEvent) {
|
||||
dragging.value = false
|
||||
const file = e.dataTransfer?.files?.[0]
|
||||
if (file && (file.type === 'application/pdf' || file.name.toLowerCase().endsWith('.pdf'))) {
|
||||
try {
|
||||
store.clearError()
|
||||
const doc = await store.upload(file)
|
||||
if (doc) emit('uploaded', doc.id)
|
||||
} catch {
|
||||
// error is already set in store.upload
|
||||
}
|
||||
}
|
||||
if (file) await handleFile(file)
|
||||
}
|
||||
</script>
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ interface HealthResponse {
|
|||
status: string
|
||||
engine: ConversionEngine
|
||||
deploymentMode?: DeploymentMode
|
||||
maxPageCount?: number
|
||||
}
|
||||
|
||||
export type FeatureFlag = 'chunking' | 'disclaimer'
|
||||
|
|
@ -37,6 +38,7 @@ const featureRegistry: Record<FeatureFlag, FeatureFlagDef> = {
|
|||
export const useFeatureFlagStore = defineStore('feature-flags', () => {
|
||||
const engine = ref<ConversionEngine | null>(null)
|
||||
const deploymentMode = ref<DeploymentMode | null>(null)
|
||||
const maxPageCount = ref<number>(0)
|
||||
const loaded = ref(false)
|
||||
const error = ref<string | null>(null)
|
||||
|
||||
|
|
@ -56,6 +58,7 @@ export const useFeatureFlagStore = defineStore('feature-flags', () => {
|
|||
const data = await apiFetch<HealthResponse>('/api/health')
|
||||
engine.value = data.engine
|
||||
deploymentMode.value = data.deploymentMode ?? 'self-hosted'
|
||||
maxPageCount.value = data.maxPageCount ?? 0
|
||||
loaded.value = true
|
||||
error.value = null
|
||||
} catch (e) {
|
||||
|
|
@ -64,5 +67,5 @@ export const useFeatureFlagStore = defineStore('feature-flags', () => {
|
|||
}
|
||||
}
|
||||
|
||||
return { engine, deploymentMode, loaded, error, isEnabled, load }
|
||||
return { engine, deploymentMode, maxPageCount, loaded, error, isEnabled, load }
|
||||
})
|
||||
|
|
|
|||
|
|
@ -94,7 +94,10 @@ const messages: Messages = {
|
|||
// Upload
|
||||
'upload.drop': 'Déposez un PDF ici ou cliquez pour importer',
|
||||
'upload.uploading': 'Import en cours...',
|
||||
'upload.maxSize': 'Max 50Mo',
|
||||
'upload.maxSize': 'Max 5Mo',
|
||||
'upload.invalidFormat': 'Format invalide — seuls les fichiers PDF sont acceptés.',
|
||||
'upload.tooLarge': 'Fichier trop volumineux (max 5 Mo).',
|
||||
'upload.maxPages': 'Max {n} pages',
|
||||
|
||||
// History
|
||||
'history.title': 'Historique',
|
||||
|
|
@ -132,7 +135,7 @@ const messages: Messages = {
|
|||
|
||||
// Disclaimer
|
||||
'disclaimer.banner':
|
||||
'Instance de d\u00e9monstration \u2014 les documents upload\u00e9s sont partag\u00e9s et temporaires (max 50 Mo). Ne pas envoyer de fichiers confidentiels.',
|
||||
'Instance de d\u00e9monstration \u2014 les documents upload\u00e9s sont partag\u00e9s et temporaires (max 5 Mo). Ne pas envoyer de fichiers confidentiels.',
|
||||
},
|
||||
en: {
|
||||
'nav.home': 'Home',
|
||||
|
|
@ -215,7 +218,10 @@ const messages: Messages = {
|
|||
|
||||
'upload.drop': 'Drop a PDF here or click to upload',
|
||||
'upload.uploading': 'Uploading...',
|
||||
'upload.maxSize': 'Max 50MB',
|
||||
'upload.maxSize': 'Max 5MB',
|
||||
'upload.invalidFormat': 'Invalid format — only PDF files are accepted.',
|
||||
'upload.tooLarge': 'File too large (max 5 MB).',
|
||||
'upload.maxPages': 'Max {n} pages',
|
||||
|
||||
'history.title': 'History',
|
||||
'history.tabAnalyses': 'Analyses',
|
||||
|
|
@ -249,7 +255,7 @@ const messages: Messages = {
|
|||
|
||||
// Disclaimer
|
||||
'disclaimer.banner':
|
||||
'Demo instance \u2014 uploaded documents are shared and temporary (max 50 MB). Do not upload confidential files.',
|
||||
'Demo instance \u2014 uploaded documents are shared and temporary (max 5 MB). Do not upload confidential files.',
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ server {
|
|||
proxy_pass http://127.0.0.1:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_read_timeout 600s;
|
||||
proxy_send_timeout 600s;
|
||||
client_max_body_size 50M;
|
||||
proxy_read_timeout 900s;
|
||||
proxy_send_timeout 900s;
|
||||
client_max_body_size 5M;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue