From 2db251182fc405f0f480d2de2a4b0171cc704410 Mon Sep 17 00:00:00 2001 From: pjmalandrino Date: Sat, 21 Mar 2026 15:03:10 +0100 Subject: [PATCH] Refacto and add cp paste btn --- Dockerfile | 9 +- docker-compose.yml | 4 +- document-parser/Dockerfile | 5 +- document-parser/api/documents.py | 4 + document-parser/data/docling_studio.db | Bin 622592 -> 622592 bytes document-parser/domain/parsing.py | 135 ++++++++---------- document-parser/main.py | 4 +- document-parser/persistence/database.py | 4 + document-parser/services/analysis_service.py | 10 +- document-parser/services/document_service.py | 3 +- .../tests/test_pipeline_options.py | 99 ++++++------- frontend/Dockerfile | 2 +- frontend/src/features/analysis/store.js | 13 +- .../src/features/analysis/ui/ResultTabs.vue | 89 +++++++++++- frontend/src/features/document/store.js | 10 +- frontend/src/features/history/api.js | 9 -- frontend/src/features/history/api.test.js | 32 ----- .../src/features/history/navigation.test.js | 7 +- frontend/src/features/history/store.js | 27 +--- frontend/src/features/history/store.test.js | 65 +-------- .../src/features/history/ui/HistoryList.vue | 4 +- frontend/src/shared/i18n.js | 4 + nginx.conf | 6 + 23 files changed, 262 insertions(+), 283 deletions(-) delete mode 100644 frontend/src/features/history/api.js delete mode 100644 frontend/src/features/history/api.test.js diff --git a/Dockerfile b/Dockerfile index 4562860..4c33624 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,13 +41,16 @@ COPY --from=frontend-build /build/dist /usr/share/nginx/html # Nginx config COPY nginx.conf /etc/nginx/sites-enabled/default +# Non-root user +RUN useradd --create-home --shell /bin/bash appuser + # Data directories -RUN mkdir -p /app/uploads /app/data +RUN mkdir -p /app/uploads /app/data && chown -R appuser:appuser /app ENV UPLOAD_DIR=/app/uploads ENV DB_PATH=/app/data/docling_studio.db EXPOSE 3000 -# Start both nginx and uvicorn -CMD ["sh", "-c", "nginx && exec uvicorn main:app --host 127.0.0.1 --port 8000"] +# nginx needs to run as root for port binding, then drops to appuser for uvicorn +CMD ["sh", "-c", "nginx && exec su appuser -c 'uvicorn main:app --host 127.0.0.1 --port 8000'"] diff --git a/docker-compose.yml b/docker-compose.yml index 86819f4..6e75d5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,8 @@ services: document-parser: build: context: ./document-parser - ports: - - "8000:8000" + expose: + - "8000" volumes: - uploads_data:/app/uploads - db_data:/app/data diff --git a/document-parser/Dockerfile b/document-parser/Dockerfile index 6ab8e94..76cc697 100644 --- a/document-parser/Dockerfile +++ b/document-parser/Dockerfile @@ -13,11 +13,14 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . -RUN mkdir -p /app/uploads /app/data +RUN useradd --create-home --shell /bin/bash appuser \ + && mkdir -p /app/uploads /app/data \ + && chown -R appuser:appuser /app EXPOSE 8000 ENV UPLOAD_DIR=/app/uploads ENV DB_PATH=/app/data/docling_studio.db +USER appuser CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/document-parser/api/documents.py b/document-parser/api/documents.py index 32e2dc3..1a41823 100644 --- a/document-parser/api/documents.py +++ b/document-parser/api/documents.py @@ -31,6 +31,10 @@ async def upload(file: UploadFile): if not file.filename: raise HTTPException(status_code=400, detail="No filename provided") + # Reject early if Content-Length exceeds limit (before reading body) + if file.size and file.size > document_service.MAX_FILE_SIZE: + raise HTTPException(status_code=413, detail="File too large (max 50 MB)") + content = await file.read() try: diff --git a/document-parser/data/docling_studio.db b/document-parser/data/docling_studio.db index ef5cdbb204cd60849ca7bf50b51fb545b3e7ab4e..a4f5f99a7625f274b46cd6a0e6b76138e0830747 100644 GIT binary patch delta 32305 zcmc(oYm6M%mEZZ8;bTOZlC`p9%d$$8teLTtRas;0Y! zY`Ukr)enXeWS8Us>o|sY?e)fO1W49SfbwgM0`AWaV!TvIz_a{*;#z zBS5f@vH6{ItEwN<^CT%t7|u*}Ro#2fJ&*r++%vDe_QY$iJ@KuF?mxa~&z|q`^&(%h ze2ss5#(C)Yp7bk?dyoF#JqPcZ-t&QP9{o2*zV?9&M;^SVd(ZS+um5NwtR6i0;ZHrX z_w~b-S~<8`Dc`(Ou9rHiL9N}oQfdZ%J1AfA+sU;mR-` z&SDw9x7Zg4?!D4#`|VC^;M##LhBl9^AJ$x-e*0@jzP$g)zI%VY=jfh$e|_Z7?)moH z_uuzl?+w4YZ{OjIi%;&~`_+5SuDD*Y5G=c?V!r66a{h{&@?HK3ie+bcxm0wEzJKNm zi&y6AS8IOh%7vihH?BNWDP0R%S5BWwL+P98MtQ|=G^&-7->%eav;4a6|GYgud$AQX zTeFSptA5q5m7A4XefDYD-|wtGa`Lk)e$g&C#g&woE9O$U;z}mv6?5fOG2<39-ttP| z1k3W`teq{SY$uh?ShjoI$sW(yX*W~MINp=CecZPH)gxnjkL)}8qdiA|cJxQ`|J(Ol ze|&U$YX8N(+gaMa@BD0bW@6u=LwhDJ9XfRM*KfW4*9XI|{`1Lw_g%dB$p72_`VZ&6 z`j1n4&YnHHvYcHBR=h$g69mOnuCP)}xf#z+M2YGPt8Xjv<|HCA)&R-=wo)!LTdxoIs2t$L-_vZ~zCsCHUbaI?{@tghYS zQk@r-1FKeVHv_8~)JkE!#*b-h?wbEnC$JjL`f8(X1+6weuD#i|*1m6*>J284D%Gpi zpqZYSV1^5Iw%h8|0&BTm<6j+WvAiu>2OX+xB&hj8|`0%9WtTbY6V%#fgOr zP5F%)@x|)#eYBgXAUqBFL;$`k=H7bnO z37{djJk#`^P26OG3+<)=#cN)hZKQh6~_8tA@o}<4y`pdWXF8qh1llSg_WN*S) z`;P2?RI}G?Qk4`_n|LorNQk6^RoTB5nMymEbw*Tzm?$7^bzZH$P z^$P~YQeW_$pFd*#y}b+X>|NNq|Ht7cZncGGZQ*#c!8OZh73=v$A$Ge`Tf1f0g z)jQX@C?7?fro2}cb`{Qr1O61QiuiTLYIG_s3x-E5S`8U~Mdoj{;LVcqw{@eADDh7< zc%{2{^3VezaxjOqAVp=K5;rDj-RV&aT2b0g@qtx8QqfRTmIjJ&q^>7@?J z(h3kPYt^sGfaS?bZKd8^HC&mBrr)+&ldvAr*@kg>&8N{ETQmN19eFjpijeCBn~+N` zz@)<)RGp0W0re2=6g%=1b8S_s)~6A9A%)Qr`kK&?rQNYgm3GjYVU1;_S=(p8P7|_C z)#|HOxr4~uY@=>snQQy(;fz(r=FeDuvt21wgXy%jSh09nZIbzks%WkKIpmrYCDTU| z9;>uc&?|UMnryPWAeA-GF zQgzx28a3Jx(cQue!fL98I@g1?GKtYpN-fB{y7uNOYEUddiQ9$R#s>454y|*ISuobr zj@1g=%}%M^5wT%!jfRKUzmNWJ2enEQTcR@3mUXPc-%U&AQ#LDQGi2xM%X)9jzpYLc z#fdyL{fg2b=Bd_=s%{B)_AzI;PAyxDOBXKAEnU1YZ)GrXXXdR7Yk$PQMp0i<(R?9o zU0U+vS@a7fpY)#3%JY-+Ule^mZ#`d!BaEgGL_N!bOA(`953C!a+K*e8j4A88k5@k9 zqX4RYMCQ^pESy>Sr5d&vYsLn#F$04;X7bORzi{@{5^GR1EaHZy5L)W^mqImQykOSC z#vurhzR{6AoVB1a(#0(Ee0H@Kw0QsNT1l+-rL|ktwOc}5wg3i2BfgL}22FLGA^^6x zx4Q}&L>N5#6S`3b3523&B751w($`m)E4APigecluWmRPj9R+P`tUL=US47*OE*o_q z!I?_!OF{XxC?Y^fXLVxYVoNmq3e!XIL=1&j_{}_r=$GoX5YFi&&%($;X|Qh#THk74 ztINppm@4I4hO>lCR{~7F=<8}^9)_+)?{hJ+~NYJgb7)7%3pVON`K;9F)jc*KT2Wo7^b6M##su zi9VKr7ErtF#x=in4a?2XvPqD!RBtx;EmGt&meads)xW7m)CY{#40nn^qG{D$ioHet zgnwU8kpyP&;Stafar*%lvr-R9a2#io2B)%6HL-&R%L*|X?I0!gyfZQJl%i-M2OC>!sW`*h zexrh76jo4UgMIiF&#rZ{RihWad%g?B$@?mc4 z*mIoZabcdJrxZw4FguYF6vDE}&FRHD8es)efmeWHh6|gKXc{C`t+atYh?qDl zYR;dj*Ghs6;dQ@*rZD_1T#a|bzw8`aakYc}(^r5TxXK0!+SHG4R9e6X*57H?Dy zG=qL9MM`$qF~dN}YyCFrOkD=?wfc~WXjx2*=>FA!n_h{;XEH<>9!0hU=ya+eRwO|f zC^F?D+TwFPs-6`JN!BQk&6;~&{kfW0l9qL9X>nrWRN03|G0^5?6=~?Do)h=1RH`6Y zCfRA%1wx7vutHsTP--GxAmXh!aD-LZ`}r5Fsd`6A3Ry&tUdQ*ch!#v+ z^KIxY7QZ@KUs>Vf@}!u3*q5zcyA_x>g`%;~OrC${nG5qT^lAKPP@T8v>~_|LGWa!fo<%6djwJ@NvrsfcF*7Q&Dqvgm@C{t)nRQ&jv?MK=d>iCq#w7KgU% zH_I(+3O7@nT~T8*(c8IWMqLb)lw5DgOJYYI6UF)vbhJW@#-KTV5Z-djh{$Kf-EW17 zTN~NlHuxQX7?)c_rw0zwPzf$m?_agsRcISjxdCmfoTZRe*i!NFOC8X;b*+OzuL-c0 zaYe5W2~ao`^~g?449gbU_HjxiV-DrkJ5G=T;I%xt85#LVA`>JXN8xhuul{7Ot1w5%%>M{)(TrgNm zjHSAxICr8s#CyXef%r^_Lt-Wol?6+(9t-@hDptTCyHUk{$jh)YVnPrDF|CyrT0j(& zD73X($moV44Ygl=W&+Fsyjn)OMdhvfgA=u;+M#-1SPry-i175FfSo_LG=J{m3&t!e zY!@&Vd3riswIY3j1KYHPm5s>51wdFrPU0;!OC62TOcAztLa>2rt%gXFS`bWBxntZc zV4+SMDD3bWfD>BNm?Se1OvqyA>Q%){kO_IlHpmtVwh>ez;$6Ci?PpsA?TB3w&zYE# z0s}|^euOtB!gcJX_G3YonQT(LM`oZZGAK{Kj23^^UtRUnm$*4yGP_uft9p;w zDV{05D{%#eR`1>~p&wcmk$srwyy0tMt9nf@@-B&olqau2spAt9sSwT5s8n%xRlFAJ ze>Z3@$VRZB*z`gLG1ig$DC1$-LI4-K$aT)Rg6vhlDFDfMGB1n5 zmu-@zToY6-+kg_;P>e&fNf1~-IA(xX9k&*OCT{D^O8e!sU+JU3#Qe>~q2&oclc3|{ zAhw|hk2#p zb_-q-4;HhlHbS17dgkd=FwS`{BPVuHTfpN=iN~aQoSXrD=op5Y!vIGoG{XfDop=_D|U% z1r2Pjf<8+GkR+7(B}9i@3zo~^w6daFWtT-vj+wFea>7hTKZv=Bf3_kuBqkS)t6-BE zvKDiUHe{cRD8&jdXsg&(Gl#` ztV;;3_kMNM;(O+*K0!Hsp#L8kAurNsWJ^qReO7j?chjrZ>E7I9W+h-2wkw(nFn2=8 zM2LIO@{^3PBvMS=QyF8~2Oc~rHY8e700ExDDJolEs<6Ld6aO5kpFA)=I*j1gD7#s0%91wC)^r!$y+O#B6RYTmW zlfLGB)EQv@B8C2Pr5Xy>xq+%MdJn~b+|eJP|{3K1CC23l3mEat>yT0!kbrO87X`({L;-t=DTgc_+% z_{Vy-UPt|=*+eLaL9cQ%yd~C0g4&T>z+}BJmozl_o-jfr4N9DN;|1gyUD4_=VF@-hubSr-oxyf-nic{RWtRX!({k-ks2=Wf zPZCHj-lX`eQxaipCGpA`tEI81H}Er5eGv};TA2(ZPV$+4nJWi3@DhnXssCm&NUR_l zUx$wMs?mEhn1)I2Ao`63l+ZF9uu>7B)g54+<_35Q$LkFoFM|ms$k4aX=hPUAdcg5A z9$^YH3K}M2P{v2^t zL?o^mz_D98Z+*sEoV)lGB7bTX71{F3;3>2u3K8QXu?54#NYvL6&N}IFSmC^crYp5H z`eC+0U4p#*I&V+)DiSQYDA&@xYoBSAI?HM7%EW|Su=1{zEn2Q?<-G36*+ZXtd^?p3 z-5<^#TI&A#fkThH^Vofd9@saA?=$&M;e&_%$-%I3GEViMm~EV#IB_j|^64JiY?R$U zG0S}uCmJW`M6@uHd^4(Zhe`erN}-^k67Jd#lKjbJ7q&?BCo>(L?T-q_j9k%a#UPv5F)GW^xcsk*hIcziR)=P(n{rT{OSJc|yowWE6 zdxTr?M<06tSk!G&Z(7`$JCQ37!Q|#W3eVf`cdk5)ljYmUl}FpysUj`ait-PJ2a5}A z+HJ}_#6Q_p&K%F~PNdAko#xC>%(C=}6SGPMl(5sEUlngqWA**n^MtEme#I6N8;O(X zAu2`d1pv|FO6)9)KE%u$@^W7~d?=kBg6gb0oliG)AKOZ&hj$~NzJqjnxK%#gn2A0$ zplUPHnqqwi7+`}0`^Ne8sJbl%T~Qhzz}eH87vf|&=5H|Xj4G8Rd%|EBx6HKzJf@<+ zs!7#{r^z7fBHupDc_VD6SQ3IlrUWHHvWCs^aWwPZ2~7;+x0QJ(^ev8$5>ip)`L(iYnrbbS_JBMvlM+I7n@}2L*uR?TH8z2 zYp^%Y)DH{6Mo=@#VZ!vtL{dSZ-Z%UQZbwY;q;{6HzcV@eaMwxuJCm~ycb&A~Y0iF# zcN!-(T~F!L&hzy^p6#UT4ST?(o22Z+rNx9zcayr805Bn?ICH;kj*Rs+_P}BJ_`A#DOX~T~B=M6h&5m;TA$o7ON&IyZ6-6?#3FB{_ zvPb^noP8n(f~xvrV-k&0h%6y3Z1q6u9`X?mm2_(yoi=gX@`joFWOAClr}S!c`aYQr zG0^R0@#CbYh~zet_$OwG|48UzBg-)?&pXNLZ!4)E4sT-55Q{rPxLe4(>m+}a;NK*_ zFTv>H1b_dWx<>L(&>|EZaYv?C&9c z5;>!@{VTVZ?oW7mi>!b5@*f?Xy>u6|{@v%SgO7DTbL7y$&-IEXMrZ%GpZxE=G`s^v z05NaG{(Y|=AkLLb@m*97kPy~X1)xmX!P_niNHo;OS|N)5>VAs-Hw6OWW)%XFDmi(_ z>jZjb0%3eO>Tc8t#Pg4;14b1GkYSR{dYoxTb4k4A?o#DT^)d5KAp^a+Z6)C z1p>|=)Ai1em5nwL$uV7tCz%zq20m)0YtPDu*R7KhS^uImu@o*rX_5M@>B!vL* zYlsO(n-N^!UNu0(!91doizN5HtC9dJ0ti5BP&O$Ehz5uQBzL}cK0ve9DB0c_>bcs4Y`^3W$6~dllgo)lgs6ujFWYF zohu}ATn`%2J;Gr;JDcG)rr~)`&h;Ef;yGS^Xwpw#?$~zV#T7gom0}!Ysolb_#-8X} zUmiOUdiiwTcG;i|q{wA##?EKFbS_hHJ!!6yXUPyTqlC@nZ8oWp&wAd_;Qe(SW*(Hq zZB^UBjc1rK6?N;nIBZhip|9t7mwxADVXv7(_q{(Ge^?0o5PTq%&E&Fa55i{QJWu}P zS{A;_c?H|k8}d$?{SijUcNcy#{@`O9)`vvHRVPn(Grt;pG8873AqttaY*PU?bv;;0 zxP$F2<_lgXLywRg!*VTy9C!r>Rw+Qj{4o7SSDTn(6B&8M6O)JK=QMz(q&~G_hY6ji zw|Uh`_pN_4_E7iTcgF6k$LnxqhlK%(c_asRm$f-*7T{)WFT+(=Il;@NA!asP$iNMD z-pLJdLgECZJg^cK4SyTqy;Gv08yXa>K<)Z1CJ2c7`q{&dmED0XW%Jm|8 z-4V+)t$HUNZGwwZC=?wR#n1g&FYUSvyT!tpU1Kr{{MYaGF zEuccw3ZxUT;T73IPV~UMV2zCw3hv!d?UXLhMV%1dpEPS768=cGLp_Aj-*Cj-J)-GI z^3I*sQkgf#x{UxI2UOzTBUd(PJ|fSoEK|O)aRZWdHC6gdZmu`zdD{WtAwg~$ECvUln!Jpik-LyRIMlXXdIN|;${P{-A0HXNKQtob z2yXNWg`zOCU=llBaEqP`2Su2~@v>MKx_YZnT@Te0{#dX_1oMPUumc%YW3ub52d zWD^|#P*=JyUGJ%|Tn?C#%b>xG3PTwAE$w0OGexHeqOb*m=JGB|lFg3Q*zm~c%yqSC zao%lIY1$_Y?c#IANlpD=mzSR6is2pl3M`EK(o>wRg1{=wkwMB++pr6Y&n#qhVexuHb++|?S=Ur?G zEBN2&l@ZV=Uj#vAam5504V%wCt+Eah<$=p6B-EUNM6hvI2DPzgMr)z^1>3%;QQfSRspxqK}29{M(e4gjMRvMAr2gVVlF z0UEGiw_oDj9DoSyg#i#B3b!bY0}RF*=^Mpmf@UFPNl2_~ z@oBWh{jGWQ47Mp1JULVVh9xBmXF!y$=590@R z*ZYT%p8zCtu2l}F_Im%A?W>t@ceV$Rz7t==xAv`6x_*Gg{K|KMCw)E;J~7Zn`~($v z+Y_8pNQ3ROIJAmW@Lr4JMH4_lpNzsOy@Ey@t|#>XgO-hK&|T<^9G5eVc=&0z?S&$O zb${^t2ObIKT!}=5mY3@{iGE#*FnfWx6n+JckckMO7e)mHWO+E~d3-$MN$%A29qH4l zneKo7$k?&&-~P&hBO`-+8dm&w-#ZWocnIVHHPt0zc-%2GY?fGJMneb%#F0ga4*94awx!@6*cqSq&n2xut8fVOCy*iZ#$ixF88-~ z3R4OeN?z%j89;hGT^?-mBp(ok8qw%U9}z?&=N#5Idm33cs?XTkBZCB3xb22dW_p~f zl*4QNd{2X%t(!lYr)dB+R7;9A+`Ivlv1N(R22drsM&JU+ON2T{Fd7*m0_+jgaN%-% zHas*YTQ-4KwV ze;v^kd=q}fwxdXhS)xt;=S$-cKDv)YUMh&@kSIb*(hf>RH-gMhdxwc!+ zYlbs@xx#ggxy+}aXbO`?H|L6`8?{SiAL0gG1M18)>RMmpt53Ueudb!L(X6Yd>?FjS zbx8C$MIyVBViWt#HE??t{Zl~2VIws_Mdq!LA*w6sB2ingx|cNxoj5-AaQDA`WbAjk zpZdZE;N&i3AhSsZ(nIZH(F+ob&LE#?BDa{6kTlMf=v*?8dGt7eVOQMET!CeffXqpn z24N<9lXbiAzB%^EZDt@r5MZk;Q4Nw>LP1`)MFw)HeA;lz=&omD()LV)+Z(1K#ksOs zMlsevVynmwk+>~lvrIN&*gqHlc{jD#o3v3g%McCaR}AJ7qjA+3jSO%wp9TDqqg4bd z0F-1YX}$uSi4iZR8r%q~xG~*-zc%&(1COI@qf4rrIHN-bR#Ltq z<^&=fwe1-)W4QE^^z`s&gu~b~Tp!0B4lMUUU?~_F>QpzheRQh}9+zI>fMR@D;iA#q z>iR;DX1ZCM`Umo6k}x(IYKfcU?|4K?L?&gvf>3a9$jVb*q0=o6#~ZXzd?$7B&yDJS zH(VHRmO0$~POsYQ>f_DY)O*&vZ&u1QD3mCe+zf#xa3fU*QtBY9QlzEIjJ=7XD8qs- zrS&?T#32Y(uJ5O3rYM0*>8WRadsrd~8#dPKzV)rKk9Pm$KT)*;H;GndDlVvSvdC(lML_~BtPrk7rah2H21i1(Sh6rDmqxx} zBA4J|9%p^XC*SH2M5b|ICX|8`-j&OU?uQNv#{2D@+7MNtIolT!SolLcT%MGSQ^rhd~I6Cn`R6w3jKvk6R(obEQx zWti{ z2}Z7iBJr^iGjv~yQIC$8t3kg%OL&_)9WO&%U!e%5h`+!B?t!zY0a15W;C2oWM_w^A zI<(){ozjfa*P(LVR7mR3dxz8iw2Cwzj%-!gz4Dv=k7zYUH6jlwAF2rf7m0T}3V{Ho z5m?FxZ@>5;hx%6H8{5rpz%7zGpqR=N6DN7`oTkr<{4uBqXpEA3agfbnnP|lucGOht z)9s!=@QEm2K@|{+1(jp0?8vkwOp-%m#)VbG*kSb98;QLQl}Ge7>Ac(tr3^7lL5mRqj?oQ zgS`q8P$nBrp`aE8AF*_KuF&L6^0`qzQFE&?#QAi*iM)iuIJIQ!Rm(k=xe2(Z~)M-?+nwO2Rlb$1D=SmKS z9nMf}1rU(?+I{V3YuW8EMB<@kL*ZaD+l zWR|IN!Wf}js76hLr3nR?h=wpbX%UG(;+AUMgG=0-&P#PC*aKN83}*x4KC9hFe>wi) zUT4*Z2Ro}??qqGh6g5}1tnO$2`}jw??|x(K$mTs(FTHxXmb6gqYqDyVC+QEwk0X!G(d7RK3@l7OPNI{(E%?9ag(9esk5$C z6Hg_9_yQvE(UOzqdZ|Ajx zCKhg^a*CZD5c{;Kytuc~PI|>gC&aqeGkf1_O>2ISYRRNMtzM8`y#_0M_|dTk6(_}^ znv9h1+U!&mT@w68{sz!1dnLQcH!3#>=K!ZRE-xK*$jwYLINLq(r3UN$oP+>4QXhB=?f@+|KS`&zO2xBz2KyNG+vRUC82z~2hP(O#ZPY9% zdZVVh$*H<%;0^oka&}p4(^idl^)*3qW=osz>NS^o$1NK0V)e}*u^H8W*V&)}ufE)i zHb&UZT+mwGUfpBF@dTl8hyZ_aK{YI{UJNEOn>C1#^9BG4L@zEy9QMz*KC{<%WY5ed z__&EO=x0NTElCcXw%lgW?}K+Bk<$qCjrgv(oQ-0X@f4pp|la!Tin}dCA_;+fimn;;K|zi z+6kMv%Ao;bLX?%Em*nh?;59;O z4wcXtQj%JBei*N9>!e|Sj6Bml_pAN)YOL)=GZTVyX{o=*GGo{?G7)@4;&;?d0xBhT zg*ruI0Jm9OSyD{}jQbFyjXK2GeRQvs0n2j9S8+y#RC3R>QwNUe&q~ZR38xf^W>HNi zg+-%+pxbDyZ1&11DpjGM$GOBhMIDxJ?@(m-(chIKN3$Xdk$@^iV6+}Q#0+J*L@t36 zO3$1-WZea-3D`{he;^G zqy7!QIyy-?w4%Wd)cvfN{qhT6eqq!f@;#G7n`JZRFZr&&LDHUti}N|-}(7_ Ihpa#N|BHQCmH+?% delta 243 zcmZo@P-|#Vn;* z8W=yX-z?C;@6gWjff)Rs~`;AXW!r4ItK>uJD0%J0}N_$qvM9?GHY% P0x^hY-~QkOheAI9dBQ bool: + return self == ConversionOptions() + + @dataclass class ConversionResult: page_count: int @@ -102,35 +123,26 @@ def _get_element_type(item: DocItem) -> str: # Pipeline factory # --------------------------------------------------------------------------- -def build_converter( - do_ocr: bool = True, - do_table_structure: bool = True, - table_mode: str = "accurate", - do_code_enrichment: bool = False, - do_formula_enrichment: bool = False, - do_picture_classification: bool = False, - do_picture_description: bool = False, - generate_picture_images: bool = False, - generate_page_images: bool = False, - images_scale: float = 1.0, -) -> DocumentConverter: +def build_converter(options: ConversionOptions | None = None) -> DocumentConverter: """Build a DocumentConverter with the given pipeline options.""" + opts = options or ConversionOptions() + table_options = TableStructureOptions( do_cell_matching=True, - mode=TableFormerMode.ACCURATE if table_mode == "accurate" else TableFormerMode.FAST, + mode=TableFormerMode.ACCURATE if opts.table_mode == "accurate" else TableFormerMode.FAST, ) pipeline_options = PdfPipelineOptions( - do_ocr=do_ocr, - do_table_structure=do_table_structure, + do_ocr=opts.do_ocr, + do_table_structure=opts.do_table_structure, table_structure_options=table_options, - do_code_enrichment=do_code_enrichment, - do_formula_enrichment=do_formula_enrichment, - do_picture_classification=do_picture_classification, - do_picture_description=do_picture_description, - generate_page_images=generate_page_images, - generate_picture_images=generate_picture_images, - images_scale=images_scale, + do_code_enrichment=opts.do_code_enrichment, + do_formula_enrichment=opts.do_formula_enrichment, + do_picture_classification=opts.do_picture_classification, + do_picture_description=opts.do_picture_description, + generate_page_images=opts.generate_page_images, + generate_picture_images=opts.generate_picture_images, + images_scale=opts.images_scale, ) return DocumentConverter( @@ -191,7 +203,7 @@ def _process_content_item( try: page_no = prov.page_no if page_no not in pages: - pages[page_no] = PageDetail(page_number=page_no, width=612.0, height=792.0) + pages[page_no] = PageDetail(page_number=page_no, width=_DEFAULT_PAGE_WIDTH, height=_DEFAULT_PAGE_HEIGHT) page_height = pages[page_no].height @@ -205,13 +217,13 @@ def _process_content_item( if isinstance(item, TableItem): try: content = item.export_to_markdown() - except Exception: + except (AttributeError, ValueError): pass pages[page_no].elements.append( PageElement(type=element_type, bbox=bbox, content=content, level=level) ) - except Exception: + except (AttributeError, KeyError, TypeError, ValueError): logger.warning( "Skipping item %s on page %s", type(item).__name__, @@ -227,77 +239,54 @@ def _process_content_item( # Main conversion entry point # --------------------------------------------------------------------------- +def _select_converter(options: ConversionOptions) -> DocumentConverter: + """Return the cached default converter or build a custom one.""" + if options.is_default(): + return get_default_converter() + return build_converter(options) + + +def _build_fallback_pages(doc, page_count: int) -> list[PageDetail]: + """Create empty PageDetail entries when extraction yields nothing.""" + return [ + PageDetail( + page_number=i + 1, + width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else _DEFAULT_PAGE_WIDTH, + height=doc.pages[i + 1].size.height if (i + 1) in doc.pages else _DEFAULT_PAGE_HEIGHT, + ) + for i in range(page_count) + ] + + def convert_document( file_path: str, - *, - do_ocr: bool = True, - do_table_structure: bool = True, - table_mode: str = "accurate", - do_code_enrichment: bool = False, - do_formula_enrichment: bool = False, - do_picture_classification: bool = False, - do_picture_description: bool = False, - generate_picture_images: bool = False, - generate_page_images: bool = False, - images_scale: float = 1.0, + options: ConversionOptions | None = None, ) -> ConversionResult: """Convert a document and return structured results. This is the main entry point for document parsing. Runs synchronously (caller should use asyncio.to_thread for non-blocking execution). """ - # Use cached default converter only when all options match defaults - is_default = ( - do_ocr and do_table_structure and table_mode == "accurate" - and not do_code_enrichment and not do_formula_enrichment - and not do_picture_classification and not do_picture_description - and not generate_picture_images and not generate_page_images - and images_scale == 1.0 - ) + opts = options or ConversionOptions() with _converter_lock: - if is_default: - conv = get_default_converter() - else: - conv = build_converter( - do_ocr=do_ocr, - do_table_structure=do_table_structure, - table_mode=table_mode, - do_code_enrichment=do_code_enrichment, - do_formula_enrichment=do_formula_enrichment, - do_picture_classification=do_picture_classification, - do_picture_description=do_picture_description, - generate_picture_images=generate_picture_images, - generate_page_images=generate_page_images, - images_scale=images_scale, - ) - + conv = _select_converter(opts) result = conv.convert(file_path) doc = result.document - content_markdown = doc.export_to_markdown() - content_html = doc.export_to_html() page_count = len(doc.pages) - pages_detail, skipped = extract_pages_detail(result) if not pages_detail and page_count > 0: - pages_detail = [ - PageDetail( - page_number=i + 1, - width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else 612.0, - height=doc.pages[i + 1].size.height if (i + 1) in doc.pages else 792.0, - ) - for i in range(page_count) - ] + pages_detail = _build_fallback_pages(doc, page_count) if skipped > 0: logger.info("Parsed: %d pages, %d items skipped", page_count, skipped) return ConversionResult( page_count=page_count or len(pages_detail) or 1, - content_markdown=content_markdown, - content_html=content_html, + content_markdown=doc.export_to_markdown(), + content_html=doc.export_to_html(), pages=pages_detail, skipped_items=skipped, ) diff --git a/document-parser/main.py b/document-parser/main.py index f1b4038..dd19148 100644 --- a/document-parser/main.py +++ b/document-parser/main.py @@ -48,8 +48,8 @@ app.add_middleware( CORSMiddleware, allow_origins=[o.strip() for o in allowed_origins], allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + allow_methods=["GET", "POST", "DELETE", "OPTIONS"], + allow_headers=["Content-Type", "Authorization"], ) # Mount routers diff --git a/document-parser/persistence/database.py b/document-parser/persistence/database.py index dc8d95d..6aaa37b 100644 --- a/document-parser/persistence/database.py +++ b/document-parser/persistence/database.py @@ -35,6 +35,10 @@ CREATE TABLE IF NOT EXISTS analysis_jobs ( completed_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); + +CREATE INDEX IF NOT EXISTS idx_analysis_jobs_status ON analysis_jobs(status); +CREATE INDEX IF NOT EXISTS idx_analysis_jobs_created_at ON analysis_jobs(created_at); +CREATE INDEX IF NOT EXISTS idx_documents_created_at ON documents(created_at); """ diff --git a/document-parser/services/analysis_service.py b/document-parser/services/analysis_service.py index f061980..9a6f5e7 100644 --- a/document-parser/services/analysis_service.py +++ b/document-parser/services/analysis_service.py @@ -8,7 +8,7 @@ import logging from dataclasses import asdict from domain.models import AnalysisJob -from domain.parsing import ConversionResult, convert_document +from domain.parsing import ConversionOptions, ConversionResult, convert_document from persistence import analysis_repo, document_repo logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ def _on_task_done(task: asyncio.Task) -> None: return exc = task.exception() if exc: - logger.error("Unhandled exception in analysis task: %s", exc, exc_info=exc) + logger.error("Unhandled exception in analysis task: %s", exc, exc_info=True) async def find_all() -> list[AnalysisJob]: @@ -72,12 +72,12 @@ async def _run_analysis( await analysis_repo.update_status(job) logger.info("Analysis started: %s (file: %s)", job_id, filename) - # Build kwargs from pipeline options - convert_kwargs = pipeline_options or {} + # Build conversion options from pipeline dict + options = ConversionOptions(**(pipeline_options or {})) # Run blocking Docling conversion in a thread with timeout result: ConversionResult = await asyncio.wait_for( - asyncio.to_thread(convert_document, file_path, **convert_kwargs), + asyncio.to_thread(convert_document, file_path, options), timeout=CONVERSION_TIMEOUT, ) diff --git a/document-parser/services/document_service.py b/document-parser/services/document_service.py index dd3b6c4..f36825d 100644 --- a/document-parser/services/document_service.py +++ b/document-parser/services/document_service.py @@ -32,7 +32,8 @@ async def upload(filename: str, content_type: str, file_content: bytes) -> Docum os.makedirs(UPLOAD_DIR, exist_ok=True) - safe_name = f"{uuid.uuid4()}_{filename}" + ext = os.path.splitext(filename)[1] or ".pdf" + safe_name = f"{uuid.uuid4()}{ext}" file_path = os.path.join(UPLOAD_DIR, safe_name) with open(file_path, "wb") as f: diff --git a/document-parser/tests/test_pipeline_options.py b/document-parser/tests/test_pipeline_options.py index 6e619e8..7da273f 100644 --- a/document-parser/tests/test_pipeline_options.py +++ b/document-parser/tests/test_pipeline_options.py @@ -12,7 +12,7 @@ from docling.datamodel.pipeline_options import ( TableFormerMode, ) -from domain.parsing import build_converter, convert_document, get_default_converter +from domain.parsing import ConversionOptions, build_converter, convert_document, get_default_converter # --------------------------------------------------------------------------- @@ -42,62 +42,62 @@ class TestBuildConverter: assert opts.images_scale == 1.0 def test_ocr_disabled(self): - conv = build_converter(do_ocr=False) + conv = build_converter(ConversionOptions(do_ocr=False)) opts = self._get_pipeline_options(conv) assert opts.do_ocr is False def test_table_mode_fast(self): - conv = build_converter(table_mode="fast") + conv = build_converter(ConversionOptions(table_mode="fast")) opts = self._get_pipeline_options(conv) assert opts.table_structure_options.mode == TableFormerMode.FAST def test_table_mode_accurate(self): - conv = build_converter(table_mode="accurate") + conv = build_converter(ConversionOptions(table_mode="accurate")) opts = self._get_pipeline_options(conv) assert opts.table_structure_options.mode == TableFormerMode.ACCURATE def test_table_structure_disabled(self): - conv = build_converter(do_table_structure=False) + conv = build_converter(ConversionOptions(do_table_structure=False)) opts = self._get_pipeline_options(conv) assert opts.do_table_structure is False def test_code_enrichment_enabled(self): - conv = build_converter(do_code_enrichment=True) + conv = build_converter(ConversionOptions(do_code_enrichment=True)) opts = self._get_pipeline_options(conv) assert opts.do_code_enrichment is True def test_formula_enrichment_enabled(self): - conv = build_converter(do_formula_enrichment=True) + conv = build_converter(ConversionOptions(do_formula_enrichment=True)) opts = self._get_pipeline_options(conv) assert opts.do_formula_enrichment is True def test_picture_classification_enabled(self): - conv = build_converter(do_picture_classification=True) + conv = build_converter(ConversionOptions(do_picture_classification=True)) opts = self._get_pipeline_options(conv) assert opts.do_picture_classification is True def test_picture_description_enabled(self): - conv = build_converter(do_picture_description=True) + conv = build_converter(ConversionOptions(do_picture_description=True)) opts = self._get_pipeline_options(conv) assert opts.do_picture_description is True def test_generate_picture_images(self): - conv = build_converter(generate_picture_images=True) + conv = build_converter(ConversionOptions(generate_picture_images=True)) opts = self._get_pipeline_options(conv) assert opts.generate_picture_images is True def test_generate_page_images(self): - conv = build_converter(generate_page_images=True) + conv = build_converter(ConversionOptions(generate_page_images=True)) opts = self._get_pipeline_options(conv) assert opts.generate_page_images is True def test_images_scale(self): - conv = build_converter(images_scale=2.0) + conv = build_converter(ConversionOptions(images_scale=2.0)) opts = self._get_pipeline_options(conv) assert opts.images_scale == 2.0 def test_all_options_combined(self): - conv = build_converter( + conv = build_converter(ConversionOptions( do_ocr=False, do_table_structure=True, table_mode="fast", @@ -108,7 +108,7 @@ class TestBuildConverter: generate_picture_images=True, generate_page_images=True, images_scale=1.5, - ) + )) opts = self._get_pipeline_options(conv) assert opts.do_ocr is False assert opts.do_table_structure is True @@ -158,7 +158,7 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", do_ocr=False) + convert_document("/tmp/test.pdf", ConversionOptions(do_ocr=False)) mock_build.assert_called_once() mock_get_default.assert_not_called() @@ -175,10 +175,10 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", table_mode="fast") + opts = ConversionOptions(table_mode="fast") + convert_document("/tmp/test.pdf", opts) - mock_build.assert_called_once() - assert mock_build.call_args.kwargs["table_mode"] == "fast" + mock_build.assert_called_once_with(opts) @patch("domain.parsing.get_default_converter") @patch("domain.parsing.build_converter") @@ -192,10 +192,10 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", do_code_enrichment=True) + opts = ConversionOptions(do_code_enrichment=True) + convert_document("/tmp/test.pdf", opts) - mock_build.assert_called_once() - assert mock_build.call_args.kwargs["do_code_enrichment"] is True + mock_build.assert_called_once_with(opts) @patch("domain.parsing.get_default_converter") @patch("domain.parsing.build_converter") @@ -209,7 +209,7 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", do_formula_enrichment=True) + convert_document("/tmp/test.pdf", ConversionOptions(do_formula_enrichment=True)) mock_build.assert_called_once() @@ -225,7 +225,7 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", do_picture_classification=True) + convert_document("/tmp/test.pdf", ConversionOptions(do_picture_classification=True)) mock_build.assert_called_once() @@ -241,7 +241,7 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", generate_picture_images=True) + convert_document("/tmp/test.pdf", ConversionOptions(generate_picture_images=True)) mock_build.assert_called_once() @@ -257,10 +257,10 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document("/tmp/test.pdf", images_scale=2.0) + opts = ConversionOptions(images_scale=2.0) + convert_document("/tmp/test.pdf", opts) - mock_build.assert_called_once() - assert mock_build.call_args.kwargs["images_scale"] == 2.0 + mock_build.assert_called_once_with(opts) @patch("domain.parsing.get_default_converter") @patch("domain.parsing.build_converter") @@ -274,8 +274,7 @@ class TestConvertDocumentRouting: mock_conv.convert.return_value = mock_result mock_build.return_value = mock_conv - convert_document( - "/tmp/test.pdf", + opts = ConversionOptions( do_ocr=False, do_table_structure=False, table_mode="fast", @@ -287,19 +286,9 @@ class TestConvertDocumentRouting: generate_page_images=True, images_scale=1.5, ) + convert_document("/tmp/test.pdf", opts) - mock_build.assert_called_once_with( - do_ocr=False, - do_table_structure=False, - table_mode="fast", - do_code_enrichment=True, - do_formula_enrichment=True, - do_picture_classification=True, - do_picture_description=True, - generate_picture_images=True, - generate_page_images=True, - images_scale=1.5, - ) + mock_build.assert_called_once_with(opts) # --------------------------------------------------------------------------- @@ -392,18 +381,15 @@ class TestServiceForwardsPipelineOptions: await _run_analysis("j1", "/tmp/test.pdf", "test.pdf", opts) - mock_convert.assert_called_once_with( - "/tmp/test.pdf", - do_ocr=False, - table_mode="fast", - do_code_enrichment=True, - do_formula_enrichment=False, - do_picture_classification=False, - do_picture_description=False, - generate_picture_images=True, - generate_page_images=False, - images_scale=2.0, - ) + mock_convert.assert_called_once() + call_args = mock_convert.call_args + assert call_args[0][0] == "/tmp/test.pdf" + conv_opts = call_args[0][1] + assert conv_opts.do_ocr is False + assert conv_opts.table_mode == "fast" + assert conv_opts.do_code_enrichment is True + assert conv_opts.generate_picture_images is True + assert conv_opts.images_scale == 2.0 @patch("services.analysis_service.analysis_repo") @patch("services.analysis_service.document_repo") @@ -428,8 +414,11 @@ class TestServiceForwardsPipelineOptions: await _run_analysis("j1", "/tmp/test.pdf", "test.pdf", None) - # Called with file_path only (no kwargs spread from empty dict) - mock_convert.assert_called_once_with("/tmp/test.pdf") + # Called with file_path and default ConversionOptions + mock_convert.assert_called_once() + call_args = mock_convert.call_args + assert call_args[0][0] == "/tmp/test.pdf" + assert call_args[0][1] == ConversionOptions() @patch("services.analysis_service.analysis_repo") @patch("services.analysis_service.document_repo") diff --git a/frontend/Dockerfile b/frontend/Dockerfile index c1c36cb..ea2b68a 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -2,7 +2,7 @@ FROM node:20-alpine AS build WORKDIR /app COPY package*.json ./ -RUN npm install +RUN npm ci COPY . . RUN npm run build diff --git a/frontend/src/features/analysis/store.js b/frontend/src/features/analysis/store.js index e01025e..3e41c89 100644 --- a/frontend/src/features/analysis/store.js +++ b/frontend/src/features/analysis/store.js @@ -6,6 +6,7 @@ export const useAnalysisStore = defineStore('analysis', () => { const analyses = ref([]) const currentAnalysis = ref(null) const running = ref(false) + const error = ref(null) const pollingInterval = ref(null) const currentPages = computed(() => { @@ -17,16 +18,21 @@ export const useAnalysisStore = defineStore('analysis', () => { } }) + function clearError() { error.value = null } + async function load() { try { + error.value = null analyses.value = await api.fetchAnalyses() } catch (e) { + error.value = e.message || 'Failed to load analyses' console.error('Failed to load analyses', e) } } async function run(documentId, pipelineOptions = null) { running.value = true + error.value = null try { const analysis = await api.createAnalysis(documentId, pipelineOptions) currentAnalysis.value = analysis @@ -35,6 +41,7 @@ export const useAnalysisStore = defineStore('analysis', () => { return analysis } catch (e) { running.value = false + error.value = e.message || 'Failed to start analysis' console.error('Failed to start analysis', e) throw e } @@ -46,7 +53,6 @@ export const useAnalysisStore = defineStore('analysis', () => { try { const updated = await api.fetchAnalysis(id) currentAnalysis.value = updated - // Update in list const idx = analyses.value.findIndex(a => a.id === id) if (idx !== -1) analyses.value[idx] = updated if (updated.status === 'COMPLETED' || updated.status === 'FAILED') { @@ -54,6 +60,7 @@ export const useAnalysisStore = defineStore('analysis', () => { running.value = false } } catch (e) { + error.value = e.message || 'Polling error' console.error('Polling error', e) stopPolling() running.value = false @@ -72,6 +79,7 @@ export const useAnalysisStore = defineStore('analysis', () => { try { currentAnalysis.value = await api.fetchAnalysis(id) } catch (e) { + error.value = e.message || 'Failed to load analysis' console.error('Failed to load analysis', e) } } @@ -82,9 +90,10 @@ export const useAnalysisStore = defineStore('analysis', () => { analyses.value = analyses.value.filter(a => a.id !== id) if (currentAnalysis.value?.id === id) currentAnalysis.value = null } catch (e) { + error.value = e.message || 'Failed to delete analysis' console.error('Failed to delete analysis', e) } } - return { analyses, currentAnalysis, currentPages, running, load, run, select, remove, stopPolling } + return { analyses, currentAnalysis, currentPages, running, error, clearError, load, run, select, remove, stopPolling } }) diff --git a/frontend/src/features/analysis/ui/ResultTabs.vue b/frontend/src/features/analysis/ui/ResultTabs.vue index 1a3a9e7..e1aab63 100644 --- a/frontend/src/features/analysis/ui/ResultTabs.vue +++ b/frontend/src/features/analysis/ui/ResultTabs.vue @@ -36,6 +36,20 @@ {{ el.type }} L{{ el.level }} +
@@ -50,6 +64,15 @@
+ 
{{ pageMarkdown }}
@@ -74,7 +97,7 @@