feat(documents): add web import and creation dialog with SSRF-safe loader
Introduce a unified upload menu dialog enabling users to upload files, create markdown/text documents, or import content directly from web URLs. Implement a secure server-side loader for fetching and parsing web pages with SSRF protection, content-type checks, and size limits. Add new API route for importing documents from URLs, and update dependencies to support HTML parsing and markdown conversion. - Add UploadMenuDialog component with file, create, and URL import tabs - Implement SSRF-mitigated web-loader using linkedom, readability, turndown - Add /api/documents/import-url endpoint for web imports - Update DocumentList to use new upload dialog - Add BrowserIcon and supporting UI elements - Update dependencies: @mozilla/readability, linkedom, turndown - Expose importUrl client API This improves document onboarding flexibility and enhances security for web-based imports.
This commit is contained in:
Richard R
parent
ea4bcb5cf0
commit
5a44a59ed1
10 changed files with 964 additions and 4 deletions
|
|
@ -35,6 +35,7 @@
|
|||
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
||||
"@headlessui/react": "^2.2.10",
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"@napi-rs/canvas": "^0.1.100",
|
||||
"@tanstack/react-query": "^5.101.0",
|
||||
"@types/archiver": "^7.0.0",
|
||||
|
|
@ -55,6 +56,7 @@
|
|||
"ffmpeg-static": "^5.3.0",
|
||||
"howler": "^2.2.4",
|
||||
"jszip": "^3.10.1",
|
||||
"linkedom": "^0.18.12",
|
||||
"lru-cache": "^11.5.1",
|
||||
"next": "^15.5.19",
|
||||
"onnxruntime-node": "^1.26.0",
|
||||
|
|
@ -75,6 +77,7 @@
|
|||
"react-reader": "^2.0.15",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"replicate": "^1.4.0",
|
||||
"turndown": "^7.2.4",
|
||||
"uuid": "^11.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -83,10 +86,12 @@
|
|||
"@tailwindcss/typography": "^0.5.19",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/howler": "^2.2.13",
|
||||
"@types/mozilla-readability": "^0.2.1",
|
||||
"@types/node": "^20.19.41",
|
||||
"@types/pg": "^8.20.0",
|
||||
"@types/react": "^19.2.16",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@types/turndown": "^5.0.6",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint-config-next": "^15.5.19",
|
||||
|
|
|
|||
160
pnpm-lock.yaml
160
pnpm-lock.yaml
|
|
@ -23,6 +23,9 @@ importers:
|
|||
'@huggingface/tokenizers':
|
||||
specifier: ^0.1.3
|
||||
version: 0.1.3
|
||||
'@mozilla/readability':
|
||||
specifier: ^0.6.0
|
||||
version: 0.6.0
|
||||
'@napi-rs/canvas':
|
||||
specifier: ^0.1.100
|
||||
version: 0.1.100
|
||||
|
|
@ -83,6 +86,9 @@ importers:
|
|||
jszip:
|
||||
specifier: ^3.10.1
|
||||
version: 3.10.1
|
||||
linkedom:
|
||||
specifier: ^0.18.12
|
||||
version: 0.18.12(canvas@3.2.3)
|
||||
lru-cache:
|
||||
specifier: ^11.5.1
|
||||
version: 11.5.1
|
||||
|
|
@ -143,6 +149,9 @@ importers:
|
|||
replicate:
|
||||
specifier: ^1.4.0
|
||||
version: 1.4.0
|
||||
turndown:
|
||||
specifier: ^7.2.4
|
||||
version: 7.2.4
|
||||
uuid:
|
||||
specifier: ^11.1.1
|
||||
version: 11.1.1
|
||||
|
|
@ -162,6 +171,9 @@ importers:
|
|||
'@types/howler':
|
||||
specifier: ^2.2.13
|
||||
version: 2.2.13
|
||||
'@types/mozilla-readability':
|
||||
specifier: ^0.2.1
|
||||
version: 0.2.1
|
||||
'@types/node':
|
||||
specifier: ^20.19.41
|
||||
version: 20.19.41
|
||||
|
|
@ -174,6 +186,9 @@ importers:
|
|||
'@types/react-dom':
|
||||
specifier: ^19.2.3
|
||||
version: 19.2.3(@types/react@19.2.16)
|
||||
'@types/turndown':
|
||||
specifier: ^5.0.6
|
||||
version: 5.0.6
|
||||
'@types/uuid':
|
||||
specifier: ^10.0.0
|
||||
version: 10.0.0
|
||||
|
|
@ -1240,6 +1255,13 @@ packages:
|
|||
'@jridgewell/trace-mapping@0.3.31':
|
||||
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
|
||||
|
||||
'@mixmark-io/domino@2.2.0':
|
||||
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
|
||||
|
||||
'@mozilla/readability@0.6.0':
|
||||
resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==}
|
||||
engines: {node: '>=14.0.0'}
|
||||
|
||||
'@napi-rs/canvas-android-arm64@0.1.100':
|
||||
resolution: {integrity: sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==}
|
||||
engines: {node: '>= 10'}
|
||||
|
|
@ -1686,6 +1708,9 @@ packages:
|
|||
'@types/mdast@4.0.4':
|
||||
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
|
||||
|
||||
'@types/mozilla-readability@0.2.1':
|
||||
resolution: {integrity: sha512-RjLpe3MIpj53L+q50DTNlhv6KjHjgajFQwAF4KuxVZTa2+6Cs2UNAxS5awxABfZEHRYjGKeRDB21Zrtnsi911w==}
|
||||
|
||||
'@types/ms@2.1.0':
|
||||
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
|
||||
|
||||
|
|
@ -1709,6 +1734,9 @@ packages:
|
|||
'@types/readdir-glob@1.1.5':
|
||||
resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
|
||||
|
||||
'@types/turndown@5.0.6':
|
||||
resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==}
|
||||
|
||||
'@types/unist@2.0.11':
|
||||
resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
|
||||
|
||||
|
|
@ -2259,6 +2287,9 @@ packages:
|
|||
bl@4.1.0:
|
||||
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
||||
|
||||
boolbase@1.0.0:
|
||||
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
|
||||
|
||||
bowser@2.14.1:
|
||||
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
|
||||
|
||||
|
|
@ -2418,11 +2449,21 @@ packages:
|
|||
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
css-select@5.2.2:
|
||||
resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==}
|
||||
|
||||
css-what@6.2.2:
|
||||
resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
cssesc@3.0.0:
|
||||
resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
|
||||
engines: {node: '>=4'}
|
||||
hasBin: true
|
||||
|
||||
cssom@0.5.0:
|
||||
resolution: {integrity: sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==}
|
||||
|
||||
csstype@3.2.3:
|
||||
resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==}
|
||||
|
||||
|
|
@ -2523,6 +2564,19 @@ packages:
|
|||
resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
dom-serializer@2.0.0:
|
||||
resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}
|
||||
|
||||
domelementtype@2.3.0:
|
||||
resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==}
|
||||
|
||||
domhandler@5.0.3:
|
||||
resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==}
|
||||
engines: {node: '>= 4'}
|
||||
|
||||
domutils@3.2.2:
|
||||
resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}
|
||||
|
||||
dotenv@17.4.2:
|
||||
resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==}
|
||||
engines: {node: '>=12'}
|
||||
|
|
@ -2643,6 +2697,14 @@ packages:
|
|||
end-of-stream@1.4.5:
|
||||
resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==}
|
||||
|
||||
entities@4.5.0:
|
||||
resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
|
||||
engines: {node: '>=0.12'}
|
||||
|
||||
entities@7.0.1:
|
||||
resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==}
|
||||
engines: {node: '>=0.12'}
|
||||
|
||||
env-paths@2.2.1:
|
||||
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
|
||||
engines: {node: '>=6'}
|
||||
|
|
@ -3111,9 +3173,15 @@ packages:
|
|||
howler@2.2.4:
|
||||
resolution: {integrity: sha512-iARIBPgcQrwtEr+tALF+rapJ8qSc+Set2GJQl7xT1MQzWaVkFebdJhR3alVlSiUf5U7nAANKuj3aWpwerocD5w==}
|
||||
|
||||
html-escaper@3.0.3:
|
||||
resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==}
|
||||
|
||||
html-url-attributes@3.0.1:
|
||||
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
|
||||
|
||||
htmlparser2@10.1.0:
|
||||
resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==}
|
||||
|
||||
http-response-object@3.0.2:
|
||||
resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}
|
||||
|
||||
|
|
@ -3463,6 +3531,15 @@ packages:
|
|||
lines-and-columns@1.2.4:
|
||||
resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
|
||||
|
||||
linkedom@0.18.12:
|
||||
resolution: {integrity: sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==}
|
||||
engines: {node: '>=16'}
|
||||
peerDependencies:
|
||||
canvas: '>= 2'
|
||||
peerDependenciesMeta:
|
||||
canvas:
|
||||
optional: true
|
||||
|
||||
localforage@1.10.0:
|
||||
resolution: {integrity: sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==}
|
||||
|
||||
|
|
@ -3752,6 +3829,9 @@ packages:
|
|||
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
nth-check@2.1.1:
|
||||
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
|
||||
|
||||
object-assign@4.1.1:
|
||||
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
|
@ -4599,6 +4679,10 @@ packages:
|
|||
tunnel-agent@0.6.0:
|
||||
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
|
||||
|
||||
turndown@7.2.4:
|
||||
resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==}
|
||||
engines: {node: '>=18', npm: '>=9'}
|
||||
|
||||
tweetnacl@1.0.3:
|
||||
resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==}
|
||||
|
||||
|
|
@ -4637,6 +4721,9 @@ packages:
|
|||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
uhyphen@0.2.0:
|
||||
resolution: {integrity: sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==}
|
||||
|
||||
unbox-primitive@1.1.0:
|
||||
resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
|
@ -5669,6 +5756,10 @@ snapshots:
|
|||
'@jridgewell/resolve-uri': 3.1.2
|
||||
'@jridgewell/sourcemap-codec': 1.5.5
|
||||
|
||||
'@mixmark-io/domino@2.2.0': {}
|
||||
|
||||
'@mozilla/readability@0.6.0': {}
|
||||
|
||||
'@napi-rs/canvas-android-arm64@0.1.100':
|
||||
optional: true
|
||||
|
||||
|
|
@ -6018,6 +6109,8 @@ snapshots:
|
|||
dependencies:
|
||||
'@types/unist': 3.0.3
|
||||
|
||||
'@types/mozilla-readability@0.2.1': {}
|
||||
|
||||
'@types/ms@2.1.0': {}
|
||||
|
||||
'@types/node@10.17.60': {}
|
||||
|
|
@ -6044,6 +6137,8 @@ snapshots:
|
|||
dependencies:
|
||||
'@types/node': 20.19.41
|
||||
|
||||
'@types/turndown@5.0.6': {}
|
||||
|
||||
'@types/unist@2.0.11': {}
|
||||
|
||||
'@types/unist@3.0.3': {}
|
||||
|
|
@ -6542,6 +6637,8 @@ snapshots:
|
|||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
boolbase@1.0.0: {}
|
||||
|
||||
bowser@2.14.1: {}
|
||||
|
||||
brace-expansion@1.1.15:
|
||||
|
|
@ -6699,8 +6796,20 @@ snapshots:
|
|||
shebang-command: 2.0.0
|
||||
which: 2.0.2
|
||||
|
||||
css-select@5.2.2:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
css-what: 6.2.2
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
nth-check: 2.1.1
|
||||
|
||||
css-what@6.2.2: {}
|
||||
|
||||
cssesc@3.0.0: {}
|
||||
|
||||
cssom@0.5.0: {}
|
||||
|
||||
csstype@3.2.3: {}
|
||||
|
||||
d@1.0.2:
|
||||
|
|
@ -6793,6 +6902,24 @@ snapshots:
|
|||
dependencies:
|
||||
esutils: 2.0.3
|
||||
|
||||
dom-serializer@2.0.0:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
entities: 4.5.0
|
||||
|
||||
domelementtype@2.3.0: {}
|
||||
|
||||
domhandler@5.0.3:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
|
||||
domutils@3.2.2:
|
||||
dependencies:
|
||||
dom-serializer: 2.0.0
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
|
||||
dotenv@17.4.2: {}
|
||||
|
||||
drizzle-kit@0.31.10:
|
||||
|
|
@ -6828,6 +6955,10 @@ snapshots:
|
|||
dependencies:
|
||||
once: 1.4.0
|
||||
|
||||
entities@4.5.0: {}
|
||||
|
||||
entities@7.0.1: {}
|
||||
|
||||
env-paths@2.2.1: {}
|
||||
|
||||
epubjs@0.3.93:
|
||||
|
|
@ -7575,8 +7706,17 @@ snapshots:
|
|||
|
||||
howler@2.2.4: {}
|
||||
|
||||
html-escaper@3.0.3: {}
|
||||
|
||||
html-url-attributes@3.0.1: {}
|
||||
|
||||
htmlparser2@10.1.0:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
entities: 7.0.1
|
||||
|
||||
http-response-object@3.0.2:
|
||||
dependencies:
|
||||
'@types/node': 10.17.60
|
||||
|
|
@ -7901,6 +8041,16 @@ snapshots:
|
|||
|
||||
lines-and-columns@1.2.4: {}
|
||||
|
||||
linkedom@0.18.12(canvas@3.2.3):
|
||||
dependencies:
|
||||
css-select: 5.2.2
|
||||
cssom: 0.5.0
|
||||
html-escaper: 3.0.3
|
||||
htmlparser2: 10.1.0
|
||||
uhyphen: 0.2.0
|
||||
optionalDependencies:
|
||||
canvas: 3.2.3
|
||||
|
||||
localforage@1.10.0:
|
||||
dependencies:
|
||||
lie: 3.1.1
|
||||
|
|
@ -8380,6 +8530,10 @@ snapshots:
|
|||
|
||||
normalize-path@3.0.0: {}
|
||||
|
||||
nth-check@2.1.1:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
|
||||
object-assign@4.1.1: {}
|
||||
|
||||
object-hash@3.0.0: {}
|
||||
|
|
@ -9421,6 +9575,10 @@ snapshots:
|
|||
dependencies:
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
turndown@7.2.4:
|
||||
dependencies:
|
||||
'@mixmark-io/domino': 2.2.0
|
||||
|
||||
tweetnacl@1.0.3: {}
|
||||
|
||||
type-check@0.4.0:
|
||||
|
|
@ -9468,6 +9626,8 @@ snapshots:
|
|||
|
||||
typescript@5.9.3: {}
|
||||
|
||||
uhyphen@0.2.0: {}
|
||||
|
||||
unbox-primitive@1.1.0:
|
||||
dependencies:
|
||||
call-bound: 1.0.4
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ export default function HTMLPage() {
|
|||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [activeSidebar, setActiveSidebar] = useState<null | 'settings' | 'segments' | 'audiobook'>(null);
|
||||
const [containerHeight, setContainerHeight] = useState<string>('auto');
|
||||
const [padPct, setPadPct] = useState<number>(100); // 0..100 (100 = full width)
|
||||
const [padPct, setPadPct] = useState<number>(50); // 0..100 (50 = 50% default width)
|
||||
const [maxPadPx, setMaxPadPx] = useState<number>(0);
|
||||
const inFlightDocIdRef = useRef<string | null>(null);
|
||||
const loadedDocIdRef = useRef<string | null>(null);
|
||||
|
|
|
|||
60
src/app/api/documents/import-url/route.ts
Normal file
60
src/app/api/documents/import-url/route.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { fetchAndParseUrl } from '@/lib/server/documents/web-loader';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
// 1. Authenticate user request
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
// 2. Validate URL body parameter
|
||||
let body;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Invalid request body' }, { status: 400 });
|
||||
}
|
||||
|
||||
const { url } = body;
|
||||
if (!url || typeof url !== 'string') {
|
||||
return NextResponse.json({ error: 'Missing or invalid url parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
// 3. Fetch webpage, scrape readable elements, convert to markdown
|
||||
const result = await fetchAndParseUrl(url);
|
||||
|
||||
// 4. Return title and parsed markdown content
|
||||
return NextResponse.json(result);
|
||||
} catch (error) {
|
||||
serverLogger.error(
|
||||
{
|
||||
event: 'documents.import-url.failed',
|
||||
error: errorToLog(error),
|
||||
},
|
||||
'Failed to import and parse document from web URL'
|
||||
);
|
||||
|
||||
const errorMessage = error instanceof Error ? error.message : 'Failed to import URL';
|
||||
|
||||
// Check if it's a validation error or network error to set suitable status codes
|
||||
const isValidationError =
|
||||
errorMessage.includes('forbidden') ||
|
||||
errorMessage.includes('Invalid URL') ||
|
||||
errorMessage.includes('Only HTTP') ||
|
||||
errorMessage.includes('exceeds the maximum') ||
|
||||
errorMessage.includes('Unsupported content type');
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: errorMessage },
|
||||
{ status: isValidationError ? 400 : 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -22,7 +22,9 @@ import { ConfirmDialog } from '@/components/ConfirmDialog';
|
|||
import { CreateFolderDialog } from '@/components/doclist/CreateFolderDialog';
|
||||
import { DocumentListSkeleton } from '@/components/doclist/DocumentListSkeleton';
|
||||
import { DocumentUploader, type UploadBatchState } from '@/components/documents/DocumentUploader';
|
||||
import { IconButton } from '@/components/ui';
|
||||
import { IconButton, SidebarNavItem } from '@/components/ui';
|
||||
import { UploadIcon } from '@/components/icons/Icons';
|
||||
import { UploadMenuDialog } from '@/components/documents/UploadMenuDialog';
|
||||
import { DocumentDndProvider } from './dnd/DocumentDndProvider';
|
||||
import {
|
||||
DocumentSelectionProvider,
|
||||
|
|
@ -209,6 +211,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
const [mobileSidebarOpen, setMobileSidebarOpen] = useState(false);
|
||||
const [query, setQuery] = useState('');
|
||||
const [activeUploadBatches, setActiveUploadBatches] = useState<Record<string, UploadBatchState>>({});
|
||||
const [isUploadDialogOpen, setIsUploadDialogOpen] = useState(false);
|
||||
|
||||
const [isInitialized, setIsInitialized] = useState(cachedState !== null);
|
||||
|
||||
|
|
@ -619,7 +622,14 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
onDropOnFolder={handleDropOnFolder}
|
||||
width={sidebarWidth}
|
||||
onWidthChange={setSidebarWidth}
|
||||
topSlot={<DocumentUploader variant="compact" onUploadBatchChange={handleUploadBatchChange} />}
|
||||
topSlot={(
|
||||
<SidebarNavItem
|
||||
compact
|
||||
onClick={() => setIsUploadDialogOpen(true)}
|
||||
icon={<UploadIcon className="w-3.5 h-3.5" />}
|
||||
label="Upload documents"
|
||||
/>
|
||||
)}
|
||||
bottomSlot={(
|
||||
<div className="flex flex-col gap-2">
|
||||
{sidebarUploadState && (
|
||||
|
|
@ -775,6 +785,12 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
confirmText="Remove Folders"
|
||||
isDangerous
|
||||
/>
|
||||
|
||||
<UploadMenuDialog
|
||||
isOpen={isUploadDialogOpen}
|
||||
onClose={() => setIsUploadDialogOpen(false)}
|
||||
onUploadBatchChange={handleUploadBatchChange}
|
||||
/>
|
||||
</FinderWindow>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -288,7 +288,7 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
|||
<div className="absolute inset-0 bg-gradient-to-br from-surface-solid to-surface-sunken" />
|
||||
<div className="absolute inset-0 opacity-[0.06] bg-[radial-gradient(circle_at_1px_1px,rgba(0,0,0,1)_1px,transparent_0)] [background-size:12px_12px]" />
|
||||
<div className="relative z-10 h-full w-full p-2 flex flex-col">
|
||||
<div className="mt-auto rounded-md bg-surface-solid backdrop-blur-[1px] shadow-elev-1 ring-1 ring-line-soft p-2.5 max-h-[70%] overflow-hidden">
|
||||
<div className="mb-auto rounded-md bg-surface-solid backdrop-blur-[1px] shadow-elev-1 ring-1 ring-line-soft p-2.5 max-h-[85%] overflow-hidden">
|
||||
{isTxtFile ? (
|
||||
<pre className="text-[10px] sm:text-[11px] leading-snug text-foreground whitespace-pre-wrap font-mono">
|
||||
{textPreview}
|
||||
|
|
|
|||
447
src/components/documents/UploadMenuDialog.tsx
Normal file
447
src/components/documents/UploadMenuDialog.tsx
Normal file
|
|
@ -0,0 +1,447 @@
|
|||
'use client';
|
||||
|
||||
import { useState, useCallback } from 'react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { useDocuments } from '@/contexts/DocumentContext';
|
||||
import { importUrl } from '@/lib/client/api/documents';
|
||||
import {
|
||||
ModalFrame,
|
||||
ModalTitle,
|
||||
SidebarNav,
|
||||
SidebarNavItem,
|
||||
SegmentedControl,
|
||||
Input,
|
||||
Textarea,
|
||||
Button,
|
||||
} from '@/components/ui';
|
||||
import {
|
||||
UploadIcon,
|
||||
FileIcon,
|
||||
RefreshIcon,
|
||||
CheckIcon,
|
||||
BrowserIcon,
|
||||
} from '@/components/icons/Icons';
|
||||
import { DocumentUploader, type UploadBatchState } from './DocumentUploader';
|
||||
|
||||
interface UploadMenuDialogProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
onUploadBatchChange?: (state: UploadBatchState) => void;
|
||||
}
|
||||
|
||||
type TabValue = 'file' | 'create' | 'url';
|
||||
|
||||
type SidebarSection = {
|
||||
id: TabValue;
|
||||
label: string;
|
||||
icon: React.ComponentType<React.SVGProps<SVGSVGElement>>;
|
||||
};
|
||||
|
||||
const SIDEBAR_SECTIONS: SidebarSection[] = [
|
||||
{ id: 'file', label: 'Upload Files', icon: UploadIcon },
|
||||
{ id: 'create', label: 'Create Document', icon: FileIcon },
|
||||
{ id: 'url', label: 'Import from Web', icon: BrowserIcon },
|
||||
];
|
||||
|
||||
export function UploadMenuDialog({
|
||||
isOpen,
|
||||
onClose,
|
||||
onUploadBatchChange,
|
||||
}: UploadMenuDialogProps) {
|
||||
const { uploadDocuments } = useDocuments();
|
||||
const [activeTab, setActiveTab] = useState<TabValue>('file');
|
||||
|
||||
// --- Create Text/Markdown State ---
|
||||
const [docName, setDocName] = useState('');
|
||||
const [docExtension, setDocExtension] = useState<'.md' | '.txt'>('.md');
|
||||
const [docContent, setDocContent] = useState('');
|
||||
const [isCreatingDoc, setIsCreatingDoc] = useState(false);
|
||||
|
||||
// --- Import URL State ---
|
||||
const [webUrl, setWebUrl] = useState('');
|
||||
const [importStep, setImportStep] = useState<
|
||||
'idle' | 'fetching' | 'converting' | 'uploading' | 'error'
|
||||
>('idle');
|
||||
const [importError, setImportError] = useState<string | null>(null);
|
||||
|
||||
const handleTabChange = useCallback((tab: TabValue) => {
|
||||
setActiveTab(tab);
|
||||
setImportError(null);
|
||||
setImportStep('idle');
|
||||
}, []);
|
||||
|
||||
// --- Create Doc Action ---
|
||||
const handleCreateDocument = async () => {
|
||||
if (!docName.trim()) {
|
||||
toast.error('Please enter a document name');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsCreatingDoc(true);
|
||||
try {
|
||||
let filename = docName.trim();
|
||||
const ext = docExtension;
|
||||
if (!filename.toLowerCase().endsWith(ext)) {
|
||||
filename += ext;
|
||||
}
|
||||
|
||||
const mimeType = ext === '.md' ? 'text/markdown' : 'text/plain';
|
||||
const file = new File([docContent], filename, { type: mimeType });
|
||||
|
||||
await uploadDocuments([file]);
|
||||
toast.success(`"${filename}" created successfully!`);
|
||||
|
||||
// Reset inputs & close
|
||||
setDocName('');
|
||||
setDocContent('');
|
||||
onClose();
|
||||
} catch (err) {
|
||||
console.error('Failed to create document:', err);
|
||||
toast.error(err instanceof Error ? err.message : 'Failed to create document');
|
||||
} finally {
|
||||
setIsCreatingDoc(false);
|
||||
}
|
||||
};
|
||||
|
||||
// --- Import URL Action ---
|
||||
const handleImportUrl = async () => {
|
||||
let cleanUrl = webUrl.trim();
|
||||
if (!cleanUrl) {
|
||||
toast.error('Please enter a URL');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!/^https?:\/\//i.test(cleanUrl)) {
|
||||
cleanUrl = 'https://' + cleanUrl;
|
||||
}
|
||||
|
||||
setImportError(null);
|
||||
setImportStep('fetching');
|
||||
|
||||
try {
|
||||
// Step 1: Connect and scrape content on the server
|
||||
const scrapeResult = await importUrl(cleanUrl);
|
||||
|
||||
setImportStep('converting');
|
||||
// Simulated processing time for micro-interaction polish
|
||||
await new Promise((resolve) => setTimeout(resolve, 800));
|
||||
|
||||
setImportStep('uploading');
|
||||
|
||||
// Create virtual file from markdown content
|
||||
const safeTitle = (scrapeResult.title || 'Imported Web Page')
|
||||
.replace(/[/\\?%*:|"<>\s]/g, '_')
|
||||
.substring(0, 80);
|
||||
const filename = `${safeTitle}.md`;
|
||||
const file = new File([scrapeResult.content], filename, {
|
||||
type: 'text/markdown',
|
||||
});
|
||||
|
||||
await uploadDocuments([file]);
|
||||
|
||||
// Success
|
||||
toast.success(`Successfully imported "${scrapeResult.title}"!`);
|
||||
setWebUrl('');
|
||||
setImportStep('idle');
|
||||
onClose();
|
||||
} catch (err) {
|
||||
console.error('Failed to import URL:', err);
|
||||
const message =
|
||||
err instanceof Error ? err.message : 'An error occurred during import';
|
||||
setImportError(message);
|
||||
setImportStep('error');
|
||||
toast.error(message);
|
||||
}
|
||||
};
|
||||
|
||||
const handleClose = () => {
|
||||
// Only allow closing if not actively processing
|
||||
if (isCreatingDoc || importStep === 'fetching' || importStep === 'converting' || importStep === 'uploading') {
|
||||
return;
|
||||
}
|
||||
onClose();
|
||||
};
|
||||
|
||||
return (
|
||||
<ModalFrame open={isOpen} onClose={handleClose} size="xl">
|
||||
<div className="flex flex-col h-[480px]">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between px-5 py-3 border-b border-line-soft shrink-0">
|
||||
<ModalTitle>Add Documents</ModalTitle>
|
||||
<button
|
||||
onClick={handleClose}
|
||||
className="rounded-md p-1 text-soft hover:bg-accent-wash hover:text-foreground transition-colors duration-base"
|
||||
aria-label="Close dialog"
|
||||
>
|
||||
<svg className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Mobile Navigation (Horizontal row of items) */}
|
||||
<SidebarNav layout="grid" className="sm:hidden border-b border-line-soft bg-background p-2 shrink-0">
|
||||
{SIDEBAR_SECTIONS.map((section) => {
|
||||
const Icon = section.icon;
|
||||
return (
|
||||
<SidebarNavItem
|
||||
compact
|
||||
key={section.id}
|
||||
onClick={() => handleTabChange(section.id)}
|
||||
active={activeTab === section.id}
|
||||
icon={<Icon className="w-3.5 h-3.5" />}
|
||||
label={section.label}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</SidebarNav>
|
||||
|
||||
{/* Main Body (Split layout on desktop) */}
|
||||
<div className="flex flex-row min-h-0 flex-1">
|
||||
{/* Desktop Navigation (Left sidebar) */}
|
||||
<nav className="hidden sm:block w-fit shrink-0 border-r border-line-soft bg-background p-2">
|
||||
<SidebarNav>
|
||||
{SIDEBAR_SECTIONS.map((section) => {
|
||||
const Icon = section.icon;
|
||||
const active = activeTab === section.id;
|
||||
return (
|
||||
<SidebarNavItem
|
||||
key={section.id}
|
||||
onClick={() => handleTabChange(section.id)}
|
||||
active={active}
|
||||
icon={<Icon className="w-4 h-4" />}
|
||||
label={section.label}
|
||||
className="whitespace-nowrap"
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</SidebarNav>
|
||||
</nav>
|
||||
|
||||
{/* Tab Panel Content (Right detail view) */}
|
||||
<div className="flex-1 min-w-0 p-4 overflow-y-auto bg-surface">
|
||||
|
||||
{/* TAB 1: File Uploader */}
|
||||
{activeTab === 'file' && (
|
||||
<div className="h-full flex flex-col gap-4 animate-fade-in">
|
||||
<p className="text-xs text-soft leading-relaxed shrink-0">
|
||||
Select files from your computer or drag and drop them anywhere. Supported formats include PDF, EPUB, TXT, and MD.
|
||||
</p>
|
||||
<DocumentUploader
|
||||
className="flex-1 flex flex-col justify-center border-2 border-dashed border-line rounded-lg bg-surface-sunken hover:bg-surface-solid transition-colors duration-base"
|
||||
onUploadBatchChange={(state) => {
|
||||
onUploadBatchChange?.(state);
|
||||
// Automatically close modal when files successfully start uploading
|
||||
if (state.isActive) {
|
||||
onClose();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* TAB 2: Create Custom Document */}
|
||||
{activeTab === 'create' && (
|
||||
<div className="h-full flex flex-col justify-between animate-fade-in">
|
||||
<div className="space-y-3 flex-1 flex flex-col min-h-0">
|
||||
<div className="flex flex-col sm:flex-row gap-2 shrink-0">
|
||||
<div className="flex-1">
|
||||
<Input
|
||||
type="text"
|
||||
value={docName}
|
||||
onChange={(e) => setDocName(e.target.value)}
|
||||
placeholder="Document title (e.g. notes)"
|
||||
className="w-full font-medium"
|
||||
controlSize="md"
|
||||
disabled={isCreatingDoc}
|
||||
/>
|
||||
</div>
|
||||
<div className="shrink-0">
|
||||
<SegmentedControl<'.md' | '.txt'>
|
||||
ariaLabel="Document format selection"
|
||||
value={docExtension}
|
||||
options={[
|
||||
{ value: '.md', label: 'Markdown (.md)' },
|
||||
{ value: '.txt', label: 'Plain Text (.txt)' },
|
||||
]}
|
||||
onChange={setDocExtension}
|
||||
className="grid-cols-2"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex-1 flex flex-col min-h-0">
|
||||
<Textarea
|
||||
value={docContent}
|
||||
onChange={(e) => setDocContent(e.target.value)}
|
||||
placeholder={
|
||||
docExtension === '.md'
|
||||
? '# Write Markdown here\n\n- Bullet points\n- **Bold text**\n- Synchronized highlighting works automatically!'
|
||||
: 'Type plain text document content here...'
|
||||
}
|
||||
className="flex-1 min-h-[160px] sm:min-h-[200px] font-mono text-xs p-3 leading-relaxed resize-none"
|
||||
controlSize="md"
|
||||
disabled={isCreatingDoc}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between pt-3 mt-4 shrink-0">
|
||||
<span className="text-[11px] text-soft">
|
||||
{docContent.length} character{docContent.length === 1 ? '' : 's'}
|
||||
</span>
|
||||
<div className="flex gap-2">
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
setDocName('');
|
||||
setDocContent('');
|
||||
}}
|
||||
disabled={isCreatingDoc}
|
||||
>
|
||||
Clear
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={handleCreateDocument}
|
||||
disabled={isCreatingDoc}
|
||||
>
|
||||
{isCreatingDoc ? 'Creating...' : 'Create'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* TAB 3: Import Web URL */}
|
||||
{activeTab === 'url' && (
|
||||
<div className="h-full flex flex-col justify-between animate-fade-in">
|
||||
<div className="space-y-4 flex-1">
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<label htmlFor="web-url" className="text-xs font-semibold text-foreground">
|
||||
Source Web URL
|
||||
</label>
|
||||
<div className="flex gap-2">
|
||||
<Input
|
||||
id="web-url"
|
||||
type="url"
|
||||
value={webUrl}
|
||||
onChange={(e) => setWebUrl(e.target.value)}
|
||||
placeholder="https://en.wikipedia.org/wiki/Speed_reading"
|
||||
className="flex-1"
|
||||
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
handleImportUrl();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={handleImportUrl}
|
||||
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||
>
|
||||
Import
|
||||
</Button>
|
||||
</div>
|
||||
<p className="text-[11px] text-soft leading-normal mt-1">
|
||||
Extracts the central article body, removes boilerplate noise (headers, sidebars, ads), and converts it into a clean Markdown document for synchrony reading.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Progress / Loading Stepper */}
|
||||
{importStep !== 'idle' && (
|
||||
<div className="rounded-lg border border-line bg-surface-sunken p-4 flex flex-col gap-3.5 transition-all duration-base">
|
||||
{importStep === 'error' ? (
|
||||
<div className="flex items-start gap-2 text-danger">
|
||||
<svg className="h-5 w-5 shrink-0 mt-0.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
|
||||
</svg>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-sm font-semibold">Import Failed</p>
|
||||
<p className="text-xs mt-1 text-danger-strong overflow-hidden text-ellipsis leading-relaxed">
|
||||
{importError || 'An unknown error occurred.'}
|
||||
</p>
|
||||
<button
|
||||
onClick={() => setImportStep('idle')}
|
||||
className="text-[11px] font-medium text-accent hover:underline mt-2 flex items-center gap-1"
|
||||
>
|
||||
<RefreshIcon className="h-3 w-3" /> Try Again
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center gap-3">
|
||||
<RefreshIcon className="h-5 w-5 text-accent animate-spin" />
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-foreground">Importing Article</p>
|
||||
<p className="text-xs text-soft">Please hold on, converting document...</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Visual Progress Steps */}
|
||||
<div className="grid grid-cols-3 gap-2 mt-2 pt-2 text-center text-[10px] font-medium">
|
||||
<div
|
||||
className={`flex flex-col items-center gap-1 ${
|
||||
importStep === 'fetching' ? 'text-accent' : 'text-soft'
|
||||
}`}
|
||||
>
|
||||
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||
importStep === 'fetching'
|
||||
? 'bg-accent text-background'
|
||||
: importStep === 'converting' || importStep === 'uploading'
|
||||
? 'bg-accent-wash text-accent'
|
||||
: 'bg-surface border border-line'
|
||||
}`}>
|
||||
{importStep === 'converting' || importStep === 'uploading' ? (
|
||||
<CheckIcon className="h-2.5 w-2.5" />
|
||||
) : '1'}
|
||||
</span>
|
||||
Scraping Page
|
||||
</div>
|
||||
<div
|
||||
className={`flex flex-col items-center gap-1 ${
|
||||
importStep === 'converting' ? 'text-accent' : 'text-soft'
|
||||
}`}
|
||||
>
|
||||
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||
importStep === 'converting'
|
||||
? 'bg-accent text-background'
|
||||
: importStep === 'uploading'
|
||||
? 'bg-accent-wash text-accent'
|
||||
: 'bg-surface border border-line'
|
||||
}`}>
|
||||
{importStep === 'uploading' ? (
|
||||
<CheckIcon className="h-2.5 w-2.5" />
|
||||
) : '2'}
|
||||
</span>
|
||||
Extracting Text
|
||||
</div>
|
||||
<div
|
||||
className={`flex flex-col items-center gap-1 ${
|
||||
importStep === 'uploading' ? 'text-accent' : 'text-soft'
|
||||
}`}
|
||||
>
|
||||
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||
importStep === 'uploading' ? 'bg-accent text-background animate-pulse' : 'bg-surface border border-line'
|
||||
}`}>
|
||||
3
|
||||
</span>
|
||||
Uploading
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ModalFrame>
|
||||
);
|
||||
}
|
||||
|
|
@ -654,3 +654,28 @@ export function SparkleIcon(props: React.SVGProps<SVGSVGElement>) {
|
|||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
export function BrowserIcon(props: React.SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth={2}
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={props.className}
|
||||
width={props.width || "1.5em"}
|
||||
height={props.height || "1.5em"}
|
||||
{...props}
|
||||
>
|
||||
<rect x="3" y="4" width="18" height="16" rx="2" />
|
||||
<line x1="3" y1="9" x2="21" y2="9" />
|
||||
<circle cx="6" cy="6.5" r="1" fill="currentColor" />
|
||||
<circle cx="9" cy="6.5" r="1" fill="currentColor" />
|
||||
<circle cx="12" cy="6.5" r="1" fill="currentColor" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -587,3 +587,23 @@ export async function getDocumentPreviewStatus(
|
|||
|
||||
throw new Error(`Failed to load preview status (status ${res.status})`);
|
||||
}
|
||||
|
||||
export async function importUrl(
|
||||
url: string,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<{ title: string; content: string }> {
|
||||
const res = await fetch('/api/documents/import-url', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url }),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to import URL');
|
||||
}
|
||||
|
||||
return (await res.json()) as { title: string; content: string };
|
||||
}
|
||||
|
||||
|
|
|
|||
227
src/lib/server/documents/web-loader.ts
Normal file
227
src/lib/server/documents/web-loader.ts
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import dns from 'dns';
|
||||
import { promisify } from 'util';
|
||||
import { parseHTML } from 'linkedom';
|
||||
import { Readability } from '@mozilla/readability';
|
||||
import TurndownService from 'turndown';
|
||||
|
||||
const dnsLookup = promisify(dns.lookup);
|
||||
|
||||
/**
|
||||
* Checks if an IP address is a private, loopback, or link-local address (SSRF mitigation).
|
||||
*/
|
||||
function isPrivateIp(ip: string): boolean {
|
||||
// IPv4 checks
|
||||
if (ip === '127.0.0.1' || ip === '0.0.0.0') return true;
|
||||
|
||||
const ipv4Match = ip.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
|
||||
if (ipv4Match) {
|
||||
const [, o1, o2, o3, o4] = ipv4Match.map(Number);
|
||||
if (o1 === 10) return true; // 10.0.0.0/8
|
||||
if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12
|
||||
if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16
|
||||
if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16
|
||||
}
|
||||
|
||||
// IPv6 checks
|
||||
const ipLower = ip.toLowerCase();
|
||||
if (
|
||||
ipLower === '::1' ||
|
||||
ipLower === '::' ||
|
||||
ipLower === '0:0:0:0:0:0:0:1'
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
ipLower.startsWith('fe80:') ||
|
||||
ipLower.startsWith('fc00:') ||
|
||||
ipLower.startsWith('fd00:')
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the URL domain and validates that it does not point to a local/private network.
|
||||
*/
|
||||
async function validateUrlForSsrf(urlString: string): Promise<string> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(urlString);
|
||||
} catch {
|
||||
throw new Error('Invalid URL format');
|
||||
}
|
||||
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
||||
throw new Error('Only HTTP and HTTPS protocols are supported');
|
||||
}
|
||||
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
|
||||
if (hostname === 'localhost') {
|
||||
throw new Error('Access to localhost is forbidden');
|
||||
}
|
||||
|
||||
try {
|
||||
const lookupResult = await dnsLookup(hostname);
|
||||
if (isPrivateIp(lookupResult.address)) {
|
||||
throw new Error(`Access to private network address is forbidden`);
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes('forbidden')) {
|
||||
throw err;
|
||||
}
|
||||
// If lookup fails entirely, let the fetch attempt fail or resolve naturally
|
||||
}
|
||||
|
||||
return urlString;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a URL with a size limit (default 2MB) and connection timeout (default 6s).
|
||||
*/
|
||||
async function fetchWithLimit(
|
||||
urlStr: string,
|
||||
maxBytes: number = 2 * 1024 * 1024,
|
||||
timeoutMs: number = 6000
|
||||
): Promise<string> {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const res = await fetch(urlStr, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
},
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to fetch webpage (status ${res.status})`);
|
||||
}
|
||||
|
||||
const contentType = res.headers.get('content-type') || '';
|
||||
const isHtml =
|
||||
contentType.includes('text/html') ||
|
||||
contentType.includes('application/xhtml+xml');
|
||||
const isText = contentType.includes('text/plain');
|
||||
|
||||
if (!isHtml && !isText) {
|
||||
throw new Error('Unsupported content type. Only web pages or text files are supported.');
|
||||
}
|
||||
|
||||
const contentLength = res.headers.get('content-length');
|
||||
if (contentLength && parseInt(contentLength, 10) > maxBytes) {
|
||||
throw new Error('Web page size exceeds the maximum allowed limit of 2MB.');
|
||||
}
|
||||
|
||||
// Read response body chunks to verify size limit dynamically in case Content-Length header is missing/spoofed
|
||||
const reader = res.body?.getReader();
|
||||
if (!reader) {
|
||||
// Fallback for environments lacking full stream reader support
|
||||
const text = await res.text();
|
||||
if (Buffer.byteLength(text) > maxBytes) {
|
||||
throw new Error('Web page size exceeds the maximum allowed limit of 2MB.');
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
let bytesReceived = 0;
|
||||
const chunks: Uint8Array[] = [];
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
if (value) {
|
||||
bytesReceived += value.length;
|
||||
if (bytesReceived > maxBytes) {
|
||||
controller.abort();
|
||||
throw new Error('Web page size exceeds the maximum allowed limit of 2MB.');
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
const merged = new Uint8Array(bytesReceived);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
merged.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
return new TextDecoder().decode(merged);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects to, extracts readable content from, and converts a web page to Markdown.
|
||||
*/
|
||||
export async function fetchAndParseUrl(
|
||||
urlStr: string
|
||||
): Promise<{ title: string; content: string }> {
|
||||
// 1. SSRF prevention check
|
||||
await validateUrlForSsrf(urlStr);
|
||||
|
||||
// 2. Fetch HTML content with limit
|
||||
const html = await fetchWithLimit(urlStr);
|
||||
|
||||
// 3. Parse HTML string to a virtual DOM document
|
||||
const { document } = parseHTML(html);
|
||||
|
||||
// Resolve relative URLs for links and images to absolute paths based on source URL base
|
||||
const resolveUrl = (relativeUrl: string) => {
|
||||
try {
|
||||
return new URL(relativeUrl, urlStr).href;
|
||||
} catch {
|
||||
return relativeUrl;
|
||||
}
|
||||
};
|
||||
|
||||
const images = document.querySelectorAll('img');
|
||||
for (const img of images) {
|
||||
const src = img.getAttribute('src');
|
||||
if (src) {
|
||||
img.setAttribute('src', resolveUrl(src));
|
||||
}
|
||||
}
|
||||
|
||||
const links = document.querySelectorAll('a');
|
||||
for (const link of links) {
|
||||
const href = link.getAttribute('href');
|
||||
if (href) {
|
||||
link.setAttribute('href', resolveUrl(href));
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Run Readability to extract core article text
|
||||
const reader = new Readability(document as unknown as Document);
|
||||
const article = reader.parse();
|
||||
|
||||
if (!article || !article.content) {
|
||||
throw new Error('Could not extract meaningful article content from this webpage.');
|
||||
}
|
||||
|
||||
// 5. Convert clean HTML content to Markdown
|
||||
const turndownService = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
hr: '---',
|
||||
bullet: '-',
|
||||
codeBlockStyle: 'fenced',
|
||||
});
|
||||
|
||||
const markdown = turndownService.turndown(article.content);
|
||||
if (!markdown.trim()) {
|
||||
throw new Error('Extracted content is empty.');
|
||||
}
|
||||
|
||||
return {
|
||||
title: article.title?.trim() || 'Web Import',
|
||||
content: markdown,
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue