Merge pull request #116 from richardr1126/feat/ui-improvements
feat(documents): add web URL import, markdown/text creator, and unified sidebar modal dialog
This commit is contained in:
Richard R
GitHub
commit
cecb2489e1
14 changed files with 1151 additions and 94 deletions
|
|
@ -35,6 +35,8 @@
|
|||
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
||||
"@headlessui/react": "^2.2.10",
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"@microsoft/antissrf": "^1.0.0",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"@napi-rs/canvas": "^0.1.100",
|
||||
"@tanstack/react-query": "^5.101.0",
|
||||
"@types/archiver": "^7.0.0",
|
||||
|
|
@ -55,6 +57,7 @@
|
|||
"ffmpeg-static": "^5.3.0",
|
||||
"howler": "^2.2.4",
|
||||
"jszip": "^3.10.1",
|
||||
"linkedom": "^0.18.12",
|
||||
"lru-cache": "^11.5.1",
|
||||
"next": "^15.5.19",
|
||||
"onnxruntime-node": "^1.26.0",
|
||||
|
|
@ -75,6 +78,7 @@
|
|||
"react-reader": "^2.0.15",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"replicate": "^1.4.0",
|
||||
"turndown": "^7.2.4",
|
||||
"uuid": "^11.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -83,10 +87,12 @@
|
|||
"@tailwindcss/typography": "^0.5.19",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/howler": "^2.2.13",
|
||||
"@types/mozilla-readability": "^0.2.1",
|
||||
"@types/node": "^20.19.41",
|
||||
"@types/pg": "^8.20.0",
|
||||
"@types/react": "^19.2.16",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@types/turndown": "^5.0.6",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint-config-next": "^15.5.19",
|
||||
|
|
|
|||
168
pnpm-lock.yaml
168
pnpm-lock.yaml
|
|
@ -23,6 +23,12 @@ importers:
|
|||
'@huggingface/tokenizers':
|
||||
specifier: ^0.1.3
|
||||
version: 0.1.3
|
||||
'@microsoft/antissrf':
|
||||
specifier: ^1.0.0
|
||||
version: 1.0.0
|
||||
'@mozilla/readability':
|
||||
specifier: ^0.6.0
|
||||
version: 0.6.0
|
||||
'@napi-rs/canvas':
|
||||
specifier: ^0.1.100
|
||||
version: 0.1.100
|
||||
|
|
@ -83,6 +89,9 @@ importers:
|
|||
jszip:
|
||||
specifier: ^3.10.1
|
||||
version: 3.10.1
|
||||
linkedom:
|
||||
specifier: ^0.18.12
|
||||
version: 0.18.12(canvas@3.2.3)
|
||||
lru-cache:
|
||||
specifier: ^11.5.1
|
||||
version: 11.5.1
|
||||
|
|
@ -143,6 +152,9 @@ importers:
|
|||
replicate:
|
||||
specifier: ^1.4.0
|
||||
version: 1.4.0
|
||||
turndown:
|
||||
specifier: ^7.2.4
|
||||
version: 7.2.4
|
||||
uuid:
|
||||
specifier: ^11.1.1
|
||||
version: 11.1.1
|
||||
|
|
@ -162,6 +174,9 @@ importers:
|
|||
'@types/howler':
|
||||
specifier: ^2.2.13
|
||||
version: 2.2.13
|
||||
'@types/mozilla-readability':
|
||||
specifier: ^0.2.1
|
||||
version: 0.2.1
|
||||
'@types/node':
|
||||
specifier: ^20.19.41
|
||||
version: 20.19.41
|
||||
|
|
@ -174,6 +189,9 @@ importers:
|
|||
'@types/react-dom':
|
||||
specifier: ^19.2.3
|
||||
version: 19.2.3(@types/react@19.2.16)
|
||||
'@types/turndown':
|
||||
specifier: ^5.0.6
|
||||
version: 5.0.6
|
||||
'@types/uuid':
|
||||
specifier: ^10.0.0
|
||||
version: 10.0.0
|
||||
|
|
@ -1240,6 +1258,16 @@ packages:
|
|||
'@jridgewell/trace-mapping@0.3.31':
|
||||
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
|
||||
|
||||
'@microsoft/antissrf@1.0.0':
|
||||
resolution: {integrity: sha512-AmO1ykSha52Ef/7UXvHgu8ElsGdgcLkF5nTl7YZGyR1AkbmlQYtiVeWp+CsjkFaJzKAH5ofXLZveMmOHwX/7Jw==}
|
||||
|
||||
'@mixmark-io/domino@2.2.0':
|
||||
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
|
||||
|
||||
'@mozilla/readability@0.6.0':
|
||||
resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==}
|
||||
engines: {node: '>=14.0.0'}
|
||||
|
||||
'@napi-rs/canvas-android-arm64@0.1.100':
|
||||
resolution: {integrity: sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==}
|
||||
engines: {node: '>= 10'}
|
||||
|
|
@ -1686,6 +1714,9 @@ packages:
|
|||
'@types/mdast@4.0.4':
|
||||
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
|
||||
|
||||
'@types/mozilla-readability@0.2.1':
|
||||
resolution: {integrity: sha512-RjLpe3MIpj53L+q50DTNlhv6KjHjgajFQwAF4KuxVZTa2+6Cs2UNAxS5awxABfZEHRYjGKeRDB21Zrtnsi911w==}
|
||||
|
||||
'@types/ms@2.1.0':
|
||||
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
|
||||
|
||||
|
|
@ -1709,6 +1740,9 @@ packages:
|
|||
'@types/readdir-glob@1.1.5':
|
||||
resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
|
||||
|
||||
'@types/turndown@5.0.6':
|
||||
resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==}
|
||||
|
||||
'@types/unist@2.0.11':
|
||||
resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
|
||||
|
||||
|
|
@ -2259,6 +2293,9 @@ packages:
|
|||
bl@4.1.0:
|
||||
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
||||
|
||||
boolbase@1.0.0:
|
||||
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
|
||||
|
||||
bowser@2.14.1:
|
||||
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
|
||||
|
||||
|
|
@ -2418,11 +2455,21 @@ packages:
|
|||
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
css-select@5.2.2:
|
||||
resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==}
|
||||
|
||||
css-what@6.2.2:
|
||||
resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
cssesc@3.0.0:
|
||||
resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
|
||||
engines: {node: '>=4'}
|
||||
hasBin: true
|
||||
|
||||
cssom@0.5.0:
|
||||
resolution: {integrity: sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==}
|
||||
|
||||
csstype@3.2.3:
|
||||
resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==}
|
||||
|
||||
|
|
@ -2523,6 +2570,19 @@ packages:
|
|||
resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
dom-serializer@2.0.0:
|
||||
resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}
|
||||
|
||||
domelementtype@2.3.0:
|
||||
resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==}
|
||||
|
||||
domhandler@5.0.3:
|
||||
resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==}
|
||||
engines: {node: '>= 4'}
|
||||
|
||||
domutils@3.2.2:
|
||||
resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}
|
||||
|
||||
dotenv@17.4.2:
|
||||
resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==}
|
||||
engines: {node: '>=12'}
|
||||
|
|
@ -2643,6 +2703,14 @@ packages:
|
|||
end-of-stream@1.4.5:
|
||||
resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==}
|
||||
|
||||
entities@4.5.0:
|
||||
resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
|
||||
engines: {node: '>=0.12'}
|
||||
|
||||
entities@7.0.1:
|
||||
resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==}
|
||||
engines: {node: '>=0.12'}
|
||||
|
||||
env-paths@2.2.1:
|
||||
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
|
||||
engines: {node: '>=6'}
|
||||
|
|
@ -3111,9 +3179,15 @@ packages:
|
|||
howler@2.2.4:
|
||||
resolution: {integrity: sha512-iARIBPgcQrwtEr+tALF+rapJ8qSc+Set2GJQl7xT1MQzWaVkFebdJhR3alVlSiUf5U7nAANKuj3aWpwerocD5w==}
|
||||
|
||||
html-escaper@3.0.3:
|
||||
resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==}
|
||||
|
||||
html-url-attributes@3.0.1:
|
||||
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
|
||||
|
||||
htmlparser2@10.1.0:
|
||||
resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==}
|
||||
|
||||
http-response-object@3.0.2:
|
||||
resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}
|
||||
|
||||
|
|
@ -3463,6 +3537,15 @@ packages:
|
|||
lines-and-columns@1.2.4:
|
||||
resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
|
||||
|
||||
linkedom@0.18.12:
|
||||
resolution: {integrity: sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==}
|
||||
engines: {node: '>=16'}
|
||||
peerDependencies:
|
||||
canvas: '>= 2'
|
||||
peerDependenciesMeta:
|
||||
canvas:
|
||||
optional: true
|
||||
|
||||
localforage@1.10.0:
|
||||
resolution: {integrity: sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==}
|
||||
|
||||
|
|
@ -3752,6 +3835,9 @@ packages:
|
|||
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
nth-check@2.1.1:
|
||||
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
|
||||
|
||||
object-assign@4.1.1:
|
||||
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
|
@ -4599,6 +4685,10 @@ packages:
|
|||
tunnel-agent@0.6.0:
|
||||
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
|
||||
|
||||
turndown@7.2.4:
|
||||
resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==}
|
||||
engines: {node: '>=18', npm: '>=9'}
|
||||
|
||||
tweetnacl@1.0.3:
|
||||
resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==}
|
||||
|
||||
|
|
@ -4637,6 +4727,9 @@ packages:
|
|||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
uhyphen@0.2.0:
|
||||
resolution: {integrity: sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==}
|
||||
|
||||
unbox-primitive@1.1.0:
|
||||
resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
|
@ -5669,6 +5762,12 @@ snapshots:
|
|||
'@jridgewell/resolve-uri': 3.1.2
|
||||
'@jridgewell/sourcemap-codec': 1.5.5
|
||||
|
||||
'@microsoft/antissrf@1.0.0': {}
|
||||
|
||||
'@mixmark-io/domino@2.2.0': {}
|
||||
|
||||
'@mozilla/readability@0.6.0': {}
|
||||
|
||||
'@napi-rs/canvas-android-arm64@0.1.100':
|
||||
optional: true
|
||||
|
||||
|
|
@ -6018,6 +6117,8 @@ snapshots:
|
|||
dependencies:
|
||||
'@types/unist': 3.0.3
|
||||
|
||||
'@types/mozilla-readability@0.2.1': {}
|
||||
|
||||
'@types/ms@2.1.0': {}
|
||||
|
||||
'@types/node@10.17.60': {}
|
||||
|
|
@ -6044,6 +6145,8 @@ snapshots:
|
|||
dependencies:
|
||||
'@types/node': 20.19.41
|
||||
|
||||
'@types/turndown@5.0.6': {}
|
||||
|
||||
'@types/unist@2.0.11': {}
|
||||
|
||||
'@types/unist@3.0.3': {}
|
||||
|
|
@ -6542,6 +6645,8 @@ snapshots:
|
|||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
boolbase@1.0.0: {}
|
||||
|
||||
bowser@2.14.1: {}
|
||||
|
||||
brace-expansion@1.1.15:
|
||||
|
|
@ -6699,8 +6804,20 @@ snapshots:
|
|||
shebang-command: 2.0.0
|
||||
which: 2.0.2
|
||||
|
||||
css-select@5.2.2:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
css-what: 6.2.2
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
nth-check: 2.1.1
|
||||
|
||||
css-what@6.2.2: {}
|
||||
|
||||
cssesc@3.0.0: {}
|
||||
|
||||
cssom@0.5.0: {}
|
||||
|
||||
csstype@3.2.3: {}
|
||||
|
||||
d@1.0.2:
|
||||
|
|
@ -6793,6 +6910,24 @@ snapshots:
|
|||
dependencies:
|
||||
esutils: 2.0.3
|
||||
|
||||
dom-serializer@2.0.0:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
entities: 4.5.0
|
||||
|
||||
domelementtype@2.3.0: {}
|
||||
|
||||
domhandler@5.0.3:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
|
||||
domutils@3.2.2:
|
||||
dependencies:
|
||||
dom-serializer: 2.0.0
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
|
||||
dotenv@17.4.2: {}
|
||||
|
||||
drizzle-kit@0.31.10:
|
||||
|
|
@ -6828,6 +6963,10 @@ snapshots:
|
|||
dependencies:
|
||||
once: 1.4.0
|
||||
|
||||
entities@4.5.0: {}
|
||||
|
||||
entities@7.0.1: {}
|
||||
|
||||
env-paths@2.2.1: {}
|
||||
|
||||
epubjs@0.3.93:
|
||||
|
|
@ -7575,8 +7714,17 @@ snapshots:
|
|||
|
||||
howler@2.2.4: {}
|
||||
|
||||
html-escaper@3.0.3: {}
|
||||
|
||||
html-url-attributes@3.0.1: {}
|
||||
|
||||
htmlparser2@10.1.0:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
entities: 7.0.1
|
||||
|
||||
http-response-object@3.0.2:
|
||||
dependencies:
|
||||
'@types/node': 10.17.60
|
||||
|
|
@ -7901,6 +8049,16 @@ snapshots:
|
|||
|
||||
lines-and-columns@1.2.4: {}
|
||||
|
||||
linkedom@0.18.12(canvas@3.2.3):
|
||||
dependencies:
|
||||
css-select: 5.2.2
|
||||
cssom: 0.5.0
|
||||
html-escaper: 3.0.3
|
||||
htmlparser2: 10.1.0
|
||||
uhyphen: 0.2.0
|
||||
optionalDependencies:
|
||||
canvas: 3.2.3
|
||||
|
||||
localforage@1.10.0:
|
||||
dependencies:
|
||||
lie: 3.1.1
|
||||
|
|
@ -8380,6 +8538,10 @@ snapshots:
|
|||
|
||||
normalize-path@3.0.0: {}
|
||||
|
||||
nth-check@2.1.1:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
|
||||
object-assign@4.1.1: {}
|
||||
|
||||
object-hash@3.0.0: {}
|
||||
|
|
@ -9421,6 +9583,10 @@ snapshots:
|
|||
dependencies:
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
turndown@7.2.4:
|
||||
dependencies:
|
||||
'@mixmark-io/domino': 2.2.0
|
||||
|
||||
tweetnacl@1.0.3: {}
|
||||
|
||||
type-check@0.4.0:
|
||||
|
|
@ -9468,6 +9634,8 @@ snapshots:
|
|||
|
||||
typescript@5.9.3: {}
|
||||
|
||||
uhyphen@0.2.0: {}
|
||||
|
||||
unbox-primitive@1.1.0:
|
||||
dependencies:
|
||||
call-bound: 1.0.4
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ export default function HTMLPage() {
|
|||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [activeSidebar, setActiveSidebar] = useState<null | 'settings' | 'segments' | 'audiobook'>(null);
|
||||
const [containerHeight, setContainerHeight] = useState<string>('auto');
|
||||
const [padPct, setPadPct] = useState<number>(100); // 0..100 (100 = full width)
|
||||
const [padPct, setPadPct] = useState<number>(50); // 0..100 (50 = 50% default width)
|
||||
const [maxPadPx, setMaxPadPx] = useState<number>(0);
|
||||
const inFlightDocIdRef = useRef<string | null>(null);
|
||||
const loadedDocIdRef = useRef<string | null>(null);
|
||||
|
|
|
|||
57
src/app/api/documents/import-url/route.ts
Normal file
57
src/app/api/documents/import-url/route.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { fetchAndParseUrl } from '@/lib/server/documents/web-loader';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
// 1. Authenticate user request
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
|
||||
// 2. Validate URL body parameter
|
||||
let body;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Invalid request body' }, { status: 400 });
|
||||
}
|
||||
|
||||
const { url } = body;
|
||||
if (!url || typeof url !== 'string') {
|
||||
return NextResponse.json({ error: 'Missing or invalid url parameter' }, { status: 400 });
|
||||
}
|
||||
|
||||
// 3. Fetch webpage, scrape readable elements, convert to markdown
|
||||
const result = await fetchAndParseUrl(url);
|
||||
|
||||
// 4. Return title and parsed markdown content
|
||||
return NextResponse.json(result);
|
||||
} catch (error) {
|
||||
serverLogger.error(
|
||||
{
|
||||
event: 'documents.import-url.failed',
|
||||
error: errorToLog(error),
|
||||
},
|
||||
'Failed to import and parse document from web URL'
|
||||
);
|
||||
|
||||
const errorMessage = error instanceof Error ? error.message : 'Failed to import URL';
|
||||
|
||||
// Check if it's a validation error or network error to set suitable status codes
|
||||
const isValidationError =
|
||||
errorMessage.includes('forbidden') ||
|
||||
errorMessage.includes('disallowed by policy') ||
|
||||
errorMessage.includes('Invalid URL') ||
|
||||
errorMessage.includes('Only HTTP') ||
|
||||
errorMessage.includes('exceeds the maximum') ||
|
||||
errorMessage.includes('Unsupported content type');
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: errorMessage },
|
||||
{ status: isValidationError ? 400 : 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -42,7 +42,7 @@ import { flushUserPreferencesSync } from '@/lib/client/api/user-state';
|
|||
import toast from 'react-hot-toast';
|
||||
import { useLibraryDocumentsQuery } from '@/hooks/useLibraryDocumentsQuery';
|
||||
import {
|
||||
SidebarNav,
|
||||
SidebarDialog,
|
||||
SidebarNavItem,
|
||||
SegmentedControl,
|
||||
Button,
|
||||
|
|
@ -50,8 +50,6 @@ import {
|
|||
IconButton,
|
||||
Input,
|
||||
Textarea,
|
||||
ModalFrame,
|
||||
ModalTitle,
|
||||
Select,
|
||||
} from '@/components/ui';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
|
|
@ -562,90 +560,55 @@ export function SettingsModal({
|
|||
|
||||
return (
|
||||
<>
|
||||
<ModalFrame
|
||||
<SidebarDialog
|
||||
open={isOpen}
|
||||
onClose={resetToCurrent}
|
||||
size="xl"
|
||||
panelTestId="settings-modal"
|
||||
className={isChangelogOpen ? 'z-[90]' : 'z-50'}
|
||||
modalClassName={isChangelogOpen ? 'z-[90]' : 'z-50'}
|
||||
headerTitle={
|
||||
<div className="flex items-baseline gap-4">
|
||||
<span>Settings</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setIsChangelogOpen(true)}
|
||||
className="text-sm font-medium leading-6 text-soft hover:text-accent transition-colors"
|
||||
>
|
||||
{displayVersion ? `v${displayVersion} · Changelog` : 'Changelog'}
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
headerRight={
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => showPrivacyModal()}
|
||||
className="text-sm font-medium text-soft hover:text-accent transition-colors"
|
||||
>
|
||||
Privacy
|
||||
</Button>
|
||||
}
|
||||
showCloseButton={false}
|
||||
sections={visibleSections}
|
||||
activeSectionId={activeSection}
|
||||
onSectionChange={setActiveSection}
|
||||
className="h-[490px]"
|
||||
contentClassName={
|
||||
activeSection === 'admin'
|
||||
? 'bg-[radial-gradient(circle_at_top_right,color-mix(in_srgb,var(--accent),transparent_92%),transparent_35%)]'
|
||||
: ''
|
||||
}
|
||||
customContent={
|
||||
isChangelogOpen ? (
|
||||
<SettingsChangelogPanel
|
||||
appVersion={runtimeConfig.appVersion}
|
||||
manifestUrl={runtimeConfig.changelogFeedUrl}
|
||||
onClose={() => setIsChangelogOpen(false)}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between px-5 py-3 border-b border-line-soft">
|
||||
<div className="flex items-baseline gap-4">
|
||||
<ModalTitle>Settings</ModalTitle>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setIsChangelogOpen(true)}
|
||||
className="text-sm font-medium leading-6 text-soft hover:text-accent transition-colors"
|
||||
>
|
||||
{displayVersion ? `v${displayVersion} · Changelog` : 'Changelog'}
|
||||
</Button>
|
||||
</div>
|
||||
<div className="flex items-center">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => showPrivacyModal()}
|
||||
className="text-sm font-medium text-soft hover:text-accent transition-colors"
|
||||
>
|
||||
Privacy
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isChangelogOpen ? (
|
||||
<SettingsChangelogPanel
|
||||
appVersion={runtimeConfig.appVersion}
|
||||
manifestUrl={runtimeConfig.changelogFeedUrl}
|
||||
onClose={() => setIsChangelogOpen(false)}
|
||||
/>
|
||||
) : (
|
||||
<>
|
||||
{/* Mobile nav */}
|
||||
<SidebarNav layout="grid" className="sm:hidden border-b border-line-soft bg-background p-2">
|
||||
{visibleSections.map((section) => {
|
||||
const Icon = section.icon;
|
||||
return (
|
||||
<SidebarNavItem
|
||||
compact
|
||||
key={section.id}
|
||||
onClick={() => setActiveSection(section.id)}
|
||||
active={activeSection === section.id}
|
||||
icon={<Icon className="w-3.5 h-3.5" />}
|
||||
label={section.label}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</SidebarNav>
|
||||
|
||||
<div className="flex flex-row h-[490px]">
|
||||
{/* Desktop: vertical sidebar */}
|
||||
<nav className="hidden sm:block w-44 shrink-0 border-r border-line-soft bg-background p-2">
|
||||
<SidebarNav>
|
||||
{visibleSections.map((section) => {
|
||||
const Icon = section.icon;
|
||||
const active = activeSection === section.id;
|
||||
return (
|
||||
<SidebarNavItem
|
||||
key={section.id}
|
||||
onClick={() => setActiveSection(section.id)}
|
||||
active={active}
|
||||
icon={<Icon className="w-4 h-4" />}
|
||||
label={section.label}
|
||||
className="whitespace-nowrap"
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</SidebarNav>
|
||||
</nav>
|
||||
|
||||
{/* Content */}
|
||||
<div className={`flex-1 min-w-0 p-3 overflow-y-auto ${
|
||||
activeSection === 'admin'
|
||||
? 'bg-[radial-gradient(circle_at_top_right,color-mix(in_srgb,var(--accent),transparent_92%),transparent_35%)]'
|
||||
: ''
|
||||
}`}>
|
||||
{/* API Section */}
|
||||
{activeSection === 'api' && (
|
||||
<div className="space-y-4">
|
||||
|
|
@ -1174,11 +1137,7 @@ export function SettingsModal({
|
|||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</ModalFrame>
|
||||
</SidebarDialog>
|
||||
|
||||
<ConfirmDialog
|
||||
isOpen={showDeleteAccountConfirm}
|
||||
|
|
@ -1388,7 +1347,7 @@ function SettingsChangelogPanel({
|
|||
{isExpanded && (
|
||||
<div className="pl-6 pr-1 pb-3 pt-1 space-y-2">
|
||||
{body ? (
|
||||
<div className="text-sm text-foreground leading-6 space-y-2 [&_h1]:text-base [&_h1]:font-semibold [&_h2]:text-sm [&_h2]:font-semibold [&_ul]:pl-5 [&_ol]:pl-5 [&_code]:bg-surface-sunken [&_code]:rounded [&_code]:px-1 [&_pre]:bg-surface-sunken [&_pre]:rounded [&_pre]:p-2 [&_pre]:overflow-x-auto">
|
||||
<div className="text-sm text-foreground leading-6 space-y-2 [&_h1]:text-base [&_h1]:font-semibold [&_h2]:text-sm [&_h2]:font-semibold [&_ul]:pl-5 [&_ol]:pl-5 [&_code]:bg-surface-sunken [&_code]:rounded [&_code]:px-1 [&_pre]:bg-surface-sunken [&_pre]:rounded [&_pre]:p-2 [&_pre]:overflow-x-auto [&_a]:text-accent [&_a]:hover:underline [&_a]:transition-colors">
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]}>
|
||||
{body.body || '_No release notes provided._'}
|
||||
</ReactMarkdown>
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import { CreateFolderDialog } from '@/components/doclist/CreateFolderDialog';
|
|||
import { DocumentListSkeleton } from '@/components/doclist/DocumentListSkeleton';
|
||||
import { DocumentUploader, type UploadBatchState } from '@/components/documents/DocumentUploader';
|
||||
import { IconButton } from '@/components/ui';
|
||||
import { UploadMenuDialog } from '@/components/documents/UploadMenuDialog';
|
||||
import { DocumentDndProvider } from './dnd/DocumentDndProvider';
|
||||
import {
|
||||
DocumentSelectionProvider,
|
||||
|
|
@ -209,6 +210,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
const [mobileSidebarOpen, setMobileSidebarOpen] = useState(false);
|
||||
const [query, setQuery] = useState('');
|
||||
const [activeUploadBatches, setActiveUploadBatches] = useState<Record<string, UploadBatchState>>({});
|
||||
const [isUploadDialogOpen, setIsUploadDialogOpen] = useState(false);
|
||||
|
||||
const [isInitialized, setIsInitialized] = useState(cachedState !== null);
|
||||
|
||||
|
|
@ -619,7 +621,13 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
onDropOnFolder={handleDropOnFolder}
|
||||
width={sidebarWidth}
|
||||
onWidthChange={setSidebarWidth}
|
||||
topSlot={<DocumentUploader variant="compact" onUploadBatchChange={handleUploadBatchChange} />}
|
||||
topSlot={(
|
||||
<DocumentUploader
|
||||
variant="compact"
|
||||
onUploadBatchChange={handleUploadBatchChange}
|
||||
onClick={() => setIsUploadDialogOpen(true)}
|
||||
/>
|
||||
)}
|
||||
bottomSlot={(
|
||||
<div className="flex flex-col gap-2">
|
||||
{sidebarUploadState && (
|
||||
|
|
@ -775,6 +783,12 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
confirmText="Remove Folders"
|
||||
isDangerous
|
||||
/>
|
||||
|
||||
<UploadMenuDialog
|
||||
isOpen={isUploadDialogOpen}
|
||||
onClose={() => setIsUploadDialogOpen(false)}
|
||||
onUploadBatchChange={handleUploadBatchChange}
|
||||
/>
|
||||
</FinderWindow>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -286,9 +286,20 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
|||
) : textPreview ? (
|
||||
<>
|
||||
<div className="absolute inset-0 bg-gradient-to-br from-surface-solid to-surface-sunken" />
|
||||
<div className="absolute inset-0 opacity-[0.06] bg-[radial-gradient(circle_at_1px_1px,rgba(0,0,0,1)_1px,transparent_0)] [background-size:12px_12px]" />
|
||||
<div
|
||||
aria-hidden
|
||||
className="pointer-events-none absolute inset-0"
|
||||
style={{
|
||||
backgroundImage:
|
||||
'radial-gradient(color-mix(in srgb, var(--foreground) 14%, transparent) 1px, transparent 1px)',
|
||||
backgroundSize: '12px 12px',
|
||||
opacity: 0.35,
|
||||
WebkitMaskImage: 'radial-gradient(120% 100% at 50% 40%, #000 55%, transparent 100%)',
|
||||
maskImage: 'radial-gradient(120% 100% at 50% 40%, #000 55%, transparent 100%)',
|
||||
}}
|
||||
/>
|
||||
<div className="relative z-10 h-full w-full p-2 flex flex-col">
|
||||
<div className="mt-auto rounded-md bg-surface-solid backdrop-blur-[1px] shadow-elev-1 ring-1 ring-line-soft p-2.5 max-h-[70%] overflow-hidden">
|
||||
<div className="mb-auto rounded-md bg-surface-solid backdrop-blur-[1px] shadow-elev-1 ring-1 ring-line-soft p-2.5 max-h-[85%] overflow-hidden">
|
||||
{isTxtFile ? (
|
||||
<pre className="text-[10px] sm:text-[11px] leading-snug text-foreground whitespace-pre-wrap font-mono">
|
||||
{textPreview}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ interface DocumentUploaderProps {
|
|||
variant?: 'default' | 'compact' | 'overlay';
|
||||
children?: ReactNode;
|
||||
onUploadBatchChange?: (state: UploadBatchState) => void;
|
||||
onClick?: () => void;
|
||||
}
|
||||
|
||||
export interface UploadBatchState {
|
||||
|
|
@ -28,6 +29,7 @@ export function DocumentUploader({
|
|||
variant = 'default',
|
||||
children,
|
||||
onUploadBatchChange,
|
||||
onClick,
|
||||
}: DocumentUploaderProps) {
|
||||
const uploaderId = useId();
|
||||
const enableDocx = useFeatureFlag('enableDocxConversion');
|
||||
|
|
@ -88,8 +90,8 @@ export function DocumentUploader({
|
|||
},
|
||||
multiple: true,
|
||||
disabled: isUploading,
|
||||
noClick: variant === 'overlay',
|
||||
noKeyboard: variant === 'overlay'
|
||||
noClick: variant === 'overlay' || !!onClick,
|
||||
noKeyboard: variant === 'overlay' || !!onClick
|
||||
});
|
||||
|
||||
const isDisabled = isUploading;
|
||||
|
|
@ -132,6 +134,12 @@ export function DocumentUploader({
|
|||
return (
|
||||
<div
|
||||
{...getRootProps()}
|
||||
onClick={(e) => {
|
||||
if (onClick) {
|
||||
e.stopPropagation();
|
||||
onClick();
|
||||
}
|
||||
}}
|
||||
className={dropzoneSurfaceClass({
|
||||
variant: variant === 'compact' ? 'compact' : 'default',
|
||||
active: isDragActive,
|
||||
|
|
|
|||
417
src/components/documents/UploadMenuDialog.tsx
Normal file
417
src/components/documents/UploadMenuDialog.tsx
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
'use client';
|
||||
|
||||
import { useState, useCallback } from 'react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { useDocuments } from '@/contexts/DocumentContext';
|
||||
import { importUrl } from '@/lib/client/api/documents';
|
||||
import {
|
||||
SidebarDialog,
|
||||
SegmentedControl,
|
||||
Input,
|
||||
Textarea,
|
||||
Button,
|
||||
} from '@/components/ui';
|
||||
import {
|
||||
UploadIcon,
|
||||
FileIcon,
|
||||
RefreshIcon,
|
||||
CheckIcon,
|
||||
BrowserIcon,
|
||||
} from '@/components/icons/Icons';
|
||||
import { DocumentUploader, type UploadBatchState } from './DocumentUploader';
|
||||
|
||||
interface UploadMenuDialogProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
onUploadBatchChange?: (state: UploadBatchState) => void;
|
||||
}
|
||||
|
||||
type TabValue = 'file' | 'create' | 'url';
|
||||
|
||||
type SidebarSection = {
|
||||
id: TabValue;
|
||||
label: string;
|
||||
icon: React.ComponentType<React.SVGProps<SVGSVGElement>>;
|
||||
};
|
||||
|
||||
const SIDEBAR_SECTIONS: SidebarSection[] = [
|
||||
{ id: 'file', label: 'Upload Files', icon: UploadIcon },
|
||||
{ id: 'create', label: 'Create Document', icon: FileIcon },
|
||||
{ id: 'url', label: 'Import from Web', icon: BrowserIcon },
|
||||
];
|
||||
|
||||
export function UploadMenuDialog({
|
||||
isOpen,
|
||||
onClose,
|
||||
onUploadBatchChange,
|
||||
}: UploadMenuDialogProps) {
|
||||
const { uploadDocuments } = useDocuments();
|
||||
const [activeTab, setActiveTab] = useState<TabValue>('file');
|
||||
|
||||
// --- Create Text/Markdown State ---
|
||||
const [docName, setDocName] = useState('');
|
||||
const [docExtension, setDocExtension] = useState<'.md' | '.txt'>('.md');
|
||||
const [docContent, setDocContent] = useState('');
|
||||
const [isCreatingDoc, setIsCreatingDoc] = useState(false);
|
||||
|
||||
// --- Import URL State ---
|
||||
const [webUrl, setWebUrl] = useState('');
|
||||
const [webTitle, setWebTitle] = useState('');
|
||||
const [importStep, setImportStep] = useState<
|
||||
'idle' | 'fetching' | 'converting' | 'uploading' | 'error'
|
||||
>('idle');
|
||||
const [importError, setImportError] = useState<string | null>(null);
|
||||
|
||||
const handleTabChange = useCallback((tab: TabValue) => {
|
||||
setActiveTab(tab);
|
||||
setImportError(null);
|
||||
setImportStep('idle');
|
||||
}, []);
|
||||
|
||||
// --- Create Doc Action ---
|
||||
const handleCreateDocument = async () => {
|
||||
if (!docName.trim()) {
|
||||
toast.error('Please enter a document name');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsCreatingDoc(true);
|
||||
try {
|
||||
let filename = docName.trim();
|
||||
const ext = docExtension;
|
||||
if (!filename.toLowerCase().endsWith(ext)) {
|
||||
filename += ext;
|
||||
}
|
||||
|
||||
const mimeType = ext === '.md' ? 'text/markdown' : 'text/plain';
|
||||
const file = new File([docContent], filename, { type: mimeType });
|
||||
|
||||
await uploadDocuments([file]);
|
||||
toast.success(`"${filename}" created successfully!`);
|
||||
|
||||
// Reset inputs & close
|
||||
setDocName('');
|
||||
setDocContent('');
|
||||
onClose();
|
||||
} catch (err) {
|
||||
console.error('Failed to create document:', err);
|
||||
toast.error(err instanceof Error ? err.message : 'Failed to create document');
|
||||
} finally {
|
||||
setIsCreatingDoc(false);
|
||||
}
|
||||
};
|
||||
|
||||
// --- Import URL Action ---
|
||||
const handleImportUrl = async () => {
|
||||
let cleanUrl = webUrl.trim();
|
||||
if (!cleanUrl) {
|
||||
toast.error('Please enter a URL');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!/^https?:\/\//i.test(cleanUrl)) {
|
||||
cleanUrl = 'https://' + cleanUrl;
|
||||
}
|
||||
|
||||
setImportError(null);
|
||||
setImportStep('fetching');
|
||||
|
||||
try {
|
||||
// Step 1: Connect and scrape content on the server
|
||||
const scrapeResult = await importUrl(cleanUrl);
|
||||
|
||||
setImportStep('converting');
|
||||
setImportStep('uploading');
|
||||
|
||||
// Create virtual file from markdown content
|
||||
const displayTitle = webTitle.trim() || scrapeResult.title || 'Imported Web Page';
|
||||
const safeTitle = displayTitle
|
||||
.replace(/[/\\?%*:|"<>\s]+/g, '_') // collapse runs of disallowed chars
|
||||
.replace(/^_+|_+$/g, '') // trim leading/trailing underscores
|
||||
.substring(0, 80);
|
||||
const filename = `${safeTitle}.md`;
|
||||
const file = new File([scrapeResult.content], filename, {
|
||||
type: 'text/markdown',
|
||||
});
|
||||
|
||||
await uploadDocuments([file]);
|
||||
|
||||
// Success
|
||||
toast.success(`Successfully imported "${displayTitle}"!`);
|
||||
setWebUrl('');
|
||||
setWebTitle('');
|
||||
setImportStep('idle');
|
||||
onClose();
|
||||
} catch (err) {
|
||||
console.error('Failed to import URL:', err);
|
||||
const message =
|
||||
err instanceof Error ? err.message : 'An error occurred during import';
|
||||
setImportError(message);
|
||||
setImportStep('error');
|
||||
toast.error(message);
|
||||
}
|
||||
};
|
||||
|
||||
const handleClose = () => {
|
||||
// Only allow closing if not actively processing
|
||||
if (isCreatingDoc || importStep === 'fetching' || importStep === 'converting' || importStep === 'uploading') {
|
||||
return;
|
||||
}
|
||||
onClose();
|
||||
};
|
||||
|
||||
return (
|
||||
<SidebarDialog
|
||||
open={isOpen}
|
||||
onClose={handleClose}
|
||||
headerTitle="Add Documents"
|
||||
sections={SIDEBAR_SECTIONS}
|
||||
activeSectionId={activeTab}
|
||||
onSectionChange={handleTabChange}
|
||||
className="h-[480px]"
|
||||
>
|
||||
{/* TAB 1: File Uploader */}
|
||||
{activeTab === 'file' && (
|
||||
<div className="h-full flex flex-col gap-4 animate-fade-in">
|
||||
<p className="text-xs text-soft leading-relaxed shrink-0">
|
||||
Select files from your computer or drag and drop them anywhere. Supported formats include PDF, EPUB, TXT, and MD.
|
||||
</p>
|
||||
<DocumentUploader
|
||||
className="flex-1 flex flex-col justify-center border-2 border-dashed border-line rounded-lg bg-surface-sunken hover:bg-surface-solid transition-colors duration-base"
|
||||
onUploadBatchChange={(state) => {
|
||||
onUploadBatchChange?.(state);
|
||||
// Automatically close modal when files successfully start uploading
|
||||
if (state.isActive) {
|
||||
onClose();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* TAB 2: Create Custom Document */}
|
||||
{activeTab === 'create' && (
|
||||
<div className="h-full flex flex-col justify-between animate-fade-in">
|
||||
<div className="space-y-3 flex-1 flex flex-col min-h-0">
|
||||
<div className="flex flex-col sm:flex-row gap-2 shrink-0">
|
||||
<div className="flex-1">
|
||||
<Input
|
||||
type="text"
|
||||
value={docName}
|
||||
onChange={(e) => setDocName(e.target.value)}
|
||||
placeholder="Document title (e.g. notes)"
|
||||
className="w-full font-medium"
|
||||
controlSize="md"
|
||||
disabled={isCreatingDoc}
|
||||
/>
|
||||
</div>
|
||||
<div className="shrink-0">
|
||||
<SegmentedControl<'.md' | '.txt'>
|
||||
ariaLabel="Document format selection"
|
||||
value={docExtension}
|
||||
options={[
|
||||
{ value: '.md', label: 'Markdown (.md)' },
|
||||
{ value: '.txt', label: 'Plain Text (.txt)' },
|
||||
]}
|
||||
onChange={setDocExtension}
|
||||
className="grid-cols-2"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex-1 flex flex-col min-h-0">
|
||||
<Textarea
|
||||
value={docContent}
|
||||
onChange={(e) => setDocContent(e.target.value)}
|
||||
placeholder={
|
||||
docExtension === '.md'
|
||||
? '# Write Markdown here\n\n- Bullet points\n- **Bold text**\n- Synchronized highlighting works automatically!'
|
||||
: 'Type plain text document content here...'
|
||||
}
|
||||
className="flex-1 min-h-[160px] sm:min-h-[200px] font-mono text-xs p-3 leading-relaxed resize-none"
|
||||
controlSize="md"
|
||||
disabled={isCreatingDoc}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between pt-3 mt-4 shrink-0">
|
||||
<span className="text-[11px] text-soft">
|
||||
{docContent.length} character{docContent.length === 1 ? '' : 's'}
|
||||
</span>
|
||||
<div className="flex gap-2">
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
setDocName('');
|
||||
setDocContent('');
|
||||
}}
|
||||
disabled={isCreatingDoc}
|
||||
>
|
||||
Clear
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={handleCreateDocument}
|
||||
disabled={isCreatingDoc}
|
||||
>
|
||||
{isCreatingDoc ? 'Creating...' : 'Create'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* TAB 3: Import Web URL */}
|
||||
{activeTab === 'url' && (
|
||||
<div className="h-full flex flex-col justify-between animate-fade-in">
|
||||
<div className="space-y-4 flex-1">
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<label htmlFor="web-url" className="text-xs font-semibold text-foreground">
|
||||
Source Web URL
|
||||
</label>
|
||||
<div className="flex gap-2">
|
||||
<Input
|
||||
id="web-url"
|
||||
type="url"
|
||||
value={webUrl}
|
||||
onChange={(e) => setWebUrl(e.target.value)}
|
||||
placeholder="https://en.wikipedia.org/wiki/Speed_reading"
|
||||
className="flex-1"
|
||||
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
handleImportUrl();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={handleImportUrl}
|
||||
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||
>
|
||||
Import
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<label htmlFor="web-title" className="text-xs font-semibold text-foreground">
|
||||
Document Title (Optional)
|
||||
</label>
|
||||
<Input
|
||||
id="web-title"
|
||||
type="text"
|
||||
value={webTitle}
|
||||
onChange={(e) => setWebTitle(e.target.value)}
|
||||
placeholder="Leave empty to use article title"
|
||||
className="w-full"
|
||||
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
handleImportUrl();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<p className="text-[11px] text-soft leading-normal mt-1">
|
||||
Extracts the central article body, removes boilerplate noise (headers, sidebars, ads), and converts it into a clean Markdown document for synchrony reading.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Progress / Loading Stepper */}
|
||||
{importStep !== 'idle' && (
|
||||
<div className="rounded-lg border border-line bg-surface-sunken p-4 flex flex-col gap-3.5 transition-colors duration-base">
|
||||
{importStep === 'error' ? (
|
||||
<div className="flex items-start gap-2 text-danger">
|
||||
<svg className="h-5 w-5 shrink-0 mt-0.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
|
||||
</svg>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-sm font-semibold">Import Failed</p>
|
||||
<p className="text-xs mt-1 text-danger-strong overflow-hidden text-ellipsis leading-relaxed">
|
||||
{importError || 'An unknown error occurred.'}
|
||||
</p>
|
||||
<button
|
||||
onClick={() => setImportStep('idle')}
|
||||
className="text-[11px] font-medium text-accent hover:underline mt-2 flex items-center gap-1"
|
||||
>
|
||||
<RefreshIcon className="h-3 w-3" /> Try Again
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center gap-3">
|
||||
<RefreshIcon className="h-5 w-5 text-accent animate-spin" />
|
||||
<div>
|
||||
<p className="text-sm font-semibold text-foreground">Importing Article</p>
|
||||
<p className="text-xs text-soft">Please hold on, converting document...</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Visual Progress Steps */}
|
||||
<div className="grid grid-cols-3 gap-2 mt-2 pt-2 text-center text-[10px] font-medium">
|
||||
<div
|
||||
className={`flex flex-col items-center gap-1 ${
|
||||
importStep === 'fetching' ? 'text-accent' : 'text-soft'
|
||||
}`}
|
||||
>
|
||||
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||
importStep === 'fetching'
|
||||
? 'bg-accent text-background'
|
||||
: importStep === 'converting' || importStep === 'uploading'
|
||||
? 'bg-accent-wash text-accent'
|
||||
: 'bg-surface border border-line'
|
||||
}`}>
|
||||
{importStep === 'converting' || importStep === 'uploading' ? (
|
||||
<CheckIcon className="h-2.5 w-2.5" />
|
||||
) : '1'}
|
||||
</span>
|
||||
Scraping Page
|
||||
</div>
|
||||
<div
|
||||
className={`flex flex-col items-center gap-1 ${
|
||||
importStep === 'converting' ? 'text-accent' : 'text-soft'
|
||||
}`}
|
||||
>
|
||||
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||
importStep === 'converting'
|
||||
? 'bg-accent text-background'
|
||||
: importStep === 'uploading'
|
||||
? 'bg-accent-wash text-accent'
|
||||
: 'bg-surface border border-line'
|
||||
}`}>
|
||||
{importStep === 'uploading' ? (
|
||||
<CheckIcon className="h-2.5 w-2.5" />
|
||||
) : '2'}
|
||||
</span>
|
||||
Extracting Text
|
||||
</div>
|
||||
<div
|
||||
className={`flex flex-col items-center gap-1 ${
|
||||
importStep === 'uploading' ? 'text-accent' : 'text-soft'
|
||||
}`}
|
||||
>
|
||||
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||
importStep === 'uploading' ? 'bg-accent text-background animate-pulse' : 'bg-surface border border-line'
|
||||
}`}>
|
||||
3
|
||||
</span>
|
||||
Uploading
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</SidebarDialog>
|
||||
);
|
||||
}
|
||||
|
|
@ -654,3 +654,28 @@ export function SparkleIcon(props: React.SVGProps<SVGSVGElement>) {
|
|||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
export function BrowserIcon(props: React.SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth={2}
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={props.className}
|
||||
width={props.width || "1.5em"}
|
||||
height={props.height || "1.5em"}
|
||||
{...props}
|
||||
>
|
||||
<rect x="3" y="4" width="18" height="16" rx="2" />
|
||||
<line x1="3" y1="9" x2="21" y2="9" />
|
||||
<circle cx="6" cy="6.5" r="1" fill="currentColor" />
|
||||
<circle cx="9" cy="6.5" r="1" fill="currentColor" />
|
||||
<circle cx="12" cy="6.5" r="1" fill="currentColor" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,3 +23,4 @@ export * from './switch';
|
|||
export * from './tokens';
|
||||
export * from './toolbar';
|
||||
export * from './variants';
|
||||
export * from './sidebar-dialog';
|
||||
|
|
|
|||
128
src/components/ui/sidebar-dialog.tsx
Normal file
128
src/components/ui/sidebar-dialog.tsx
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
'use client';
|
||||
|
||||
import { ReactNode } from 'react';
|
||||
import { ModalFrame, ModalTitle, type DialogSize } from './dialog';
|
||||
import { SidebarNav, SidebarNavItem } from './sidebar-nav';
|
||||
import { cn } from './cn';
|
||||
|
||||
export type SidebarDialogSection<T extends string = string> = {
|
||||
id: T;
|
||||
label: string;
|
||||
icon: React.ComponentType<React.SVGProps<SVGSVGElement>>;
|
||||
};
|
||||
|
||||
export function SidebarDialog<T extends string = string>({
|
||||
open,
|
||||
onClose,
|
||||
size = 'xl',
|
||||
panelTestId,
|
||||
modalClassName,
|
||||
headerTitle,
|
||||
headerRight,
|
||||
showCloseButton = true,
|
||||
sections,
|
||||
activeSectionId,
|
||||
onSectionChange,
|
||||
children,
|
||||
className,
|
||||
contentClassName,
|
||||
customContent,
|
||||
}: {
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
size?: DialogSize;
|
||||
panelTestId?: string;
|
||||
modalClassName?: string;
|
||||
headerTitle: ReactNode;
|
||||
headerRight?: ReactNode;
|
||||
showCloseButton?: boolean;
|
||||
sections: SidebarDialogSection<T>[];
|
||||
activeSectionId: T;
|
||||
onSectionChange: (id: T) => void;
|
||||
children: ReactNode;
|
||||
className?: string;
|
||||
contentClassName?: string;
|
||||
customContent?: ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<ModalFrame
|
||||
open={open}
|
||||
onClose={onClose}
|
||||
size={size}
|
||||
panelTestId={panelTestId}
|
||||
className={modalClassName}
|
||||
>
|
||||
{customContent ? (
|
||||
customContent
|
||||
) : (
|
||||
<div className={cn('flex flex-col h-[480px]', className)}>
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between px-5 py-3 border-b border-line-soft shrink-0">
|
||||
<div className="flex items-baseline gap-4">
|
||||
<ModalTitle>{headerTitle}</ModalTitle>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{headerRight}
|
||||
{showCloseButton && (
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="rounded-md p-1 text-soft hover:bg-accent-wash hover:text-foreground transition-colors duration-base"
|
||||
aria-label="Close dialog"
|
||||
>
|
||||
<svg className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Mobile Navigation (Horizontal row of items) */}
|
||||
<SidebarNav layout="grid" className="sm:hidden border-b border-line-soft bg-background p-2 shrink-0">
|
||||
{sections.map((section) => {
|
||||
const Icon = section.icon;
|
||||
return (
|
||||
<SidebarNavItem
|
||||
compact
|
||||
key={section.id}
|
||||
onClick={() => onSectionChange(section.id)}
|
||||
active={activeSectionId === section.id}
|
||||
icon={<Icon className="w-3.5 h-3.5" />}
|
||||
label={section.label}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</SidebarNav>
|
||||
|
||||
{/* Main Body (Split layout on desktop) */}
|
||||
<div className="flex flex-row min-h-0 flex-1">
|
||||
{/* Desktop Navigation (Left sidebar) */}
|
||||
<nav className="hidden sm:block w-fit shrink-0 border-r border-line-soft bg-background p-2">
|
||||
<SidebarNav>
|
||||
{sections.map((section) => {
|
||||
const Icon = section.icon;
|
||||
const active = activeSectionId === section.id;
|
||||
return (
|
||||
<SidebarNavItem
|
||||
key={section.id}
|
||||
onClick={() => onSectionChange(section.id)}
|
||||
active={active}
|
||||
icon={<Icon className="w-4 h-4" />}
|
||||
label={section.label}
|
||||
className="whitespace-nowrap"
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</SidebarNav>
|
||||
</nav>
|
||||
|
||||
{/* Tab Panel Content (Right detail view) */}
|
||||
<div className={cn('flex-1 min-w-0 p-4 overflow-y-auto bg-surface', contentClassName)}>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</ModalFrame>
|
||||
);
|
||||
}
|
||||
|
|
@ -587,3 +587,23 @@ export async function getDocumentPreviewStatus(
|
|||
|
||||
throw new Error(`Failed to load preview status (status ${res.status})`);
|
||||
}
|
||||
|
||||
export async function importUrl(
|
||||
url: string,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<{ title: string; content: string }> {
|
||||
const res = await fetch('/api/documents/import-url', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url }),
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to import URL');
|
||||
}
|
||||
|
||||
return (await res.json()) as { title: string; content: string };
|
||||
}
|
||||
|
||||
|
|
|
|||
243
src/lib/server/documents/web-loader.ts
Normal file
243
src/lib/server/documents/web-loader.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import http from 'http';
|
||||
import https from 'https';
|
||||
import { parseHTML } from 'linkedom';
|
||||
import { Readability } from '@mozilla/readability';
|
||||
import TurndownService from 'turndown';
|
||||
import { AntiSSRFPolicy, PolicyConfigOptions } from '@microsoft/antissrf';
|
||||
|
||||
const MAX_BYTES = 2 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 6000;
|
||||
const MAX_REDIRECTS = 5;
|
||||
const USER_AGENT =
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
|
||||
/**
|
||||
* SSRF protection is delegated to Microsoft's AntiSSRF library. The policy's
|
||||
* agents inject a connection-time DNS lookup that resolves the host and rejects
|
||||
* any request whose resolved address falls in an internal/special-purpose range
|
||||
* (loopback, private, link-local, CGNAT, cloud metadata, multicast, …). Because
|
||||
* the check happens on the address the socket actually connects to, it is
|
||||
* inherently safe against DNS rebinding, and it re-runs on every redirect hop
|
||||
* (each hop is a fresh request through the agent).
|
||||
*
|
||||
* `ExternalOnlyLatest` blocks the `recommendedLatest` range set and stays
|
||||
* up to date without code changes. We allow plain HTTP because article URLs are
|
||||
* not always HTTPS; remove `allowPlainTextHttp` to require HTTPS.
|
||||
*/
|
||||
const ssrfPolicy = new AntiSSRFPolicy(PolicyConfigOptions.ExternalOnlyLatest);
|
||||
ssrfPolicy.allowPlainTextHttp = true;
|
||||
const httpAgent = ssrfPolicy.getHttpAgent();
|
||||
const httpsAgent = ssrfPolicy.getHttpsAgent();
|
||||
|
||||
/**
|
||||
* Performs a single GET request through the AntiSSRF agent. Resolves with either
|
||||
* the response body or a redirect location to follow. Enforces a size limit and
|
||||
* a hard timeout (resource limits, independent of the SSRF policy).
|
||||
*/
|
||||
function requestOnce(
|
||||
url: URL,
|
||||
maxBytes: number,
|
||||
timeoutMs: number
|
||||
): Promise<{ body?: string; redirectLocation?: string }> {
|
||||
const isHttps = url.protocol === 'https:';
|
||||
const lib = isHttps ? https : http;
|
||||
const agent = isHttps ? httpsAgent : httpAgent;
|
||||
const port = url.port ? Number(url.port) : isHttps ? 443 : 80;
|
||||
|
||||
const options: https.RequestOptions = {
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname, // connect by hostname so the agent's safe lookup runs
|
||||
port,
|
||||
path: `${url.pathname}${url.search}`,
|
||||
method: 'GET',
|
||||
agent, // AntiSSRF agent enforces the SSRF policy at connection time
|
||||
headers: {
|
||||
'User-Agent': USER_AGENT,
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
},
|
||||
timeout: timeoutMs,
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const ok = (v: { body?: string; redirectLocation?: string }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(hardTimer);
|
||||
resolve(v);
|
||||
};
|
||||
const fail = (e: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(hardTimer);
|
||||
reject(e);
|
||||
};
|
||||
|
||||
const req = lib.request(options, (res) => {
|
||||
const status = res.statusCode ?? 0;
|
||||
|
||||
if (status >= 300 && status < 400 && res.headers.location) {
|
||||
res.resume(); // drain so the socket can be reused/closed
|
||||
ok({ redirectLocation: res.headers.location });
|
||||
return;
|
||||
}
|
||||
|
||||
if (status < 200 || status >= 300) {
|
||||
res.resume();
|
||||
fail(new Error(`Failed to fetch webpage (status ${status})`));
|
||||
return;
|
||||
}
|
||||
|
||||
const contentType = (res.headers['content-type'] || '').toString();
|
||||
const isHtml =
|
||||
contentType.includes('text/html') ||
|
||||
contentType.includes('application/xhtml+xml');
|
||||
const isText = contentType.includes('text/plain');
|
||||
if (!isHtml && !isText) {
|
||||
res.resume();
|
||||
fail(
|
||||
new Error('Unsupported content type. Only web pages or text files are supported.')
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const contentLength = res.headers['content-length'];
|
||||
if (contentLength && parseInt(contentLength.toString(), 10) > maxBytes) {
|
||||
res.resume();
|
||||
fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Stream and enforce the size limit dynamically (header may be absent/spoofed).
|
||||
let received = 0;
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (chunk: Buffer) => {
|
||||
received += chunk.length;
|
||||
if (received > maxBytes) {
|
||||
req.destroy();
|
||||
fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.'));
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
res.on('end', () => ok({ body: Buffer.concat(chunks).toString('utf-8') }));
|
||||
res.on('error', fail);
|
||||
});
|
||||
|
||||
// Hard deadline covering the whole request (connect + transfer), in addition
|
||||
// to the socket idle timeout, to bound slow-read/slowloris responses.
|
||||
const hardTimer = setTimeout(() => {
|
||||
req.destroy(new Error('Request timed out'));
|
||||
}, timeoutMs);
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy(new Error('Request timed out'));
|
||||
});
|
||||
req.on('error', fail);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates and fetches a URL with a size limit (default 2MB) and connection
|
||||
* timeout (default 6s). Redirects are followed manually so each hop is re-checked
|
||||
* by the AntiSSRF agent and counted against the redirect cap. Returns the body
|
||||
* and the final (post-redirect) URL.
|
||||
*/
|
||||
async function fetchWithLimit(
|
||||
urlStr: string,
|
||||
maxBytes: number = MAX_BYTES,
|
||||
timeoutMs: number = TIMEOUT_MS
|
||||
): Promise<{ html: string; finalUrl: string }> {
|
||||
let currentUrl = urlStr;
|
||||
|
||||
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(currentUrl);
|
||||
} catch {
|
||||
throw new Error('Invalid URL format');
|
||||
}
|
||||
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
||||
throw new Error('Only HTTP and HTTPS protocols are supported');
|
||||
}
|
||||
|
||||
const result = await requestOnce(url, maxBytes, timeoutMs);
|
||||
|
||||
if (result.redirectLocation) {
|
||||
currentUrl = new URL(result.redirectLocation, url).href;
|
||||
continue;
|
||||
}
|
||||
|
||||
return { html: result.body ?? '', finalUrl: currentUrl };
|
||||
}
|
||||
|
||||
throw new Error('Too many redirects while fetching the webpage.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects to, extracts readable content from, and converts a web page to Markdown.
|
||||
*/
|
||||
export async function fetchAndParseUrl(
|
||||
urlStr: string
|
||||
): Promise<{ title: string; content: string }> {
|
||||
// 1. Fetch HTML content with limit. The AntiSSRF agent validates the
|
||||
// destination (and every redirect hop) at connection time.
|
||||
const { html, finalUrl } = await fetchWithLimit(urlStr);
|
||||
|
||||
// 2. Parse HTML string to a virtual DOM document
|
||||
const { document } = parseHTML(html);
|
||||
|
||||
// Resolve relative URLs against the final (post-redirect) location so links
|
||||
// and images point at the page we actually fetched.
|
||||
const resolveUrl = (relativeUrl: string) => {
|
||||
try {
|
||||
return new URL(relativeUrl, finalUrl).href;
|
||||
} catch {
|
||||
return relativeUrl;
|
||||
}
|
||||
};
|
||||
|
||||
const images = document.querySelectorAll('img');
|
||||
for (const img of images) {
|
||||
const src = img.getAttribute('src');
|
||||
if (src) {
|
||||
img.setAttribute('src', resolveUrl(src));
|
||||
}
|
||||
}
|
||||
|
||||
const links = document.querySelectorAll('a');
|
||||
for (const link of links) {
|
||||
const href = link.getAttribute('href');
|
||||
if (href) {
|
||||
link.setAttribute('href', resolveUrl(href));
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Run Readability to extract core article text
|
||||
const reader = new Readability(document as unknown as Document);
|
||||
const article = reader.parse();
|
||||
|
||||
if (!article || !article.content) {
|
||||
throw new Error('Could not extract meaningful article content from this webpage.');
|
||||
}
|
||||
|
||||
// 4. Convert clean HTML content to Markdown
|
||||
const turndownService = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
hr: '---',
|
||||
bulletListMarker: '-',
|
||||
codeBlockStyle: 'fenced',
|
||||
});
|
||||
|
||||
const markdown = turndownService.turndown(article.content);
|
||||
if (!markdown.trim()) {
|
||||
throw new Error('Extracted content is empty.');
|
||||
}
|
||||
|
||||
return {
|
||||
title: article.title?.trim() || 'Web Import',
|
||||
content: markdown,
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue