Merge pull request #116 from richardr1126/feat/ui-improvements
feat(documents): add web URL import, markdown/text creator, and unified sidebar modal dialog
This commit is contained in:
Richard R
GitHub
commit
cecb2489e1
14 changed files with 1151 additions and 94 deletions
|
|
@ -35,6 +35,8 @@
|
||||||
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
||||||
"@headlessui/react": "^2.2.10",
|
"@headlessui/react": "^2.2.10",
|
||||||
"@huggingface/tokenizers": "^0.1.3",
|
"@huggingface/tokenizers": "^0.1.3",
|
||||||
|
"@microsoft/antissrf": "^1.0.0",
|
||||||
|
"@mozilla/readability": "^0.6.0",
|
||||||
"@napi-rs/canvas": "^0.1.100",
|
"@napi-rs/canvas": "^0.1.100",
|
||||||
"@tanstack/react-query": "^5.101.0",
|
"@tanstack/react-query": "^5.101.0",
|
||||||
"@types/archiver": "^7.0.0",
|
"@types/archiver": "^7.0.0",
|
||||||
|
|
@ -55,6 +57,7 @@
|
||||||
"ffmpeg-static": "^5.3.0",
|
"ffmpeg-static": "^5.3.0",
|
||||||
"howler": "^2.2.4",
|
"howler": "^2.2.4",
|
||||||
"jszip": "^3.10.1",
|
"jszip": "^3.10.1",
|
||||||
|
"linkedom": "^0.18.12",
|
||||||
"lru-cache": "^11.5.1",
|
"lru-cache": "^11.5.1",
|
||||||
"next": "^15.5.19",
|
"next": "^15.5.19",
|
||||||
"onnxruntime-node": "^1.26.0",
|
"onnxruntime-node": "^1.26.0",
|
||||||
|
|
@ -75,6 +78,7 @@
|
||||||
"react-reader": "^2.0.15",
|
"react-reader": "^2.0.15",
|
||||||
"remark-gfm": "^4.0.1",
|
"remark-gfm": "^4.0.1",
|
||||||
"replicate": "^1.4.0",
|
"replicate": "^1.4.0",
|
||||||
|
"turndown": "^7.2.4",
|
||||||
"uuid": "^11.1.1"
|
"uuid": "^11.1.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
@ -83,10 +87,12 @@
|
||||||
"@tailwindcss/typography": "^0.5.19",
|
"@tailwindcss/typography": "^0.5.19",
|
||||||
"@types/better-sqlite3": "^7.6.13",
|
"@types/better-sqlite3": "^7.6.13",
|
||||||
"@types/howler": "^2.2.13",
|
"@types/howler": "^2.2.13",
|
||||||
|
"@types/mozilla-readability": "^0.2.1",
|
||||||
"@types/node": "^20.19.41",
|
"@types/node": "^20.19.41",
|
||||||
"@types/pg": "^8.20.0",
|
"@types/pg": "^8.20.0",
|
||||||
"@types/react": "^19.2.16",
|
"@types/react": "^19.2.16",
|
||||||
"@types/react-dom": "^19.2.3",
|
"@types/react-dom": "^19.2.3",
|
||||||
|
"@types/turndown": "^5.0.6",
|
||||||
"@types/uuid": "^10.0.0",
|
"@types/uuid": "^10.0.0",
|
||||||
"eslint": "^9.39.4",
|
"eslint": "^9.39.4",
|
||||||
"eslint-config-next": "^15.5.19",
|
"eslint-config-next": "^15.5.19",
|
||||||
|
|
|
||||||
168
pnpm-lock.yaml
168
pnpm-lock.yaml
|
|
@ -23,6 +23,12 @@ importers:
|
||||||
'@huggingface/tokenizers':
|
'@huggingface/tokenizers':
|
||||||
specifier: ^0.1.3
|
specifier: ^0.1.3
|
||||||
version: 0.1.3
|
version: 0.1.3
|
||||||
|
'@microsoft/antissrf':
|
||||||
|
specifier: ^1.0.0
|
||||||
|
version: 1.0.0
|
||||||
|
'@mozilla/readability':
|
||||||
|
specifier: ^0.6.0
|
||||||
|
version: 0.6.0
|
||||||
'@napi-rs/canvas':
|
'@napi-rs/canvas':
|
||||||
specifier: ^0.1.100
|
specifier: ^0.1.100
|
||||||
version: 0.1.100
|
version: 0.1.100
|
||||||
|
|
@ -83,6 +89,9 @@ importers:
|
||||||
jszip:
|
jszip:
|
||||||
specifier: ^3.10.1
|
specifier: ^3.10.1
|
||||||
version: 3.10.1
|
version: 3.10.1
|
||||||
|
linkedom:
|
||||||
|
specifier: ^0.18.12
|
||||||
|
version: 0.18.12(canvas@3.2.3)
|
||||||
lru-cache:
|
lru-cache:
|
||||||
specifier: ^11.5.1
|
specifier: ^11.5.1
|
||||||
version: 11.5.1
|
version: 11.5.1
|
||||||
|
|
@ -143,6 +152,9 @@ importers:
|
||||||
replicate:
|
replicate:
|
||||||
specifier: ^1.4.0
|
specifier: ^1.4.0
|
||||||
version: 1.4.0
|
version: 1.4.0
|
||||||
|
turndown:
|
||||||
|
specifier: ^7.2.4
|
||||||
|
version: 7.2.4
|
||||||
uuid:
|
uuid:
|
||||||
specifier: ^11.1.1
|
specifier: ^11.1.1
|
||||||
version: 11.1.1
|
version: 11.1.1
|
||||||
|
|
@ -162,6 +174,9 @@ importers:
|
||||||
'@types/howler':
|
'@types/howler':
|
||||||
specifier: ^2.2.13
|
specifier: ^2.2.13
|
||||||
version: 2.2.13
|
version: 2.2.13
|
||||||
|
'@types/mozilla-readability':
|
||||||
|
specifier: ^0.2.1
|
||||||
|
version: 0.2.1
|
||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: ^20.19.41
|
specifier: ^20.19.41
|
||||||
version: 20.19.41
|
version: 20.19.41
|
||||||
|
|
@ -174,6 +189,9 @@ importers:
|
||||||
'@types/react-dom':
|
'@types/react-dom':
|
||||||
specifier: ^19.2.3
|
specifier: ^19.2.3
|
||||||
version: 19.2.3(@types/react@19.2.16)
|
version: 19.2.3(@types/react@19.2.16)
|
||||||
|
'@types/turndown':
|
||||||
|
specifier: ^5.0.6
|
||||||
|
version: 5.0.6
|
||||||
'@types/uuid':
|
'@types/uuid':
|
||||||
specifier: ^10.0.0
|
specifier: ^10.0.0
|
||||||
version: 10.0.0
|
version: 10.0.0
|
||||||
|
|
@ -1240,6 +1258,16 @@ packages:
|
||||||
'@jridgewell/trace-mapping@0.3.31':
|
'@jridgewell/trace-mapping@0.3.31':
|
||||||
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
|
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
|
||||||
|
|
||||||
|
'@microsoft/antissrf@1.0.0':
|
||||||
|
resolution: {integrity: sha512-AmO1ykSha52Ef/7UXvHgu8ElsGdgcLkF5nTl7YZGyR1AkbmlQYtiVeWp+CsjkFaJzKAH5ofXLZveMmOHwX/7Jw==}
|
||||||
|
|
||||||
|
'@mixmark-io/domino@2.2.0':
|
||||||
|
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
|
||||||
|
|
||||||
|
'@mozilla/readability@0.6.0':
|
||||||
|
resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==}
|
||||||
|
engines: {node: '>=14.0.0'}
|
||||||
|
|
||||||
'@napi-rs/canvas-android-arm64@0.1.100':
|
'@napi-rs/canvas-android-arm64@0.1.100':
|
||||||
resolution: {integrity: sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==}
|
resolution: {integrity: sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==}
|
||||||
engines: {node: '>= 10'}
|
engines: {node: '>= 10'}
|
||||||
|
|
@ -1686,6 +1714,9 @@ packages:
|
||||||
'@types/mdast@4.0.4':
|
'@types/mdast@4.0.4':
|
||||||
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
|
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
|
||||||
|
|
||||||
|
'@types/mozilla-readability@0.2.1':
|
||||||
|
resolution: {integrity: sha512-RjLpe3MIpj53L+q50DTNlhv6KjHjgajFQwAF4KuxVZTa2+6Cs2UNAxS5awxABfZEHRYjGKeRDB21Zrtnsi911w==}
|
||||||
|
|
||||||
'@types/ms@2.1.0':
|
'@types/ms@2.1.0':
|
||||||
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
|
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
|
||||||
|
|
||||||
|
|
@ -1709,6 +1740,9 @@ packages:
|
||||||
'@types/readdir-glob@1.1.5':
|
'@types/readdir-glob@1.1.5':
|
||||||
resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
|
resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
|
||||||
|
|
||||||
|
'@types/turndown@5.0.6':
|
||||||
|
resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==}
|
||||||
|
|
||||||
'@types/unist@2.0.11':
|
'@types/unist@2.0.11':
|
||||||
resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
|
resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
|
||||||
|
|
||||||
|
|
@ -2259,6 +2293,9 @@ packages:
|
||||||
bl@4.1.0:
|
bl@4.1.0:
|
||||||
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
||||||
|
|
||||||
|
boolbase@1.0.0:
|
||||||
|
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
|
||||||
|
|
||||||
bowser@2.14.1:
|
bowser@2.14.1:
|
||||||
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
|
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
|
||||||
|
|
||||||
|
|
@ -2418,11 +2455,21 @@ packages:
|
||||||
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
||||||
engines: {node: '>= 8'}
|
engines: {node: '>= 8'}
|
||||||
|
|
||||||
|
css-select@5.2.2:
|
||||||
|
resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==}
|
||||||
|
|
||||||
|
css-what@6.2.2:
|
||||||
|
resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==}
|
||||||
|
engines: {node: '>= 6'}
|
||||||
|
|
||||||
cssesc@3.0.0:
|
cssesc@3.0.0:
|
||||||
resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
|
resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
|
||||||
engines: {node: '>=4'}
|
engines: {node: '>=4'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
cssom@0.5.0:
|
||||||
|
resolution: {integrity: sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==}
|
||||||
|
|
||||||
csstype@3.2.3:
|
csstype@3.2.3:
|
||||||
resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==}
|
resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==}
|
||||||
|
|
||||||
|
|
@ -2523,6 +2570,19 @@ packages:
|
||||||
resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
|
resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
|
||||||
engines: {node: '>=0.10.0'}
|
engines: {node: '>=0.10.0'}
|
||||||
|
|
||||||
|
dom-serializer@2.0.0:
|
||||||
|
resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}
|
||||||
|
|
||||||
|
domelementtype@2.3.0:
|
||||||
|
resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==}
|
||||||
|
|
||||||
|
domhandler@5.0.3:
|
||||||
|
resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==}
|
||||||
|
engines: {node: '>= 4'}
|
||||||
|
|
||||||
|
domutils@3.2.2:
|
||||||
|
resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}
|
||||||
|
|
||||||
dotenv@17.4.2:
|
dotenv@17.4.2:
|
||||||
resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==}
|
resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==}
|
||||||
engines: {node: '>=12'}
|
engines: {node: '>=12'}
|
||||||
|
|
@ -2643,6 +2703,14 @@ packages:
|
||||||
end-of-stream@1.4.5:
|
end-of-stream@1.4.5:
|
||||||
resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==}
|
resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==}
|
||||||
|
|
||||||
|
entities@4.5.0:
|
||||||
|
resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
|
||||||
|
engines: {node: '>=0.12'}
|
||||||
|
|
||||||
|
entities@7.0.1:
|
||||||
|
resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==}
|
||||||
|
engines: {node: '>=0.12'}
|
||||||
|
|
||||||
env-paths@2.2.1:
|
env-paths@2.2.1:
|
||||||
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
|
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
|
||||||
engines: {node: '>=6'}
|
engines: {node: '>=6'}
|
||||||
|
|
@ -3111,9 +3179,15 @@ packages:
|
||||||
howler@2.2.4:
|
howler@2.2.4:
|
||||||
resolution: {integrity: sha512-iARIBPgcQrwtEr+tALF+rapJ8qSc+Set2GJQl7xT1MQzWaVkFebdJhR3alVlSiUf5U7nAANKuj3aWpwerocD5w==}
|
resolution: {integrity: sha512-iARIBPgcQrwtEr+tALF+rapJ8qSc+Set2GJQl7xT1MQzWaVkFebdJhR3alVlSiUf5U7nAANKuj3aWpwerocD5w==}
|
||||||
|
|
||||||
|
html-escaper@3.0.3:
|
||||||
|
resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==}
|
||||||
|
|
||||||
html-url-attributes@3.0.1:
|
html-url-attributes@3.0.1:
|
||||||
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
|
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
|
||||||
|
|
||||||
|
htmlparser2@10.1.0:
|
||||||
|
resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==}
|
||||||
|
|
||||||
http-response-object@3.0.2:
|
http-response-object@3.0.2:
|
||||||
resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}
|
resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}
|
||||||
|
|
||||||
|
|
@ -3463,6 +3537,15 @@ packages:
|
||||||
lines-and-columns@1.2.4:
|
lines-and-columns@1.2.4:
|
||||||
resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
|
resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
|
||||||
|
|
||||||
|
linkedom@0.18.12:
|
||||||
|
resolution: {integrity: sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==}
|
||||||
|
engines: {node: '>=16'}
|
||||||
|
peerDependencies:
|
||||||
|
canvas: '>= 2'
|
||||||
|
peerDependenciesMeta:
|
||||||
|
canvas:
|
||||||
|
optional: true
|
||||||
|
|
||||||
localforage@1.10.0:
|
localforage@1.10.0:
|
||||||
resolution: {integrity: sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==}
|
resolution: {integrity: sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==}
|
||||||
|
|
||||||
|
|
@ -3752,6 +3835,9 @@ packages:
|
||||||
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
|
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
|
||||||
engines: {node: '>=0.10.0'}
|
engines: {node: '>=0.10.0'}
|
||||||
|
|
||||||
|
nth-check@2.1.1:
|
||||||
|
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
|
||||||
|
|
||||||
object-assign@4.1.1:
|
object-assign@4.1.1:
|
||||||
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
|
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
|
||||||
engines: {node: '>=0.10.0'}
|
engines: {node: '>=0.10.0'}
|
||||||
|
|
@ -4599,6 +4685,10 @@ packages:
|
||||||
tunnel-agent@0.6.0:
|
tunnel-agent@0.6.0:
|
||||||
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
|
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
|
||||||
|
|
||||||
|
turndown@7.2.4:
|
||||||
|
resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==}
|
||||||
|
engines: {node: '>=18', npm: '>=9'}
|
||||||
|
|
||||||
tweetnacl@1.0.3:
|
tweetnacl@1.0.3:
|
||||||
resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==}
|
resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==}
|
||||||
|
|
||||||
|
|
@ -4637,6 +4727,9 @@ packages:
|
||||||
engines: {node: '>=14.17'}
|
engines: {node: '>=14.17'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
uhyphen@0.2.0:
|
||||||
|
resolution: {integrity: sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==}
|
||||||
|
|
||||||
unbox-primitive@1.1.0:
|
unbox-primitive@1.1.0:
|
||||||
resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==}
|
resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==}
|
||||||
engines: {node: '>= 0.4'}
|
engines: {node: '>= 0.4'}
|
||||||
|
|
@ -5669,6 +5762,12 @@ snapshots:
|
||||||
'@jridgewell/resolve-uri': 3.1.2
|
'@jridgewell/resolve-uri': 3.1.2
|
||||||
'@jridgewell/sourcemap-codec': 1.5.5
|
'@jridgewell/sourcemap-codec': 1.5.5
|
||||||
|
|
||||||
|
'@microsoft/antissrf@1.0.0': {}
|
||||||
|
|
||||||
|
'@mixmark-io/domino@2.2.0': {}
|
||||||
|
|
||||||
|
'@mozilla/readability@0.6.0': {}
|
||||||
|
|
||||||
'@napi-rs/canvas-android-arm64@0.1.100':
|
'@napi-rs/canvas-android-arm64@0.1.100':
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
|
@ -6018,6 +6117,8 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/unist': 3.0.3
|
'@types/unist': 3.0.3
|
||||||
|
|
||||||
|
'@types/mozilla-readability@0.2.1': {}
|
||||||
|
|
||||||
'@types/ms@2.1.0': {}
|
'@types/ms@2.1.0': {}
|
||||||
|
|
||||||
'@types/node@10.17.60': {}
|
'@types/node@10.17.60': {}
|
||||||
|
|
@ -6044,6 +6145,8 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 20.19.41
|
'@types/node': 20.19.41
|
||||||
|
|
||||||
|
'@types/turndown@5.0.6': {}
|
||||||
|
|
||||||
'@types/unist@2.0.11': {}
|
'@types/unist@2.0.11': {}
|
||||||
|
|
||||||
'@types/unist@3.0.3': {}
|
'@types/unist@3.0.3': {}
|
||||||
|
|
@ -6542,6 +6645,8 @@ snapshots:
|
||||||
inherits: 2.0.4
|
inherits: 2.0.4
|
||||||
readable-stream: 3.6.2
|
readable-stream: 3.6.2
|
||||||
|
|
||||||
|
boolbase@1.0.0: {}
|
||||||
|
|
||||||
bowser@2.14.1: {}
|
bowser@2.14.1: {}
|
||||||
|
|
||||||
brace-expansion@1.1.15:
|
brace-expansion@1.1.15:
|
||||||
|
|
@ -6699,8 +6804,20 @@ snapshots:
|
||||||
shebang-command: 2.0.0
|
shebang-command: 2.0.0
|
||||||
which: 2.0.2
|
which: 2.0.2
|
||||||
|
|
||||||
|
css-select@5.2.2:
|
||||||
|
dependencies:
|
||||||
|
boolbase: 1.0.0
|
||||||
|
css-what: 6.2.2
|
||||||
|
domhandler: 5.0.3
|
||||||
|
domutils: 3.2.2
|
||||||
|
nth-check: 2.1.1
|
||||||
|
|
||||||
|
css-what@6.2.2: {}
|
||||||
|
|
||||||
cssesc@3.0.0: {}
|
cssesc@3.0.0: {}
|
||||||
|
|
||||||
|
cssom@0.5.0: {}
|
||||||
|
|
||||||
csstype@3.2.3: {}
|
csstype@3.2.3: {}
|
||||||
|
|
||||||
d@1.0.2:
|
d@1.0.2:
|
||||||
|
|
@ -6793,6 +6910,24 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
esutils: 2.0.3
|
esutils: 2.0.3
|
||||||
|
|
||||||
|
dom-serializer@2.0.0:
|
||||||
|
dependencies:
|
||||||
|
domelementtype: 2.3.0
|
||||||
|
domhandler: 5.0.3
|
||||||
|
entities: 4.5.0
|
||||||
|
|
||||||
|
domelementtype@2.3.0: {}
|
||||||
|
|
||||||
|
domhandler@5.0.3:
|
||||||
|
dependencies:
|
||||||
|
domelementtype: 2.3.0
|
||||||
|
|
||||||
|
domutils@3.2.2:
|
||||||
|
dependencies:
|
||||||
|
dom-serializer: 2.0.0
|
||||||
|
domelementtype: 2.3.0
|
||||||
|
domhandler: 5.0.3
|
||||||
|
|
||||||
dotenv@17.4.2: {}
|
dotenv@17.4.2: {}
|
||||||
|
|
||||||
drizzle-kit@0.31.10:
|
drizzle-kit@0.31.10:
|
||||||
|
|
@ -6828,6 +6963,10 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
once: 1.4.0
|
once: 1.4.0
|
||||||
|
|
||||||
|
entities@4.5.0: {}
|
||||||
|
|
||||||
|
entities@7.0.1: {}
|
||||||
|
|
||||||
env-paths@2.2.1: {}
|
env-paths@2.2.1: {}
|
||||||
|
|
||||||
epubjs@0.3.93:
|
epubjs@0.3.93:
|
||||||
|
|
@ -7575,8 +7714,17 @@ snapshots:
|
||||||
|
|
||||||
howler@2.2.4: {}
|
howler@2.2.4: {}
|
||||||
|
|
||||||
|
html-escaper@3.0.3: {}
|
||||||
|
|
||||||
html-url-attributes@3.0.1: {}
|
html-url-attributes@3.0.1: {}
|
||||||
|
|
||||||
|
htmlparser2@10.1.0:
|
||||||
|
dependencies:
|
||||||
|
domelementtype: 2.3.0
|
||||||
|
domhandler: 5.0.3
|
||||||
|
domutils: 3.2.2
|
||||||
|
entities: 7.0.1
|
||||||
|
|
||||||
http-response-object@3.0.2:
|
http-response-object@3.0.2:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 10.17.60
|
'@types/node': 10.17.60
|
||||||
|
|
@ -7901,6 +8049,16 @@ snapshots:
|
||||||
|
|
||||||
lines-and-columns@1.2.4: {}
|
lines-and-columns@1.2.4: {}
|
||||||
|
|
||||||
|
linkedom@0.18.12(canvas@3.2.3):
|
||||||
|
dependencies:
|
||||||
|
css-select: 5.2.2
|
||||||
|
cssom: 0.5.0
|
||||||
|
html-escaper: 3.0.3
|
||||||
|
htmlparser2: 10.1.0
|
||||||
|
uhyphen: 0.2.0
|
||||||
|
optionalDependencies:
|
||||||
|
canvas: 3.2.3
|
||||||
|
|
||||||
localforage@1.10.0:
|
localforage@1.10.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
lie: 3.1.1
|
lie: 3.1.1
|
||||||
|
|
@ -8380,6 +8538,10 @@ snapshots:
|
||||||
|
|
||||||
normalize-path@3.0.0: {}
|
normalize-path@3.0.0: {}
|
||||||
|
|
||||||
|
nth-check@2.1.1:
|
||||||
|
dependencies:
|
||||||
|
boolbase: 1.0.0
|
||||||
|
|
||||||
object-assign@4.1.1: {}
|
object-assign@4.1.1: {}
|
||||||
|
|
||||||
object-hash@3.0.0: {}
|
object-hash@3.0.0: {}
|
||||||
|
|
@ -9421,6 +9583,10 @@ snapshots:
|
||||||
dependencies:
|
dependencies:
|
||||||
safe-buffer: 5.2.1
|
safe-buffer: 5.2.1
|
||||||
|
|
||||||
|
turndown@7.2.4:
|
||||||
|
dependencies:
|
||||||
|
'@mixmark-io/domino': 2.2.0
|
||||||
|
|
||||||
tweetnacl@1.0.3: {}
|
tweetnacl@1.0.3: {}
|
||||||
|
|
||||||
type-check@0.4.0:
|
type-check@0.4.0:
|
||||||
|
|
@ -9468,6 +9634,8 @@ snapshots:
|
||||||
|
|
||||||
typescript@5.9.3: {}
|
typescript@5.9.3: {}
|
||||||
|
|
||||||
|
uhyphen@0.2.0: {}
|
||||||
|
|
||||||
unbox-primitive@1.1.0:
|
unbox-primitive@1.1.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
call-bound: 1.0.4
|
call-bound: 1.0.4
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,7 @@ export default function HTMLPage() {
|
||||||
const [isLoading, setIsLoading] = useState(true);
|
const [isLoading, setIsLoading] = useState(true);
|
||||||
const [activeSidebar, setActiveSidebar] = useState<null | 'settings' | 'segments' | 'audiobook'>(null);
|
const [activeSidebar, setActiveSidebar] = useState<null | 'settings' | 'segments' | 'audiobook'>(null);
|
||||||
const [containerHeight, setContainerHeight] = useState<string>('auto');
|
const [containerHeight, setContainerHeight] = useState<string>('auto');
|
||||||
const [padPct, setPadPct] = useState<number>(100); // 0..100 (100 = full width)
|
const [padPct, setPadPct] = useState<number>(50); // 0..100 (50 = 50% default width)
|
||||||
const [maxPadPx, setMaxPadPx] = useState<number>(0);
|
const [maxPadPx, setMaxPadPx] = useState<number>(0);
|
||||||
const inFlightDocIdRef = useRef<string | null>(null);
|
const inFlightDocIdRef = useRef<string | null>(null);
|
||||||
const loadedDocIdRef = useRef<string | null>(null);
|
const loadedDocIdRef = useRef<string | null>(null);
|
||||||
|
|
|
||||||
57
src/app/api/documents/import-url/route.ts
Normal file
57
src/app/api/documents/import-url/route.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||||
|
import { fetchAndParseUrl } from '@/lib/server/documents/web-loader';
|
||||||
|
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
export async function POST(req: NextRequest) {
|
||||||
|
try {
|
||||||
|
// 1. Authenticate user request
|
||||||
|
const ctxOrRes = await requireAuthContext(req);
|
||||||
|
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||||
|
|
||||||
|
// 2. Validate URL body parameter
|
||||||
|
let body;
|
||||||
|
try {
|
||||||
|
body = await req.json();
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json({ error: 'Invalid request body' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const { url } = body;
|
||||||
|
if (!url || typeof url !== 'string') {
|
||||||
|
return NextResponse.json({ error: 'Missing or invalid url parameter' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Fetch webpage, scrape readable elements, convert to markdown
|
||||||
|
const result = await fetchAndParseUrl(url);
|
||||||
|
|
||||||
|
// 4. Return title and parsed markdown content
|
||||||
|
return NextResponse.json(result);
|
||||||
|
} catch (error) {
|
||||||
|
serverLogger.error(
|
||||||
|
{
|
||||||
|
event: 'documents.import-url.failed',
|
||||||
|
error: errorToLog(error),
|
||||||
|
},
|
||||||
|
'Failed to import and parse document from web URL'
|
||||||
|
);
|
||||||
|
|
||||||
|
const errorMessage = error instanceof Error ? error.message : 'Failed to import URL';
|
||||||
|
|
||||||
|
// Check if it's a validation error or network error to set suitable status codes
|
||||||
|
const isValidationError =
|
||||||
|
errorMessage.includes('forbidden') ||
|
||||||
|
errorMessage.includes('disallowed by policy') ||
|
||||||
|
errorMessage.includes('Invalid URL') ||
|
||||||
|
errorMessage.includes('Only HTTP') ||
|
||||||
|
errorMessage.includes('exceeds the maximum') ||
|
||||||
|
errorMessage.includes('Unsupported content type');
|
||||||
|
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: errorMessage },
|
||||||
|
{ status: isValidationError ? 400 : 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -42,7 +42,7 @@ import { flushUserPreferencesSync } from '@/lib/client/api/user-state';
|
||||||
import toast from 'react-hot-toast';
|
import toast from 'react-hot-toast';
|
||||||
import { useLibraryDocumentsQuery } from '@/hooks/useLibraryDocumentsQuery';
|
import { useLibraryDocumentsQuery } from '@/hooks/useLibraryDocumentsQuery';
|
||||||
import {
|
import {
|
||||||
SidebarNav,
|
SidebarDialog,
|
||||||
SidebarNavItem,
|
SidebarNavItem,
|
||||||
SegmentedControl,
|
SegmentedControl,
|
||||||
Button,
|
Button,
|
||||||
|
|
@ -50,8 +50,6 @@ import {
|
||||||
IconButton,
|
IconButton,
|
||||||
Input,
|
Input,
|
||||||
Textarea,
|
Textarea,
|
||||||
ModalFrame,
|
|
||||||
ModalTitle,
|
|
||||||
Select,
|
Select,
|
||||||
} from '@/components/ui';
|
} from '@/components/ui';
|
||||||
import ReactMarkdown from 'react-markdown';
|
import ReactMarkdown from 'react-markdown';
|
||||||
|
|
@ -562,90 +560,55 @@ export function SettingsModal({
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<ModalFrame
|
<SidebarDialog
|
||||||
open={isOpen}
|
open={isOpen}
|
||||||
onClose={resetToCurrent}
|
onClose={resetToCurrent}
|
||||||
size="xl"
|
size="xl"
|
||||||
panelTestId="settings-modal"
|
panelTestId="settings-modal"
|
||||||
className={isChangelogOpen ? 'z-[90]' : 'z-50'}
|
modalClassName={isChangelogOpen ? 'z-[90]' : 'z-50'}
|
||||||
|
headerTitle={
|
||||||
|
<div className="flex items-baseline gap-4">
|
||||||
|
<span>Settings</span>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
onClick={() => setIsChangelogOpen(true)}
|
||||||
|
className="text-sm font-medium leading-6 text-soft hover:text-accent transition-colors"
|
||||||
|
>
|
||||||
|
{displayVersion ? `v${displayVersion} · Changelog` : 'Changelog'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
headerRight={
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
onClick={() => showPrivacyModal()}
|
||||||
|
className="text-sm font-medium text-soft hover:text-accent transition-colors"
|
||||||
|
>
|
||||||
|
Privacy
|
||||||
|
</Button>
|
||||||
|
}
|
||||||
|
showCloseButton={false}
|
||||||
|
sections={visibleSections}
|
||||||
|
activeSectionId={activeSection}
|
||||||
|
onSectionChange={setActiveSection}
|
||||||
|
className="h-[490px]"
|
||||||
|
contentClassName={
|
||||||
|
activeSection === 'admin'
|
||||||
|
? 'bg-[radial-gradient(circle_at_top_right,color-mix(in_srgb,var(--accent),transparent_92%),transparent_35%)]'
|
||||||
|
: ''
|
||||||
|
}
|
||||||
|
customContent={
|
||||||
|
isChangelogOpen ? (
|
||||||
|
<SettingsChangelogPanel
|
||||||
|
appVersion={runtimeConfig.appVersion}
|
||||||
|
manifestUrl={runtimeConfig.changelogFeedUrl}
|
||||||
|
onClose={() => setIsChangelogOpen(false)}
|
||||||
|
/>
|
||||||
|
) : undefined
|
||||||
|
}
|
||||||
>
|
>
|
||||||
{/* Header */}
|
|
||||||
<div className="flex items-center justify-between px-5 py-3 border-b border-line-soft">
|
|
||||||
<div className="flex items-baseline gap-4">
|
|
||||||
<ModalTitle>Settings</ModalTitle>
|
|
||||||
<Button
|
|
||||||
variant="ghost"
|
|
||||||
size="sm"
|
|
||||||
onClick={() => setIsChangelogOpen(true)}
|
|
||||||
className="text-sm font-medium leading-6 text-soft hover:text-accent transition-colors"
|
|
||||||
>
|
|
||||||
{displayVersion ? `v${displayVersion} · Changelog` : 'Changelog'}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
<div className="flex items-center">
|
|
||||||
<Button
|
|
||||||
variant="ghost"
|
|
||||||
size="sm"
|
|
||||||
onClick={() => showPrivacyModal()}
|
|
||||||
className="text-sm font-medium text-soft hover:text-accent transition-colors"
|
|
||||||
>
|
|
||||||
Privacy
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{isChangelogOpen ? (
|
|
||||||
<SettingsChangelogPanel
|
|
||||||
appVersion={runtimeConfig.appVersion}
|
|
||||||
manifestUrl={runtimeConfig.changelogFeedUrl}
|
|
||||||
onClose={() => setIsChangelogOpen(false)}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<>
|
|
||||||
{/* Mobile nav */}
|
|
||||||
<SidebarNav layout="grid" className="sm:hidden border-b border-line-soft bg-background p-2">
|
|
||||||
{visibleSections.map((section) => {
|
|
||||||
const Icon = section.icon;
|
|
||||||
return (
|
|
||||||
<SidebarNavItem
|
|
||||||
compact
|
|
||||||
key={section.id}
|
|
||||||
onClick={() => setActiveSection(section.id)}
|
|
||||||
active={activeSection === section.id}
|
|
||||||
icon={<Icon className="w-3.5 h-3.5" />}
|
|
||||||
label={section.label}
|
|
||||||
/>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
</SidebarNav>
|
|
||||||
|
|
||||||
<div className="flex flex-row h-[490px]">
|
|
||||||
{/* Desktop: vertical sidebar */}
|
|
||||||
<nav className="hidden sm:block w-44 shrink-0 border-r border-line-soft bg-background p-2">
|
|
||||||
<SidebarNav>
|
|
||||||
{visibleSections.map((section) => {
|
|
||||||
const Icon = section.icon;
|
|
||||||
const active = activeSection === section.id;
|
|
||||||
return (
|
|
||||||
<SidebarNavItem
|
|
||||||
key={section.id}
|
|
||||||
onClick={() => setActiveSection(section.id)}
|
|
||||||
active={active}
|
|
||||||
icon={<Icon className="w-4 h-4" />}
|
|
||||||
label={section.label}
|
|
||||||
className="whitespace-nowrap"
|
|
||||||
/>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
</SidebarNav>
|
|
||||||
</nav>
|
|
||||||
|
|
||||||
{/* Content */}
|
|
||||||
<div className={`flex-1 min-w-0 p-3 overflow-y-auto ${
|
|
||||||
activeSection === 'admin'
|
|
||||||
? 'bg-[radial-gradient(circle_at_top_right,color-mix(in_srgb,var(--accent),transparent_92%),transparent_35%)]'
|
|
||||||
: ''
|
|
||||||
}`}>
|
|
||||||
{/* API Section */}
|
{/* API Section */}
|
||||||
{activeSection === 'api' && (
|
{activeSection === 'api' && (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
|
|
@ -1174,11 +1137,7 @@ export function SettingsModal({
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</SidebarDialog>
|
||||||
</div>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</ModalFrame>
|
|
||||||
|
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
isOpen={showDeleteAccountConfirm}
|
isOpen={showDeleteAccountConfirm}
|
||||||
|
|
@ -1388,7 +1347,7 @@ function SettingsChangelogPanel({
|
||||||
{isExpanded && (
|
{isExpanded && (
|
||||||
<div className="pl-6 pr-1 pb-3 pt-1 space-y-2">
|
<div className="pl-6 pr-1 pb-3 pt-1 space-y-2">
|
||||||
{body ? (
|
{body ? (
|
||||||
<div className="text-sm text-foreground leading-6 space-y-2 [&_h1]:text-base [&_h1]:font-semibold [&_h2]:text-sm [&_h2]:font-semibold [&_ul]:pl-5 [&_ol]:pl-5 [&_code]:bg-surface-sunken [&_code]:rounded [&_code]:px-1 [&_pre]:bg-surface-sunken [&_pre]:rounded [&_pre]:p-2 [&_pre]:overflow-x-auto">
|
<div className="text-sm text-foreground leading-6 space-y-2 [&_h1]:text-base [&_h1]:font-semibold [&_h2]:text-sm [&_h2]:font-semibold [&_ul]:pl-5 [&_ol]:pl-5 [&_code]:bg-surface-sunken [&_code]:rounded [&_code]:px-1 [&_pre]:bg-surface-sunken [&_pre]:rounded [&_pre]:p-2 [&_pre]:overflow-x-auto [&_a]:text-accent [&_a]:hover:underline [&_a]:transition-colors">
|
||||||
<ReactMarkdown remarkPlugins={[remarkGfm]}>
|
<ReactMarkdown remarkPlugins={[remarkGfm]}>
|
||||||
{body.body || '_No release notes provided._'}
|
{body.body || '_No release notes provided._'}
|
||||||
</ReactMarkdown>
|
</ReactMarkdown>
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ import { CreateFolderDialog } from '@/components/doclist/CreateFolderDialog';
|
||||||
import { DocumentListSkeleton } from '@/components/doclist/DocumentListSkeleton';
|
import { DocumentListSkeleton } from '@/components/doclist/DocumentListSkeleton';
|
||||||
import { DocumentUploader, type UploadBatchState } from '@/components/documents/DocumentUploader';
|
import { DocumentUploader, type UploadBatchState } from '@/components/documents/DocumentUploader';
|
||||||
import { IconButton } from '@/components/ui';
|
import { IconButton } from '@/components/ui';
|
||||||
|
import { UploadMenuDialog } from '@/components/documents/UploadMenuDialog';
|
||||||
import { DocumentDndProvider } from './dnd/DocumentDndProvider';
|
import { DocumentDndProvider } from './dnd/DocumentDndProvider';
|
||||||
import {
|
import {
|
||||||
DocumentSelectionProvider,
|
DocumentSelectionProvider,
|
||||||
|
|
@ -209,6 +210,7 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
const [mobileSidebarOpen, setMobileSidebarOpen] = useState(false);
|
const [mobileSidebarOpen, setMobileSidebarOpen] = useState(false);
|
||||||
const [query, setQuery] = useState('');
|
const [query, setQuery] = useState('');
|
||||||
const [activeUploadBatches, setActiveUploadBatches] = useState<Record<string, UploadBatchState>>({});
|
const [activeUploadBatches, setActiveUploadBatches] = useState<Record<string, UploadBatchState>>({});
|
||||||
|
const [isUploadDialogOpen, setIsUploadDialogOpen] = useState(false);
|
||||||
|
|
||||||
const [isInitialized, setIsInitialized] = useState(cachedState !== null);
|
const [isInitialized, setIsInitialized] = useState(cachedState !== null);
|
||||||
|
|
||||||
|
|
@ -619,7 +621,13 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
onDropOnFolder={handleDropOnFolder}
|
onDropOnFolder={handleDropOnFolder}
|
||||||
width={sidebarWidth}
|
width={sidebarWidth}
|
||||||
onWidthChange={setSidebarWidth}
|
onWidthChange={setSidebarWidth}
|
||||||
topSlot={<DocumentUploader variant="compact" onUploadBatchChange={handleUploadBatchChange} />}
|
topSlot={(
|
||||||
|
<DocumentUploader
|
||||||
|
variant="compact"
|
||||||
|
onUploadBatchChange={handleUploadBatchChange}
|
||||||
|
onClick={() => setIsUploadDialogOpen(true)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
bottomSlot={(
|
bottomSlot={(
|
||||||
<div className="flex flex-col gap-2">
|
<div className="flex flex-col gap-2">
|
||||||
{sidebarUploadState && (
|
{sidebarUploadState && (
|
||||||
|
|
@ -775,6 +783,12 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
||||||
confirmText="Remove Folders"
|
confirmText="Remove Folders"
|
||||||
isDangerous
|
isDangerous
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
<UploadMenuDialog
|
||||||
|
isOpen={isUploadDialogOpen}
|
||||||
|
onClose={() => setIsUploadDialogOpen(false)}
|
||||||
|
onUploadBatchChange={handleUploadBatchChange}
|
||||||
|
/>
|
||||||
</FinderWindow>
|
</FinderWindow>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -286,9 +286,20 @@ export function DocumentPreview({ doc }: DocumentPreviewProps) {
|
||||||
) : textPreview ? (
|
) : textPreview ? (
|
||||||
<>
|
<>
|
||||||
<div className="absolute inset-0 bg-gradient-to-br from-surface-solid to-surface-sunken" />
|
<div className="absolute inset-0 bg-gradient-to-br from-surface-solid to-surface-sunken" />
|
||||||
<div className="absolute inset-0 opacity-[0.06] bg-[radial-gradient(circle_at_1px_1px,rgba(0,0,0,1)_1px,transparent_0)] [background-size:12px_12px]" />
|
<div
|
||||||
|
aria-hidden
|
||||||
|
className="pointer-events-none absolute inset-0"
|
||||||
|
style={{
|
||||||
|
backgroundImage:
|
||||||
|
'radial-gradient(color-mix(in srgb, var(--foreground) 14%, transparent) 1px, transparent 1px)',
|
||||||
|
backgroundSize: '12px 12px',
|
||||||
|
opacity: 0.35,
|
||||||
|
WebkitMaskImage: 'radial-gradient(120% 100% at 50% 40%, #000 55%, transparent 100%)',
|
||||||
|
maskImage: 'radial-gradient(120% 100% at 50% 40%, #000 55%, transparent 100%)',
|
||||||
|
}}
|
||||||
|
/>
|
||||||
<div className="relative z-10 h-full w-full p-2 flex flex-col">
|
<div className="relative z-10 h-full w-full p-2 flex flex-col">
|
||||||
<div className="mt-auto rounded-md bg-surface-solid backdrop-blur-[1px] shadow-elev-1 ring-1 ring-line-soft p-2.5 max-h-[70%] overflow-hidden">
|
<div className="mb-auto rounded-md bg-surface-solid backdrop-blur-[1px] shadow-elev-1 ring-1 ring-line-soft p-2.5 max-h-[85%] overflow-hidden">
|
||||||
{isTxtFile ? (
|
{isTxtFile ? (
|
||||||
<pre className="text-[10px] sm:text-[11px] leading-snug text-foreground whitespace-pre-wrap font-mono">
|
<pre className="text-[10px] sm:text-[11px] leading-snug text-foreground whitespace-pre-wrap font-mono">
|
||||||
{textPreview}
|
{textPreview}
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ interface DocumentUploaderProps {
|
||||||
variant?: 'default' | 'compact' | 'overlay';
|
variant?: 'default' | 'compact' | 'overlay';
|
||||||
children?: ReactNode;
|
children?: ReactNode;
|
||||||
onUploadBatchChange?: (state: UploadBatchState) => void;
|
onUploadBatchChange?: (state: UploadBatchState) => void;
|
||||||
|
onClick?: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface UploadBatchState {
|
export interface UploadBatchState {
|
||||||
|
|
@ -28,6 +29,7 @@ export function DocumentUploader({
|
||||||
variant = 'default',
|
variant = 'default',
|
||||||
children,
|
children,
|
||||||
onUploadBatchChange,
|
onUploadBatchChange,
|
||||||
|
onClick,
|
||||||
}: DocumentUploaderProps) {
|
}: DocumentUploaderProps) {
|
||||||
const uploaderId = useId();
|
const uploaderId = useId();
|
||||||
const enableDocx = useFeatureFlag('enableDocxConversion');
|
const enableDocx = useFeatureFlag('enableDocxConversion');
|
||||||
|
|
@ -88,8 +90,8 @@ export function DocumentUploader({
|
||||||
},
|
},
|
||||||
multiple: true,
|
multiple: true,
|
||||||
disabled: isUploading,
|
disabled: isUploading,
|
||||||
noClick: variant === 'overlay',
|
noClick: variant === 'overlay' || !!onClick,
|
||||||
noKeyboard: variant === 'overlay'
|
noKeyboard: variant === 'overlay' || !!onClick
|
||||||
});
|
});
|
||||||
|
|
||||||
const isDisabled = isUploading;
|
const isDisabled = isUploading;
|
||||||
|
|
@ -132,6 +134,12 @@ export function DocumentUploader({
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
{...getRootProps()}
|
{...getRootProps()}
|
||||||
|
onClick={(e) => {
|
||||||
|
if (onClick) {
|
||||||
|
e.stopPropagation();
|
||||||
|
onClick();
|
||||||
|
}
|
||||||
|
}}
|
||||||
className={dropzoneSurfaceClass({
|
className={dropzoneSurfaceClass({
|
||||||
variant: variant === 'compact' ? 'compact' : 'default',
|
variant: variant === 'compact' ? 'compact' : 'default',
|
||||||
active: isDragActive,
|
active: isDragActive,
|
||||||
|
|
|
||||||
417
src/components/documents/UploadMenuDialog.tsx
Normal file
417
src/components/documents/UploadMenuDialog.tsx
Normal file
|
|
@ -0,0 +1,417 @@
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState, useCallback } from 'react';
|
||||||
|
import toast from 'react-hot-toast';
|
||||||
|
import { useDocuments } from '@/contexts/DocumentContext';
|
||||||
|
import { importUrl } from '@/lib/client/api/documents';
|
||||||
|
import {
|
||||||
|
SidebarDialog,
|
||||||
|
SegmentedControl,
|
||||||
|
Input,
|
||||||
|
Textarea,
|
||||||
|
Button,
|
||||||
|
} from '@/components/ui';
|
||||||
|
import {
|
||||||
|
UploadIcon,
|
||||||
|
FileIcon,
|
||||||
|
RefreshIcon,
|
||||||
|
CheckIcon,
|
||||||
|
BrowserIcon,
|
||||||
|
} from '@/components/icons/Icons';
|
||||||
|
import { DocumentUploader, type UploadBatchState } from './DocumentUploader';
|
||||||
|
|
||||||
|
interface UploadMenuDialogProps {
|
||||||
|
isOpen: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
onUploadBatchChange?: (state: UploadBatchState) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
type TabValue = 'file' | 'create' | 'url';
|
||||||
|
|
||||||
|
type SidebarSection = {
|
||||||
|
id: TabValue;
|
||||||
|
label: string;
|
||||||
|
icon: React.ComponentType<React.SVGProps<SVGSVGElement>>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const SIDEBAR_SECTIONS: SidebarSection[] = [
|
||||||
|
{ id: 'file', label: 'Upload Files', icon: UploadIcon },
|
||||||
|
{ id: 'create', label: 'Create Document', icon: FileIcon },
|
||||||
|
{ id: 'url', label: 'Import from Web', icon: BrowserIcon },
|
||||||
|
];
|
||||||
|
|
||||||
|
export function UploadMenuDialog({
|
||||||
|
isOpen,
|
||||||
|
onClose,
|
||||||
|
onUploadBatchChange,
|
||||||
|
}: UploadMenuDialogProps) {
|
||||||
|
const { uploadDocuments } = useDocuments();
|
||||||
|
const [activeTab, setActiveTab] = useState<TabValue>('file');
|
||||||
|
|
||||||
|
// --- Create Text/Markdown State ---
|
||||||
|
const [docName, setDocName] = useState('');
|
||||||
|
const [docExtension, setDocExtension] = useState<'.md' | '.txt'>('.md');
|
||||||
|
const [docContent, setDocContent] = useState('');
|
||||||
|
const [isCreatingDoc, setIsCreatingDoc] = useState(false);
|
||||||
|
|
||||||
|
// --- Import URL State ---
|
||||||
|
const [webUrl, setWebUrl] = useState('');
|
||||||
|
const [webTitle, setWebTitle] = useState('');
|
||||||
|
const [importStep, setImportStep] = useState<
|
||||||
|
'idle' | 'fetching' | 'converting' | 'uploading' | 'error'
|
||||||
|
>('idle');
|
||||||
|
const [importError, setImportError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const handleTabChange = useCallback((tab: TabValue) => {
|
||||||
|
setActiveTab(tab);
|
||||||
|
setImportError(null);
|
||||||
|
setImportStep('idle');
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// --- Create Doc Action ---
|
||||||
|
const handleCreateDocument = async () => {
|
||||||
|
if (!docName.trim()) {
|
||||||
|
toast.error('Please enter a document name');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setIsCreatingDoc(true);
|
||||||
|
try {
|
||||||
|
let filename = docName.trim();
|
||||||
|
const ext = docExtension;
|
||||||
|
if (!filename.toLowerCase().endsWith(ext)) {
|
||||||
|
filename += ext;
|
||||||
|
}
|
||||||
|
|
||||||
|
const mimeType = ext === '.md' ? 'text/markdown' : 'text/plain';
|
||||||
|
const file = new File([docContent], filename, { type: mimeType });
|
||||||
|
|
||||||
|
await uploadDocuments([file]);
|
||||||
|
toast.success(`"${filename}" created successfully!`);
|
||||||
|
|
||||||
|
// Reset inputs & close
|
||||||
|
setDocName('');
|
||||||
|
setDocContent('');
|
||||||
|
onClose();
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to create document:', err);
|
||||||
|
toast.error(err instanceof Error ? err.message : 'Failed to create document');
|
||||||
|
} finally {
|
||||||
|
setIsCreatingDoc(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// --- Import URL Action ---
|
||||||
|
const handleImportUrl = async () => {
|
||||||
|
let cleanUrl = webUrl.trim();
|
||||||
|
if (!cleanUrl) {
|
||||||
|
toast.error('Please enter a URL');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!/^https?:\/\//i.test(cleanUrl)) {
|
||||||
|
cleanUrl = 'https://' + cleanUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
setImportError(null);
|
||||||
|
setImportStep('fetching');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Step 1: Connect and scrape content on the server
|
||||||
|
const scrapeResult = await importUrl(cleanUrl);
|
||||||
|
|
||||||
|
setImportStep('converting');
|
||||||
|
setImportStep('uploading');
|
||||||
|
|
||||||
|
// Create virtual file from markdown content
|
||||||
|
const displayTitle = webTitle.trim() || scrapeResult.title || 'Imported Web Page';
|
||||||
|
const safeTitle = displayTitle
|
||||||
|
.replace(/[/\\?%*:|"<>\s]+/g, '_') // collapse runs of disallowed chars
|
||||||
|
.replace(/^_+|_+$/g, '') // trim leading/trailing underscores
|
||||||
|
.substring(0, 80);
|
||||||
|
const filename = `${safeTitle}.md`;
|
||||||
|
const file = new File([scrapeResult.content], filename, {
|
||||||
|
type: 'text/markdown',
|
||||||
|
});
|
||||||
|
|
||||||
|
await uploadDocuments([file]);
|
||||||
|
|
||||||
|
// Success
|
||||||
|
toast.success(`Successfully imported "${displayTitle}"!`);
|
||||||
|
setWebUrl('');
|
||||||
|
setWebTitle('');
|
||||||
|
setImportStep('idle');
|
||||||
|
onClose();
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to import URL:', err);
|
||||||
|
const message =
|
||||||
|
err instanceof Error ? err.message : 'An error occurred during import';
|
||||||
|
setImportError(message);
|
||||||
|
setImportStep('error');
|
||||||
|
toast.error(message);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleClose = () => {
|
||||||
|
// Only allow closing if not actively processing
|
||||||
|
if (isCreatingDoc || importStep === 'fetching' || importStep === 'converting' || importStep === 'uploading') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onClose();
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<SidebarDialog
|
||||||
|
open={isOpen}
|
||||||
|
onClose={handleClose}
|
||||||
|
headerTitle="Add Documents"
|
||||||
|
sections={SIDEBAR_SECTIONS}
|
||||||
|
activeSectionId={activeTab}
|
||||||
|
onSectionChange={handleTabChange}
|
||||||
|
className="h-[480px]"
|
||||||
|
>
|
||||||
|
{/* TAB 1: File Uploader */}
|
||||||
|
{activeTab === 'file' && (
|
||||||
|
<div className="h-full flex flex-col gap-4 animate-fade-in">
|
||||||
|
<p className="text-xs text-soft leading-relaxed shrink-0">
|
||||||
|
Select files from your computer or drag and drop them anywhere. Supported formats include PDF, EPUB, TXT, and MD.
|
||||||
|
</p>
|
||||||
|
<DocumentUploader
|
||||||
|
className="flex-1 flex flex-col justify-center border-2 border-dashed border-line rounded-lg bg-surface-sunken hover:bg-surface-solid transition-colors duration-base"
|
||||||
|
onUploadBatchChange={(state) => {
|
||||||
|
onUploadBatchChange?.(state);
|
||||||
|
// Automatically close modal when files successfully start uploading
|
||||||
|
if (state.isActive) {
|
||||||
|
onClose();
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* TAB 2: Create Custom Document */}
|
||||||
|
{activeTab === 'create' && (
|
||||||
|
<div className="h-full flex flex-col justify-between animate-fade-in">
|
||||||
|
<div className="space-y-3 flex-1 flex flex-col min-h-0">
|
||||||
|
<div className="flex flex-col sm:flex-row gap-2 shrink-0">
|
||||||
|
<div className="flex-1">
|
||||||
|
<Input
|
||||||
|
type="text"
|
||||||
|
value={docName}
|
||||||
|
onChange={(e) => setDocName(e.target.value)}
|
||||||
|
placeholder="Document title (e.g. notes)"
|
||||||
|
className="w-full font-medium"
|
||||||
|
controlSize="md"
|
||||||
|
disabled={isCreatingDoc}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="shrink-0">
|
||||||
|
<SegmentedControl<'.md' | '.txt'>
|
||||||
|
ariaLabel="Document format selection"
|
||||||
|
value={docExtension}
|
||||||
|
options={[
|
||||||
|
{ value: '.md', label: 'Markdown (.md)' },
|
||||||
|
{ value: '.txt', label: 'Plain Text (.txt)' },
|
||||||
|
]}
|
||||||
|
onChange={setDocExtension}
|
||||||
|
className="grid-cols-2"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex-1 flex flex-col min-h-0">
|
||||||
|
<Textarea
|
||||||
|
value={docContent}
|
||||||
|
onChange={(e) => setDocContent(e.target.value)}
|
||||||
|
placeholder={
|
||||||
|
docExtension === '.md'
|
||||||
|
? '# Write Markdown here\n\n- Bullet points\n- **Bold text**\n- Synchronized highlighting works automatically!'
|
||||||
|
: 'Type plain text document content here...'
|
||||||
|
}
|
||||||
|
className="flex-1 min-h-[160px] sm:min-h-[200px] font-mono text-xs p-3 leading-relaxed resize-none"
|
||||||
|
controlSize="md"
|
||||||
|
disabled={isCreatingDoc}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center justify-between pt-3 mt-4 shrink-0">
|
||||||
|
<span className="text-[11px] text-soft">
|
||||||
|
{docContent.length} character{docContent.length === 1 ? '' : 's'}
|
||||||
|
</span>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button
|
||||||
|
variant="secondary"
|
||||||
|
onClick={() => {
|
||||||
|
setDocName('');
|
||||||
|
setDocContent('');
|
||||||
|
}}
|
||||||
|
disabled={isCreatingDoc}
|
||||||
|
>
|
||||||
|
Clear
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={handleCreateDocument}
|
||||||
|
disabled={isCreatingDoc}
|
||||||
|
>
|
||||||
|
{isCreatingDoc ? 'Creating...' : 'Create'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* TAB 3: Import Web URL */}
|
||||||
|
{activeTab === 'url' && (
|
||||||
|
<div className="h-full flex flex-col justify-between animate-fade-in">
|
||||||
|
<div className="space-y-4 flex-1">
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
<div className="flex flex-col gap-1.5">
|
||||||
|
<label htmlFor="web-url" className="text-xs font-semibold text-foreground">
|
||||||
|
Source Web URL
|
||||||
|
</label>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Input
|
||||||
|
id="web-url"
|
||||||
|
type="url"
|
||||||
|
value={webUrl}
|
||||||
|
onChange={(e) => setWebUrl(e.target.value)}
|
||||||
|
placeholder="https://en.wikipedia.org/wiki/Speed_reading"
|
||||||
|
className="flex-1"
|
||||||
|
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||||
|
onKeyDown={(e) => {
|
||||||
|
if (e.key === 'Enter') {
|
||||||
|
e.preventDefault();
|
||||||
|
handleImportUrl();
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={handleImportUrl}
|
||||||
|
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||||
|
>
|
||||||
|
Import
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex flex-col gap-1.5">
|
||||||
|
<label htmlFor="web-title" className="text-xs font-semibold text-foreground">
|
||||||
|
Document Title (Optional)
|
||||||
|
</label>
|
||||||
|
<Input
|
||||||
|
id="web-title"
|
||||||
|
type="text"
|
||||||
|
value={webTitle}
|
||||||
|
onChange={(e) => setWebTitle(e.target.value)}
|
||||||
|
placeholder="Leave empty to use article title"
|
||||||
|
className="w-full"
|
||||||
|
disabled={importStep !== 'idle' && importStep !== 'error'}
|
||||||
|
onKeyDown={(e) => {
|
||||||
|
if (e.key === 'Enter') {
|
||||||
|
e.preventDefault();
|
||||||
|
handleImportUrl();
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-[11px] text-soft leading-normal mt-1">
|
||||||
|
Extracts the central article body, removes boilerplate noise (headers, sidebars, ads), and converts it into a clean Markdown document for synchrony reading.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Progress / Loading Stepper */}
|
||||||
|
{importStep !== 'idle' && (
|
||||||
|
<div className="rounded-lg border border-line bg-surface-sunken p-4 flex flex-col gap-3.5 transition-colors duration-base">
|
||||||
|
{importStep === 'error' ? (
|
||||||
|
<div className="flex items-start gap-2 text-danger">
|
||||||
|
<svg className="h-5 w-5 shrink-0 mt-0.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
|
||||||
|
</svg>
|
||||||
|
<div className="flex-1 min-w-0">
|
||||||
|
<p className="text-sm font-semibold">Import Failed</p>
|
||||||
|
<p className="text-xs mt-1 text-danger-strong overflow-hidden text-ellipsis leading-relaxed">
|
||||||
|
{importError || 'An unknown error occurred.'}
|
||||||
|
</p>
|
||||||
|
<button
|
||||||
|
onClick={() => setImportStep('idle')}
|
||||||
|
className="text-[11px] font-medium text-accent hover:underline mt-2 flex items-center gap-1"
|
||||||
|
>
|
||||||
|
<RefreshIcon className="h-3 w-3" /> Try Again
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<RefreshIcon className="h-5 w-5 text-accent animate-spin" />
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-semibold text-foreground">Importing Article</p>
|
||||||
|
<p className="text-xs text-soft">Please hold on, converting document...</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Visual Progress Steps */}
|
||||||
|
<div className="grid grid-cols-3 gap-2 mt-2 pt-2 text-center text-[10px] font-medium">
|
||||||
|
<div
|
||||||
|
className={`flex flex-col items-center gap-1 ${
|
||||||
|
importStep === 'fetching' ? 'text-accent' : 'text-soft'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||||
|
importStep === 'fetching'
|
||||||
|
? 'bg-accent text-background'
|
||||||
|
: importStep === 'converting' || importStep === 'uploading'
|
||||||
|
? 'bg-accent-wash text-accent'
|
||||||
|
: 'bg-surface border border-line'
|
||||||
|
}`}>
|
||||||
|
{importStep === 'converting' || importStep === 'uploading' ? (
|
||||||
|
<CheckIcon className="h-2.5 w-2.5" />
|
||||||
|
) : '1'}
|
||||||
|
</span>
|
||||||
|
Scraping Page
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className={`flex flex-col items-center gap-1 ${
|
||||||
|
importStep === 'converting' ? 'text-accent' : 'text-soft'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||||
|
importStep === 'converting'
|
||||||
|
? 'bg-accent text-background'
|
||||||
|
: importStep === 'uploading'
|
||||||
|
? 'bg-accent-wash text-accent'
|
||||||
|
: 'bg-surface border border-line'
|
||||||
|
}`}>
|
||||||
|
{importStep === 'uploading' ? (
|
||||||
|
<CheckIcon className="h-2.5 w-2.5" />
|
||||||
|
) : '2'}
|
||||||
|
</span>
|
||||||
|
Extracting Text
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className={`flex flex-col items-center gap-1 ${
|
||||||
|
importStep === 'uploading' ? 'text-accent' : 'text-soft'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<span className={`h-4 w-4 rounded-full flex items-center justify-center text-[9px] ${
|
||||||
|
importStep === 'uploading' ? 'bg-accent text-background animate-pulse' : 'bg-surface border border-line'
|
||||||
|
}`}>
|
||||||
|
3
|
||||||
|
</span>
|
||||||
|
Uploading
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</SidebarDialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -654,3 +654,28 @@ export function SparkleIcon(props: React.SVGProps<SVGSVGElement>) {
|
||||||
</svg>
|
</svg>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function BrowserIcon(props: React.SVGProps<SVGSVGElement>) {
|
||||||
|
return (
|
||||||
|
<svg
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
strokeWidth={2}
|
||||||
|
strokeLinecap="round"
|
||||||
|
strokeLinejoin="round"
|
||||||
|
className={props.className}
|
||||||
|
width={props.width || "1.5em"}
|
||||||
|
height={props.height || "1.5em"}
|
||||||
|
{...props}
|
||||||
|
>
|
||||||
|
<rect x="3" y="4" width="18" height="16" rx="2" />
|
||||||
|
<line x1="3" y1="9" x2="21" y2="9" />
|
||||||
|
<circle cx="6" cy="6.5" r="1" fill="currentColor" />
|
||||||
|
<circle cx="9" cy="6.5" r="1" fill="currentColor" />
|
||||||
|
<circle cx="12" cy="6.5" r="1" fill="currentColor" />
|
||||||
|
</svg>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -23,3 +23,4 @@ export * from './switch';
|
||||||
export * from './tokens';
|
export * from './tokens';
|
||||||
export * from './toolbar';
|
export * from './toolbar';
|
||||||
export * from './variants';
|
export * from './variants';
|
||||||
|
export * from './sidebar-dialog';
|
||||||
|
|
|
||||||
128
src/components/ui/sidebar-dialog.tsx
Normal file
128
src/components/ui/sidebar-dialog.tsx
Normal file
|
|
@ -0,0 +1,128 @@
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import { ReactNode } from 'react';
|
||||||
|
import { ModalFrame, ModalTitle, type DialogSize } from './dialog';
|
||||||
|
import { SidebarNav, SidebarNavItem } from './sidebar-nav';
|
||||||
|
import { cn } from './cn';
|
||||||
|
|
||||||
|
export type SidebarDialogSection<T extends string = string> = {
|
||||||
|
id: T;
|
||||||
|
label: string;
|
||||||
|
icon: React.ComponentType<React.SVGProps<SVGSVGElement>>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function SidebarDialog<T extends string = string>({
|
||||||
|
open,
|
||||||
|
onClose,
|
||||||
|
size = 'xl',
|
||||||
|
panelTestId,
|
||||||
|
modalClassName,
|
||||||
|
headerTitle,
|
||||||
|
headerRight,
|
||||||
|
showCloseButton = true,
|
||||||
|
sections,
|
||||||
|
activeSectionId,
|
||||||
|
onSectionChange,
|
||||||
|
children,
|
||||||
|
className,
|
||||||
|
contentClassName,
|
||||||
|
customContent,
|
||||||
|
}: {
|
||||||
|
open: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
size?: DialogSize;
|
||||||
|
panelTestId?: string;
|
||||||
|
modalClassName?: string;
|
||||||
|
headerTitle: ReactNode;
|
||||||
|
headerRight?: ReactNode;
|
||||||
|
showCloseButton?: boolean;
|
||||||
|
sections: SidebarDialogSection<T>[];
|
||||||
|
activeSectionId: T;
|
||||||
|
onSectionChange: (id: T) => void;
|
||||||
|
children: ReactNode;
|
||||||
|
className?: string;
|
||||||
|
contentClassName?: string;
|
||||||
|
customContent?: ReactNode;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<ModalFrame
|
||||||
|
open={open}
|
||||||
|
onClose={onClose}
|
||||||
|
size={size}
|
||||||
|
panelTestId={panelTestId}
|
||||||
|
className={modalClassName}
|
||||||
|
>
|
||||||
|
{customContent ? (
|
||||||
|
customContent
|
||||||
|
) : (
|
||||||
|
<div className={cn('flex flex-col h-[480px]', className)}>
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center justify-between px-5 py-3 border-b border-line-soft shrink-0">
|
||||||
|
<div className="flex items-baseline gap-4">
|
||||||
|
<ModalTitle>{headerTitle}</ModalTitle>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{headerRight}
|
||||||
|
{showCloseButton && (
|
||||||
|
<button
|
||||||
|
onClick={onClose}
|
||||||
|
className="rounded-md p-1 text-soft hover:bg-accent-wash hover:text-foreground transition-colors duration-base"
|
||||||
|
aria-label="Close dialog"
|
||||||
|
>
|
||||||
|
<svg className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M6 18L18 6M6 6l12 12" />
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Mobile Navigation (Horizontal row of items) */}
|
||||||
|
<SidebarNav layout="grid" className="sm:hidden border-b border-line-soft bg-background p-2 shrink-0">
|
||||||
|
{sections.map((section) => {
|
||||||
|
const Icon = section.icon;
|
||||||
|
return (
|
||||||
|
<SidebarNavItem
|
||||||
|
compact
|
||||||
|
key={section.id}
|
||||||
|
onClick={() => onSectionChange(section.id)}
|
||||||
|
active={activeSectionId === section.id}
|
||||||
|
icon={<Icon className="w-3.5 h-3.5" />}
|
||||||
|
label={section.label}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</SidebarNav>
|
||||||
|
|
||||||
|
{/* Main Body (Split layout on desktop) */}
|
||||||
|
<div className="flex flex-row min-h-0 flex-1">
|
||||||
|
{/* Desktop Navigation (Left sidebar) */}
|
||||||
|
<nav className="hidden sm:block w-fit shrink-0 border-r border-line-soft bg-background p-2">
|
||||||
|
<SidebarNav>
|
||||||
|
{sections.map((section) => {
|
||||||
|
const Icon = section.icon;
|
||||||
|
const active = activeSectionId === section.id;
|
||||||
|
return (
|
||||||
|
<SidebarNavItem
|
||||||
|
key={section.id}
|
||||||
|
onClick={() => onSectionChange(section.id)}
|
||||||
|
active={active}
|
||||||
|
icon={<Icon className="w-4 h-4" />}
|
||||||
|
label={section.label}
|
||||||
|
className="whitespace-nowrap"
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</SidebarNav>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
{/* Tab Panel Content (Right detail view) */}
|
||||||
|
<div className={cn('flex-1 min-w-0 p-4 overflow-y-auto bg-surface', contentClassName)}>
|
||||||
|
{children}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</ModalFrame>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -587,3 +587,23 @@ export async function getDocumentPreviewStatus(
|
||||||
|
|
||||||
throw new Error(`Failed to load preview status (status ${res.status})`);
|
throw new Error(`Failed to load preview status (status ${res.status})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function importUrl(
|
||||||
|
url: string,
|
||||||
|
options?: { signal?: AbortSignal }
|
||||||
|
): Promise<{ title: string; content: string }> {
|
||||||
|
const res = await fetch('/api/documents/import-url', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ url }),
|
||||||
|
signal: options?.signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||||
|
throw new Error(data?.error || 'Failed to import URL');
|
||||||
|
}
|
||||||
|
|
||||||
|
return (await res.json()) as { title: string; content: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
|
|
||||||
243
src/lib/server/documents/web-loader.ts
Normal file
243
src/lib/server/documents/web-loader.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
||||||
|
import http from 'http';
|
||||||
|
import https from 'https';
|
||||||
|
import { parseHTML } from 'linkedom';
|
||||||
|
import { Readability } from '@mozilla/readability';
|
||||||
|
import TurndownService from 'turndown';
|
||||||
|
import { AntiSSRFPolicy, PolicyConfigOptions } from '@microsoft/antissrf';
|
||||||
|
|
||||||
|
const MAX_BYTES = 2 * 1024 * 1024;
|
||||||
|
const TIMEOUT_MS = 6000;
|
||||||
|
const MAX_REDIRECTS = 5;
|
||||||
|
const USER_AGENT =
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SSRF protection is delegated to Microsoft's AntiSSRF library. The policy's
|
||||||
|
* agents inject a connection-time DNS lookup that resolves the host and rejects
|
||||||
|
* any request whose resolved address falls in an internal/special-purpose range
|
||||||
|
* (loopback, private, link-local, CGNAT, cloud metadata, multicast, …). Because
|
||||||
|
* the check happens on the address the socket actually connects to, it is
|
||||||
|
* inherently safe against DNS rebinding, and it re-runs on every redirect hop
|
||||||
|
* (each hop is a fresh request through the agent).
|
||||||
|
*
|
||||||
|
* `ExternalOnlyLatest` blocks the `recommendedLatest` range set and stays
|
||||||
|
* up to date without code changes. We allow plain HTTP because article URLs are
|
||||||
|
* not always HTTPS; remove `allowPlainTextHttp` to require HTTPS.
|
||||||
|
*/
|
||||||
|
const ssrfPolicy = new AntiSSRFPolicy(PolicyConfigOptions.ExternalOnlyLatest);
|
||||||
|
ssrfPolicy.allowPlainTextHttp = true;
|
||||||
|
const httpAgent = ssrfPolicy.getHttpAgent();
|
||||||
|
const httpsAgent = ssrfPolicy.getHttpsAgent();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Performs a single GET request through the AntiSSRF agent. Resolves with either
|
||||||
|
* the response body or a redirect location to follow. Enforces a size limit and
|
||||||
|
* a hard timeout (resource limits, independent of the SSRF policy).
|
||||||
|
*/
|
||||||
|
function requestOnce(
|
||||||
|
url: URL,
|
||||||
|
maxBytes: number,
|
||||||
|
timeoutMs: number
|
||||||
|
): Promise<{ body?: string; redirectLocation?: string }> {
|
||||||
|
const isHttps = url.protocol === 'https:';
|
||||||
|
const lib = isHttps ? https : http;
|
||||||
|
const agent = isHttps ? httpsAgent : httpAgent;
|
||||||
|
const port = url.port ? Number(url.port) : isHttps ? 443 : 80;
|
||||||
|
|
||||||
|
const options: https.RequestOptions = {
|
||||||
|
protocol: url.protocol,
|
||||||
|
hostname: url.hostname, // connect by hostname so the agent's safe lookup runs
|
||||||
|
port,
|
||||||
|
path: `${url.pathname}${url.search}`,
|
||||||
|
method: 'GET',
|
||||||
|
agent, // AntiSSRF agent enforces the SSRF policy at connection time
|
||||||
|
headers: {
|
||||||
|
'User-Agent': USER_AGENT,
|
||||||
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||||
|
},
|
||||||
|
timeout: timeoutMs,
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let settled = false;
|
||||||
|
const ok = (v: { body?: string; redirectLocation?: string }) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(hardTimer);
|
||||||
|
resolve(v);
|
||||||
|
};
|
||||||
|
const fail = (e: Error) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(hardTimer);
|
||||||
|
reject(e);
|
||||||
|
};
|
||||||
|
|
||||||
|
const req = lib.request(options, (res) => {
|
||||||
|
const status = res.statusCode ?? 0;
|
||||||
|
|
||||||
|
if (status >= 300 && status < 400 && res.headers.location) {
|
||||||
|
res.resume(); // drain so the socket can be reused/closed
|
||||||
|
ok({ redirectLocation: res.headers.location });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status < 200 || status >= 300) {
|
||||||
|
res.resume();
|
||||||
|
fail(new Error(`Failed to fetch webpage (status ${status})`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = (res.headers['content-type'] || '').toString();
|
||||||
|
const isHtml =
|
||||||
|
contentType.includes('text/html') ||
|
||||||
|
contentType.includes('application/xhtml+xml');
|
||||||
|
const isText = contentType.includes('text/plain');
|
||||||
|
if (!isHtml && !isText) {
|
||||||
|
res.resume();
|
||||||
|
fail(
|
||||||
|
new Error('Unsupported content type. Only web pages or text files are supported.')
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = res.headers['content-length'];
|
||||||
|
if (contentLength && parseInt(contentLength.toString(), 10) > maxBytes) {
|
||||||
|
res.resume();
|
||||||
|
fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream and enforce the size limit dynamically (header may be absent/spoofed).
|
||||||
|
let received = 0;
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
res.on('data', (chunk: Buffer) => {
|
||||||
|
received += chunk.length;
|
||||||
|
if (received > maxBytes) {
|
||||||
|
req.destroy();
|
||||||
|
fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
chunks.push(chunk);
|
||||||
|
});
|
||||||
|
res.on('end', () => ok({ body: Buffer.concat(chunks).toString('utf-8') }));
|
||||||
|
res.on('error', fail);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Hard deadline covering the whole request (connect + transfer), in addition
|
||||||
|
// to the socket idle timeout, to bound slow-read/slowloris responses.
|
||||||
|
const hardTimer = setTimeout(() => {
|
||||||
|
req.destroy(new Error('Request timed out'));
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
req.on('timeout', () => {
|
||||||
|
req.destroy(new Error('Request timed out'));
|
||||||
|
});
|
||||||
|
req.on('error', fail);
|
||||||
|
req.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates and fetches a URL with a size limit (default 2MB) and connection
|
||||||
|
* timeout (default 6s). Redirects are followed manually so each hop is re-checked
|
||||||
|
* by the AntiSSRF agent and counted against the redirect cap. Returns the body
|
||||||
|
* and the final (post-redirect) URL.
|
||||||
|
*/
|
||||||
|
async function fetchWithLimit(
|
||||||
|
urlStr: string,
|
||||||
|
maxBytes: number = MAX_BYTES,
|
||||||
|
timeoutMs: number = TIMEOUT_MS
|
||||||
|
): Promise<{ html: string; finalUrl: string }> {
|
||||||
|
let currentUrl = urlStr;
|
||||||
|
|
||||||
|
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
|
||||||
|
let url: URL;
|
||||||
|
try {
|
||||||
|
url = new URL(currentUrl);
|
||||||
|
} catch {
|
||||||
|
throw new Error('Invalid URL format');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
||||||
|
throw new Error('Only HTTP and HTTPS protocols are supported');
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await requestOnce(url, maxBytes, timeoutMs);
|
||||||
|
|
||||||
|
if (result.redirectLocation) {
|
||||||
|
currentUrl = new URL(result.redirectLocation, url).href;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { html: result.body ?? '', finalUrl: currentUrl };
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error('Too many redirects while fetching the webpage.');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connects to, extracts readable content from, and converts a web page to Markdown.
|
||||||
|
*/
|
||||||
|
export async function fetchAndParseUrl(
|
||||||
|
urlStr: string
|
||||||
|
): Promise<{ title: string; content: string }> {
|
||||||
|
// 1. Fetch HTML content with limit. The AntiSSRF agent validates the
|
||||||
|
// destination (and every redirect hop) at connection time.
|
||||||
|
const { html, finalUrl } = await fetchWithLimit(urlStr);
|
||||||
|
|
||||||
|
// 2. Parse HTML string to a virtual DOM document
|
||||||
|
const { document } = parseHTML(html);
|
||||||
|
|
||||||
|
// Resolve relative URLs against the final (post-redirect) location so links
|
||||||
|
// and images point at the page we actually fetched.
|
||||||
|
const resolveUrl = (relativeUrl: string) => {
|
||||||
|
try {
|
||||||
|
return new URL(relativeUrl, finalUrl).href;
|
||||||
|
} catch {
|
||||||
|
return relativeUrl;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const images = document.querySelectorAll('img');
|
||||||
|
for (const img of images) {
|
||||||
|
const src = img.getAttribute('src');
|
||||||
|
if (src) {
|
||||||
|
img.setAttribute('src', resolveUrl(src));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const links = document.querySelectorAll('a');
|
||||||
|
for (const link of links) {
|
||||||
|
const href = link.getAttribute('href');
|
||||||
|
if (href) {
|
||||||
|
link.setAttribute('href', resolveUrl(href));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Run Readability to extract core article text
|
||||||
|
const reader = new Readability(document as unknown as Document);
|
||||||
|
const article = reader.parse();
|
||||||
|
|
||||||
|
if (!article || !article.content) {
|
||||||
|
throw new Error('Could not extract meaningful article content from this webpage.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Convert clean HTML content to Markdown
|
||||||
|
const turndownService = new TurndownService({
|
||||||
|
headingStyle: 'atx',
|
||||||
|
hr: '---',
|
||||||
|
bulletListMarker: '-',
|
||||||
|
codeBlockStyle: 'fenced',
|
||||||
|
});
|
||||||
|
|
||||||
|
const markdown = turndownService.turndown(article.content);
|
||||||
|
if (!markdown.trim()) {
|
||||||
|
throw new Error('Extracted content is empty.');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: article.title?.trim() || 'Web Import',
|
||||||
|
content: markdown,
|
||||||
|
};
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue