danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #74 from arabcoders/dev

Browse source 
Locally check id before requesting extract_info
This commit is contained in:
Abdulmohsen 2024-03-08 16:57:36 +03:00 committed by GitHub
commit 8b7be3a366
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 158 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.vscode/launch.json vendored
View file

@ -27,6 +27,22 @@
"request": "launch", "request": "launch",
"program": "app/main.py", "program": "app/main.py",
"console": "internalConsole", "console": "internalConsole",
"justMyCode": true,
"env": {
"YTP_CONFIG_PATH": "${workspaceFolder}/var/config",
"YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",
"YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
"YTP_URL_HOST": "http://localhost:8081",
"YTP_LOG_LEVEL": "DEBUG"
}
},
{
"name": "Python: main.py (Deep)",
"type": "debugpy",
"request": "launch",
"program": "app/main.py",
"console": "internalConsole",
"justMyCode": false,
"env": { "env": {
"YTP_CONFIG_PATH": "${workspaceFolder}/var/config", "YTP_CONFIG_PATH": "${workspaceFolder}/var/config",
"YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads", "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",

2
app/Config.py
View file

@ -45,6 +45,8 @@ class Config:
new_version_available: bool = False new_version_available: bool = False
extract_info_timeout: int = 70
_int_vars: tuple = ('port', 'max_workers',) _int_vars: tuple = ('port', 'max_workers',)
_immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available') _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available')
_boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',) _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',)

87
app/DownloadQueue.py
View file

@ -211,57 +211,43 @@ class DownloadQueue:
LOG.info(f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}') LOG.info(f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}')
already = set() if already is None else already already = set() if already is None else already
if url in already: if url in already:
LOG.info('recursion detected, skipping') LOG.info('recursion detected, skipping.')
return {'status': 'ok'} return {'status': 'ok'}
else:
already.add(url) already.add(url)
try: try:
with ThreadPoolExecutor(thread_name_prefix='extract_info') as pool: downloaded, id_dict = self.isDownloaded(url)
LOG.debug(f'extracting info from {url=}') if downloaded is True:
entry = await asyncio.get_running_loop().run_in_executor( message = f"[ { id_dict.get('id') } ]: has been downloaded already."
pool, LOG.info(message)
return {'status': 'error', 'msg': message}
started = time.perf_counter()
LOG.debug(f'extract_info: checking {url=}')
entry = await asyncio.wait_for(
fut=asyncio.get_running_loop().run_in_executor(
None,
ExtractInfo, ExtractInfo,
mergeConfig(self.config.ytdl_options, ytdlp_config), mergeConfig(self.config.ytdl_options, ytdlp_config),
url, url,
bool(self.config.ytdl_debug) bool(self.config.ytdl_debug)
) ),
timeout=self.config.extract_info_timeout)
if not entry: if not entry:
if not self.config.keep_archive: return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
return {
'status': 'error',
'msg': 'No metadata, most likely video has been downloaded before.' if self.config.keep_archive else 'Unable to extract info check logs.'
}
LOG.debug(f'No metadata, Rechecking with archive disabled. {url=}') LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}')
entry = await asyncio.get_running_loop().run_in_executor(
pool,
ExtractInfo,
mergeConfig(self.config.ytdl_options, ytdlp_config),
url,
bool(self.config.ytdl_debug),
True
)
if not entry:
return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
if self.isDownloaded(entry):
LOG.info(f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.')
return {
'status': 'error',
'msg': f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.'
}
if self.isDownloaded(entry):
raise yt_dlp.utils.ExistingVideoReached()
LOG.debug(f'extract_info length: {len(entry)}')
except yt_dlp.utils.ExistingVideoReached: except yt_dlp.utils.ExistingVideoReached:
return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'} return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
except yt_dlp.utils.YoutubeDLError as exc: except yt_dlp.utils.YoutubeDLError as exc:
return {'status': 'error', 'msg': str(exc)} return {'status': 'error', 'msg': str(exc)}
except asyncio.exceptions.TimeoutError as exc:
return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'}
return await self.__add_entry( return await self.__add_entry(
entry=entry, entry=entry,
@ -282,31 +268,37 @@ class DownloadQueue:
LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}') LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}')
continue continue
itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
if item.started() is True: if item.started() is True:
LOG.info(f'Canceling {id=} {item.info.title=}') LOG.debug(f'Canceling {itemMessage}')
item.cancel() item.cancel()
LOG.info(f'Cancelled {itemMessage}')
else: else:
item.close() item.close()
LOG.info(f'Deleting from queue {id=} {item.info.title=}') LOG.debug(f'Deleting from queue {itemMessage}')
self.queue.delete(id) self.queue.delete(id)
await self.notifier.canceled(id) await self.notifier.canceled(id)
self.done.put(item) self.done.put(item)
await self.notifier.completed(item) await self.notifier.completed(item)
LOG.info(f'Deleted from queue {itemMessage}')
return {'status': 'ok'} return {'status': 'ok'}
async def clear(self, ids): async def clear(self, ids):
for id in ids: for id in ids:
try: try:
item = self.done.get(key=id) item = self.done.get(key=id)
except KeyError as e: except KeyError as e:
LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}') LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}')
continue continue
LOG.info(f'Deleting completed download {id=} {item.info.title=}') itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
LOG.debug(f'Deleting completed download {itemMessage}')
self.done.delete(id) self.done.delete(id)
await self.notifier.cleared(id) await self.notifier.cleared(id)
LOG.info(f'Deleted completed download {itemMessage}')
return {'status': 'ok'} return {'status': 'ok'}
@ -348,10 +340,8 @@ class DownloadQueue:
LOG.info(f'Waiting for worker to be free.') LOG.info(f'Waiting for worker to be free.')
await asyncio.sleep(1) await asyncio.sleep(1)
LOG.debug(f"Has '{executor.get_available_workers()}' free workers.")
while not self.queue.hasDownloads(): while not self.queue.hasDownloads():
LOG.info('Waiting for item to download.') LOG.info(f"Waiting for item to download. '{executor.get_available_workers()}' free workers.")
await self.event.wait() await self.event.wait()
self.event.clear() self.event.clear()
LOG.debug(f"Cleared wait event.") LOG.debug(f"Cleared wait event.")
@ -409,5 +399,8 @@ class DownloadQueue:
self.event.set() self.event.set()
def isDownloaded(self, info: dict) -> bool: def isDownloaded(self, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]:
return self.config.keep_archive and isDownloaded(self.config.ytdl_options.get('download_archive', None), info) if not url or not self.config.keep_archive:
return False, None
return isDownloaded(self.config.ytdl_options.get('download_archive', None), url)

59
app/Utils.py
View file

@ -14,6 +14,8 @@ LOG = logging.getLogger('Utils')
AUDIO_FORMATS: tuple = ('m4a', 'mp3', 'opus', 'wav') AUDIO_FORMATS: tuple = ('m4a', 'mp3', 'opus', 'wav')
IGNORED_KEYS: tuple = ('cookiefile', 'paths', 'outtmpl', 'progress_hooks', 'postprocessor_hooks',) IGNORED_KEYS: tuple = ('cookiefile', 'paths', 'outtmpl', 'progress_hooks', 'postprocessor_hooks',)
YTDLP_INFO_CLS = None
def get_format(format: str, quality: str) -> str: def get_format(format: str, quality: str) -> str:
""" """
@ -197,24 +199,53 @@ def mergeConfig(config: dict, new_config: dict) -> dict:
return mergeDict(new_config, config) return mergeDict(new_config, config)
def isDownloaded(archive_file: str, info: dict) -> bool: def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]:
if not info or not archive_file or not os.path.exists(archive_file): global YTDLP_INFO_CLS
return False
try: idDict = {
id = yt_dlp.YoutubeDL()._make_archive_id(info) 'id': None,
if not id: 'ie_key': None,
return False 'archive_id': None,
except Exception as e: }
LOG.error(f'Error generating archive id: {e}')
return False if not url or not archive_file or not os.path.exists(archive_file):
return False, idDict,
if not YTDLP_INFO_CLS:
YTDLP_INFO_CLS = yt_dlp.YoutubeDL(params={
'color': 'no_color',
'extract_flat': True,
'skip_download': True,
'ignoreerrors': True,
'ignore_no_formats_error': True,
'quiet': True,
})
for key, ie in YTDLP_INFO_CLS._ies.items():
if not ie.suitable(url):
continue
if not ie.working():
break
temp_id = ie.get_temp_id(url)
if not temp_id:
break
idDict['id'] = temp_id
idDict['ie_key'] = key
idDict['archive_id'] = YTDLP_INFO_CLS._make_archive_id(idDict)
break
if not idDict['archive_id']:
return False, idDict,
with open(archive_file, 'r') as f: with open(archive_file, 'r') as f:
for line in f.readlines(): for line in f.readlines():
if id in line: if idDict['archive_id'] in line:
return True return True, idDict,
return False return False, idDict,
def jsonCookie(cookies: dict[dict[str, any]]) -> str | None: def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
@ -241,7 +272,7 @@ def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
cookieDict = cookies[domain][subDomain][cookie] cookieDict = cookies[domain][subDomain][cookie]
if 0 == int(cookieDict['expirationDate']): if 0 == int(cookieDict['expirationDate']):
cookieDict['expirationDate']: float = datetime.now(timezone.utc).timestamp() + (86400 * 1000) cookieDict['expirationDate'] = datetime.now(timezone.utc).timestamp() + (86400 * 1000)
hasCookies = True hasCookies = True
netscapeCookies += "\t".join([ netscapeCookies += "\t".join([

6
frontend/src/App.vue
View file

@ -1,5 +1,6 @@
<template> <template>
<PageHeader :config="config" @toggleForm="addForm = !addForm" @toggleTasks="showTasks = !showTasks" /> <PageHeader :config="config" @toggleForm="addForm = !addForm" @toggleTasks="showTasks = !showTasks"
@reload="reloadWindow" />
<formAdd v-if="addForm" :config="config" @addItem="addItem" /> <formAdd v-if="addForm" :config="config" @addItem="addItem" />
<pageTasks v-if="showTasks" :tasks="config.tasks" /> <pageTasks v-if="showTasks" :tasks="config.tasks" />
<DownloadingList :config="config" :queue="downloading" @deleteItem="deleteItem" /> <DownloadingList :config="config" :queue="downloading" @deleteItem="deleteItem" />
@ -176,6 +177,8 @@ const playItem = (item) => {
video_link.value = config.app.url_host + config.app.url_prefix + baseDir + encodeURIComponent(item.filename); video_link.value = config.app.url_host + config.app.url_prefix + baseDir + encodeURIComponent(item.filename);
}; };
const reloadWindow = () => window.location.reload();
bus.on((event, data) => { bus.on((event, data) => {
if (!['show_form'].includes(event)) { if (!['show_form'].includes(event)) {
return true; return true;
@ -187,4 +190,3 @@ bus.on((event, data) => {
}); });
</script> </script>

78
frontend/src/components/Page-Header.vue
View file

@ -17,11 +17,24 @@
<font-awesome-icon icon="fa-solid fa-tasks" /> <font-awesome-icon icon="fa-solid fa-tasks" />
</button> </button>
</div> </div>
<div class="navbar-item">
<button v-tooltip="'Reload window'" class="button is-dark has-tooltip-bottom" @click="$emit('reload')">
<font-awesome-icon icon="fa-solid fa-rotate-right" />
</button>
</div>
<div class="navbar-item"> <div class="navbar-item">
<button v-tooltip="'Switch to Light theme'" class="button is-dark has-tooltip-bottom" <button v-tooltip="'Switch to Light theme'" class="button is-dark has-tooltip-bottom"
@click="selectedTheme = 'light'" v-if="selectedTheme == 'dark'">🌞</button> @click="selectedTheme = 'light'" v-if="selectedTheme == 'dark'">
<span class="icon is-small is-left has-text-warning">
<font-awesome-icon icon="fa-solid fa-sun" />
</span>
</button>
<button v-tooltip="'Switch to Dark theme'" class="button is-dark has-tooltip-bottom" <button v-tooltip="'Switch to Dark theme'" class="button is-dark has-tooltip-bottom"
@click="selectedTheme = 'dark'" v-if="selectedTheme == 'light'">🌚</button> @click="selectedTheme = 'dark'" v-if="selectedTheme == 'light'">
<span class="icon is-small is-left">
<font-awesome-icon icon="fa-solid fa-moon" />
</span>
</button>
</div> </div>
</div> </div>
</nav> </nav>
@ -45,36 +58,39 @@ defineProps({
const applyPreferredColorScheme = (scheme) => { const applyPreferredColorScheme = (scheme) => {
for (var s = 0; s < document.styleSheets.length; s++) { for (var s = 0; s < document.styleSheets.length; s++) {
for (var i = 0; i < document.styleSheets[s].cssRules.length; i++) { for (var i = 0; i < document.styleSheets[s].cssRules.length; i++) {
const rule = document.styleSheets[s].cssRules[i]; try {
const rule = document.styleSheets[s].cssRules[i];
if (rule && rule.media && rule.media.mediaText.includes("prefers-color-scheme")) { if (rule && rule.media && rule.media.mediaText.includes("prefers-color-scheme")) {
switch (scheme) { switch (scheme) {
case "light": case "light":
rule.media.appendMedium("original-prefers-color-scheme"); rule.media.appendMedium("original-prefers-color-scheme");
if (rule.media.mediaText.includes("light")) { if (rule.media.mediaText.includes("light")) {
rule.media.deleteMedium("(prefers-color-scheme: light)"); rule.media.deleteMedium("(prefers-color-scheme: light)");
} }
if (rule.media.mediaText.includes("dark")) { if (rule.media.mediaText.includes("dark")) {
rule.media.deleteMedium("(prefers-color-scheme: dark)"); rule.media.deleteMedium("(prefers-color-scheme: dark)");
} }
break; break;
case "dark": case "dark":
rule.media.appendMedium("(prefers-color-scheme: light)"); rule.media.appendMedium("(prefers-color-scheme: light)");
rule.media.appendMedium("(prefers-color-scheme: dark)"); rule.media.appendMedium("(prefers-color-scheme: dark)");
if (rule.media.mediaText.includes("original")) { if (rule.media.mediaText.includes("original")) {
rule.media.deleteMedium("original-prefers-color-scheme"); rule.media.deleteMedium("original-prefers-color-scheme");
} }
break; break;
default: default:
rule.media.appendMedium("(prefers-color-scheme: dark)"); rule.media.appendMedium("(prefers-color-scheme: dark)");
if (rule.media.mediaText.includes("light")) { if (rule.media.mediaText.includes("light")) {
rule.media.deleteMedium("(prefers-color-scheme: light)"); rule.media.deleteMedium("(prefers-color-scheme: light)");
} }
if (rule.media.mediaText.includes("original")) { if (rule.media.mediaText.includes("original")) {
rule.media.deleteMedium("original-prefers-color-scheme"); rule.media.deleteMedium("original-prefers-color-scheme");
} }
break; break;
}
} }
} catch (e) {
console.debug(e);
} }
} }
} }

7
frontend/src/main.js
View file

@ -6,7 +6,7 @@ import { library } from '@fortawesome/fontawesome-svg-core'
import { FontAwesomeIcon } from '@fortawesome/vue-fontawesome' import { FontAwesomeIcon } from '@fortawesome/vue-fontawesome'
import { import {
faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload, faUpRightFromSquare, faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload, faUpRightFromSquare,
faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ, faArrowDownAZ, faEject, faGlobe faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ, faArrowDownAZ, faEject, faGlobe, faMoon, faSun
} from '@fortawesome/free-solid-svg-icons' } from '@fortawesome/free-solid-svg-icons'
import { faSquare, faSquareCheck } from '@fortawesome/free-regular-svg-icons' import { faSquare, faSquareCheck } from '@fortawesome/free-regular-svg-icons'
@ -17,8 +17,9 @@ import './assets/css/bulma-dark.css'
import './assets/css/style.css' import './assets/css/style.css'
import 'floating-vue/dist/style.css' import 'floating-vue/dist/style.css'
library.add(faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload, faUpRightFromSquare, library.add(faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload,
faSquare, faSquareCheck, faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ, faArrowDownAZ, faEject, faGlobe) faUpRightFromSquare, faSquare, faSquareCheck, faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ,
faArrowDownAZ, faEject, faGlobe, faMoon, faSun)
const app = createApp(App); const app = createApp(App);
app.config.globalProperties.capitalize = s => s && s[0].toUpperCase() + s.slice(1); app.config.globalProperties.capitalize = s => s && s[0].toUpperCase() + s.slice(1);