From 708e058b01493d392188447f6d05fbed3c3e0bf0 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Mon, 11 Mar 2024 19:39:32 +0300 Subject: [PATCH 1/5] Added socket_timeout. --- app/Config.py | 7 ++++++- app/Download.py | 3 +++ app/DownloadQueue.py | 5 ++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/Config.py b/app/Config.py index 0847dc70..7d905d06 100644 --- a/app/Config.py +++ b/app/Config.py @@ -8,6 +8,7 @@ from version import APP_VERSION from dotenv import load_dotenv from yt_dlp.version import __version__ as YTDLP_VERSION + class Config: __instance = None config_path: str = '.' @@ -47,9 +48,11 @@ class Config: extract_info_timeout: int = 70 + socket_timeout: int = 30 + ytdlp_version: str = YTDLP_VERSION - _int_vars: tuple = ('port', 'max_workers',) + _int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout',) _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available', 'ytdlp_version',) _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',) @@ -153,6 +156,8 @@ class Config: else: LOG.info(f'No custom yt-dlp options found in "{self.config_path}"') + self.ytdl_options['socket_timeout'] = self.socket_timeout + if self.keep_archive: LOG.info(f'keep archive: {self.keep_archive}') self.ytdl_options['download_archive'] = os.path.join( diff --git a/app/Download.py b/app/Download.py index cd10785a..1da4a8fd 100644 --- a/app/Download.py +++ b/app/Download.py @@ -261,6 +261,9 @@ class Download: if 'filename' in status: self.info.filename = os.path.relpath(status.get('filename'), self.download_dir) + if os.path.exists(status.get('filename')): + self.info.file_size = os.path.getsize(status.get('filename')) + # Set correct file extension for thumbnails if self.info.format == 'thumbnail': self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) diff --git a/app/DownloadQueue.py b/app/DownloadQueue.py index 46e6619e..4d89ae29 100644 --- a/app/DownloadQueue.py +++ b/app/DownloadQueue.py @@ -242,11 +242,14 @@ class DownloadQueue: return {'status': 'error', 'msg': 'Unable to extract info check logs.'} LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}') - except yt_dlp.utils.ExistingVideoReached: + except yt_dlp.utils.ExistingVideoReached as exc: + LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}') return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'} except yt_dlp.utils.YoutubeDLError as exc: + LOG.error(f'YoutubeDLError: Unable to extract info. {str(exc)}') return {'status': 'error', 'msg': str(exc)} except asyncio.exceptions.TimeoutError as exc: + LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}') return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'} return await self.__add_entry( From 8042f5ac45e99becababb4599c7706f05acfc2fd Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Mon, 11 Mar 2024 19:45:51 +0300 Subject: [PATCH 2/5] Serve static content from memory. --- app/main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/main.py b/app/main.py index ec360acc..8866dbf0 100644 --- a/app/main.py +++ b/app/main.py @@ -397,9 +397,13 @@ class Main: if req.path not in self.staticHolder: return web.HTTPNotFound() - return web.FileResponse(path=self.staticHolder[req.path].get('file'), headers={ - 'Cache-Control': 'public, max-age=31536000', + item: dict = self.staticHolder[req.path] + + return web.Response(body=item.get('content'), headers={ 'Pragma': 'public', + 'Cache-Control': 'public, max-age=31536000', + 'Content-Type': item.get('content_type'), + 'X-Via': 'memory' if not item.get('file', None) else 'disk', }) staticDir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'frontend/dist') @@ -412,8 +416,8 @@ class Main: urlPath = f"{self.config.url_prefix}{file.replace(f'{staticDir}/', '')}" self.staticHolder[urlPath] = { - 'file': file, - # 'content': open(file, 'rb').read(), + # 'file': file, + 'content': open(file, 'rb').read(), 'content_type': self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)), } From d24a24157982292db7f17b55e667dbe8a16d087a Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Wed, 13 Mar 2024 22:41:39 +0300 Subject: [PATCH 3/5] Cleanup code. --- app/DownloadQueue.py | 24 +++++++++++++----------- app/Utils.py | 5 +---- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/app/DownloadQueue.py b/app/DownloadQueue.py index 4d89ae29..2799a063 100644 --- a/app/DownloadQueue.py +++ b/app/DownloadQueue.py @@ -11,7 +11,8 @@ from ItemDTO import ItemDTO from DataStore import DataStore from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig from AsyncPool import AsyncPool -from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor +from concurrent.futures import ThreadPoolExecutor + LOG = logging.getLogger('DownloadQueue') TYPE_DONE: str = 'done' TYPE_QUEUE: str = 'queue' @@ -228,15 +229,16 @@ class DownloadQueue: started = time.perf_counter() LOG.debug(f'extract_info: checking {url=}') - entry = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - ExtractInfo, - mergeConfig(self.config.ytdl_options, ytdlp_config), - url, - bool(self.config.ytdl_debug) - ), - timeout=self.config.extract_info_timeout) + with ThreadPoolExecutor as executor: + entry = await asyncio.wait_for( + fut=asyncio.get_running_loop().run_in_executor( + executor, + ExtractInfo, + mergeConfig(self.config.ytdl_options, ytdlp_config), + url, + bool(self.config.ytdl_debug) + ), + timeout=self.config.extract_info_timeout) if not entry: return {'status': 'error', 'msg': 'Unable to extract info check logs.'} @@ -250,7 +252,7 @@ class DownloadQueue: return {'status': 'error', 'msg': str(exc)} except asyncio.exceptions.TimeoutError as exc: LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}') - return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'} + return {'status': 'error', 'msg': f'TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.'} return await self.__add_entry( entry=entry, diff --git a/app/Utils.py b/app/Utils.py index 8111807b..2794ec30 100644 --- a/app/Utils.py +++ b/app/Utils.py @@ -137,7 +137,7 @@ def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) return download_path -def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = False) -> dict: +def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict: params: dict = { 'color': 'no_color', 'extract_flat': True, @@ -158,9 +158,6 @@ def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = else: params['quiet'] = True - if forceLookup and 'download_archive' in params: - params.pop('download_archive') - return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) From 5098244193ce42b32c571e899d8b5f8961903194 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Wed, 13 Mar 2024 22:49:40 +0300 Subject: [PATCH 4/5] fix --- app/DownloadQueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/DownloadQueue.py b/app/DownloadQueue.py index 2799a063..a8d32228 100644 --- a/app/DownloadQueue.py +++ b/app/DownloadQueue.py @@ -229,7 +229,7 @@ class DownloadQueue: started = time.perf_counter() LOG.debug(f'extract_info: checking {url=}') - with ThreadPoolExecutor as executor: + with ThreadPoolExecutor(1) as executor: entry = await asyncio.wait_for( fut=asyncio.get_running_loop().run_in_executor( executor, From 439cc55b2a3e9a452d9b7e20925302f88f435ed3 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Wed, 13 Mar 2024 23:37:07 +0300 Subject: [PATCH 5/5] updated dockerfile --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5efe8c34..6847f9bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,8 @@ ENV YTP_CONFIG_PATH=/config ENV YTP_TEMP_PATH=/tmp ENV YTP_DOWNLOAD_PATH=/downloads ENV YTP_PORT=8081 +ENV XDG_CONFIG_HOME=/config +ENV XDG_CACHE_HOME=/tmp # removed ffmpeg as 6.1.0 is broken with DASH protocal downloads COPY --from=mwader/static-ffmpeg:6.1.1 /ffmpeg /usr/bin/ @@ -47,10 +49,10 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh COPY --chown=app:app ./app /app/app COPY --chown=app:app --from=npm_builder /ytptube/dist /app/frontend/dist -COPY --chown=app:app --from=python_builder /app/.venv /app/.venv +COPY --chown=app:app --from=python_builder /app/.venv /opt/python COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck -ENV PATH="/app/.venv/bin:$PATH" +ENV PATH="/opt/python/bin:$PATH" RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck @@ -69,4 +71,4 @@ ENTRYPOINT ["/entrypoint.sh"] ENV PYDEVD_DISABLE_FILE_VALIDATION=1 -CMD ["/app/.venv/bin/python", "/app/app/main.py", "--ytptube-mp"] +CMD ["/opt/python/bin/python", "/app/app/main.py", "--ytptube-mp"]