diff --git a/Dockerfile b/Dockerfile index 5efe8c34..6847f9bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,8 @@ ENV YTP_CONFIG_PATH=/config ENV YTP_TEMP_PATH=/tmp ENV YTP_DOWNLOAD_PATH=/downloads ENV YTP_PORT=8081 +ENV XDG_CONFIG_HOME=/config +ENV XDG_CACHE_HOME=/tmp # removed ffmpeg as 6.1.0 is broken with DASH protocal downloads COPY --from=mwader/static-ffmpeg:6.1.1 /ffmpeg /usr/bin/ @@ -47,10 +49,10 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh COPY --chown=app:app ./app /app/app COPY --chown=app:app --from=npm_builder /ytptube/dist /app/frontend/dist -COPY --chown=app:app --from=python_builder /app/.venv /app/.venv +COPY --chown=app:app --from=python_builder /app/.venv /opt/python COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck -ENV PATH="/app/.venv/bin:$PATH" +ENV PATH="/opt/python/bin:$PATH" RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck @@ -69,4 +71,4 @@ ENTRYPOINT ["/entrypoint.sh"] ENV PYDEVD_DISABLE_FILE_VALIDATION=1 -CMD ["/app/.venv/bin/python", "/app/app/main.py", "--ytptube-mp"] +CMD ["/opt/python/bin/python", "/app/app/main.py", "--ytptube-mp"] diff --git a/app/Config.py b/app/Config.py index 0847dc70..7d905d06 100644 --- a/app/Config.py +++ b/app/Config.py @@ -8,6 +8,7 @@ from version import APP_VERSION from dotenv import load_dotenv from yt_dlp.version import __version__ as YTDLP_VERSION + class Config: __instance = None config_path: str = '.' @@ -47,9 +48,11 @@ class Config: extract_info_timeout: int = 70 + socket_timeout: int = 30 + ytdlp_version: str = YTDLP_VERSION - _int_vars: tuple = ('port', 'max_workers',) + _int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout',) _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available', 'ytdlp_version',) _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',) @@ -153,6 +156,8 @@ class Config: else: LOG.info(f'No custom yt-dlp options found in "{self.config_path}"') + self.ytdl_options['socket_timeout'] = self.socket_timeout + if self.keep_archive: LOG.info(f'keep archive: {self.keep_archive}') self.ytdl_options['download_archive'] = os.path.join( diff --git a/app/Download.py b/app/Download.py index cd10785a..1da4a8fd 100644 --- a/app/Download.py +++ b/app/Download.py @@ -261,6 +261,9 @@ class Download: if 'filename' in status: self.info.filename = os.path.relpath(status.get('filename'), self.download_dir) + if os.path.exists(status.get('filename')): + self.info.file_size = os.path.getsize(status.get('filename')) + # Set correct file extension for thumbnails if self.info.format == 'thumbnail': self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) diff --git a/app/DownloadQueue.py b/app/DownloadQueue.py index 46e6619e..a8d32228 100644 --- a/app/DownloadQueue.py +++ b/app/DownloadQueue.py @@ -11,7 +11,8 @@ from ItemDTO import ItemDTO from DataStore import DataStore from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig from AsyncPool import AsyncPool -from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor +from concurrent.futures import ThreadPoolExecutor + LOG = logging.getLogger('DownloadQueue') TYPE_DONE: str = 'done' TYPE_QUEUE: str = 'queue' @@ -228,26 +229,30 @@ class DownloadQueue: started = time.perf_counter() LOG.debug(f'extract_info: checking {url=}') - entry = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - ExtractInfo, - mergeConfig(self.config.ytdl_options, ytdlp_config), - url, - bool(self.config.ytdl_debug) - ), - timeout=self.config.extract_info_timeout) + with ThreadPoolExecutor(1) as executor: + entry = await asyncio.wait_for( + fut=asyncio.get_running_loop().run_in_executor( + executor, + ExtractInfo, + mergeConfig(self.config.ytdl_options, ytdlp_config), + url, + bool(self.config.ytdl_debug) + ), + timeout=self.config.extract_info_timeout) if not entry: return {'status': 'error', 'msg': 'Unable to extract info check logs.'} LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}') - except yt_dlp.utils.ExistingVideoReached: + except yt_dlp.utils.ExistingVideoReached as exc: + LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}') return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'} except yt_dlp.utils.YoutubeDLError as exc: + LOG.error(f'YoutubeDLError: Unable to extract info. {str(exc)}') return {'status': 'error', 'msg': str(exc)} except asyncio.exceptions.TimeoutError as exc: - return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'} + LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}') + return {'status': 'error', 'msg': f'TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.'} return await self.__add_entry( entry=entry, diff --git a/app/Utils.py b/app/Utils.py index 8111807b..2794ec30 100644 --- a/app/Utils.py +++ b/app/Utils.py @@ -137,7 +137,7 @@ def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) return download_path -def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = False) -> dict: +def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict: params: dict = { 'color': 'no_color', 'extract_flat': True, @@ -158,9 +158,6 @@ def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = else: params['quiet'] = True - if forceLookup and 'download_archive' in params: - params.pop('download_archive') - return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) diff --git a/app/main.py b/app/main.py index ec360acc..8866dbf0 100644 --- a/app/main.py +++ b/app/main.py @@ -397,9 +397,13 @@ class Main: if req.path not in self.staticHolder: return web.HTTPNotFound() - return web.FileResponse(path=self.staticHolder[req.path].get('file'), headers={ - 'Cache-Control': 'public, max-age=31536000', + item: dict = self.staticHolder[req.path] + + return web.Response(body=item.get('content'), headers={ 'Pragma': 'public', + 'Cache-Control': 'public, max-age=31536000', + 'Content-Type': item.get('content_type'), + 'X-Via': 'memory' if not item.get('file', None) else 'disk', }) staticDir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'frontend/dist') @@ -412,8 +416,8 @@ class Main: urlPath = f"{self.config.url_prefix}{file.replace(f'{staticDir}/', '')}" self.staticHolder[urlPath] = { - 'file': file, - # 'content': open(file, 'rb').read(), + # 'file': file, + 'content': open(file, 'rb').read(), 'content_type': self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)), }