Merge pull request #77 from arabcoders/dev

Added socket_timeout option to downloads

Dockerfile

@@ -31,6 +31,8 @@ ENV YTP_CONFIG_PATH=/config
 ENV YTP_TEMP_PATH=/tmp
 ENV YTP_DOWNLOAD_PATH=/downloads
 ENV YTP_PORT=8081
+ENV XDG_CONFIG_HOME=/config
+ENV XDG_CACHE_HOME=/tmp
 
 # removed ffmpeg as 6.1.0 is broken with DASH protocal downloads
 COPY --from=mwader/static-ffmpeg:6.1.1 /ffmpeg /usr/bin/
@@ -47,10 +49,10 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
 
 COPY --chown=app:app ./app /app/app
 COPY --chown=app:app --from=npm_builder /ytptube/dist /app/frontend/dist
-COPY --chown=app:app --from=python_builder /app/.venv /app/.venv
+COPY --chown=app:app --from=python_builder /app/.venv /opt/python
 COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck
 
-ENV PATH="/app/.venv/bin:$PATH"
+ENV PATH="/opt/python/bin:$PATH"
 
 RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck
 
@@ -69,4 +71,4 @@ ENTRYPOINT ["/entrypoint.sh"]
 
 ENV PYDEVD_DISABLE_FILE_VALIDATION=1
 
-CMD ["/app/.venv/bin/python", "/app/app/main.py", "--ytptube-mp"]
+CMD ["/opt/python/bin/python", "/app/app/main.py", "--ytptube-mp"]

app/Config.py

@@ -8,6 +8,7 @@ from version import APP_VERSION
 from dotenv import load_dotenv
 from yt_dlp.version import __version__ as YTDLP_VERSION
 
+
 class Config:
     __instance = None
     config_path: str = '.'
@@ -47,9 +48,11 @@ class Config:
 
     extract_info_timeout: int = 70
 
+    socket_timeout: int = 30
+
     ytdlp_version: str = YTDLP_VERSION
 
-    _int_vars: tuple = ('port', 'max_workers',)
+    _int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout',)
     _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available', 'ytdlp_version',)
     _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',)
 
@@ -153,6 +156,8 @@ class Config:
         else:
             LOG.info(f'No custom yt-dlp options found in "{self.config_path}"')
 
+        self.ytdl_options['socket_timeout'] = self.socket_timeout
+
         if self.keep_archive:
             LOG.info(f'keep archive: {self.keep_archive}')
             self.ytdl_options['download_archive'] = os.path.join(

app/Download.py

@@ -261,6 +261,9 @@ class Download:
             if 'filename' in status:
                 self.info.filename = os.path.relpath(status.get('filename'), self.download_dir)
 
+                if os.path.exists(status.get('filename')):
+                    self.info.file_size = os.path.getsize(status.get('filename'))
+
                 # Set correct file extension for thumbnails
                 if self.info.format == 'thumbnail':
                     self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)

app/DownloadQueue.py

@@ -11,7 +11,8 @@ from ItemDTO import ItemDTO
 from DataStore import DataStore
 from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig
 from AsyncPool import AsyncPool
-from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
+
 LOG = logging.getLogger('DownloadQueue')
 TYPE_DONE: str = 'done'
 TYPE_QUEUE: str = 'queue'
@@ -228,26 +229,30 @@ class DownloadQueue:
             started = time.perf_counter()
             LOG.debug(f'extract_info: checking {url=}')
 
-            entry = await asyncio.wait_for(
+            with ThreadPoolExecutor(1) as executor:
-                fut=asyncio.get_running_loop().run_in_executor(
+                entry = await asyncio.wait_for(
-                    None,
+                    fut=asyncio.get_running_loop().run_in_executor(
-                    ExtractInfo,
+                        executor,
-                    mergeConfig(self.config.ytdl_options, ytdlp_config),
+                        ExtractInfo,
-                    url,
+                        mergeConfig(self.config.ytdl_options, ytdlp_config),
-                    bool(self.config.ytdl_debug)
+                        url,
-                ),
+                        bool(self.config.ytdl_debug)
-                timeout=self.config.extract_info_timeout)
+                    ),
+                    timeout=self.config.extract_info_timeout)
 
             if not entry:
                 return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
 
             LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}')
-        except yt_dlp.utils.ExistingVideoReached:
+        except yt_dlp.utils.ExistingVideoReached as exc:
+            LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}')
             return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
         except yt_dlp.utils.YoutubeDLError as exc:
+            LOG.error(f'YoutubeDLError: Unable to extract info. {str(exc)}')
             return {'status': 'error', 'msg': str(exc)}
         except asyncio.exceptions.TimeoutError as exc:
-            return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'}
+            LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}')
+            return {'status': 'error', 'msg': f'TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.'}
 
         return await self.__add_entry(
             entry=entry,

app/Utils.py

@@ -137,7 +137,7 @@ def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True)
     return download_path
 
 
-def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = False) -> dict:
+def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict:
     params: dict = {
         'color': 'no_color',
         'extract_flat': True,
@@ -158,9 +158,6 @@ def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool =
     else:
         params['quiet'] = True
 
-    if forceLookup and 'download_archive' in params:
-        params.pop('download_archive')
-
     return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)
 
 

app/main.py

@@ -397,9 +397,13 @@ class Main:
             if req.path not in self.staticHolder:
                 return web.HTTPNotFound()
 
-            return web.FileResponse(path=self.staticHolder[req.path].get('file'), headers={
+            item: dict = self.staticHolder[req.path]
-                'Cache-Control': 'public, max-age=31536000',
+
+            return web.Response(body=item.get('content'), headers={
                 'Pragma': 'public',
+                'Cache-Control': 'public, max-age=31536000',
+                'Content-Type': item.get('content_type'),
+                'X-Via': 'memory' if not item.get('file', None) else 'disk',
             })
 
         staticDir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'frontend/dist')
@@ -412,8 +416,8 @@ class Main:
                 urlPath = f"{self.config.url_prefix}{file.replace(f'{staticDir}/', '')}"
 
                 self.staticHolder[urlPath] = {
-                    'file': file,
+                    # 'file': file,
-                    # 'content': open(file, 'rb').read(),
+                    'content': open(file, 'rb').read(),
                     'content_type': self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)),
                 }