danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #77 from arabcoders/dev

Browse source 
Added socket_timeout option to downloads
This commit is contained in:
Abdulmohsen 2024-03-14 00:21:43 +03:00 committed by GitHub
commit 6e6150c9d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 40 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Dockerfile
View file

@ -31,6 +31,8 @@ ENV YTP_CONFIG_PATH=/config
ENV YTP_TEMP_PATH=/tmp
ENV YTP_DOWNLOAD_PATH=/downloads
ENV YTP_PORT=8081
ENV XDG_CONFIG_HOME=/config
ENV XDG_CACHE_HOME=/tmp
# removed ffmpeg as 6.1.0 is broken with DASH protocal downloads
COPY --from=mwader/static-ffmpeg:6.1.1 /ffmpeg /usr/bin/
@ -47,10 +49,10 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
COPY --chown=app:app ./app /app/app
COPY --chown=app:app --from=npm_builder /ytptube/dist /app/frontend/dist
COPY --chown=app:app --from=python_builder /app/.venv /app/.venv
COPY --chown=app:app --from=python_builder /app/.venv /opt/python
COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck
ENV PATH="/app/.venv/bin:$PATH"
ENV PATH="/opt/python/bin:$PATH"
RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck
@ -69,4 +71,4 @@ ENTRYPOINT ["/entrypoint.sh"]
ENV PYDEVD_DISABLE_FILE_VALIDATION=1
CMD ["/app/.venv/bin/python", "/app/app/main.py", "--ytptube-mp"]
CMD ["/opt/python/bin/python", "/app/app/main.py", "--ytptube-mp"]

7
app/Config.py
View file

@ -8,6 +8,7 @@ from version import APP_VERSION
from dotenv import load_dotenv
from yt_dlp.version import __version__ as YTDLP_VERSION
class Config:
__instance = None
config_path: str = '.'
@ -47,9 +48,11 @@ class Config:
extract_info_timeout: int = 70
socket_timeout: int = 30
ytdlp_version: str = YTDLP_VERSION
_int_vars: tuple = ('port', 'max_workers',)
_int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout',)
_immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available', 'ytdlp_version',)
_boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',)
@ -153,6 +156,8 @@ class Config:
else:
LOG.info(f'No custom yt-dlp options found in "{self.config_path}"')
self.ytdl_options['socket_timeout'] = self.socket_timeout
if self.keep_archive:
LOG.info(f'keep archive: {self.keep_archive}')
self.ytdl_options['download_archive'] = os.path.join(

3
app/Download.py
View file

@ -261,6 +261,9 @@ class Download:
if 'filename' in status:
self.info.filename = os.path.relpath(status.get('filename'), self.download_dir)
if os.path.exists(status.get('filename')):
self.info.file_size = os.path.getsize(status.get('filename'))
# Set correct file extension for thumbnails
if self.info.format == 'thumbnail':
self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)

29
app/DownloadQueue.py
View file

@ -11,7 +11,8 @@ from ItemDTO import ItemDTO
from DataStore import DataStore
from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig
from AsyncPool import AsyncPool
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from concurrent.futures import ThreadPoolExecutor
LOG = logging.getLogger('DownloadQueue')
TYPE_DONE: str = 'done'
TYPE_QUEUE: str = 'queue'
@ -228,26 +229,30 @@ class DownloadQueue:
started = time.perf_counter()
LOG.debug(f'extract_info: checking {url=}')
entry = await asyncio.wait_for(
fut=asyncio.get_running_loop().run_in_executor(
None,
ExtractInfo,
mergeConfig(self.config.ytdl_options, ytdlp_config),
url,
bool(self.config.ytdl_debug)
),
timeout=self.config.extract_info_timeout)
with ThreadPoolExecutor(1) as executor:
entry = await asyncio.wait_for(
fut=asyncio.get_running_loop().run_in_executor(
executor,
ExtractInfo,
mergeConfig(self.config.ytdl_options, ytdlp_config),
url,
bool(self.config.ytdl_debug)
),
timeout=self.config.extract_info_timeout)
if not entry:
return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}')
except yt_dlp.utils.ExistingVideoReached:
except yt_dlp.utils.ExistingVideoReached as exc:
LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}')
return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
except yt_dlp.utils.YoutubeDLError as exc:
LOG.error(f'YoutubeDLError: Unable to extract info. {str(exc)}')
return {'status': 'error', 'msg': str(exc)}
except asyncio.exceptions.TimeoutError as exc:
return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'}
LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}')
return {'status': 'error', 'msg': f'TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.'}
return await self.__add_entry(
entry=entry,

5
app/Utils.py
View file

@ -137,7 +137,7 @@ def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True)
return download_path
def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = False) -> dict:
def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict:
params: dict = {
'color': 'no_color',
'extract_flat': True,
@ -158,9 +158,6 @@ def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool =
else:
params['quiet'] = True
if forceLookup and 'download_archive' in params:
params.pop('download_archive')
return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)

12
app/main.py
View file

@ -397,9 +397,13 @@ class Main:
if req.path not in self.staticHolder:
return web.HTTPNotFound()
return web.FileResponse(path=self.staticHolder[req.path].get('file'), headers={
'Cache-Control': 'public, max-age=31536000',
item: dict = self.staticHolder[req.path]
return web.Response(body=item.get('content'), headers={
'Pragma': 'public',
'Cache-Control': 'public, max-age=31536000',
'Content-Type': item.get('content_type'),
'X-Via': 'memory' if not item.get('file', None) else 'disk',
})
staticDir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'frontend/dist')
@ -412,8 +416,8 @@ class Main:
urlPath = f"{self.config.url_prefix}{file.replace(f'{staticDir}/', '')}"
self.staticHolder[urlPath] = {
'file': file,
# 'content': open(file, 'rb').read(),
# 'file': file,
'content': open(file, 'rb').read(),
'content_type': self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)),
}