Merge pull request #213 from arabcoders/dev

Fix date range conversion

.vscode/launch.json

@@ -35,12 +35,7 @@
         "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",
         "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
         "PYDEVD_DISABLE_FILE_VALIDATION": "1",
-        "YTP_IGNORE_UI": "true",
+        "YTP_IGNORE_UI": "true"
-        "YTP_PIP_IGNORE_UPDATES": "true",
-        "YTP_LOG_LEVEL": "DEBUG",
-        "YTP_DEBUG": "true",
-        "YTP_YTDL_DEBUG": "false",
-        "YTP_ACCESS_LOG": "true",
       }
     },
     {

.vscode/settings.json

@@ -17,10 +17,13 @@
     "aiocron",
     "arrowless",
     "copyts",
+    "daterange",
     "dotenv",
     "finaldir",
     "getpid",
+    "gpac",
     "httpx",
+    "levelno",
     "libcurl",
     "libx",
     "matroska",

Dockerfile

@@ -13,7 +13,7 @@ ENV PYTHONFAULTHANDLER=1
 
 # Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows)
 # Install dependencies
-RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev && pip install pipenv
+RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev curl make && pip install pipenv
 
 WORKDIR /app
 
@@ -21,6 +21,13 @@ ARG PIPENV_FLAGS="--deploy"
 COPY ./Pipfile* .
 RUN PIPENV_VENV_IN_PROJECT=1 pipenv install ${PIPENV_FLAGS}
 
+# Bundle mp4box into the image
+WORKDIR /tmp
+RUN mkdir /gpac-master && curl -k -L https://github.com/gpac/gpac/archive/refs/tags/v2.4.0.zip -o /tmp/gpac.zip && \
+    unzip /tmp/gpac.zip && mv /tmp/gpac-*/* /gpac-master
+WORKDIR /gpac-master
+RUN ./configure --static-mp4box --use-zlib=no && make -j4
+
 FROM python:3.11-alpine
 
 ARG TZ=UTC
@@ -46,13 +53,15 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
 COPY --chown=app:app ./app /app/app
 COPY --chown=app:app --from=node_builder /app/exported /app/ui/exported
 COPY --chown=app:app --from=python_builder /app/.venv /opt/python
+COPY --chown=app:app --from=python_builder /gpac-master/bin/gcc/MP4Box /usr/bin/mp4box
 COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck
 
 ENV PATH="/opt/python/bin:$PATH"
 
 RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck && \
   sed -i 's$#!\/app\/\.venv\/bin\/python$#!/opt/python/bin/python$' /opt/python/bin/* && \
-  sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate*
+  sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate* && \
+  chmod +x /usr/bin/mp4box
 
 VOLUME /config
 VOLUME /downloads

README.md

@@ -21,6 +21,7 @@ YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since
 * Basic Authentication support.
 * Support for curl_cffi, see [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation)
 * Support for both advanced and basic mode for WebUI.
+* Bundled tools in container: curl-cffi, ffmpeg, ffprobe, aria2, rtmpdump, mkvtoolsnix, mp4box.
 
 For more API endpoints, please refer to the [API documentation](API.md).
     

app/library/Download.py

@@ -3,6 +3,7 @@ import hashlib
 import logging
 import multiprocessing
 import os
+import re
 import shutil
 import time
 from email.utils import formatdate
@@ -16,7 +17,24 @@ from .ffprobe import ffprobe
 from .ItemDTO import ItemDTO
 from .YTDLPOpts import YTDLPOpts
 
-LOG = logging.getLogger("download")
+LOG = logging.getLogger("Download")
+
+
+class NestedLogger:
+    debug_messages = ["[debug] ", "[download] "]
+
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+
+    def debug(self, msg: str):
+        levelno = logging.DEBUG if any(msg.startswith(x) for x in self.debug_messages) else logging.INFO
+        self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE))
+
+    def error(self, msg):
+        self.logger.error(msg)
+
+    def warning(self, msg):
+        self.logger.warning(msg)
 
 
 class Download:
@@ -66,7 +84,7 @@ class Download:
     temp_keep: bool = False
     "Keep temp folder after download."
 
-    def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False):
+    def __init__(self, info: ItemDTO, info_dict: dict = None):
         config = Config.get_instance()
 
         self.download_dir = info.download_dir
@@ -78,7 +96,8 @@ class Download:
         self.info = info
         self.id = info._id
         self.default_ytdl_opts = config.ytdl_options
-        self.debug = debug
+        self.debug = bool(config.debug)
+        self.debug_ytdl = bool(config.ytdl_debug)
         self.cancelled = False
         self.tmpfilename = None
         self.status_queue = None
@@ -89,6 +108,7 @@ class Download:
         self.is_live = bool(info.is_live) or info.live_in is not None
         self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
         self.info_dict = info_dict
+        self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
 
     def _progress_hook(self, data: dict):
         dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
@@ -99,14 +119,14 @@ class Download:
         self.status_queue.put({"id": self.id, **dataDict})
 
     def _postprocessor_hook(self, data: dict):
+        if self.debug:
+            self.logger.debug(f"Postprocessor hook: {data}")
+
         if "MoveFiles" != data.get("postprocessor") or "finished" != data.get("status"):
             dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
             self.status_queue.put({"id": self.id, **dataDict, "status": "postprocessing"})
             return
 
-        if self.debug:
-            LOG.debug(f"Postprocessor hook: {data}")
-
         if "__finaldir" in data["info_dict"]:
             filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"]))
         else:
@@ -147,19 +167,21 @@ class Download:
                 }
             )
 
-            if self.debug:
+            if self.debug_ytdl:
                 params["verbose"] = True
                 params["noprogress"] = False
 
             if self.info.cookies:
                 try:
                     cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt")
-                    LOG.debug(f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'.")
+                    self.logger.debug(
+                        f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
+                    )
                     with open(cookie_file, "w") as f:
                         f.write(self.info.cookies)
                         params["cookiefile"] = f.name
                 except ValueError as e:
-                    LOG.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.")
+                    self.logger.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.")
 
             if self.is_live or self.is_manifestless:
                 hasDeletedOptions = False
@@ -171,37 +193,39 @@ class Download:
                         deletedOpts.append(opt)
 
                 if hasDeletedOptions:
-                    LOG.warning(
+                    self.logger.warning(
                         f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
                     )
 
-            LOG.info(
+            self.logger.info(
                 f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.'
             )
 
-            LOG.debug("Params before passing to yt-dlp.", extra=params)
+            self.logger.debug("Params before passing to yt-dlp.", extra=params)
 
             if "impersonate" in params:
                 from yt_dlp.networking.impersonate import ImpersonateTarget
 
                 params["impersonate"] = ImpersonateTarget.from_str(params["impersonate"])
 
+            params["logger"] = NestedLogger(self.logger)
+
             cls = yt_dlp.YoutubeDL(params=params)
 
             if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
-                LOG.debug(f"Downloading '{self.info.url}' using pre-info.")
+                self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
                 cls.process_ie_result(self.info_dict, download=True)
                 ret = cls._download_retcode
             else:
-                LOG.debug(f"Downloading using url: {self.info.url}")
+                self.logger.debug(f"Downloading using url: {self.info.url}")
                 ret = cls.download([self.info.url])
 
             self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"})
         except Exception as exc:
-            LOG.exception(exc)
+            self.logger.exception(exc)
             self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)})
 
-        LOG.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.')
+        self.logger.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.')
 
     async def start(self, emitter: Emitter):
         self.emitter = emitter
@@ -244,28 +268,28 @@ class Download:
         self.cancel_in_progress = True
         procId = self.proc.ident
 
-        LOG.info(f"Closing PID='{procId}' download process.")
+        self.logger.info(f"Closing PID='{procId}' download process.")
 
         try:
             try:
                 if "update_task" in self.__dict__ and self.update_task.done() is False:
                     self.update_task.cancel()
             except Exception as e:
-                LOG.error(f"Failed to close status queue: '{procId}'. {e}")
+                self.logger.error(f"Failed to close status queue: '{procId}'. {e}")
 
             self.kill()
 
             loop = asyncio.get_running_loop()
 
             if self.proc.is_alive():
-                LOG.debug(f"Waiting for PID='{procId}' to close.")
+                self.logger.debug(f"Waiting for PID='{procId}' to close.")
                 await loop.run_in_executor(None, self.proc.join)
-                LOG.debug(f"PID='{procId}' closed.")
+                self.logger.debug(f"PID='{procId}' closed.")
 
             if self.status_queue:
-                LOG.debug(f"Closing status queue for PID='{procId}'.")
+                self.logger.debug(f"Closing status queue for PID='{procId}'.")
                 self.status_queue.put(Terminator())
-                LOG.debug(f"Closed status queue for PID='{procId}'.")
+                self.logger.debug(f"Closed status queue for PID='{procId}'.")
 
             if self.proc:
                 self.proc.close()
@@ -273,11 +297,11 @@ class Download:
 
             self.delete_temp()
 
-            LOG.debug(f"Closed PID='{procId}' download process.")
+            self.logger.debug(f"Closed PID='{procId}' download process.")
 
             return True
         except Exception as e:
-            LOG.error(f"Failed to close process: '{procId}'. {e}")
+            self.logger.error(f"Failed to close process: '{procId}'. {e}")
 
         return False
 
@@ -295,11 +319,11 @@ class Download:
             return False
 
         try:
-            LOG.info(f"Killing download process: '{self.proc.ident}'.")
+            self.logger.info(f"Killing download process: '{self.proc.ident}'.")
             self.proc.kill()
             return True
         except Exception as e:
-            LOG.error(f"Failed to kill process: '{self.proc.ident}'. {e}")
+            self.logger.error(f"Failed to kill process: '{self.proc.ident}'. {e}")
 
         return False
 
@@ -308,7 +332,7 @@ class Download:
             return
 
         if "finished" != self.info.status and self.is_live:
-            LOG.warning(
+            self.logger.warning(
                 f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'."
             )
             return
@@ -317,12 +341,12 @@ class Download:
             return
 
         if self.temp_path == self.temp_dir:
-            LOG.warning(
+            self.logger.warning(
                 f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder."
             )
             return
 
-        LOG.info(f"Deleting Temp folder '{self.temp_path}'.")
+        self.logger.info(f"Deleting Temp folder '{self.temp_path}'.")
         shutil.rmtree(self.temp_path, ignore_errors=True)
 
     async def progress_update(self):
@@ -343,7 +367,7 @@ class Download:
                 continue
 
             if self.debug:
-                LOG.debug(f"Status Update: {self.info._id=} {status=}")
+                self.logger.debug(f"Status Update: {self.info._id=} {status=}")
 
             if isinstance(status, str):
                 asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")
@@ -400,7 +424,7 @@ class Download:
                 except Exception as e:
                     self.info.extras["is_video"] = True
                     self.info.extras["is_audio"] = True
-                    LOG.exception(e)
+                    self.logger.exception(e)
-                    LOG.error(f"Failed to ffprobe: {status.get}. {e}")
+                    self.logger.error(f"Failed to ffprobe: {status.get}. {e}")
 
             asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")

app/library/DownloadQueue.py

@@ -310,7 +310,7 @@ class DownloadQueue(metaclass=Singleton):
                 if property.startswith("playlist"):
                     dl.template = str(dl.template).replace(f"%({property})s", str(value))
 
-            dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug))
+            dlInfo: Download = Download(info=dl, info_dict=entry)
 
             if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"):
                 dlInfo.info.status = "not_live"

app/library/Utils.py

@@ -424,6 +424,7 @@ def arg_converter(args: str) -> dict:
 
     """
     import yt_dlp.options
+    from yt_dlp.utils import DateRange
 
     create_parser = yt_dlp.options.create_parser
 
@@ -443,7 +444,9 @@ def arg_converter(args: str) -> dict:
     if "postprocessors" in diff:
         diff["postprocessors"] = [pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]]
 
-    return json.loads(json.dumps(diff))
+    from .encoder import Encoder
+
+    return json.loads(json.dumps(diff, cls=Encoder))
 
 
 def validate_uuid(uuid_str: str, version: int = 4) -> bool:

app/library/YTDLPOpts.py

@@ -132,4 +132,9 @@ class YTDLPOpts(metaclass=Singleton):
         if "format" in data and data["format"] in ["not_set", "default"]:
             data["format"] = None
 
+        if "daterange" in data and isinstance(data["daterange"], dict):
+            from yt_dlp.utils import DateRange
+
+            data["daterange"] = DateRange(data["daterange"]["start"], data["daterange"]["end"])
+
         return data

app/library/encoder.py

@@ -1,6 +1,9 @@
 import json
+from datetime import date
 from pathlib import Path
 
+from yt_dlp.utils import DateRange
+
 
 class Encoder(json.JSONEncoder):
     """
@@ -13,6 +16,12 @@ class Encoder(json.JSONEncoder):
         if isinstance(o, Path):
             return str(o)
 
+        if isinstance(o, DateRange):
+            return {"start": str(o.start).replace("-", ""), "end": str(o.end).replace("-", "")}
+
+        if isinstance(o, date):
+            return str(o)
+
         if isinstance(o, object):
             if hasattr(o, "serialize"):
                 return o.serialize()