updated the conversion process of cli to config

.vscode/settings.json

@@ -19,11 +19,16 @@
     "attl",
     "autonumber",
     "changeslog",
+    "consoletitle",
+    "cookiesfrombrowser",
     "copyts",
     "daterange",
     "dotenv",
     "finaldir",
     "flac",
+    "forcejson",
+    "forceprint",
+    "fribidi",
     "getpid",
     "gpac",
     "httpx",
@@ -41,6 +46,7 @@
     "postprocessor",
     "preferredcodec",
     "preferredquality",
+    "printtraffic",
     "quicktime",
     "tmpfilename",
     "urandom",
@@ -56,7 +62,6 @@
     "en"
   ],
   "spellright.documentTypes": [
-    "latex",
+    "latex"
-    "plaintext"
   ],
 }

Dockerfile

@@ -35,7 +35,7 @@ ENV XDG_CONFIG_HOME=/config
 ENV XDG_CACHE_HOME=/tmp
 
 RUN mkdir /config /downloads && ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone && \
-  apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump && \
+  apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump fribidi && \
   useradd -u ${USER_ID:-1000} -U -d /app -s /bin/bash app && \
   rm -rf /var/cache/apk/*
 

app/library/Download.py

@@ -114,6 +114,8 @@ class Download:
     def _progress_hook(self, data: dict):
         dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
 
+        LOG.debug("Status: %s", data)
+
         if "finished" == data.get("status") and data.get("info_dict", {}).get("filename", None):
             dataDict["filename"] = data["info_dict"]["filename"]
 

app/library/Events.py

@@ -90,7 +90,6 @@ class Events:
     LOG_SUCCESS = "log_success"
 
     INITIAL_DATA = "initial_data"
-    YTDLP_CONVERT = "ytdlp_convert"
     ITEM_DELETE = "item_delete"
     ITEM_CANCEL = "item_cancel"
     STATUS = "status"

app/library/HttpAPI.py

@@ -35,7 +35,7 @@ from .Segments import Segments
 from .Subtitle import Subtitle
 from .Tasks import Task, Tasks
 from .Utils import (
-    IGNORED_KEYS,
+    REMOVE_KEYS,
     StreamingError,
     arg_converter,
     decrypt_data,
@@ -445,7 +445,7 @@ class HttpAPI(Common):
         try:
             response = {"opts": {}, "output_template": None, "download_path": None}
 
-            data = arg_converter(args)
+            data = arg_converter(args, dumps=True)
 
             if "outtmpl" in data and "default" in data["outtmpl"]:
                 response["output_template"] = data["outtmpl"]["default"]
@@ -456,12 +456,19 @@ class HttpAPI(Common):
             if "format" in data:
                 response["format"] = data["format"]
 
+            bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()}
+            removed_options = []
+
             for key in data:
-                if key in IGNORED_KEYS:
+                if key in bad_options.items():
+                    removed_options.append(bad_options[key])
                     continue
                 if not key.startswith("_"):
                     response["opts"][key] = data[key]
 
+            if len(removed_options) > 0:
+                response["opts"]["removed"] = ", ".join(removed_options)
+
             return web.json_response(data=response, status=web.HTTPOk.status_code)
         except Exception as e:
             err = str(e).strip()

app/library/HttpSocket.py

@@ -18,7 +18,7 @@ from .DownloadQueue import DownloadQueue
 from .encoder import Encoder
 from .Events import Event, EventBus, Events, error
 from .Presets import Presets
-from .Utils import arg_converter, is_downloaded
+from .Utils import is_downloaded
 
 LOG = logging.getLogger("socket_api")
 
@@ -285,25 +285,3 @@ class HttpSocket(Common):
     async def resume(self, *_, **__):
         self.queue.resume()
         await self._notify.emit(Events.PAUSED, data={"paused": False, "at": time.time()})
-
-    @ws_event
-    async def ytdlp_convert(self, sid: str, data: dict):
-        if not isinstance(data, dict) or "args" not in data:
-            await self._notify.emit(Events.ERROR, data=error("Invalid request or no options were given."), to=sid)
-            return
-
-        args: str | None = data.get("args")
-
-        if not args:
-            await self._notify.emit(Events.ERROR, data=error("no options were given."), to=sid)
-            return
-
-        try:
-            await self._notify.emit(Events.YTDLP_CONVERT, data=arg_converter(args), to=sid)
-        except Exception as e:
-            err = str(e).strip()
-            err = err.split("\n")[-1] if "\n" in err else err
-            LOG.error(f"Failed to convert args. '{err}'.")
-            await self._notify.emit(Events.ERROR, data=error(f"Failed to convert options. '{err}'."), to=sid)
-
-        return

app/library/Utils.py

@@ -21,7 +21,39 @@ from .LogWrapper import LogWrapper
 
 LOG = logging.getLogger("Utils")
 
-IGNORED_KEYS: tuple[str] = ("paths", "outtmpl", "progress_hooks", "postprocessor_hooks", "download_archive")
+REMOVE_KEYS: list = [
+    {
+        "paths": "-P, --paths",
+        "outtmpl": "-o, --output",
+        "progress_hooks": "--progress_hooks",
+        "postprocessor_hooks": "--postprocessor_hooks",
+        "download_archive": "--download_archive",
+    },
+    {
+        "quiet": "-q, --quiet",
+        "no_warnings": "--no-warnings",
+        "skip_download": "--skip-download",
+        "forceprint": "-O, --print",
+        "simulate": "--simulate",
+        "noprogress": "--no-progress",
+        "wait_for_video": "--wait-for-video",
+        "mark_watched": "--mark-watched",
+        "color": "--color",
+        "verbose": "-v, --verbose",
+        "debug_printtraffic": "--print-traffic",
+        "write_pages": "--write-pages",
+        "dump_intermediate_pages": "--dump-pages",
+        "progress_delta": " --progress-delta",
+        "progress_template": "--progress-template",
+        "consoletitle": "--console-title",
+        "progress_with_newline": "--newline",
+        "forcejson": "-j, --dump-json",
+        "print_to_file": "--print-to-file",
+        "cookiesfrombrowser": "--cookies-from-browser",
+        "cookiefile": "--cookies",
+    },
+]
+
 YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None
 
 ALLOWED_SUBS_EXTENSIONS: tuple[str] = (".srt", ".vtt", ".ass")
@@ -434,13 +466,20 @@ def validate_url(url: str) -> bool:
     return True
 
 
-def arg_converter(args: str, remove_options: bool = False) -> dict:
+def arg_converter(
+    args: str,
+    level: int | bool | None = None,
+    dumps: bool = False,
+    removed_options: list | None = None,
+) -> dict:
     """
     Convert yt-dlp options to a dictionary.
 
     Args:
         args (str): yt-dlp options string.
-        remove_options (bool): Remove default options.
+        level (int|bool|None): Level of options to remove, True for all.
+        dumps (bool): Dump options as JSON.
+        removed_options (list|None): List of removed options.
 
     Returns:
         dict: yt-dlp options dictionary.
@@ -473,17 +512,33 @@ def arg_converter(args: str, remove_options: bool = False) -> dict:
         if isinstance(matchFilter, set):
             diff["match_filter"] = {"filters": list(matchFilter)}
 
-    if remove_options:
-        for key in diff.copy():
-            if key in IGNORED_KEYS:
-                diff.pop(key, None)
-
     if "_warnings" in diff:
         diff.pop("_warnings", None)
 
-    from .encoder import Encoder
+    if level:
+        bad_options = {}
+        level = len(REMOVE_KEYS) if not isinstance(level, int) else level
 
-    return json.loads(json.dumps(diff, cls=Encoder))
+        for i, item in enumerate(REMOVE_KEYS):
+            if i > level:
+                break
+            bad_options.update(item.items())
+
+        for key in diff.copy():
+            if key not in bad_options:
+                continue
+
+            if isinstance(removed_options, list):
+                removed_options.append(bad_options[key])
+
+            diff.pop(key, None)
+
+    if dumps is True:
+        from .encoder import Encoder
+
+        return json.loads(json.dumps(diff, cls=Encoder))
+
+    return diff
 
 
 def validate_uuid(uuid_str: str, version: int = 4) -> bool:

app/library/YTDLPOpts.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from .config import Config
 from .Presets import Presets
 from .Singleton import Singleton
-from .Utils import IGNORED_KEYS, arg_converter, calc_download_path, merge_dict
+from .Utils import REMOVE_KEYS, arg_converter, calc_download_path, merge_dict
 
 LOG = logging.getLogger("YTDLPOpts")
 
@@ -48,11 +48,22 @@ class YTDLPOpts(metaclass=Singleton):
             YTDLPOpts: The instance of the class
 
         """
+        bad_options = {}
+        if from_user:
+            bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()}
+
+        removed_options = []
+
         for key, value in config.items():
-            if key in IGNORED_KEYS and from_user:
+            if from_user and key in bad_options:
+                removed_options.append(bad_options[key])
                 continue
+
             self._item_opts[key] = value
 
+        if len(removed_options) > 0:
+            LOG.warning("Removed the following options: '%s'.", ", ".join(removed_options))
+
         return self
 
     def preset(self, name: str, with_cookies: bool = False) -> "YTDLPOpts":
@@ -73,7 +84,13 @@ class YTDLPOpts(metaclass=Singleton):
 
         if preset.cli:
             try:
-                self._preset_opts = arg_converter(args=preset.cli, remove_options=True)
+                removed_options = []
+                self._preset_opts = arg_converter(args=preset.cli, level=True, removed_options=removed_options)
+                if len(removed_options) > 0:
+                    LOG.warning(
+                        "Removed the following options '%s' from preset '%s'.", ", ".join(removed_options), preset.name
+                    )
+
             except Exception as e:
                 msg = f"Invalid cli options for preset '{preset.name}'. '{e!s}'."
                 raise ValueError(msg) from e
@@ -106,11 +123,19 @@ class YTDLPOpts(metaclass=Singleton):
                 self._preset_opts["postprocessors"] = preset.postprocessors
 
             if preset.args and isinstance(preset.args, dict) and len(preset.args) > 0:
+                bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()}
+                removed_options = []
                 for key, value in preset.args.items():
-                    if key in IGNORED_KEYS:
+                    if key in bad_options:
+                        removed_options.append(bad_options[key])
                         continue
                     self._preset_opts[key] = value
 
+                if len(removed_options) > 0:
+                    LOG.warning(
+                        "Removed the following options '%s' from '%s' args.", ", ".join(removed_options), preset.name
+                    )
+
         return self
 
     def get_all(self, keep: bool = False) -> dict:

app/library/common.py

@@ -98,7 +98,11 @@ class Common:
         cli = item.get("cli")
         if cli and len(cli) > 1:
             try:
-                config = arg_converter(args=cli, remove_options=True)
+                removed_options = []
+                config = arg_converter(args=cli, level=True, removed_options=removed_options)
+                if len(removed_options) > 0:
+                    LOG.warning("Removed the following options '%s'.", ", ".join(removed_options))
+
             except Exception as e:
                 msg = f"Failed to parse yt-dlp cli options. {e!s}"
                 raise ValueError(msg) from e

app/library/config.py

@@ -12,7 +12,7 @@ import coloredlogs
 from dotenv import load_dotenv
 from yt_dlp.version import __version__ as YTDLP_VERSION
 
-from .Utils import IGNORED_KEYS, load_file, merge_dict
+from .Utils import REMOVE_KEYS, arg_converter, load_file, merge_dict
 from .version import APP_VERSION
 
 
@@ -218,7 +218,7 @@ class Config:
         "instance_title",
         "sentry_dsn",
         "console_enabled",
-        "browser_enabled"
+        "browser_enabled",
     )
     "The variables that are relevant to the frontend."
 
@@ -321,8 +321,38 @@ class Config:
             except Exception as e:
                 LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}")
 
+        ytdlp_cli: str = os.path.join(self.config_path, "ytdlp.cli")
         optsFile: str = os.path.join(self.config_path, "ytdlp.json")
-        if os.path.exists(optsFile) and os.path.getsize(optsFile) > 5:
+        if os.path.exists(ytdlp_cli) and os.path.getsize(ytdlp_cli) > 2:
+            LOG.info(f"Loading yt-dlp custom options from '{ytdlp_cli}'.")
+            with open(ytdlp_cli) as f:
+                ytdlp_cli_opts = f.read().strip()
+                if ytdlp_cli_opts:
+                    try:
+                        removed_options = []
+                        self.ytdl_options = arg_converter(
+                            args=ytdlp_cli_opts,
+                            level=True,
+                            removed_options=removed_options,
+                        )
+
+                        try:
+                            LOG.debug("Parsed yt-dlp cli options '%s'.", self.ytdl_options)
+                        except Exception:
+                            pass
+
+                        if len(removed_options) > 0:
+                            LOG.warning(
+                                "Removed the following options: '%s' from '%s'", ", ".join(removed_options), ytdlp_cli
+                            )
+                    except Exception as e:
+                        msg = f"Failed to parse yt-dlp cli options from '{ytdlp_cli}'. '{e!s}'."
+                        raise ValueError(msg) from e
+                else:
+                    LOG.warning(f"Empty yt-dlp custom options file '{ytdlp_cli}'.")
+        # @Deprecated - To be removed in future versions.
+        elif os.path.exists(optsFile) and os.path.getsize(optsFile) > 5:
+            LOG.warning("The JSON ytdlp.json options file is deprecated, please use 'ytdlp.cli' file instead.")
             LOG.info(f"Loading yt-dlp custom options from '{optsFile}'.")
 
             (opts, status, error) = load_file(optsFile, dict)
@@ -330,16 +360,24 @@ class Config:
                 LOG.error(f"Could not load yt-dlp custom options from '{optsFile}'. {error}")
                 sys.exit(1)
             if isinstance(opts, dict):
-                for key in IGNORED_KEYS:
+                bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()}
-                    if key in opts:
+                removed_options = []
-                        LOG.error(f"Key '{key}' is not allowed to be loaded via 'ytdlp.json' file.")
+                for key in opts.copy():
-                        del opts[key]
+                    if key not in bad_options:
+                        continue
 
+                    removed_options.append(bad_options[key])
+                    opts.pop(key, None)
+
+                if len(removed_options) > 0:
+                    LOG.warning(
+                        "Removed the following options: '%s' from 'ytdlp.json' file.", ", ".join(removed_options)
+                    )
                 self.ytdl_options = merge_dict(self.ytdl_options, opts)
             else:
                 LOG.error(f"Invalid yt-dlp custom options file '{optsFile}'.")
         else:
-            LOG.info(f"No yt-dlp custom options found at '{optsFile}'.")
+            LOG.info(f"No yt-dlp custom options found at '{ytdlp_cli}'.")
 
         self.ytdl_options["socket_timeout"] = self.socket_timeout
 
@@ -373,7 +411,6 @@ class Config:
 
         key_file: str = os.path.join(self.config_path, "secret.key")
 
-        # save key as bytes.
         if os.path.exists(key_file) and os.path.getsize(key_file) > 5:
             with open(key_file, "rb") as f:
                 self.secret_key = f.read().strip()

ui/components/History.vue

@@ -141,17 +141,17 @@
                 </td>
                 <td class="is-vcentered is-items-center">
                   <div class="field is-grouped is-grouped-centered">
-                    <div class="control" v-if="'finished' === item.status || isEmbedable(item.url)">
+                    <div class="control" v-if="('finished' === item.status && item.filename) || isEmbedable(item.url)">
-                      <button v-if="'finished' === item.status" @click="playVideo(item)" v-tooltip="'Play video'"
+                      <button v-if="'finished' === item.status && item.filename" @click="playVideo(item)"
-                        class="button is-danger is-light is-small">
+                        v-tooltip="'Play video'" class="button is-danger is-light is-small">
                         <span class="icon"><i class="fa-solid fa-play" /></span>
                       </button>
                       <button v-else @click="embed_url = getEmbedable(item.url)" v-tooltip="'Play video'"
-                        class="button is-danger is-light is-small">
+                        class="button is-danger is-small">
                         <span class="icon"><i class="fa-solid fa-play" /></span>
                       </button>
                     </div>
-                    <div class="control" v-if="item.status != 'finished'">
+                    <div class="control" v-if="item.status != 'finished' || !item.filename">
                       <button class="button is-warning is-fullwidth is-small" v-tooltip="'Re-queue video'"
                         @click="reQueueItem(item)">
                         <span class="icon"><i class="fa-solid fa-rotate-right" /></span>
@@ -203,7 +203,7 @@
 
             <div class="card-header-icon">
               <span v-if="hideThumbnail">
-                <a v-if="'finished' === item.status" href="#" @click.prevent="playVideo(item)"
+                <a v-if="'finished' === item.status && item.filename" href="#" @click.prevent="playVideo(item)"
                   v-tooltip="'Play video.'">
                   <span class="icon"><i class="fa-solid fa-play" /></span>
                 </a>
@@ -223,7 +223,7 @@
           </header>
           <div v-if="false === hideThumbnail" class="card-image">
             <figure class="image is-3by1">
-              <span v-if="'finished' === item.status" @click="playVideo(item)" class="play-overlay">
+              <span v-if="'finished' === item.status && item.filename" @click="playVideo(item)" class="play-overlay">
                 <div class="play-icon"></div>
                 <img @load="e => pImg(e)" :src="'/api/thumbnail?url=' + encodePath(item.extras.thumbnail)"
                   v-if="item.extras?.thumbnail" />
@@ -282,7 +282,7 @@
               </div>
             </div>
             <div class="columns is-mobile is-multiline">
-              <div class="column is-half-mobile" v-if="item.status != 'finished'">
+              <div class="column is-half-mobile" v-if="item.status != 'finished' || !item.filename">
                 <a class="button is-warning is-fullwidth" @click="reQueueItem(item)">
                   <span class="icon-text is-block">
                     <span class="icon"><i class="fa-solid fa-rotate-right" /></span>
@@ -512,6 +512,9 @@ const clearIncomplete = () => {
 
 const setIcon = item => {
   if ('finished' === item.status) {
+    if (!item.filename) {
+      return 'fa-solid fa-exclamation'
+    }
     return item.is_live ? 'fa-solid fa-globe' : 'fa-solid fa-circle-check'
   }
 
@@ -524,7 +527,7 @@ const setIcon = item => {
   }
 
   if ('not_live' === item.status) {
-    return 'fa-solid fa-hourglass-half fa-spin'
+    return 'fa-solid fa-headset'
   }
 
   return 'fa-solid fa-circle'
@@ -532,6 +535,9 @@ const setIcon = item => {
 
 const setIconColor = item => {
   if ('finished' === item.status) {
+    if (!item.filename) {
+      return 'has-text-warning'
+    }
     return 'has-text-success'
   }
 
@@ -548,6 +554,9 @@ const setIconColor = item => {
 
 const setStatus = item => {
   if ('finished' === item.status) {
+    if (!item.filename) {
+      return 'Skipped?'
+    }
     return item.is_live ? 'Live Ended' : 'Completed'
   }
 

ui/components/NewDownload.vue

@@ -276,9 +276,8 @@ const hasFormatInConfig = computed(() => {
   if (!ytdlp_cli.value) {
     return false
   }
-  if (ytdlp_cli.value.includes('-f') || ytdlp_cli.value.includes('--format')) {
+
-    return true
+  return /(?<!\w)(-f|--format)(=|:)?(?!\w)/.test(ytdlp_cli.value)
-  }
 })
 
 const filter_presets = (flag = true) => config.presets.filter(item => item.default === flag)

ui/components/TaskForm.vue

@@ -388,9 +388,8 @@ const hasFormatInConfig = computed(() => {
   if (!form?.cli) {
     return false
   }
-  if (form.cli.includes('-f') || form.cli.includes('--format')) {
+
-    return true
+  return /(?<!\w)(-f|--format)(=|:)?(?!\w)/.test(form.cli)
-  }
 })
 
 const filter_presets = (flag = true) => config.presets.filter(item => item.default === flag)