Merge pull request #229 from arabcoders/dev

Maintain playlist keys when we are extracting a playlist/channel

.vscode/settings.json

@@ -16,6 +16,8 @@
     "ahash",
     "aiocron",
     "arrowless",
+    "attl",
+    "autonumber",
     "changeslog",
     "copyts",
     "daterange",

Pipfile.lock

@@ -1027,12 +1027,12 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca",
+                "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5",
-                "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
+                "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.8'",
+            "markers": "python_version >= '3.9'",
-            "version": "==1.0.1"
+            "version": "==1.1.0"
         },
         "python-engineio": {
             "hashes": [

app/library/Download.py

@@ -15,6 +15,7 @@ from .config import Config
 from .Events import EventBus, Events
 from .ffprobe import ffprobe
 from .ItemDTO import ItemDTO
+from .Utils import extract_info
 from .YTDLPOpts import YTDLPOpts
 
 LOG = logging.getLogger("Download")
@@ -96,6 +97,7 @@ class Download:
         self.id = info._id
         self.default_ytdl_opts = config.ytdl_options
         self.debug = bool(config.debug)
+        self.archive = bool(config.keep_archive)
         self.debug_ytdl = bool(config.ytdl_debug)
         self.cancelled = False
         self.tmpfilename = None
@@ -159,6 +161,19 @@ class Download:
                 .get_all()
             )
 
+            if not self.info_dict:
+                self.logger.info(f"Extracting info for '{self.info.url}'.")
+                info = extract_info(
+                    config=params,
+                    url=self.info.url,
+                    debug=self.debug,
+                    no_archive=not self.archive,
+                    follow_redirect=True,
+                )
+
+                if info:
+                    self.info_dict = info
+
             params.update(
                 {
                     "progress_hooks": [self._progress_hook],
@@ -208,11 +223,15 @@ class Download:
 
             if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
                 self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
-                cls.process_ie_result(self.info_dict, download=True)
+                cls.process_ie_result(
+                    ie_result=self.info_dict,
+                    download=True,
+                    extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")},
+                )
                 ret = cls._download_retcode
             else:
                 self.logger.debug(f"Downloading using url: {self.info.url}")
-                ret = cls.download([self.info.url])
+                ret = cls.download(url_list=[self.info.url])
 
             self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"})
         except Exception as exc:

app/library/DownloadQueue.py

@@ -176,6 +176,7 @@ class DownloadQueue(metaclass=Singleton):
         config: dict | None = None,
         cookies: str = "",
         template: str = "",
+        extras: dict | None = None,
         already=None,
     ):
         """
@@ -188,8 +189,10 @@ class DownloadQueue(metaclass=Singleton):
             config (dict): The yt-dlp configuration to use for the download.
             cookies (str): The cookies to use for the download.
             template (str): The output template to use for the download.
+            extras (dict): The extra information to add to the download.
             already (set): The set of already downloaded items.
 
+
         Returns:
             dict: The status of the operation.
 
@@ -197,6 +200,13 @@ class DownloadQueue(metaclass=Singleton):
         if not config:
             config = {}
 
+        if not extras:
+            extras = {}
+        else:
+            for key in extras.copy():
+                if not str(key).startswith("playlist"):
+                    extras.pop(key, None)
+
         if not entry:
             return {"status": "error", "msg": "Invalid/empty data was given."}
 
@@ -208,12 +218,16 @@ class DownloadQueue(metaclass=Singleton):
         eventType = entry.get("_type") or "video"
 
         if "playlist" == eventType:
+            LOG.info("Processing playlist")
             entries = entry.get("entries", [])
-            playlist_index_digits = len(str(len(entries)))
+            playlistCount = int(entry.get("playlist_count", len(entries)))
             results = []
+
             for i, etr in enumerate(entries, start=1):
                 etr["playlist"] = entry.get("id")
-                etr["playlist_index"] = f"{{0:0{playlist_index_digits:d}d}}".format(i)
+                etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i)
+                etr["playlist_autonumber"] = i
+
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
                         etr[f"playlist_{property}"] = entry.get(property)
@@ -229,6 +243,7 @@ class DownloadQueue(metaclass=Singleton):
                         config=config,
                         cookies=cookies,
                         template=template,
+                        extras=extras,
                         already=already,
                     )
                 )
@@ -279,12 +294,15 @@ class DownloadQueue(metaclass=Singleton):
                 LOG.exception(e)
                 return {"status": "error", "msg": str(e)}
 
-            extras: dict = {}
             fields: tuple = ("uploader", "channel", "thumbnail")
             for field in fields:
                 if entry.get(field):
                     extras[field] = entry.get(field)
 
+            for key in entry:
+                if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
+                    extras[key] = entry.get(key)
+
             dl = ItemDTO(
                 id=str(entry.get("id")),
                 title=str(entry.get("title")),
@@ -305,10 +323,6 @@ class DownloadQueue(metaclass=Singleton):
                 extras=extras,
             )
 
-            for property, value in entry.items():
-                if property.startswith("playlist"):
-                    dl.template = str(dl.template).replace(f"%({property})s", str(value))
-
             dlInfo: Download = Download(info=dl, info_dict=entry)
 
             if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"):
@@ -337,6 +351,7 @@ class DownloadQueue(metaclass=Singleton):
                 config=config,
                 cookies=cookies,
                 template=template,
+                extras=extras,
                 already=already,
             )
 
@@ -350,12 +365,15 @@ class DownloadQueue(metaclass=Singleton):
         config: dict | None = None,
         cookies: str = "",
         template: str = "",
+        extras: dict | None = None,
         already=None,
     ):
         _preset = Presets.get_instance().get(name=preset)
 
         config = config if config else {}
         folder = str(folder) if folder else ""
+        if not extras:
+            extras = {}
 
         if _preset:
             if _preset.folder and not folder:
@@ -372,7 +390,7 @@ class DownloadQueue(metaclass=Singleton):
         cookie_file = os.path.join(self.config.temp_path, f"c_{uuid.uuid4().hex}.txt")
 
         LOG.info(
-            f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {len(cookies)}/chars' 'YTConfig: {config}'."
+            f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {len(cookies)}/chars' 'YTConfig: {config}' 'Extras: {extras}'."
         )
 
         if isinstance(config, str):
@@ -467,6 +485,7 @@ class DownloadQueue(metaclass=Singleton):
             cookies=cookies,
             template=template,
             already=already,
+            extras=extras,
         )
 
     async def cancel(self, ids: list[str]) -> dict[str, str]:

app/library/HttpAPI.py

@@ -509,6 +509,7 @@ class HttpAPI(Common):
                 debug=False,
                 no_archive=True,
                 follow_redirect=True,
+                sanitize_info=True,
             )
 
             if "formats" in data:
@@ -531,8 +532,8 @@ class HttpAPI(Common):
 
             return web.Response(body=json.dumps(data, indent=4), status=web.HTTPOk.status_code)
         except Exception as e:
-            LOG.error(f"Error encountered while grabbing video info '{url}'. '{e}'.")
             LOG.exception(e)
+            LOG.error(f"Error encountered while getting video info for '{url}'. '{e!s}'.")
             return web.json_response(
                 data={
                     "error": "failed to get video info.",
@@ -1407,6 +1408,7 @@ class HttpAPI(Common):
             "https://imageipsum.com/1920x1080",
             "https://placedog.net/1920/1080",
         ]
+        backend = random.choice(backends)  # noqa: S311
 
         try:
             CACHE_KEY = "random_background"
@@ -1430,7 +1432,6 @@ class HttpAPI(Common):
                 },
             }
 
-            backend = random.choice(backends)  # noqa: S311
             logging.getLogger("httpx").setLevel(logging.WARNING)
             async with httpx.AsyncClient(**opts) as client:
                 response = await client.request(method="GET", url=backend, follow_redirects=True)
@@ -1463,7 +1464,7 @@ class HttpAPI(Common):
                 )
         except Exception as e:
             LOG.exception(e)
-            LOG.error(f"Failed to request random background image.'. '{e!s}'.")
+            LOG.error(f"Failed to request random background image from '{backend!s}'.'. '{e!s}'.")
             return web.json_response(
                 data={"error": "failed to retrieve the random background image."},
                 status=web.HTTPInternalServerError.status_code,
@@ -1579,8 +1580,8 @@ class HttpAPI(Common):
             cookies = http.cookiejar.MozillaCookieJar(cookie_file, None, None)
             cookies.load()
         except Exception as e:
-            LOG.error(f"failed to load cookies from '{cookie_file}'. '{e}'.")
             LOG.exception(e)
+            LOG.error(f"failed to load cookies from '{cookie_file}'. '{e!s}'.")
             return web.json_response(
                 data={"message": "Failed to load cookies"},
                 status=web.HTTPInternalServerError.status_code,

app/library/HttpSocket.py

@@ -79,7 +79,7 @@ class HttpSocket(Common):
 
         self._notify.subscribe(
             Events.ADD_URL,
-            lambda data, _, **kwargs: self.add(**data.data),  # noqa: ARG005
+            lambda data, _, **kwargs: self.add(**self.format_item(data.data)),  # noqa: ARG005
             f"{__class__.__name__}.socket_add_url",
         )
 

app/library/Utils.py

@@ -68,6 +68,7 @@ def extract_info(
     debug: bool = False,
     no_archive: bool = False,
     follow_redirect: bool = False,
+    sanitize_info: bool = False,
 ) -> dict:
     """
     Extracts video information from the given URL.
@@ -78,6 +79,7 @@ def extract_info(
         debug (bool): Enable debug logging.
         no_archive (bool): Disable download archive.
         follow_redirect (bool): Follow URL redirects.
+        sanitize_info (bool): Sanitize the extracted information
 
     Returns:
         dict: Video information.
@@ -127,9 +129,19 @@ def extract_info(
     data = yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)
 
     if data and follow_redirect and "_type" in data and "url" == data["_type"]:
-        return extract_info(config, data["url"], debug=debug, no_archive=no_archive, follow_redirect=follow_redirect)
+        return extract_info(
+            config,
+            data["url"],
+            debug=debug,
+            no_archive=no_archive,
+            follow_redirect=follow_redirect,
+            sanitize_info=sanitize_info,
+        )
 
-    return data
+    if not data:
+        return data
+
+    return yt_dlp.YoutubeDL.sanitize_info(data) if sanitize_info else data
 
 
 def merge_dict(source: dict, destination: dict) -> dict:

app/library/common.py

@@ -27,7 +27,7 @@ class Common:
         self.default_preset = config.default_preset
 
     async def add(
-        self, url: str, preset: str, folder: str, cookies: str, config: dict, template: str
+        self, url: str, preset: str, folder: str, cookies: str, config: dict, template: str, extras: dict | None = None
     ) -> dict[str, str]:
         """
         Add an item to the download queue.
@@ -39,6 +39,7 @@ class Common:
             cookies (str): The cookies to be used for the download.
             config (dict): The yt-dlp config to be used for the download.
             template (str): The template to be used for the download.
+            extras (dict): Extra data to be added to the download
 
         Returns:
             dict[str, str]: The status of the download.
@@ -55,6 +56,7 @@ class Common:
             cookies=cookies,
             config=config if isinstance(config, dict) else {},
             template=template,
+            extras=extras,
         )
 
     def format_item(self, item: dict) -> dict:
@@ -82,6 +84,7 @@ class Common:
         folder: str = str(item.get("folder")) if item.get("folder") else ""
         cookies: str = str(item.get("cookies")) if item.get("cookies") else ""
         template: str = str(item.get("template")) if item.get("template") else ""
+        extras = item.get("extras", {})
 
         config = item.get("config")
         if isinstance(config, str) and config:
@@ -98,4 +101,5 @@ class Common:
             "cookies": cookies,
             "config": config if isinstance(config, dict) else {},
             "template": template,
+            "extras": extras if isinstance(extras, dict) else {},
         }

ui/components/History.vue

@@ -485,6 +485,17 @@ const removeItem = item => {
 
 const reQueueItem = item => {
   socket.emit('item_delete', { id: item._id, remove_file: false })
+
+  let extras = {}
+
+  if (item.extras) {
+    Object.keys(item.extras).forEach(k => {
+      if (k && true === k.startsWith('playlist')) {
+        extras[k] = item.extras[k]
+      }
+    })
+  }
+
   socket.emit('add_url', {
     url: item.url,
     preset: item.preset,
@@ -492,6 +503,7 @@ const reQueueItem = item => {
     config: item.config,
     cookies: item.cookies,
     template: item.template,
+    extras: extras
   })
 }
 
@@ -501,6 +513,6 @@ watch(video_item, v => {
     return
   }
 
-  document.querySelector('body').setAttribute("style", `opacity: ${ v ? 1 : bg_opacity.value}`)
+  document.querySelector('body').setAttribute("style", `opacity: ${v ? 1 : bg_opacity.value}`)
 })
 </script>

ui/package.json

@@ -12,17 +12,17 @@
   "web-types": "./web-types.json",
   "dependencies": {
     "@pinia/nuxt": "^0.10.1",
-    "@sentry/nuxt": "^9.6.0",
+    "@sentry/nuxt": "^9.9.0",
     "@vueuse/core": "^13.0.0",
     "@vueuse/nuxt": "^13.0.0",
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.5.0",
-    "cron-parser": "^5.0.5",
+    "cron-parser": "^5.0.6",
     "cronstrue": "^2.49.0",
     "floating-vue": "^5.2.2",
     "hls.js": "^1.4.12",
     "moment": "^2.30.1",
-    "nuxt": "^3.16",
+    "nuxt": "^3.16.1",
     "pinia": "^3.0.1",
     "socket.io-client": "^4.7.2",
     "vue": "^3.4",

ui/pages/index.vue

@@ -75,6 +75,7 @@
 
 <script setup>
 import { request } from '~/utils/index'
+import { useStorage } from '@vueuse/core'
 
 const emitter = defineEmits(['getInfo'])
 const config = useConfigStore()
@@ -83,6 +84,8 @@ const socket = useSocketStore()
 const toast = useToast()
 const isChecking = ref(false)
 const get_info = ref('')
+const bg_enable = useStorage('random_bg', true)
+const bg_opacity = useStorage('random_bg_opacity', 0.85)
 
 onMounted(() => {
   if (!config.app.ui_update_title) {
@@ -138,4 +141,12 @@ const pauseDownload = () => {
 
   socket.emit('pause', {})
 }
+
+watch(get_info, v => {
+  if (!bg_enable.value) {
+    return
+  }
+
+  document.querySelector('body').setAttribute("style", `opacity: ${v ? 1 : bg_opacity.value}`)
+})
 </script>