Fix problem with processing playlist

README.md

@@ -270,7 +270,6 @@ Certain configuration values can be set via environment variables, using the `-e
 | YTP_TEMP_KEEP            | Whether to keep the Individual video temp directory or remove it | `false`                            |
 | YTP_KEEP_ARCHIVE         | Keep history of downloaded videos                                | `true`                             |
 | YTP_YTDL_DEBUG           | Whether to turn debug logging for the internal `yt-dlp` package  | `false`                            |
-| YTP_ALLOW_MANIFESTLESS   | Allow `yt-dlp` to download unprocessed streams                   | `false`                            |
 | YTP_HOST                 | Which IP address to bind to                                      | `0.0.0.0`                          |
 | YTP_PORT                 | Which port to bind to                                            | `8081`                             |
 | YTP_LOG_LEVEL            | Log level                                                        | `info`                             |

app/library/Download.py

@@ -63,7 +63,7 @@ class Download:
         "fragment_retries",
         "skip_unavailable_fragments",
     ]
-    "Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode."
+    "Bad yt-dlp options which are known to cause issues with live stream."
 
     _ytdlp_fields: tuple = (
         "tmpfilename",
@@ -113,7 +113,6 @@ class Download:
         self.max_workers = int(config.max_workers)
         self.temp_keep = bool(config.temp_keep)
         self.is_live = bool(info.is_live) or info.live_in is not None
-        self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
         self.info_dict = info_dict
         self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
         self.started_time = 0
@@ -225,7 +224,7 @@ class Download:
                 if info:
                     self.info_dict = info
 
-            if self.is_live or self.is_manifestless:
+            if self.is_live:
                 hasDeletedOptions = False
                 deletedOpts: list = []
                 for opt in self.bad_live_options:
@@ -236,7 +235,7 @@ class Download:
 
                 if hasDeletedOptions:
                     self.logger.warning(
-                        f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
+                        f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted."
                     )
 
             if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1:
@@ -263,7 +262,7 @@ class Download:
                 cls.process_ie_result(
                     ie_result=self.info_dict,
                     download=True,
-                    extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")},
+                    extra_info={k: v for k, v in self.info.extras.items() if k not in self.info_dict},
                 )
                 ret = cls._download_retcode
             else:

app/library/DownloadQueue.py

@@ -193,62 +193,29 @@ class DownloadQueue(metaclass=Singleton):
             except Exception as e:
                 LOG.error(f"Failed to cancel downloads. {e!s}")
 
-    async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None):
+    async def _process_playlist(self, entry: dict, item: Item, already=None):
-        """
-        Add an entry to the download queue.
-
-        Args:
-            entry (dict): The entry to add to the download queue.
-            item (Item): The item to add to the download queue.
-            already (set): The set of already downloaded items.
-            logs (list): The list of logs generated during information extraction.
-
-        Returns:
-            dict: The status of the operation.
-
-        """
-        if not logs:
-            logs = []
-
-        if item.has_extras():
-            for key in item.extras.copy():
-                if not self.keep_extra_key(key):
-                    item.extras.pop(key)
-        else:
-            item.extras = {}
-
-        if not entry:
-            return {"status": "error", "msg": "Invalid/empty data was given."}
-
-        options: dict = {}
-
-        error: str | None = None
-        live_in: str | None = None
-
-        eventType = entry.get("_type") or "video"
-
-        if "playlist" == eventType:
         LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
         entries = entry.get("entries", [])
         playlistCount = int(entry.get("playlist_count", len(entries)))
         results = []
 
         for i, etr in enumerate(entries, start=1):
-                etr["playlist"] = entry.get("id")
+            extras = {
-                etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i)
+                "playlist": entry.get("id"),
-                etr["playlist_autonumber"] = i
+                "playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i),
+                "playlist_autonumber": i,
+            }
 
             for property in ("id", "title", "uploader", "uploader_id"):
                 if property in entry:
-                        etr[f"playlist_{property}"] = entry.get(property)
+                    extras[f"playlist_{property}"] = entry.get(property)
 
             if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
-                    etr["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
+                extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
 
             results.append(
-                    await self.__add_entry(
+                await self.add(
-                        entry=etr,
+                    item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras),
-                        item=item.new_with(url=etr.get("url") or etr.get("webpage_url")),
                     already=already,
                 )
             )
@@ -261,7 +228,14 @@ class DownloadQueue(metaclass=Singleton):
 
         return {"status": "ok"}
 
-        if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
+    async def _add_video(self, entry: dict, item: Item, logs: list[str] | None = None):
+        if not logs:
+            logs: list[str] = []
+
+        options: dict = {}
+        error: str | None = None
+        live_in: str | None = None
+
         # check if the video is live stream.
         if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
             if entry.get("release_timestamp"):
@@ -292,9 +266,6 @@ class DownloadQueue(metaclass=Singleton):
         except KeyError:
             pass
 
-            is_manifestless = entry.get("is_manifestless", False)
-            options.update({"is_manifestless": is_manifestless})
-
         live_status: list = ["is_live", "is_upcoming"]
         is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
 
@@ -334,6 +305,7 @@ class DownloadQueue(metaclass=Singleton):
 
         try:
             dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
+
             text_logs = ""
             if filtered_logs := extract_ytdlp_logs(logs):
                 text_logs = f" Logs: {', '.join(filtered_logs)}"
@@ -354,12 +326,6 @@ class DownloadQueue(metaclass=Singleton):
                 itemDownload = self.done.put(dlInfo)
                 NotifyEvent = Events.COMPLETED
                 await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
-                elif self.config.allow_manifestless is False and is_manifestless is True:
-                    dlInfo.info.status = "error"
-                    dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs
-
-                    itemDownload = self.done.put(dlInfo)
-                    NotifyEvent = Events.COMPLETED
             else:
                 NotifyEvent = Events.ADDED
                 itemDownload = self.queue.put(dlInfo)
@@ -373,10 +339,35 @@ class DownloadQueue(metaclass=Singleton):
             LOG.error(f"Failed to download item. '{e!s}'")
             return {"status": "error", "msg": str(e)}
 
-        if eventType.startswith("url"):
+    async def _add_item(self, entry: dict, item: Item, already=None, logs: list | None = None):
+        """
+        Add an entry to the download queue.
+
+        Args:
+            entry (dict): The entry to add to the download queue.
+            item (Item): The item to add to the download queue.
+            already (set): The set of already downloaded items.
+            logs (list): The list of logs generated during information extraction.
+
+        Returns:
+            dict: The status of the operation.
+
+        """
+        if not entry:
+            return {"status": "error", "msg": "Invalid/empty data was given."}
+
+        event_type = entry.get("_type", "video")
+
+        if event_type.startswith("playlist"):
+            return await self._process_playlist(entry=entry, item=item, already=already)
+
+        if event_type.startswith("url"):
             return await self.add(item=item.new_with(url=entry.get("url")), already=already)
 
-        return {"status": "error", "msg": f'Unsupported resource "{eventType}"'}
+        if not event_type.startswith("video"):
+            return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
+
+        return await self._add_video(entry=entry, item=item, logs=logs)
 
     async def add(self, item: Item, already: set | None = None):
         """
@@ -485,7 +476,7 @@ class DownloadQueue(metaclass=Singleton):
                 if condition is not None:
                     already.pop()
                     LOG.info(f"Condition '{condition}' matched for '{item.url}'.")
-                    return await self.add(item=item.new_with(requeued=True, cli=condition.cli),already=already)
+                    return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already)
 
             end_time = time.perf_counter() - started
             LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.")
@@ -504,12 +495,12 @@ class DownloadQueue(metaclass=Singleton):
         finally:
             if cookie_file and os.path.exists(cookie_file):
                 try:
-                    os.remove(yt_conf["cookiefile"])
+                    os.remove(cookie_file)
                     del yt_conf["cookiefile"]
                 except Exception as e:
                     LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
 
-        return await self.__add_entry(entry=entry, item=item, already=already, logs=logs)
+        return await self._add_item(entry=entry, item=item, already=already, logs=logs)
 
     async def cancel(self, ids: list[str]) -> dict[str, str]:
         """
@@ -767,20 +758,6 @@ class DownloadQueue(metaclass=Singleton):
 
         return is_downloaded(file, url)
 
-    def keep_extra_key(self, key: str) -> bool:
-        """
-        Check if the extra key should be kept.
-
-        Args:
-            key (str): The extra key to check.
-
-        Returns:
-            bool: True if the extra key should be kept, False otherwise.
-
-        """
-        keys = ("playlist", "external_downloader", "live_in")
-        return any(key == k or key.startswith(k) for k in keys)
-
     async def monitor_stale(self):
         """
         Monitor the queue and pool for stale downloads and cancel them if needed.

app/library/HttpAPI.py

@@ -268,6 +268,8 @@ class HttpAPI(Common):
             HttpAPI: The instance of the HttpAPI.
 
         """
+        registered_options = []
+
         for attr_name in dir(self):
             method = getattr(self, attr_name)
             if hasattr(method, "_http_method") and hasattr(method, "_http_path"):
@@ -277,6 +279,15 @@ class HttpAPI(Common):
 
                 self.routes.route(method._http_method, f"/{http_path}")(method)
 
+                if http_path in registered_options:
+                    continue
+
+                async def options_handler(_: Request) -> Response:
+                    return web.Response(status=204)
+
+                self.routes.route("OPTIONS", f"/{http_path}")(options_handler)
+                registered_options.append(http_path)
+
         self.routes.static("/api/download/", self.config.download_path)
         self._preload_static(app)
 
@@ -391,7 +402,16 @@ class HttpAPI(Common):
                         },
                     )
 
+            try:
                 response = await handler(request)
+            except web.HTTPException as e:
+                return web.json_response(data={"error": str(e)}, status=e.status_code)
+            except Exception as e:
+                LOG.exception(e)
+                response = web.json_response(
+                    data={"error": "Internal Server Error"},
+                    status=web.HTTPInternalServerError.status_code,
+                )
 
             if isinstance(response, web.FileResponse):
                 try:
@@ -405,20 +425,6 @@ class HttpAPI(Common):
 
         return middleware_handler
 
-    @route("OPTIONS", "/{path:.*}")
-    async def add_coors(self, _: Request) -> Response:
-        """
-        Add CORS headers to the response.
-
-        Args:
-            _: The request object.
-
-        Returns:
-            Response: The response object.
-
-        """
-        return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
-
     @route("GET", "api/ping")
     async def ping(self, _: Request) -> Response:
         """

app/library/ItemDTO.py

@@ -68,6 +68,7 @@ class Item:
         """
         return self.cli and len(self.cli) > 2
 
+    @staticmethod
     def _default_preset() -> str:
         from .config import Config
 

app/library/Utils.py

@@ -941,7 +941,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
 
     from anyio import open_file
 
-    if not pathlib(file).exists():
+    if not pathlib.Path(file).exists():
         return
 
     async with await open_file(file, "rb") as f:
@@ -1020,6 +1020,7 @@ def get_archive_id(url: str) -> tuple[bool, dict[str | None, str | None, str | N
         )
 
     for key, ie in YTDLP_INFO_CLS._ies.items():
+        try:
             if not ie.suitable(url):
                 continue
 
@@ -1034,6 +1035,8 @@ def get_archive_id(url: str) -> tuple[bool, dict[str | None, str | None, str | N
             idDict["ie_key"] = key
             idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
             break
+        except Exception as e:
+            LOG.error(f"Error getting archive ID: {e}")
 
     return idDict
 

app/library/YTDLPOpts.py

@@ -131,7 +131,7 @@ class YTDLPOpts(metaclass=Singleton):
             with open(file, "w") as f:
                 f.write(preset.cookies)
 
-            load_cookies(file)
+            load_cookies(str(file))
 
             self._preset_opts["cookiefile"] = str(file)
 

app/library/config.py

@@ -43,9 +43,6 @@ class Config:
     ytdl_debug: bool = False
     """Enable yt-dlp debugging."""
 
-    allow_manifestless: bool = False
-    """Allow downloading videos without manifest."""
-
     host: str = "0.0.0.0"
     """The host to bind the server to."""
 
@@ -194,7 +191,6 @@ class Config:
         "ytdl_debug",
         "debug",
         "temp_keep",
-        "allow_manifestless",
         "access_log",
         "remove_files",
         "ignore_ui",

ui/components/History.vue

@@ -639,7 +639,7 @@ const reQueueItem = (item, event = null) => {
     cookies: item.cookies,
     template: item.template,
     cli: item?.cli,
-    extras: item?.extras && Object.keys(item.extras) > 0 ? item.extras : {},
+    extras: toRaw(item.extras ?? {}) ?? {},
   };
 
   socket.emit('item_delete', { id: item._id, remove_file: false })

ui/pages/tasks.vue

@@ -81,7 +81,7 @@ div.is-centered {
                     <span class="icon"><i class="fa-solid fa-tv" /></span>
                     <span>{{ item.preset ?? config.app.default_preset }}</span>
                   </p>
-                  <p class="is-text-overflow">
+                  <p class="is-text-overflow" v-if="item.cli">
                     <span class="icon"><i class="fa-solid fa-terminal" /></span>
                     <span>{{ item.cli }}</span>
                   </p>