Fix problem with processing playlist

README.md

@@ -270,7 +270,6 @@ Certain configuration values can be set via environment variables, using the `-e
 | YTP_TEMP_KEEP            | Whether to keep the Individual video temp directory or remove it | `false`                            |
 | YTP_KEEP_ARCHIVE         | Keep history of downloaded videos                                | `true`                             |
 | YTP_YTDL_DEBUG           | Whether to turn debug logging for the internal `yt-dlp` package  | `false`                            |
-| YTP_ALLOW_MANIFESTLESS   | Allow `yt-dlp` to download unprocessed streams                   | `false`                            |
 | YTP_HOST                 | Which IP address to bind to                                      | `0.0.0.0`                          |
 | YTP_PORT                 | Which port to bind to                                            | `8081`                             |
 | YTP_LOG_LEVEL            | Log level                                                        | `info`                             |

app/library/Download.py

@@ -63,7 +63,7 @@ class Download:
         "fragment_retries",
         "skip_unavailable_fragments",
     ]
-    "Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode."
+    "Bad yt-dlp options which are known to cause issues with live stream."
 
     _ytdlp_fields: tuple = (
         "tmpfilename",
@@ -113,7 +113,6 @@ class Download:
         self.max_workers = int(config.max_workers)
         self.temp_keep = bool(config.temp_keep)
         self.is_live = bool(info.is_live) or info.live_in is not None
-        self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
         self.info_dict = info_dict
         self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
         self.started_time = 0
@@ -225,7 +224,7 @@ class Download:
                 if info:
                     self.info_dict = info
 
-            if self.is_live or self.is_manifestless:
+            if self.is_live:
                 hasDeletedOptions = False
                 deletedOpts: list = []
                 for opt in self.bad_live_options:
@@ -236,7 +235,7 @@ class Download:
 
                 if hasDeletedOptions:
                     self.logger.warning(
-                        f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
+                        f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted."
                     )
 
             if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1:
@@ -263,7 +262,7 @@ class Download:
                 cls.process_ie_result(
                     ie_result=self.info_dict,
                     download=True,
-                    extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")},
+                    extra_info={k: v for k, v in self.info.extras.items() if k not in self.info_dict},
                 )
                 ret = cls._download_retcode
             else:

app/library/DownloadQueue.py

@@ -193,7 +193,153 @@ class DownloadQueue(metaclass=Singleton):
             except Exception as e:
                 LOG.error(f"Failed to cancel downloads. {e!s}")
 
-    async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None):
+    async def _process_playlist(self, entry: dict, item: Item, already=None):
+        LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
+        entries = entry.get("entries", [])
+        playlistCount = int(entry.get("playlist_count", len(entries)))
+        results = []
+
+        for i, etr in enumerate(entries, start=1):
+            extras = {
+                "playlist": entry.get("id"),
+                "playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i),
+                "playlist_autonumber": i,
+            }
+
+            for property in ("id", "title", "uploader", "uploader_id"):
+                if property in entry:
+                    extras[f"playlist_{property}"] = entry.get(property)
+
+            if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
+                extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
+
+            results.append(
+                await self.add(
+                    item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras),
+                    already=already,
+                )
+            )
+
+        if any("error" == res["status"] for res in results):
+            return {
+                "status": "error",
+                "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
+            }
+
+        return {"status": "ok"}
+
+    async def _add_video(self, entry: dict, item: Item, logs: list[str] | None = None):
+        if not logs:
+            logs: list[str] = []
+
+        options: dict = {}
+        error: str | None = None
+        live_in: str | None = None
+
+        # check if the video is live stream.
+        if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
+            if entry.get("release_timestamp"):
+                live_in = formatdate(entry.get("release_timestamp"), usegmt=True)
+                item.extras.update({"live_in": live_in})
+            else:
+                error = "Live stream not yet started. And no date is set."
+        else:
+            error = entry.get("msg")
+
+        LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
+
+        try:
+            _item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
+            if _item is not None:
+                err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history."
+                LOG.warning(err_msg)
+                await self.clear([_item.info._id], remove_file=False)
+        except KeyError:
+            pass
+
+        try:
+            _item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")))
+            if _item is not None:
+                err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue."
+                LOG.info(err_msg)
+                return {"status": "error", "msg": err_msg}
+        except KeyError:
+            pass
+
+        live_status: list = ["is_live", "is_upcoming"]
+        is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
+
+        try:
+            download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder)
+        except Exception as e:
+            LOG.exception(e)
+            return {"status": "error", "msg": str(e)}
+
+        for field in ("uploader", "channel", "thumbnail"):
+            if entry.get(field):
+                item.extras[field] = entry.get(field)
+
+        for key in entry:
+            if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
+                item.extras[key] = entry.get(key)
+
+        dl = ItemDTO(
+            id=str(entry.get("id")),
+            title=str(entry.get("title")),
+            url=str(entry.get("webpage_url") or entry.get("url")),
+            preset=item.preset,
+            folder=item.folder,
+            download_dir=download_dir,
+            temp_dir=self.config.temp_path,
+            cookies=item.cookies,
+            template=item.template if item.template else self.config.output_template,
+            template_chapter=self.config.output_template_chapter,
+            datetime=formatdate(time.time()),
+            error=error,
+            is_live=is_live,
+            live_in=live_in if live_in else item.extras.get("live_in", None),
+            options=options,
+            cli=item.cli,
+            extras=item.extras,
+        )
+
+        try:
+            dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
+
+            text_logs = ""
+            if filtered_logs := extract_ytdlp_logs(logs):
+                text_logs = f" Logs: {', '.join(filtered_logs)}"
+
+            if live_in or "is_upcoming" == entry.get("live_status"):
+                NotifyEvent = Events.COMPLETED
+                dlInfo.info.status = "not_live"
+                dlInfo.info.msg = "Stream is not live yet." + text_logs
+                itemDownload = self.done.put(dlInfo)
+            elif len(entry.get("formats", [])) < 1:
+                availability = entry.get("availability", "public")
+                msg = "No formats found."
+                if availability and availability not in ("public",):
+                    msg += f" Availability is set for '{availability}'."
+
+                dlInfo.info.error = msg + text_logs
+                dlInfo.info.status = "error"
+                itemDownload = self.done.put(dlInfo)
+                NotifyEvent = Events.COMPLETED
+                await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
+            else:
+                NotifyEvent = Events.ADDED
+                itemDownload = self.queue.put(dlInfo)
+                self.event.set()
+
+            await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize())
+
+            return {"status": "ok"}
+        except Exception as e:
+            LOG.exception(e)
+            LOG.error(f"Failed to download item. '{e!s}'")
+            return {"status": "error", "msg": str(e)}
+
+    async def _add_item(self, entry: dict, item: Item, already=None, logs: list | None = None):
         """
         Add an entry to the download queue.
 
@@ -207,176 +353,21 @@ class DownloadQueue(metaclass=Singleton):
             dict: The status of the operation.
 
         """
-        if not logs:
-            logs = []
-
-        if item.has_extras():
-            for key in item.extras.copy():
-                if not self.keep_extra_key(key):
-                    item.extras.pop(key)
-        else:
-            item.extras = {}
-
         if not entry:
             return {"status": "error", "msg": "Invalid/empty data was given."}
 
-        options: dict = {}
+        event_type = entry.get("_type", "video")
 
-        error: str | None = None
+        if event_type.startswith("playlist"):
-        live_in: str | None = None
+            return await self._process_playlist(entry=entry, item=item, already=already)
 
-        eventType = entry.get("_type") or "video"
+        if event_type.startswith("url"):
-
-        if "playlist" == eventType:
-            LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
-            entries = entry.get("entries", [])
-            playlistCount = int(entry.get("playlist_count", len(entries)))
-            results = []
-
-            for i, etr in enumerate(entries, start=1):
-                etr["playlist"] = entry.get("id")
-                etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i)
-                etr["playlist_autonumber"] = i
-
-                for property in ("id", "title", "uploader", "uploader_id"):
-                    if property in entry:
-                        etr[f"playlist_{property}"] = entry.get(property)
-
-                if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
-                    etr["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
-
-                results.append(
-                    await self.__add_entry(
-                        entry=etr,
-                        item=item.new_with(url=etr.get("url") or etr.get("webpage_url")),
-                        already=already,
-                    )
-                )
-
-            if any("error" == res["status"] for res in results):
-                return {
-                    "status": "error",
-                    "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
-                }
-
-            return {"status": "ok"}
-
-        if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
-            # check if the video is live stream.
-            if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
-                if entry.get("release_timestamp"):
-                    live_in = formatdate(entry.get("release_timestamp"), usegmt=True)
-                    item.extras.update({"live_in": live_in})
-                else:
-                    error = "Live stream not yet started. And no date is set."
-            else:
-                error = entry.get("msg")
-
-            LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
-
-            try:
-                _item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
-                if _item is not None:
-                    err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history."
-                    LOG.warning(err_msg)
-                    await self.clear([_item.info._id], remove_file=False)
-            except KeyError:
-                pass
-
-            try:
-                _item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")))
-                if _item is not None:
-                    err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue."
-                    LOG.info(err_msg)
-                    return {"status": "error", "msg": err_msg}
-            except KeyError:
-                pass
-
-            is_manifestless = entry.get("is_manifestless", False)
-            options.update({"is_manifestless": is_manifestless})
-
-            live_status: list = ["is_live", "is_upcoming"]
-            is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
-
-            try:
-                download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder)
-            except Exception as e:
-                LOG.exception(e)
-                return {"status": "error", "msg": str(e)}
-
-            for field in ("uploader", "channel", "thumbnail"):
-                if entry.get(field):
-                    item.extras[field] = entry.get(field)
-
-            for key in entry:
-                if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
-                    item.extras[key] = entry.get(key)
-
-            dl = ItemDTO(
-                id=str(entry.get("id")),
-                title=str(entry.get("title")),
-                url=str(entry.get("webpage_url") or entry.get("url")),
-                preset=item.preset,
-                folder=item.folder,
-                download_dir=download_dir,
-                temp_dir=self.config.temp_path,
-                cookies=item.cookies,
-                template=item.template if item.template else self.config.output_template,
-                template_chapter=self.config.output_template_chapter,
-                datetime=formatdate(time.time()),
-                error=error,
-                is_live=is_live,
-                live_in=live_in if live_in else item.extras.get("live_in", None),
-                options=options,
-                cli=item.cli,
-                extras=item.extras,
-            )
-
-            try:
-                dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
-                text_logs = ""
-                if filtered_logs := extract_ytdlp_logs(logs):
-                    text_logs = f" Logs: {', '.join(filtered_logs)}"
-
-                if live_in or "is_upcoming" == entry.get("live_status"):
-                    NotifyEvent = Events.COMPLETED
-                    dlInfo.info.status = "not_live"
-                    dlInfo.info.msg = "Stream is not live yet." + text_logs
-                    itemDownload = self.done.put(dlInfo)
-                elif len(entry.get("formats", [])) < 1:
-                    availability = entry.get("availability", "public")
-                    msg = "No formats found."
-                    if availability and availability not in ("public",):
-                        msg += f" Availability is set for '{availability}'."
-
-                    dlInfo.info.error = msg + text_logs
-                    dlInfo.info.status = "error"
-                    itemDownload = self.done.put(dlInfo)
-                    NotifyEvent = Events.COMPLETED
-                    await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
-                elif self.config.allow_manifestless is False and is_manifestless is True:
-                    dlInfo.info.status = "error"
-                    dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs
-
-                    itemDownload = self.done.put(dlInfo)
-                    NotifyEvent = Events.COMPLETED
-                else:
-                    NotifyEvent = Events.ADDED
-                    itemDownload = self.queue.put(dlInfo)
-                    self.event.set()
-
-                await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize())
-
-                return {"status": "ok"}
-            except Exception as e:
-                LOG.exception(e)
-                LOG.error(f"Failed to download item. '{e!s}'")
-                return {"status": "error", "msg": str(e)}
-
-        if eventType.startswith("url"):
             return await self.add(item=item.new_with(url=entry.get("url")), already=already)
 
-        return {"status": "error", "msg": f'Unsupported resource "{eventType}"'}
+        if not event_type.startswith("video"):
+            return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
+
+        return await self._add_video(entry=entry, item=item, logs=logs)
 
     async def add(self, item: Item, already: set | None = None):
         """
@@ -485,7 +476,7 @@ class DownloadQueue(metaclass=Singleton):
                 if condition is not None:
                     already.pop()
                     LOG.info(f"Condition '{condition}' matched for '{item.url}'.")
-                    return await self.add(item=item.new_with(requeued=True, cli=condition.cli),already=already)
+                    return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already)
 
             end_time = time.perf_counter() - started
             LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.")
@@ -504,12 +495,12 @@ class DownloadQueue(metaclass=Singleton):
         finally:
             if cookie_file and os.path.exists(cookie_file):
                 try:
-                    os.remove(yt_conf["cookiefile"])
+                    os.remove(cookie_file)
                     del yt_conf["cookiefile"]
                 except Exception as e:
                     LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
 
-        return await self.__add_entry(entry=entry, item=item, already=already, logs=logs)
+        return await self._add_item(entry=entry, item=item, already=already, logs=logs)
 
     async def cancel(self, ids: list[str]) -> dict[str, str]:
         """
@@ -767,20 +758,6 @@ class DownloadQueue(metaclass=Singleton):
 
         return is_downloaded(file, url)
 
-    def keep_extra_key(self, key: str) -> bool:
-        """
-        Check if the extra key should be kept.
-
-        Args:
-            key (str): The extra key to check.
-
-        Returns:
-            bool: True if the extra key should be kept, False otherwise.
-
-        """
-        keys = ("playlist", "external_downloader", "live_in")
-        return any(key == k or key.startswith(k) for k in keys)
-
     async def monitor_stale(self):
         """
         Monitor the queue and pool for stale downloads and cancel them if needed.

app/library/HttpAPI.py

@@ -268,6 +268,8 @@ class HttpAPI(Common):
             HttpAPI: The instance of the HttpAPI.
 
         """
+        registered_options = []
+
         for attr_name in dir(self):
             method = getattr(self, attr_name)
             if hasattr(method, "_http_method") and hasattr(method, "_http_path"):
@@ -277,6 +279,15 @@ class HttpAPI(Common):
 
                 self.routes.route(method._http_method, f"/{http_path}")(method)
 
+                if http_path in registered_options:
+                    continue
+
+                async def options_handler(_: Request) -> Response:
+                    return web.Response(status=204)
+
+                self.routes.route("OPTIONS", f"/{http_path}")(options_handler)
+                registered_options.append(http_path)
+
         self.routes.static("/api/download/", self.config.download_path)
         self._preload_static(app)
 
@@ -391,7 +402,16 @@ class HttpAPI(Common):
                         },
                     )
 
-            response = await handler(request)
+            try:
+                response = await handler(request)
+            except web.HTTPException as e:
+                return web.json_response(data={"error": str(e)}, status=e.status_code)
+            except Exception as e:
+                LOG.exception(e)
+                response = web.json_response(
+                    data={"error": "Internal Server Error"},
+                    status=web.HTTPInternalServerError.status_code,
+                )
 
             if isinstance(response, web.FileResponse):
                 try:
@@ -405,20 +425,6 @@ class HttpAPI(Common):
 
         return middleware_handler
 
-    @route("OPTIONS", "/{path:.*}")
-    async def add_coors(self, _: Request) -> Response:
-        """
-        Add CORS headers to the response.
-
-        Args:
-            _: The request object.
-
-        Returns:
-            Response: The response object.
-
-        """
-        return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
-
     @route("GET", "api/ping")
     async def ping(self, _: Request) -> Response:
         """

app/library/ItemDTO.py

@@ -68,6 +68,7 @@ class Item:
         """
         return self.cli and len(self.cli) > 2
 
+    @staticmethod
     def _default_preset() -> str:
         from .config import Config
 

app/library/Utils.py

@@ -941,7 +941,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
 
     from anyio import open_file
 
-    if not pathlib(file).exists():
+    if not pathlib.Path(file).exists():
         return
 
     async with await open_file(file, "rb") as f:
@@ -1020,20 +1020,23 @@ def get_archive_id(url: str) -> tuple[bool, dict[str | None, str | None, str | N
         )
 
     for key, ie in YTDLP_INFO_CLS._ies.items():
-        if not ie.suitable(url):
+        try:
-            continue
+            if not ie.suitable(url):
+                continue
 
-        if not ie.working():
+            if not ie.working():
+                break
+
+            temp_id = ie.get_temp_id(url)
+            if not temp_id:
+                break
+
+            idDict["id"] = temp_id
+            idDict["ie_key"] = key
+            idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
             break
-
+        except Exception as e:
-        temp_id = ie.get_temp_id(url)
+            LOG.error(f"Error getting archive ID: {e}")
-        if not temp_id:
-            break
-
-        idDict["id"] = temp_id
-        idDict["ie_key"] = key
-        idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
-        break
 
     return idDict
 

app/library/YTDLPOpts.py

@@ -131,7 +131,7 @@ class YTDLPOpts(metaclass=Singleton):
             with open(file, "w") as f:
                 f.write(preset.cookies)
 
-            load_cookies(file)
+            load_cookies(str(file))
 
             self._preset_opts["cookiefile"] = str(file)
 

app/library/config.py

@@ -43,9 +43,6 @@ class Config:
     ytdl_debug: bool = False
     """Enable yt-dlp debugging."""
 
-    allow_manifestless: bool = False
-    """Allow downloading videos without manifest."""
-
     host: str = "0.0.0.0"
     """The host to bind the server to."""
 
@@ -194,7 +191,6 @@ class Config:
         "ytdl_debug",
         "debug",
         "temp_keep",
-        "allow_manifestless",
         "access_log",
         "remove_files",
         "ignore_ui",

ui/components/History.vue

@@ -639,7 +639,7 @@ const reQueueItem = (item, event = null) => {
     cookies: item.cookies,
     template: item.template,
     cli: item?.cli,
-    extras: item?.extras && Object.keys(item.extras) > 0 ? item.extras : {},
+    extras: toRaw(item.extras ?? {}) ?? {},
   };
 
   socket.emit('item_delete', { id: item._id, remove_file: false })

ui/pages/tasks.vue

@@ -81,7 +81,7 @@ div.is-centered {
                     <span class="icon"><i class="fa-solid fa-tv" /></span>
                     <span>{{ item.preset ?? config.app.default_preset }}</span>
                   </p>
-                  <p class="is-text-overflow">
+                  <p class="is-text-overflow" v-if="item.cli">
                     <span class="icon"><i class="fa-solid fa-terminal" /></span>
                     <span>{{ item.cli }}</span>
                   </p>