From 647acc488d91c01a9458a67aa598d968bac4eb5e Mon Sep 17 00:00:00 2001 From: arabcoders Date: Fri, 23 May 2025 18:19:06 +0300 Subject: [PATCH] Fix problem with processing playlist --- README.md | 1 - app/library/Download.py | 9 +- app/library/DownloadQueue.py | 339 ++++++++++++++++------------------- app/library/HttpAPI.py | 36 ++-- app/library/ItemDTO.py | 1 + app/library/Utils.py | 29 +-- app/library/YTDLPOpts.py | 2 +- app/library/config.py | 4 - ui/components/History.vue | 2 +- ui/pages/tasks.vue | 2 +- 10 files changed, 203 insertions(+), 222 deletions(-) diff --git a/README.md b/README.md index a236a028..ff9f493a 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,6 @@ Certain configuration values can be set via environment variables, using the `-e | YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | | YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | | YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_ALLOW_MANIFESTLESS | Allow `yt-dlp` to download unprocessed streams | `false` | | YTP_HOST | Which IP address to bind to | `0.0.0.0` | | YTP_PORT | Which port to bind to | `8081` | | YTP_LOG_LEVEL | Log level | `info` | diff --git a/app/library/Download.py b/app/library/Download.py index fc705f93..7bfaf4b4 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -63,7 +63,7 @@ class Download: "fragment_retries", "skip_unavailable_fragments", ] - "Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode." + "Bad yt-dlp options which are known to cause issues with live stream." _ytdlp_fields: tuple = ( "tmpfilename", @@ -113,7 +113,6 @@ class Download: self.max_workers = int(config.max_workers) self.temp_keep = bool(config.temp_keep) self.is_live = bool(info.is_live) or info.live_in is not None - self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True self.info_dict = info_dict self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}") self.started_time = 0 @@ -225,7 +224,7 @@ class Download: if info: self.info_dict = info - if self.is_live or self.is_manifestless: + if self.is_live: hasDeletedOptions = False deletedOpts: list = [] for opt in self.bad_live_options: @@ -236,7 +235,7 @@ class Download: if hasDeletedOptions: self.logger.warning( - f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode." + f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted." ) if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1: @@ -263,7 +262,7 @@ class Download: cls.process_ie_result( ie_result=self.info_dict, download=True, - extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")}, + extra_info={k: v for k, v in self.info.extras.items() if k not in self.info_dict}, ) ret = cls._download_retcode else: diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 18c1d11b..fc15b78f 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -193,7 +193,153 @@ class DownloadQueue(metaclass=Singleton): except Exception as e: LOG.error(f"Failed to cancel downloads. {e!s}") - async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None): + async def _process_playlist(self, entry: dict, item: Item, already=None): + LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.") + entries = entry.get("entries", []) + playlistCount = int(entry.get("playlist_count", len(entries))) + results = [] + + for i, etr in enumerate(entries, start=1): + extras = { + "playlist": entry.get("id"), + "playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i), + "playlist_autonumber": i, + } + + for property in ("id", "title", "uploader", "uploader_id"): + if property in entry: + extras[f"playlist_{property}"] = entry.get(property) + + if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""): + extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg" + + results.append( + await self.add( + item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras), + already=already, + ) + ) + + if any("error" == res["status"] for res in results): + return { + "status": "error", + "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res), + } + + return {"status": "ok"} + + async def _add_video(self, entry: dict, item: Item, logs: list[str] | None = None): + if not logs: + logs: list[str] = [] + + options: dict = {} + error: str | None = None + live_in: str | None = None + + # check if the video is live stream. + if "live_status" in entry and "is_upcoming" == entry.get("live_status"): + if entry.get("release_timestamp"): + live_in = formatdate(entry.get("release_timestamp"), usegmt=True) + item.extras.update({"live_in": live_in}) + else: + error = "Live stream not yet started. And no date is set." + else: + error = entry.get("msg") + + LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.") + + try: + _item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url")) + if _item is not None: + err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history." + LOG.warning(err_msg) + await self.clear([_item.info._id], remove_file=False) + except KeyError: + pass + + try: + _item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url"))) + if _item is not None: + err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue." + LOG.info(err_msg) + return {"status": "error", "msg": err_msg} + except KeyError: + pass + + live_status: list = ["is_live", "is_upcoming"] + is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) + + try: + download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder) + except Exception as e: + LOG.exception(e) + return {"status": "error", "msg": str(e)} + + for field in ("uploader", "channel", "thumbnail"): + if entry.get(field): + item.extras[field] = entry.get(field) + + for key in entry: + if isinstance(key, str) and key.startswith("playlist") and entry.get(key): + item.extras[key] = entry.get(key) + + dl = ItemDTO( + id=str(entry.get("id")), + title=str(entry.get("title")), + url=str(entry.get("webpage_url") or entry.get("url")), + preset=item.preset, + folder=item.folder, + download_dir=download_dir, + temp_dir=self.config.temp_path, + cookies=item.cookies, + template=item.template if item.template else self.config.output_template, + template_chapter=self.config.output_template_chapter, + datetime=formatdate(time.time()), + error=error, + is_live=is_live, + live_in=live_in if live_in else item.extras.get("live_in", None), + options=options, + cli=item.cli, + extras=item.extras, + ) + + try: + dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs) + + text_logs = "" + if filtered_logs := extract_ytdlp_logs(logs): + text_logs = f" Logs: {', '.join(filtered_logs)}" + + if live_in or "is_upcoming" == entry.get("live_status"): + NotifyEvent = Events.COMPLETED + dlInfo.info.status = "not_live" + dlInfo.info.msg = "Stream is not live yet." + text_logs + itemDownload = self.done.put(dlInfo) + elif len(entry.get("formats", [])) < 1: + availability = entry.get("availability", "public") + msg = "No formats found." + if availability and availability not in ("public",): + msg += f" Availability is set for '{availability}'." + + dlInfo.info.error = msg + text_logs + dlInfo.info.status = "error" + itemDownload = self.done.put(dlInfo) + NotifyEvent = Events.COMPLETED + await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg)) + else: + NotifyEvent = Events.ADDED + itemDownload = self.queue.put(dlInfo) + self.event.set() + + await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize()) + + return {"status": "ok"} + except Exception as e: + LOG.exception(e) + LOG.error(f"Failed to download item. '{e!s}'") + return {"status": "error", "msg": str(e)} + + async def _add_item(self, entry: dict, item: Item, already=None, logs: list | None = None): """ Add an entry to the download queue. @@ -207,176 +353,21 @@ class DownloadQueue(metaclass=Singleton): dict: The status of the operation. """ - if not logs: - logs = [] - - if item.has_extras(): - for key in item.extras.copy(): - if not self.keep_extra_key(key): - item.extras.pop(key) - else: - item.extras = {} - if not entry: return {"status": "error", "msg": "Invalid/empty data was given."} - options: dict = {} + event_type = entry.get("_type", "video") - error: str | None = None - live_in: str | None = None + if event_type.startswith("playlist"): + return await self._process_playlist(entry=entry, item=item, already=already) - eventType = entry.get("_type") or "video" - - if "playlist" == eventType: - LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.") - entries = entry.get("entries", []) - playlistCount = int(entry.get("playlist_count", len(entries))) - results = [] - - for i, etr in enumerate(entries, start=1): - etr["playlist"] = entry.get("id") - etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i) - etr["playlist_autonumber"] = i - - for property in ("id", "title", "uploader", "uploader_id"): - if property in entry: - etr[f"playlist_{property}"] = entry.get(property) - - if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""): - etr["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg" - - results.append( - await self.__add_entry( - entry=etr, - item=item.new_with(url=etr.get("url") or etr.get("webpage_url")), - already=already, - ) - ) - - if any("error" == res["status"] for res in results): - return { - "status": "error", - "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res), - } - - return {"status": "ok"} - - if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry: - # check if the video is live stream. - if "live_status" in entry and "is_upcoming" == entry.get("live_status"): - if entry.get("release_timestamp"): - live_in = formatdate(entry.get("release_timestamp"), usegmt=True) - item.extras.update({"live_in": live_in}) - else: - error = "Live stream not yet started. And no date is set." - else: - error = entry.get("msg") - - LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.") - - try: - _item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url")) - if _item is not None: - err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history." - LOG.warning(err_msg) - await self.clear([_item.info._id], remove_file=False) - except KeyError: - pass - - try: - _item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url"))) - if _item is not None: - err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue." - LOG.info(err_msg) - return {"status": "error", "msg": err_msg} - except KeyError: - pass - - is_manifestless = entry.get("is_manifestless", False) - options.update({"is_manifestless": is_manifestless}) - - live_status: list = ["is_live", "is_upcoming"] - is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) - - try: - download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder) - except Exception as e: - LOG.exception(e) - return {"status": "error", "msg": str(e)} - - for field in ("uploader", "channel", "thumbnail"): - if entry.get(field): - item.extras[field] = entry.get(field) - - for key in entry: - if isinstance(key, str) and key.startswith("playlist") and entry.get(key): - item.extras[key] = entry.get(key) - - dl = ItemDTO( - id=str(entry.get("id")), - title=str(entry.get("title")), - url=str(entry.get("webpage_url") or entry.get("url")), - preset=item.preset, - folder=item.folder, - download_dir=download_dir, - temp_dir=self.config.temp_path, - cookies=item.cookies, - template=item.template if item.template else self.config.output_template, - template_chapter=self.config.output_template_chapter, - datetime=formatdate(time.time()), - error=error, - is_live=is_live, - live_in=live_in if live_in else item.extras.get("live_in", None), - options=options, - cli=item.cli, - extras=item.extras, - ) - - try: - dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs) - text_logs = "" - if filtered_logs := extract_ytdlp_logs(logs): - text_logs = f" Logs: {', '.join(filtered_logs)}" - - if live_in or "is_upcoming" == entry.get("live_status"): - NotifyEvent = Events.COMPLETED - dlInfo.info.status = "not_live" - dlInfo.info.msg = "Stream is not live yet." + text_logs - itemDownload = self.done.put(dlInfo) - elif len(entry.get("formats", [])) < 1: - availability = entry.get("availability", "public") - msg = "No formats found." - if availability and availability not in ("public",): - msg += f" Availability is set for '{availability}'." - - dlInfo.info.error = msg + text_logs - dlInfo.info.status = "error" - itemDownload = self.done.put(dlInfo) - NotifyEvent = Events.COMPLETED - await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg)) - elif self.config.allow_manifestless is False and is_manifestless is True: - dlInfo.info.status = "error" - dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs - - itemDownload = self.done.put(dlInfo) - NotifyEvent = Events.COMPLETED - else: - NotifyEvent = Events.ADDED - itemDownload = self.queue.put(dlInfo) - self.event.set() - - await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize()) - - return {"status": "ok"} - except Exception as e: - LOG.exception(e) - LOG.error(f"Failed to download item. '{e!s}'") - return {"status": "error", "msg": str(e)} - - if eventType.startswith("url"): + if event_type.startswith("url"): return await self.add(item=item.new_with(url=entry.get("url")), already=already) - return {"status": "error", "msg": f'Unsupported resource "{eventType}"'} + if not event_type.startswith("video"): + return {"status": "error", "msg": f'Unsupported event type "{event_type}".'} + + return await self._add_video(entry=entry, item=item, logs=logs) async def add(self, item: Item, already: set | None = None): """ @@ -485,7 +476,7 @@ class DownloadQueue(metaclass=Singleton): if condition is not None: already.pop() LOG.info(f"Condition '{condition}' matched for '{item.url}'.") - return await self.add(item=item.new_with(requeued=True, cli=condition.cli),already=already) + return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) end_time = time.perf_counter() - started LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.") @@ -504,12 +495,12 @@ class DownloadQueue(metaclass=Singleton): finally: if cookie_file and os.path.exists(cookie_file): try: - os.remove(yt_conf["cookiefile"]) + os.remove(cookie_file) del yt_conf["cookiefile"] except Exception as e: LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}") - return await self.__add_entry(entry=entry, item=item, already=already, logs=logs) + return await self._add_item(entry=entry, item=item, already=already, logs=logs) async def cancel(self, ids: list[str]) -> dict[str, str]: """ @@ -767,20 +758,6 @@ class DownloadQueue(metaclass=Singleton): return is_downloaded(file, url) - def keep_extra_key(self, key: str) -> bool: - """ - Check if the extra key should be kept. - - Args: - key (str): The extra key to check. - - Returns: - bool: True if the extra key should be kept, False otherwise. - - """ - keys = ("playlist", "external_downloader", "live_in") - return any(key == k or key.startswith(k) for k in keys) - async def monitor_stale(self): """ Monitor the queue and pool for stale downloads and cancel them if needed. diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index d824354c..00337da0 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -268,6 +268,8 @@ class HttpAPI(Common): HttpAPI: The instance of the HttpAPI. """ + registered_options = [] + for attr_name in dir(self): method = getattr(self, attr_name) if hasattr(method, "_http_method") and hasattr(method, "_http_path"): @@ -277,6 +279,15 @@ class HttpAPI(Common): self.routes.route(method._http_method, f"/{http_path}")(method) + if http_path in registered_options: + continue + + async def options_handler(_: Request) -> Response: + return web.Response(status=204) + + self.routes.route("OPTIONS", f"/{http_path}")(options_handler) + registered_options.append(http_path) + self.routes.static("/api/download/", self.config.download_path) self._preload_static(app) @@ -391,7 +402,16 @@ class HttpAPI(Common): }, ) - response = await handler(request) + try: + response = await handler(request) + except web.HTTPException as e: + return web.json_response(data={"error": str(e)}, status=e.status_code) + except Exception as e: + LOG.exception(e) + response = web.json_response( + data={"error": "Internal Server Error"}, + status=web.HTTPInternalServerError.status_code, + ) if isinstance(response, web.FileResponse): try: @@ -405,20 +425,6 @@ class HttpAPI(Common): return middleware_handler - @route("OPTIONS", "/{path:.*}") - async def add_coors(self, _: Request) -> Response: - """ - Add CORS headers to the response. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code) - @route("GET", "api/ping") async def ping(self, _: Request) -> Response: """ diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index 7934aa8f..cd90223f 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -68,6 +68,7 @@ class Item: """ return self.cli and len(self.cli) > 2 + @staticmethod def _default_preset() -> str: from .config import Config diff --git a/app/library/Utils.py b/app/library/Utils.py index e901f3a3..cc5f5018 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -941,7 +941,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5): from anyio import open_file - if not pathlib(file).exists(): + if not pathlib.Path(file).exists(): return async with await open_file(file, "rb") as f: @@ -1020,20 +1020,23 @@ def get_archive_id(url: str) -> tuple[bool, dict[str | None, str | None, str | N ) for key, ie in YTDLP_INFO_CLS._ies.items(): - if not ie.suitable(url): - continue + try: + if not ie.suitable(url): + continue - if not ie.working(): + if not ie.working(): + break + + temp_id = ie.get_temp_id(url) + if not temp_id: + break + + idDict["id"] = temp_id + idDict["ie_key"] = key + idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict) break - - temp_id = ie.get_temp_id(url) - if not temp_id: - break - - idDict["id"] = temp_id - idDict["ie_key"] = key - idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict) - break + except Exception as e: + LOG.error(f"Error getting archive ID: {e}") return idDict diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py index 4c4c1894..da61ea70 100644 --- a/app/library/YTDLPOpts.py +++ b/app/library/YTDLPOpts.py @@ -131,7 +131,7 @@ class YTDLPOpts(metaclass=Singleton): with open(file, "w") as f: f.write(preset.cookies) - load_cookies(file) + load_cookies(str(file)) self._preset_opts["cookiefile"] = str(file) diff --git a/app/library/config.py b/app/library/config.py index da933847..805dc11a 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -43,9 +43,6 @@ class Config: ytdl_debug: bool = False """Enable yt-dlp debugging.""" - allow_manifestless: bool = False - """Allow downloading videos without manifest.""" - host: str = "0.0.0.0" """The host to bind the server to.""" @@ -194,7 +191,6 @@ class Config: "ytdl_debug", "debug", "temp_keep", - "allow_manifestless", "access_log", "remove_files", "ignore_ui", diff --git a/ui/components/History.vue b/ui/components/History.vue index 3e6bb0fe..a5954de4 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -639,7 +639,7 @@ const reQueueItem = (item, event = null) => { cookies: item.cookies, template: item.template, cli: item?.cli, - extras: item?.extras && Object.keys(item.extras) > 0 ? item.extras : {}, + extras: toRaw(item.extras ?? {}) ?? {}, }; socket.emit('item_delete', { id: item._id, remove_file: false }) diff --git a/ui/pages/tasks.vue b/ui/pages/tasks.vue index 2cf46b27..0f965f67 100644 --- a/ui/pages/tasks.vue +++ b/ui/pages/tasks.vue @@ -81,7 +81,7 @@ div.is-centered { {{ item.preset ?? config.app.default_preset }}

-

+

{{ item.cli }}