diff --git a/app/library/Download.py b/app/library/Download.py index 8be556c2..68e33bf9 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -15,6 +15,7 @@ from .config import Config from .Events import EventBus, Events from .ffprobe import ffprobe from .ItemDTO import ItemDTO +from .Utils import extract_info from .YTDLPOpts import YTDLPOpts LOG = logging.getLogger("Download") @@ -96,6 +97,7 @@ class Download: self.id = info._id self.default_ytdl_opts = config.ytdl_options self.debug = bool(config.debug) + self.archive = bool(config.keep_archive) self.debug_ytdl = bool(config.ytdl_debug) self.cancelled = False self.tmpfilename = None @@ -159,6 +161,19 @@ class Download: .get_all() ) + if not self.info_dict: + self.logger.info(f"Extracting info for '{self.info.url}'.") + info = extract_info( + config=params, + url=self.info.url, + debug=self.debug, + no_archive=not self.archive, + follow_redirect=True, + ) + + if info: + self.info_dict = info + params.update( { "progress_hooks": [self._progress_hook], @@ -208,11 +223,15 @@ class Download: if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: self.logger.debug(f"Downloading '{self.info.url}' using pre-info.") - cls.process_ie_result(self.info_dict, download=True) + cls.process_ie_result( + ie_result=self.info_dict, + download=True, + extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")}, + ) ret = cls._download_retcode else: self.logger.debug(f"Downloading using url: {self.info.url}") - ret = cls.download([self.info.url]) + ret = cls.download(url_list=[self.info.url]) self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"}) except Exception as exc: diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index d41271ef..7aac9fba 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -176,6 +176,7 @@ class DownloadQueue(metaclass=Singleton): config: dict | None = None, cookies: str = "", template: str = "", + extras: dict | None = None, already=None, ): """ @@ -188,8 +189,10 @@ class DownloadQueue(metaclass=Singleton): config (dict): The yt-dlp configuration to use for the download. cookies (str): The cookies to use for the download. template (str): The output template to use for the download. + extras (dict): The extra information to add to the download. already (set): The set of already downloaded items. + Returns: dict: The status of the operation. @@ -197,6 +200,13 @@ class DownloadQueue(metaclass=Singleton): if not config: config = {} + if not extras: + extras = {} + else: + for key in extras.copy(): + if not str(key).startswith("playlist"): + extras.pop(key, None) + if not entry: return {"status": "error", "msg": "Invalid/empty data was given."} @@ -208,12 +218,16 @@ class DownloadQueue(metaclass=Singleton): eventType = entry.get("_type") or "video" if "playlist" == eventType: + LOG.info("Processing playlist") entries = entry.get("entries", []) - playlist_index_digits = len(str(len(entries))) + playlistCount = int(entry.get("playlist_count", len(entries))) results = [] + for i, etr in enumerate(entries, start=1): etr["playlist"] = entry.get("id") - etr["playlist_index"] = f"{{0:0{playlist_index_digits:d}d}}".format(i) + etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i) + etr["playlist_autonumber"] = i + for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry.get(property) @@ -229,6 +243,7 @@ class DownloadQueue(metaclass=Singleton): config=config, cookies=cookies, template=template, + extras=extras, already=already, ) ) @@ -279,12 +294,15 @@ class DownloadQueue(metaclass=Singleton): LOG.exception(e) return {"status": "error", "msg": str(e)} - extras: dict = {} fields: tuple = ("uploader", "channel", "thumbnail") for field in fields: if entry.get(field): extras[field] = entry.get(field) + for key in entry: + if isinstance(key, str) and key.startswith("playlist") and entry.get(key): + extras[key] = entry.get(key) + dl = ItemDTO( id=str(entry.get("id")), title=str(entry.get("title")), @@ -305,10 +323,6 @@ class DownloadQueue(metaclass=Singleton): extras=extras, ) - for property, value in entry.items(): - if property.startswith("playlist"): - dl.template = str(dl.template).replace(f"%({property})s", str(value)) - dlInfo: Download = Download(info=dl, info_dict=entry) if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"): @@ -337,6 +351,7 @@ class DownloadQueue(metaclass=Singleton): config=config, cookies=cookies, template=template, + extras=extras, already=already, ) @@ -350,12 +365,15 @@ class DownloadQueue(metaclass=Singleton): config: dict | None = None, cookies: str = "", template: str = "", + extras: dict | None = None, already=None, ): _preset = Presets.get_instance().get(name=preset) config = config if config else {} folder = str(folder) if folder else "" + if not extras: + extras = {} if _preset: if _preset.folder and not folder: @@ -372,7 +390,7 @@ class DownloadQueue(metaclass=Singleton): cookie_file = os.path.join(self.config.temp_path, f"c_{uuid.uuid4().hex}.txt") LOG.info( - f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {len(cookies)}/chars' 'YTConfig: {config}'." + f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {len(cookies)}/chars' 'YTConfig: {config}' 'Extras: {extras}'." ) if isinstance(config, str): @@ -467,6 +485,7 @@ class DownloadQueue(metaclass=Singleton): cookies=cookies, template=template, already=already, + extras=extras, ) async def cancel(self, ids: list[str]) -> dict[str, str]: diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 3a624dd9..9884fb07 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -1408,6 +1408,7 @@ class HttpAPI(Common): "https://imageipsum.com/1920x1080", "https://placedog.net/1920/1080", ] + backend = random.choice(backends) # noqa: S311 try: CACHE_KEY = "random_background" @@ -1431,7 +1432,6 @@ class HttpAPI(Common): }, } - backend = random.choice(backends) # noqa: S311 logging.getLogger("httpx").setLevel(logging.WARNING) async with httpx.AsyncClient(**opts) as client: response = await client.request(method="GET", url=backend, follow_redirects=True) @@ -1464,7 +1464,7 @@ class HttpAPI(Common): ) except Exception as e: LOG.exception(e) - LOG.error(f"Failed to request random background image.'. '{e!s}'.") + LOG.error(f"Failed to request random background image from '{backend!s}'.'. '{e!s}'.") return web.json_response( data={"error": "failed to retrieve the random background image."}, status=web.HTTPInternalServerError.status_code, @@ -1580,8 +1580,8 @@ class HttpAPI(Common): cookies = http.cookiejar.MozillaCookieJar(cookie_file, None, None) cookies.load() except Exception as e: - LOG.error(f"failed to load cookies from '{cookie_file}'. '{e}'.") LOG.exception(e) + LOG.error(f"failed to load cookies from '{cookie_file}'. '{e!s}'.") return web.json_response( data={"message": "Failed to load cookies"}, status=web.HTTPInternalServerError.status_code, diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index e266d308..12d379cc 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -79,7 +79,7 @@ class HttpSocket(Common): self._notify.subscribe( Events.ADD_URL, - lambda data, _, **kwargs: self.add(**data.data), # noqa: ARG005 + lambda data, _, **kwargs: self.add(**self.format_item(data.data)), # noqa: ARG005 f"{__class__.__name__}.socket_add_url", ) diff --git a/app/library/Utils.py b/app/library/Utils.py index bd0dcd5b..3f843f2e 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -138,6 +138,9 @@ def extract_info( sanitize_info=sanitize_info, ) + if not data: + return data + return yt_dlp.YoutubeDL.sanitize_info(data) if sanitize_info else data diff --git a/app/library/common.py b/app/library/common.py index 440aa263..371ca22a 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -27,7 +27,7 @@ class Common: self.default_preset = config.default_preset async def add( - self, url: str, preset: str, folder: str, cookies: str, config: dict, template: str + self, url: str, preset: str, folder: str, cookies: str, config: dict, template: str, extras: dict | None = None ) -> dict[str, str]: """ Add an item to the download queue. @@ -39,6 +39,7 @@ class Common: cookies (str): The cookies to be used for the download. config (dict): The yt-dlp config to be used for the download. template (str): The template to be used for the download. + extras (dict): Extra data to be added to the download Returns: dict[str, str]: The status of the download. @@ -55,6 +56,7 @@ class Common: cookies=cookies, config=config if isinstance(config, dict) else {}, template=template, + extras=extras, ) def format_item(self, item: dict) -> dict: @@ -82,6 +84,7 @@ class Common: folder: str = str(item.get("folder")) if item.get("folder") else "" cookies: str = str(item.get("cookies")) if item.get("cookies") else "" template: str = str(item.get("template")) if item.get("template") else "" + extras = item.get("extras", {}) config = item.get("config") if isinstance(config, str) and config: @@ -98,4 +101,5 @@ class Common: "cookies": cookies, "config": config if isinstance(config, dict) else {}, "template": template, + "extras": extras if isinstance(extras, dict) else {}, } diff --git a/ui/components/History.vue b/ui/components/History.vue index 000eaf27..d01a19fa 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -485,6 +485,17 @@ const removeItem = item => { const reQueueItem = item => { socket.emit('item_delete', { id: item._id, remove_file: false }) + + let extras = {} + + if (item.extras) { + Object.keys(item.extras).forEach(k => { + if (k && true === k.startsWith('playlist')) { + extras[k] = item.extras[k] + } + }) + } + socket.emit('add_url', { url: item.url, preset: item.preset, @@ -492,6 +503,7 @@ const reQueueItem = item => { config: item.config, cookies: item.cookies, template: item.template, + extras: extras }) } @@ -501,6 +513,6 @@ watch(video_item, v => { return } - document.querySelector('body').setAttribute("style", `opacity: ${ v ? 1 : bg_opacity.value}`) + document.querySelector('body').setAttribute("style", `opacity: ${v ? 1 : bg_opacity.value}`) })