From 3f2b9feb7da1fc1c2337f89467c803ebeb399a3f Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sat, 18 Jan 2025 18:27:30 +0300 Subject: [PATCH 1/3] Cancel the status update if exception happens during await update_task --- app/library/Download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/library/Download.py b/app/library/Download.py index 8786e95d..a8d63b32 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -321,7 +321,8 @@ class Download: status = await self.update_task except Exception as e: LOG.error(f"Failed to get status update for: {self.info._id=}. {e}") - pass + LOG.exception(e) + return if status is None or status.__class__ is Terminator: LOG.debug(f"Closing progress update for: {self.info._id=}.") From 35421540d7fe0f003331deae7c7d17e8754715a9 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sat, 18 Jan 2025 19:09:48 +0300 Subject: [PATCH 2/3] another attempt to fix multi-processing #173 --- app/library/Download.py | 4 +++- app/library/DownloadQueue.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/library/Download.py b/app/library/Download.py index a8d63b32..d94691a2 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -28,7 +28,6 @@ class Download: """ id: str = None - manager = None download_dir: str = None temp_dir: str = None output_template: str = None @@ -319,6 +318,9 @@ class Download: return try: status = await self.update_task + except FileNotFoundError: + LOG.debug(f"Closing progress update for: {self.info._id=} {status=}.") + return except Exception as e: LOG.error(f"Failed to get status update for: {self.info._id=}. {e}") LOG.exception(e) diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 323aec9e..b6882a97 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -468,6 +468,7 @@ class DownloadQueue: await entry.close() if self.queue.exists(key=id): + LOG.debug(f"Download '{id}' is done. Removing from queue.") self.queue.delete(key=id) if entry.is_canceled() is True: @@ -477,6 +478,8 @@ class DownloadQueue: self.done.put(value=entry) asyncio.create_task(self.emitter.completed(dl=entry.info.serialize()), name=f"notifier-d-{id}") + else: + LOG.warning(f"Download '{id}' not found in queue.") if self.event: self.event.set() From 0d9f0ded88ac7a8d6016d6de590dacb25af6c077 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sat, 18 Jan 2025 20:57:23 +0300 Subject: [PATCH 3/3] Another attempt at fixing #173 --- app/library/Download.py | 37 ++++++++++++++++++++++-------------- app/library/DownloadQueue.py | 4 ++-- app/library/HttpSocket.py | 6 +++--- app/library/Utils.py | 1 - ui/stores/SocketStore.js | 6 ++++++ 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/app/library/Download.py b/app/library/Download.py index d94691a2..9d77a58a 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -94,6 +94,10 @@ class Download: def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} + + if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", False): + dataDict["filename"] = data["info_dict"]["filename"] + self.status_queue.put({"id": self.id, **dataDict}) def _postprocessor_hook(self, data: dict): @@ -242,6 +246,9 @@ class Download: if self.proc.is_alive(): tasks.append(loop.run_in_executor(None, self.proc.join)) + if self.status_queue: + self.status_queue.put(Terminator()) + if self.manager: tasks.append(loop.run_in_executor(None, self.manager.shutdown)) @@ -313,17 +320,9 @@ class Download: while True: try: self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get) - except asyncio.CancelledError: - LOG.debug(f"Closing progress update for: {self.info._id=}.") - return - try: status = await self.update_task - except FileNotFoundError: - LOG.debug(f"Closing progress update for: {self.info._id=} {status=}.") - return - except Exception as e: - LOG.error(f"Failed to get status update for: {self.info._id=}. {e}") - LOG.exception(e) + except (asyncio.CancelledError, OSError, FileNotFoundError): + LOG.debug(f"Closing progress update for: {self.info._id=}.") return if status is None or status.__class__ is Terminator: @@ -370,16 +369,26 @@ class Download: self.info.speed = status.get("speed") self.info.eta = status.get("eta") - if self.info.status == "finished" and "filename" in status and os.path.exists(status.get("filename")): + if ( + "finished" == self.info.status + and "filename" in status + and os.path.isfile(status.get("filename")) + and os.path.exists(status.get("filename")) + ): try: self.info.file_size = os.path.getsize(status.get("filename")) + self.info.datetime = str(formatdate(time.time())) + except FileNotFoundError: + pass + + try: ff = await ffprobe(status.get("filename")) self.info.extras["is_video"] = ff.has_video() self.info.extras["is_audio"] = ff.has_audio() - self.info.datetime = str(formatdate(time.time())) - except (FileNotFoundError, Exception) as e: + except Exception as e: self.info.extras["is_video"] = True self.info.extras["is_audio"] = True - LOG.exception(f"Failed to ffprobe: {status.get('filename')}. {e}") + LOG.exception(f"Failed to ffprobe: {status.get}. {e}") + LOG.exception(e) asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index b6882a97..27db6d8d 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -429,7 +429,7 @@ class DownloadQueue: self.event.clear() LOG.debug("Cleared wait event.") - if self.paused and isinstance(self.paused, asyncio.Event): + if self.paused and isinstance(self.paused, asyncio.Event) and self.isPaused(): LOG.info("Download pool is paused.") await self.paused.wait() LOG.info("Download pool resumed downloading.") @@ -449,7 +449,7 @@ class DownloadQueue: async def __downloadFile(self, id: str, entry: Download): LOG.info( - f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'folder: {entry.info.folder}'." + f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {entry.info.folder}'." ) try: diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index 2b3ea60c..e401d6f5 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -161,9 +161,9 @@ class HttpSocket(common): await self.emitter.warning("No URL provided.", to=sid) return - preset: str = data.get("preset", "default") - folder: str = data.get("folder") - ytdlp_cookies: str = data.get("ytdlp_cookies") + preset: str = str(data.get("preset", "default")) + folder: str = str(data.get("folder")) + ytdlp_cookies: str = str(data.get("ytdlp_cookies")) ytdlp_config: dict | None = data.get("ytdlp_config") output_template: str = data.get("output_template") if ytdlp_config is None: diff --git a/app/library/Utils.py b/app/library/Utils.py index 84ccbde8..bf33cbcf 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -17,7 +17,6 @@ from yt_dlp.networking.impersonate import ImpersonateTarget LOG = logging.getLogger("Utils") IGNORED_KEYS: tuple[str] = ( - "cookiefile", "paths", "outtmpl", "progress_hooks", diff --git a/ui/stores/SocketStore.js b/ui/stores/SocketStore.js index dcba21b0..c48cb9a2 100644 --- a/ui/stores/SocketStore.js +++ b/ui/stores/SocketStore.js @@ -78,6 +78,12 @@ export const useSocketStore = defineStore('socket', () => { socket.value.on("updated", stream => { const data = JSON.parse(stream); + + if (true === stateStore.has('history', item._id)) { + stateStore.update('history', item._id, item); + return; + } + let dl = stateStore.get('queue', data._id, {}); data.deleting = dl?.deleting; stateStore.update('queue', data._id, data);