diff --git a/app/Download.py b/app/Download.py index 261b37d9..09e28a23 100644 --- a/app/Download.py +++ b/app/Download.py @@ -129,7 +129,7 @@ class Download: f'Invalid cookies: was provided for {self.info.title} - {str(e)}') log.info( - f'Downloading {self.info._id=} {self.info.title=}... {params=}') + f'Downloading id="{self.info.id}" title="{self.info.title}".') ret = yt_dlp.YoutubeDL(params=params).download([self.info.url]) self.status_queue.put( @@ -141,10 +141,10 @@ class Download: 'msg': str(exc), 'error': str(exc) }) - - if self.tempPath and self.info._id and os.path.exists(self.tempPath): - log.debug(f'Deleting Temp directory: {self.tempPath}') - shutil.rmtree(self.tempPath, ignore_errors=True) + finally: + if self.tempPath and self.info._id and os.path.exists(self.tempPath): + log.debug(f'Deleting Temp directory: {self.tempPath}') + shutil.rmtree(self.tempPath, ignore_errors=True) async def start(self, notifier: Notifier): if self.manager is None: @@ -153,11 +153,12 @@ class Download: self.status_queue = self.manager.Queue() self.loop = asyncio.get_running_loop() self.notifier = notifier + self.proc = multiprocessing.Process(target=self._download) self.proc.start() self.info.status = 'preparing' await self.notifier.updated(self.info) - asyncio.create_task(self.update_status()) + asyncio.create_task(self.progress_update()) return await self.loop.run_in_executor(None, self.proc.join) def cancel(self): @@ -179,10 +180,20 @@ class Download: def started(self): return self.proc is not None - async def update_status(self): + async def progress_update(self): + """ + Update status of download task and notify the client. + """ while True: status = await self.loop.run_in_executor(None, self.status_queue.get) - if status is None: + if not status: + return + + if self.debug: + log.debug(f'Status Update: {self.info._id=} {status=}') + + if isinstance(status, str): + await self.notifier.updated(self.info) return self.tmpfilename = status.get('tmpfilename') @@ -196,9 +207,13 @@ class Download: self.info.filename = re.sub( r'\.webm$', '.jpg', self.info.filename) - self.info.status = status['status'] + self.info.status = status.get('status', self.info.status) self.info.msg = status.get('msg') + if self.info.status == 'error' and 'error' in status: + self.info.error = status.get('error') + await self.notifier.error(self.info, self.info.error) + if 'downloaded_bytes' in status: total = status.get('total_bytes') or status.get( 'total_bytes_estimate') @@ -210,4 +225,7 @@ class Download: self.info.speed = status.get('speed') self.info.eta = status.get('eta') + if self.info.status == 'finished' and 'filename' in status and self.info.format != 'thumbnail': + self.info.file_size = os.path.getsize(status.get('filename')) + await self.notifier.updated(self.info) diff --git a/app/DownloadQueue.py b/app/DownloadQueue.py index 3f68d3f3..ff8d9a19 100644 --- a/app/DownloadQueue.py +++ b/app/DownloadQueue.py @@ -57,14 +57,11 @@ class DownloadQueue: if 'live_status' in entry and entry.get('live_status') == 'is_upcoming': if 'release_timestamp' in entry and entry.get('release_timestamp'): - dt_ts = datetime.fromtimestamp( - entry.get('release_timestamp'), tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S %z') - error = f"Live stream is scheduled to start at {dt_ts}" live_in = formatdate(entry.get('release_timestamp')) else: error = 'Live stream not yet started. And no date is set.' else: - error = entry['msg'] if 'msg' in entry else None + error = entry.get('msg', None) etype = entry.get('_type') or 'video' if etype == 'playlist': @@ -73,12 +70,12 @@ class DownloadQueue: playlist_index_digits = len(str(len(entries))) results = [] for index, etr in enumerate(entries, start=1): - etr['playlist'] = entry['id'] + etr['playlist'] = entry.get('id') etr['playlist_index'] = '{{0:0{0:d}d}}'.format( playlist_index_digits).format(index) for property in ('id', 'title', 'uploader', 'uploader_id'): if property in entry: - etr[f'playlist_{property}'] = entry[property] + etr[f'playlist_{property}'] = entry.get(property) results.append( await self.__add_entry( @@ -98,11 +95,10 @@ class DownloadQueue: return {'status': 'ok'} elif (etype == 'video' or etype.startswith('url')) and 'id' in entry and 'title' in entry: - log.debug( f"entry: {entry.get('id', None)} - {entry.get('webpage_url', None)} - {entry.get('url', None) }") - if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry['url']): + if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry.get('url')): item = self.done.get(key=entry['id'], url=entry.get( 'webpage_url') or entry['url']) @@ -111,15 +107,15 @@ class DownloadQueue: await self.clear([item.info._id]) - if self.queue.exists(key=entry['id'], url=entry.get('webpage_url') or entry['url']): + if self.queue.exists(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url')): log.info( f'Item [{item.info.title}] already in download queue') return {'status': 'error', 'msg': 'Link already queued for downloading.'} dl = ItemDTO( - id=entry['id'], - title=entry['title'], - url=entry.get('webpage_url') or entry['url'], + id=entry.get('id'), + title=entry.get('title'), + url=entry.get('webpage_url') or entry.get('url'), quality=quality, format=format, folder=folder, @@ -128,7 +124,7 @@ class DownloadQueue: output_template=output_template if output_template else self.config.output_template, datetime=formatdate(time.time()), error=error, - is_live=entry['is_live'] if 'is_live' in entry else None, + is_live=entry.get('is_live', None), live_in=live_in, ) @@ -164,7 +160,7 @@ class DownloadQueue: } elif etype.startswith('url'): return await self.add( - entry=entry['url'], + entry=entry.get('url'), quality=quality, format=format, folder=folder, @@ -193,7 +189,7 @@ class DownloadQueue: ytdlp_config = ytdlp_config if ytdlp_config else {} log.info( - f'adding {url}: {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}') + f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}') already = set() if already is None else already if url in already: @@ -232,7 +228,7 @@ class DownloadQueue: async def cancel(self, ids): for id in ids: if not self.queue.exists(id): - log.warn(f'requested cancel for non-existent download {id}') + log.warn(f'Requested cancel for non-existent download {id=}') continue item = self.queue.get(key=id) @@ -242,7 +238,7 @@ class DownloadQueue: self.queue.get(id).cancel() else: self.queue.delete(id) - log.info(f'deleting {id=} {item.info.title=}') + log.info(f'Deleting {id=} {item.info.title=}') await self.notifier.canceled(id) return {'status': 'ok'} @@ -250,10 +246,10 @@ class DownloadQueue: async def clear(self, ids): for id in ids: if not self.done.exists(key=id): - log.warn(f'requested delete for non-existent download {id}') + log.warn(f'Requested delete for non-existent download {id}') continue item = self.done.get(key=id) - log.info(f'deleting {id=} {item.info.title=}') + log.info(f'Deleting {id=} {item.info.title=}') self.done.delete(id) await self.notifier.cleared(id) @@ -287,7 +283,7 @@ class DownloadQueue: id, entry = self.queue.next() log.info( - f'downloading {id=} - {entry.info.title=} - {entry.info.url=} - {entry.info.folder=}') + f'Queuing {id=} - {entry.info.title=} - {entry.info.url=} - {entry.info.folder=}.') await entry.start(self.notifier) diff --git a/app/ItemDTO.py b/app/ItemDTO.py index 423a5326..2cc5013f 100644 --- a/app/ItemDTO.py +++ b/app/ItemDTO.py @@ -24,6 +24,7 @@ class ItemDTO: is_live: bool = None datetime: str = field(default_factory=lambda: str(formatdate(time.time()))) live_in: str = None + file_size: int = None # yt-dlp injected fields. tmpfilename: str = None diff --git a/app/Utils.py b/app/Utils.py index 7584079b..fd6bd1a8 100644 --- a/app/Utils.py +++ b/app/Utils.py @@ -286,20 +286,23 @@ class Notifier: self.sio = sio self.serializer = serializer - async def added(self, dl): + async def added(self, dl: dict): await self.emit('added', dl) - async def updated(self, dl): + async def updated(self, dl: dict): await self.emit('updated', dl) - async def completed(self, dl): + async def completed(self, dl: dict): await self.emit('completed', dl) - async def canceled(self, id): + async def canceled(self, id: str): await self.emit('canceled', id) - async def cleared(self, id): + async def cleared(self, id: str): await self.emit('cleared', id) + async def error(self, dl: dict, message: str): + await self.emit('error', (dl, message)) + async def emit(self, event: str, data): await self.sio.emit(event, self.serializer.encode(data)) diff --git a/app/main.py b/app/main.py index 62f8941c..680aa4f0 100644 --- a/app/main.py +++ b/app/main.py @@ -107,7 +107,6 @@ class Main: ) async def connect(self, sid, _): - self.logger.info(f'Config [{self.config.__dict__}].') data: dict = { **self.dqueue.get(), "config": self.config, diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 40e51642..81d309d7 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -73,6 +73,11 @@ onMounted(() => { toast.success(`Item queued successfully: ${downloading[item._id]?.title}`); }); + socket.on('error', stream => { + const [item, error] = JSON.parse(stream); + toast.error(`${item?.id}: Error: ${error}`); + }); + socket.on('completed', stream => { const item = JSON.parse(stream); if (item._id in downloading) { diff --git a/frontend/src/components/Page-Completed.vue b/frontend/src/components/Page-Completed.vue index 55909b70..75aba9ec 100644 --- a/frontend/src/components/Page-Completed.vue +++ b/frontend/src/components/Page-Completed.vue @@ -79,34 +79,52 @@
+
+ + LIVE stream is scheduled to start at {{ moment(item.datetime).format() }} + +
{{ item.error }}
{{ item.msg }}
-
- - - +
+ + + + + + {{ capitalize(item.status) }} + + + + + + + + Live Stream - {{ capitalize(item.status) }}
-
+
{{ moment(item.datetime).fromNow() }}
-
+
{{ moment(item.live_in).fromNow() }}
-
+
+ {{ formatBytes(item.file_size) }} +
+