From d063cf0f52e2614320b88ac19efb53571fe0c176 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 28 Jan 2025 00:09:46 +0300 Subject: [PATCH] use same name cross client and server for variables and config files. --- app/library/Download.py | 16 +++--- app/library/DownloadQueue.py | 58 +++++++++++----------- app/library/EventsSubscriber.py | 12 +++-- app/library/HttpAPI.py | 3 +- app/library/HttpSocket.py | 3 +- app/library/ItemDTO.py | 36 +++++++++++--- app/library/Tasks.py | 86 ++++++++++++++++++++------------- app/library/common.py | 55 ++++++++++++++------- app/main.py | 2 +- ui/components/History.vue | 6 +-- ui/components/TaskForm.vue | 4 +- ui/pages/tasks.vue | 2 +- 12 files changed, 175 insertions(+), 108 deletions(-) diff --git a/app/library/Download.py b/app/library/Download.py index f8c6e316..b60bbf77 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -28,8 +28,8 @@ class Download: id: str = None download_dir: str = None temp_dir: str = None - output_template: str = None - output_template_chapter: str = None + template: str = None + template_chapter: str = None ytdl_opts: dict = None info: ItemDTO = None default_ytdl_opts: dict = None @@ -72,10 +72,10 @@ class Download: self.download_dir = info.download_dir self.temp_dir = info.temp_dir - self.output_template_chapter = info.output_template_chapter - self.output_template = info.output_template + self.template = info.template + self.template_chapter = info.template_chapter self.preset = info.preset - self.ytdl_opts = info.ytdlp_config if info.ytdlp_config else {} + self.ytdl_opts = info.config if info.config else {} self.info = info self.id = info._id self.default_ytdl_opts = config.ytdl_options @@ -120,7 +120,7 @@ class Download: { "color": "no_color", "paths": {"home": self.download_dir, "temp": self.tempPath}, - "outtmpl": {"default": self.output_template, "chapter": self.output_template_chapter}, + "outtmpl": {"default": self.template, "chapter": self.template_chapter}, "noprogress": True, "break_on_existing": True, "progress_hooks": [self._progress_hook], @@ -136,9 +136,9 @@ class Download: params["verbose"] = True params["noprogress"] = False - if self.info.ytdlp_cookies: + if self.info.cookies: try: - data = jsonCookie(json.loads(self.info.ytdlp_cookies)) + data = jsonCookie(json.loads(self.info.cookies)) if not data: LOG.warning( f"The cookie string that was provided for {self.info.title} is empty or not in expected spec." diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 6171290a..fd0cd2e0 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -132,9 +132,9 @@ class DownloadQueue(metaclass=Singleton): entry: dict, preset: str, folder: str, - ytdlp_config: dict = {}, - ytdlp_cookies: str = "", - output_template: str = "", + config: dict = {}, + cookies: str = "", + template: str = "", already=None, ): """ @@ -144,9 +144,9 @@ class DownloadQueue(metaclass=Singleton): entry (dict): The entry to add to the download queue. preset (str): The preset to use for the download. folder (str): The folder to save the download to. - ytdlp_config (dict): The yt-dlp configuration to use for the download. - ytdlp_cookies (str): The cookies to use for the download. - output_template (str): The output template to use for the download. + config (dict): The yt-dlp configuration to use for the download. + cookies (str): The cookies to use for the download. + template (str): The output template to use for the download. already (set): The set of already downloaded items. Returns: @@ -180,9 +180,9 @@ class DownloadQueue(metaclass=Singleton): entry=etr, preset=preset, folder=folder, - ytdlp_config=ytdlp_config, - ytdlp_cookies=ytdlp_cookies, - output_template=output_template, + config=config, + cookies=cookies, + template=template, already=already, ) ) @@ -249,10 +249,10 @@ class DownloadQueue(metaclass=Singleton): folder=folder, download_dir=download_dir, temp_dir=self.config.temp_path, - ytdlp_cookies=ytdlp_cookies, - ytdlp_config=ytdlp_config, - output_template=output_template if output_template else self.config.output_template, - output_template_chapter=self.config.output_template_chapter, + cookies=cookies, + config=config, + template=template if template else self.config.output_template, + template_chapter=self.config.output_template_chapter, datetime=formatdate(time.time()), error=error, is_live=is_live, @@ -263,7 +263,7 @@ class DownloadQueue(metaclass=Singleton): for property, value in entry.items(): if property.startswith("playlist"): - dl.output_template = str(dl.output_template).replace(f"%({property})s", str(value)) + dl.template = str(dl.template).replace(f"%({property})s", str(value)) dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug)) @@ -291,9 +291,9 @@ class DownloadQueue(metaclass=Singleton): url=str(entry.get("url")), preset=preset, folder=folder, - ytdlp_config=ytdlp_config, - ytdlp_cookies=ytdlp_cookies, - output_template=output_template, + config=config, + cookies=cookies, + template=template, already=already, ) @@ -304,25 +304,25 @@ class DownloadQueue(metaclass=Singleton): url: str, preset: str, folder: str, - ytdlp_config: dict = {}, - ytdlp_cookies: str = "", - output_template: str = "", + config: dict = {}, + cookies: str = "", + template: str = "", already=None, ): - ytdlp_config = ytdlp_config if ytdlp_config else {} + config = config if config else {} folder = str(folder) if folder else "" filePath = calcDownloadPath(basePath=self.config.download_path, folder=folder) LOG.info( - f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'." + f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {cookies}' 'YTConfig: {config}'." ) - if isinstance(ytdlp_config, str): + if isinstance(config, str): try: - ytdlp_config = json.loads(ytdlp_config) + config = json.loads(config) except Exception as e: - LOG.error(f"Unable to load '{ytdlp_config=}'. {str(e)}") + LOG.error(f"Unable to load '{config=}'. {str(e)}") return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {str(e)}"} already = set() if already is None else already @@ -347,7 +347,7 @@ class DownloadQueue(metaclass=Singleton): fut=asyncio.get_running_loop().run_in_executor( None, ExtractInfo, - get_opts(preset, mergeConfig(self.config.ytdl_options, ytdlp_config)), + get_opts(preset, mergeConfig(self.config.ytdl_options, config)), url, bool(self.config.ytdl_debug), ), @@ -377,9 +377,9 @@ class DownloadQueue(metaclass=Singleton): entry=entry, preset=preset, folder=folder, - ytdlp_config=ytdlp_config, - ytdlp_cookies=ytdlp_cookies, - output_template=output_template, + config=config, + cookies=cookies, + template=template, already=already, ) diff --git a/app/library/EventsSubscriber.py b/app/library/EventsSubscriber.py index 7d5bb1a5..19c9ae9e 100644 --- a/app/library/EventsSubscriber.py +++ b/app/library/EventsSubscriber.py @@ -40,6 +40,9 @@ class Events: TEST = "test" TASKS_ADD = "task_add" + TASK_DISPATCHED = "task_dispatched" + TASK_FINISHED = "task_finished" + TASK_ERROR = "task_error" @dataclass(kw_only=True) @@ -169,10 +172,13 @@ class EventsSubscriber(metaclass=Singleton): tasks = [] for id, callback in self._listeners[event].items(): try: - if "data" not in kwargs or not isinstance(kwargs["data"], Event): - data = Event(id=id, data={"args": args if args else [], **kwargs}) - else: + if args and isinstance(args[0], Event): + data = args[0] + elif "data" in kwargs and isinstance(kwargs["data"], Event): data = kwargs["data"] + else: + data = Event(id=id, data={"args": args if args else [], **kwargs}) + tasks.append(asyncio.create_task(callback(event, data))) except Exception as e: LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.") diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 2f6dfc75..f13351fb 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -63,7 +63,7 @@ class HttpAPI(common): self.routes = web.RouteTableDef() self.cache = Cache() - super().__init__(queue=self.queue, encoder=self.encoder) + super().__init__(queue=self.queue, encoder=self.encoder, config=self.config) def route(method: str, path: str) -> Awaitable: """ @@ -980,7 +980,6 @@ class HttpAPI(common): ) except Exception as e: LOG.error(f"Error fetching thumbnail from '{url}'. '{e}'.") - LOG.exception(e) return web.json_response( data={"error": "failed to retrieve the thumbnail."}, status=web.HTTPInternalServerError.status_code ) diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index c7c74123..f2907240 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -52,7 +52,7 @@ class HttpSocket(common): self.emitter.add_emitter([emit], local=False) - super().__init__(queue=queue, encoder=encoder) + super().__init__(queue=queue, encoder=encoder, config=config) def ws_event(func): # type: ignore """ @@ -77,6 +77,7 @@ class HttpSocket(common): # self.sio.on("*", es.emit) async def handle_event(_: str, data: Event): + LOG.debug(f"Event received. '{data}'") await self.add(**data.data) EventsSubscriber.get_instance().subscribe(Events.ADD_URL, "socket_add_url", handle_event) diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index aa6e513e..29045fe2 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from email.utils import formatdate from typing import Any + @dataclass(kw_only=True) class ItemDTO: """ @@ -14,22 +15,21 @@ class ItemDTO: _id: str = field(default_factory=lambda: str(uuid.uuid4()), init=False) - error: str|None = None + error: str | None = None id: str title: str url: str quality: str | None = None format: str | None = None - thumbnail: str | None = None preset: str = "default" folder: str download_dir: str | None = None temp_dir: str | None = None status: str | None = None - ytdlp_cookies: str | None = None - ytdlp_config: dict = field(default_factory=dict) - output_template: str | None = None - output_template_chapter: str | None = None + cookies: str | None = None + config: dict = field(default_factory=dict) + template: str | None = None + template_chapter: str | None = None timestamp: float = time.time_ns() is_live: bool | None = None datetime: str = field(default_factory=lambda: str(formatdate(time.time()))) @@ -49,19 +49,41 @@ class ItemDTO: speed: str | None = None eta: str | None = None + # DEPRECATED: These fields are deprecated and will be removed in the future. + thumbnail: str | None = None + ytdlp_cookies: str | None = None + ytdlp_config: dict = field(default_factory=dict) + output_template: str | None = None + output_template_chapter: str | None = None + def serialize(self) -> dict: deprecated: tuple = ( "thumbnail", "quality", "format", + "ytdlp_cookies", + "ytdlp_config", + "output_template", + "output_template_chapter", ) if self.thumbnail and "thumbnail" not in self.extras: self.extras["thumbnail"] = self.thumbnail + mapper: dict = { + "cookies": "ytdlp_cookies", + "config": "ytdlp_config", + "template": "output_template", + "template_chapter": "output_template_chapter", + } + + for k, v in mapper.items(): + if not self.get(k) and self.get(v): + self.__dict__[k] = self.get(v) + dump = self.__dict__.copy() for f in deprecated: - dump.pop(f) + dump.pop(f, None) return dump diff --git a/app/library/Tasks.py b/app/library/Tasks.py index a102060f..37bfe4c2 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -1,14 +1,15 @@ import asyncio -import datetime +from datetime import datetime import json import logging import os import time +from dataclasses import dataclass, field from typing import Any, List import httpx from aiocron import Cron, crontab -from dataclasses import dataclass, field +from aiohttp import web from .config import Config from .Emitter import Emitter @@ -81,16 +82,12 @@ class Tasks(metaclass=Singleton): self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_event_loop() self._emitter: Emitter = emitter or Emitter.get_instance() - if os.path.exists(self._file): + if os.path.exists(self._file) and "600" != oct(os.stat(self._file).st_mode)[-3:]: try: - if "600" != oct(os.stat(self._file).st_mode)[-3:]: - os.chmod(self._file, 0o600) + os.chmod(self._file, 0o600) except Exception: pass - if os.path.getsize(self._file) > 10: - self.load() - def handle_event(_, e: Event): self.save(**e.data) @@ -109,6 +106,24 @@ class Tasks(metaclass=Singleton): Tasks._instance = Tasks() return Tasks._instance + def attach(self, _: web.Application): + """ + Attach the tasks to the aiohttp application. + + Args: + _ (web.Application): The aiohttp application. + + Returns: + None + """ + self.load() + + def shutdown(self): + """ + Shutdown the tasks service. + """ + self.clear() + def getTasks(self) -> List[Task]: """Return the tasks.""" return [job.task for job in self._jobs] @@ -266,7 +281,7 @@ class Tasks(metaclass=Singleton): None """ try: - timeNow = datetime.datetime().isoformat() + timeNow = datetime.now().isoformat() started = time.time() if not task.url: LOG.error(f"Failed to dispatch task '{task.id}: {task.name}'. No URL found.") @@ -274,43 +289,48 @@ class Tasks(metaclass=Singleton): preset: str = str(task.preset or self._default_preset) folder: str = task.folder if task.folder else "" - ytdlp_cookies: str = str(task.cookies) if task.cookies else "" - output_template: str = task.template if task.template else "" + cookies: str = str(task.cookies) if task.cookies else "" + template: str = task.template if task.template else "" - ytdlp_config = task.config if task.config else {} - if isinstance(ytdlp_config, str) and ytdlp_config: + config = task.config if task.config else {} + if isinstance(config, str) and config: try: - ytdlp_config = json.loads(ytdlp_config) + config = json.loads(config) except Exception as e: LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {str(e)}") return - await self.emitter.info(f"Task '{task.name}' dispatched at '{timeNow}'.") LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.") - await self._emitter.emit( - event=Events.ADD_URL, - data=Event( - id=task.id, - data={ - "url": task.url, - "preset": preset, - "folder": folder, - "ytdlp_cookies": ytdlp_cookies, - "ytdlp_config": ytdlp_config, - "output_template": output_template, - }, - ), - local=True, + tasks = [] + tasks.append(self._emitter.info(f"Task '{task.name}' dispatched at '{timeNow}'.")) + tasks.append( + self._emitter.emit( + event=Events.ADD_URL, + data=Event( + id=task.id, + data={ + "url": task.url, + "preset": preset, + "folder": folder, + "cookies": cookies, + "config": config, + "template": template, + }, + ), + local=True, + ) ) - timeNow = datetime.datetime().isoformat() + await asyncio.wait_for(asyncio.gather(*tasks), timeout=None) + + timeNow = datetime.now().isoformat() ended = time.time() LOG.info(f"Task '{task.id}: {task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") - await self.emitter.success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds.") + await self._emitter.success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds.") except Exception as e: - timeNow = datetime.datetime().isoformat() + timeNow = datetime.now().isoformat() LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{str(e)}'.") - await self.emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{str(e)}'.") + await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{str(e)}'.") diff --git a/app/library/common.py b/app/library/common.py index 47a9d32b..d02955d3 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -3,6 +3,7 @@ import logging from .DownloadQueue import DownloadQueue from .encoder import Encoder +from .config import Config LOG = logging.getLogger("common") @@ -12,31 +13,49 @@ class common: This class is used to share common methods between the socket and the API gateways. """ - queue: DownloadQueue - encoder: Encoder - - def __init__(self, queue: DownloadQueue, encoder: Encoder): + def __init__( + self, + queue: DownloadQueue | None = None, + encoder: Encoder | None = None, + config: Config | None = None, + ): super().__init__() - self.queue = queue - self.encoder = encoder + self.queue = queue or DownloadQueue.get_instance() + self.encoder = encoder or Encoder() + + config = config or Config.get_instance() + self.default_preset = config.default_preset async def add( - self, url: str, preset: str, folder: str, ytdlp_cookies: str, ytdlp_config: dict, output_template: str + self, url: str, preset: str, folder: str, cookies: str, config: dict, template: str ) -> dict[str, str]: - if ytdlp_cookies and isinstance(ytdlp_cookies, dict): - ytdlp_cookies = self.encoder.encode(ytdlp_cookies) + """ + Add an item to the download queue. - status = await self.queue.add( + Args: + url (str): The url to be added to the queue. + preset (str): The preset to be used for the download. + folder (str): The folder to save the download to. + cookies (str): The cookies to be used for the download. + config (dict): The yt-dlp config to be used for the download. + template (str): The template to be used for the download. + + Returns: + dict[str, str]: The status of the download. + { "status": "text" } + """ + if cookies and isinstance(cookies, dict): + cookies = self.encoder.encode(cookies) + + return await self.queue.add( url=url, preset=preset if preset else "default", folder=folder, - ytdlp_cookies=ytdlp_cookies, - ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {}, - output_template=output_template, + cookies=cookies, + config=config if isinstance(config, dict) else {}, + template=template, ) - return status - def format_item(self, item: dict) -> dict: """ Format the item to be added to the download queue. @@ -72,9 +91,9 @@ class common: "url": url, "preset": preset, "folder": folder, - "ytdlp_cookies": cookies, - "ytdlp_config": config if isinstance(config, dict) else {}, - "output_template": template, + "cookies": cookies, + "config": config if isinstance(config, dict) else {}, + "template": template, } return item diff --git a/app/main.py b/app/main.py index 8a4faca3..7af485e5 100644 --- a/app/main.py +++ b/app/main.py @@ -112,7 +112,7 @@ class Main: """ self.socket.attach(self.app) self.http.attach(self.app) - Tasks.get_instance().load() + Tasks.get_instance().attach(self.app) def started(_): LOG.info("=" * 40) diff --git a/ui/components/History.vue b/ui/components/History.vue index 4dba73c8..d3008801 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -468,9 +468,9 @@ const reQueueItem = item => { url: item.url, preset: item.preset, folder: item.folder, - ytdlp_config: item.ytdlp_config, - ytdlp_cookies: item.ytdlp_cookies, - output_template: item.output_template, + config: item.config, + cookies: item.cookies, + template: item.template, }) } diff --git a/ui/components/TaskForm.vue b/ui/components/TaskForm.vue index 0d870a5c..2395122e 100644 --- a/ui/components/TaskForm.vue +++ b/ui/components/TaskForm.vue @@ -113,13 +113,13 @@
+ v-model="form.template" :disabled="addInProgress">
The output template to use, if not set, it will defaults to - {{ config.app.output_template }} + {{ config.app.template }} diff --git a/ui/pages/tasks.vue b/ui/pages/tasks.vue index 931bb28b..86dacbdc 100644 --- a/ui/pages/tasks.vue +++ b/ui/pages/tasks.vue @@ -83,7 +83,7 @@ div.is-centered {

- {{ item.output_template ?? config.app.output_template }} + {{ item.template ?? config.app.output_template }}