diff --git a/.vscode/settings.json b/.vscode/settings.json index 0323161e..f21e8522 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,10 @@ ], "cSpell.words": [ "aconvert", + "ahas", + "ahash", "aiocron", + "arrowless", "copyts", "dotenv", "finaldir", diff --git a/Dockerfile b/Dockerfile index a4e0e41d..767b716e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:lts-alpine AS node_builder WORKDIR /app COPY ui ./ -RUN if [ ! -d /app/exported ]; then yarn install --production --prefer-offline --frozen-lockfile && yarn run generate; else echo "Skipping UI build, already built."; fi +RUN if [ ! -f "/app/exported/index.html" ]; then yarn install --production --prefer-offline --frozen-lockfile && yarn run generate; else echo "Skipping UI build, already built."; fi FROM python:3.11-alpine AS python_builder @@ -26,7 +26,7 @@ FROM python:3.11-alpine ARG TZ=UTC ARG USER_ID=1000 ENV IN_CONTAINER=1 -ENV UMASK=022 +ENV UMASK=0002 ENV YTP_CONFIG_PATH=/config ENV YTP_TEMP_PATH=/tmp ENV YTP_DOWNLOAD_PATH=/downloads diff --git a/Pipfile b/Pipfile index 2a392ea8..a7693427 100644 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ mutagen = "*" brotli = "*" brotlicffi = "*" yt-dlp = "*" +anyio = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index f85c38f2..47831ff2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "cd9a56440fe36b901993ced28ee6ef8d301a5125344cda86519fb7c54ede1449" + "sha256": "17668280c8e863033313b83f64240f6837e733957fbc2613767d297fee6c6fc2" }, "pipfile-spec": 6, "requires": { @@ -128,6 +128,7 @@ "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a", "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a" ], + "index": "pypi", "markers": "python_version >= '3.9'", "version": "==4.8.0" }, diff --git a/app/library/AsyncPool.py b/app/library/AsyncPool.py index 81f72263..f6b58a9a 100644 --- a/app/library/AsyncPool.py +++ b/app/library/AsyncPool.py @@ -1,6 +1,6 @@ import asyncio import logging -from datetime import datetime, timezone +from datetime import UTC, datetime class Terminator: @@ -14,9 +14,9 @@ class AsyncPool: worker_co, name: str, logger: logging.Logger, - loop: asyncio.AbstractEventLoop = None, + loop: asyncio.AbstractEventLoop | None = None, load_factor: int = 1, - max_task_time: int = None, + max_task_time: int | None = None, return_futures: bool = False, raise_on_join: bool = False, ): @@ -54,6 +54,12 @@ class AsyncPool: self._status: dict[str, dict | None] = {} async def _worker_loop(self, worker_id: str): + """ + This is the main loop for each worker. It will pull from the queue and call the worker_co + coroutine with the arguments passed to `push`. + :param worker_id: id of the worker + :return: None + """ while True: got_obj = False future = None @@ -84,7 +90,7 @@ class AsyncPool: raise except BaseException as e: if isinstance(e, asyncio.exceptions.CancelledError): - raise e + raise self._exceptions = True @@ -92,7 +98,7 @@ class AsyncPool: # don't log the failure when the client is receiving the future future.set_exception(e) else: - self._logger.exception(f"Worker call failed. {str(e)}") + self._logger.exception(f"Worker call failed. {e!s}") finally: self._status[worker_id] = None @@ -150,10 +156,15 @@ class AsyncPool: for worker_number in range(self._num_workers): worker_id = f"worker_{worker_number+1}" - self._createWorker(worker_id) + self._create_worker(worker_id) async def restart(self, worker_id: str, msg: str = None) -> bool: - """Will restart the worker pool""" + """ + Will restart the worker pool + :param worker_id: worker id to restart + :param msg: message to send to the worker + :return: True if worker was restarted + """ if worker_id not in self._workers: self._logger.warning(f"Worker {worker_id} does not exist.") return False @@ -162,19 +173,31 @@ class AsyncPool: self._workers[worker_id].cancel(msg) await self._workers[worker_id] except asyncio.exceptions.CancelledError as e: - self._logger.warning(f"Worker {worker_id} restarted. {str(e)}") + self._logger.warning(f"Worker {worker_id} restarted. {e!s}") if worker_id in self._status: self._status.pop(worker_id) if worker_id in self._workers: self._workers.pop(worker_id) - self._createWorker(worker_id) + self._create_worker(worker_id) return True + async def on_shutdown(self, _) -> None: + """Will shutdown the worker pool""" + try: + await asyncio.wait_for(asyncio.gather(*[self.stop(worker_id) for worker_id in self._workers]), timeout=10) + except Exception: + self._logger.error(f"Exception shutting down {self._name}") + async def stop(self, worker_id: str, msg: str = None) -> bool: - """Will stop the worker""" + """ + Will stop the worker + :param worker_id: worker id to stop + :param msg: message to send to the worker + :return: True if worker was stopped + """ if worker_id not in self._workers: self._logger.warning(f"Worker {worker_id} does not exist.") return False @@ -183,7 +206,7 @@ class AsyncPool: try: await self._workers[worker_id] except asyncio.exceptions.CancelledError as e: - self._logger.warning(f"Worker {worker_id} stopped. {str(e)}") + self._logger.warning(f"Worker {worker_id} stopped. {e!s}") if worker_id in self._status: self._status.pop(worker_id) @@ -214,9 +237,10 @@ class AsyncPool: self._logger.info(f"Completed {self._name}") if self._exceptions and self._raise_on_join: - raise Exception(f"Exception occurred in {self._name} pool") + msg = f"Exception occurred in {self._name} pool" + raise Exception(msg) - def _createWorker(self, worker_id: str) -> asyncio.Future: + def _create_worker(self, worker_id: str) -> asyncio.Future: if worker_id in self._workers: self._logger.debug(f"Worker {worker_id} already exists.") return self._workers[worker_id] @@ -231,4 +255,4 @@ class AsyncPool: return self._workers[worker_id] def _time(self) -> datetime: - return datetime.now(tz=timezone.utc) + return datetime.now(tz=UTC) diff --git a/app/library/DataStore.py b/app/library/DataStore.py index a59f81d7..372e6569 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -1,7 +1,7 @@ import copy import json from collections import OrderedDict -from datetime import datetime, timezone +from datetime import UTC, datetime from email.utils import formatdate from sqlite3 import Connection @@ -31,31 +31,30 @@ class DataStore: for id, item in self.saved_items(): self.dict.update({id: Download(info=item)}) - def exists(self, key: str = None, url: str = None) -> bool: + def exists(self, key: str | None = None, url: str | None = None) -> bool: if not key and not url: - raise KeyError("key or url must be provided.") + msg = "key or url must be provided." + raise KeyError(msg) if key and key in self.dict: return True - for i in self.dict: - if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url): - return True + return any((key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url) for i in self.dict) - return False - - def get(self, key: str, url: str = None) -> Download: + def get(self, key: str, url: str | None = None) -> Download: if not key and not url: - raise KeyError("key or url must be provided.") + msg = "key or url must be provided." + raise KeyError(msg) for i in self.dict: if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url): return self.dict[i] - raise KeyError(f"{key=} or {url=} not found.") + msg = f"{key=} or {url=} not found." + raise KeyError(msg) - def getById(self, id: str) -> Download | None: - return self.dict[id] if id in self.dict else None + def get_by_id(self, id: str) -> Download | None: + return self.dict.get(id, None) def items(self) -> list[tuple[str, Download]]: return self.dict.items() @@ -68,25 +67,25 @@ class DataStore: ) for row in cursor: - rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") + rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 data: dict = json.loads(row["data"]) key: str = data.pop("_id") item: ItemDTO = ItemDTO(**data) item._id = key - item.datetime = formatdate(rowDate.replace(tzinfo=timezone.utc).timestamp()) + item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp()) items.append((row["id"], item)) return items def put(self, value: Download) -> Download: self.dict.update({value.info._id: value}) - self._updateStoreItem(self.type, value.info) + self._update_store_item(self.type, value.info) return self.dict[value.info._id] def delete(self, key: str) -> None: self.dict.pop(key, None) - self._deleteStoreItem(key) + self._delete_store_item(key) def next(self) -> tuple[str, Download]: return next(iter(self.dict.items())) @@ -94,17 +93,13 @@ class DataStore: def empty(self): return not bool(self.dict) - def hasDownloads(self): + def has_downloads(self): if 0 == len(self.dict): return False - for key in self.dict: - if self.dict[key].started() is False: - return True + return any(self.dict[key].started() is False for key in self.dict) - return False - - def getNextDownload(self) -> Download: + def get_next_download(self) -> Download: for key in self.dict: if self.dict[key].started() is False and self.dict[key].is_cancelled() is False: return self.dict[key] @@ -115,7 +110,7 @@ class DataStore: self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() return True - def _updateStoreItem(self, type: str, item: ItemDTO) -> None: + def _update_store_item(self, type: str, item: ItemDTO) -> None: sqlStatement = """ INSERT INTO "history" ("id", "type", "url", "data") VALUES (?, ?, ?, ?) @@ -146,11 +141,11 @@ class DataStore: type, stored.url, stored.json(), - datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - def _deleteStoreItem(self, key: str) -> None: + def _delete_store_item(self, key: str) -> None: self.connection.execute( 'DELETE FROM "history" WHERE "type" = ? AND "id" = ?', ( diff --git a/app/library/Download.py b/app/library/Download.py index b60bbf77..c2b041a5 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -15,7 +15,7 @@ from .config import Config from .Emitter import Emitter from .ffprobe import ffprobe from .ItemDTO import ItemDTO -from .Utils import get_opts, jsonCookie, mergeConfig +from .Utils import get_opts, json_cookie, merge_config LOG = logging.getLogger("download") @@ -34,7 +34,7 @@ class Download: info: ItemDTO = None default_ytdl_opts: dict = None debug: bool = False - tempPath: str = None + temp_path: str = None emitter: Emitter = None cancelled: bool = False is_live: bool = False @@ -64,7 +64,7 @@ class Download: ) "Fields to be extracted from yt-dlp progress hook." - tempKeep: bool = False + temp_keep: bool = False "Keep temp folder after download." def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False): @@ -86,7 +86,7 @@ class Download: self.proc = None self.emitter = None self.max_workers = int(config.max_workers) - self.tempKeep = bool(config.temp_keep) + self.temp_keep = bool(config.temp_keep) self.is_live = bool(info.is_live) or info.live_in is not None self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True self.info_dict = info_dict @@ -94,7 +94,7 @@ class Download: def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} - if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", None): + if "finished" == data.get("status") and data.get("info_dict", {}).get("filename", None): dataDict["filename"] = data["info_dict"]["filename"] self.status_queue.put({"id": self.id, **dataDict}) @@ -115,11 +115,11 @@ class Download: def _download(self): try: - params: dict = get_opts(self.preset, mergeConfig(self.default_ytdl_opts, self.ytdl_opts)) + params: dict = get_opts(self.preset, merge_config(self.default_ytdl_opts, self.ytdl_opts)) params.update( { "color": "no_color", - "paths": {"home": self.download_dir, "temp": self.tempPath}, + "paths": {"home": self.download_dir, "temp": self.temp_path}, "outtmpl": {"default": self.template, "chapter": self.template_chapter}, "noprogress": True, "break_on_existing": True, @@ -138,17 +138,17 @@ class Download: if self.info.cookies: try: - data = jsonCookie(json.loads(self.info.cookies)) + data = json_cookie(json.loads(self.info.cookies)) if not data: LOG.warning( f"The cookie string that was provided for {self.info.title} is empty or not in expected spec." ) - with open(os.path.join(self.tempPath, f"cookie_{self.info._id}.txt"), "w") as f: + with open(os.path.join(self.temp_path, f"cookie_{self.info._id}.txt"), "w") as f: f.write(data) params["cookiefile"] = f.name except ValueError as e: - LOG.error(f"Invalid cookies: was provided for '{self.info.title}'. '{str(e)}'.") + LOG.error(f"Invalid cookies: was provided for '{self.info.title}'. '{e!s}'.") if self.is_live or self.is_manifestless: hasDeletedOptions = False @@ -191,10 +191,10 @@ class Download: self.status_queue = Config.get_manager().Queue() # Create temp dir for each download. - self.tempPath = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode("utf-8")).hexdigest(5)) + self.temp_path = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5)) - if not os.path.exists(self.tempPath): - os.makedirs(self.tempPath, exist_ok=True) + if not os.path.exists(self.temp_path): + os.makedirs(self.temp_path, exist_ok=True) self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download) self.proc.start() @@ -235,7 +235,6 @@ class Download: self.update_task.cancel() except Exception as e: LOG.error(f"Failed to close status queue: '{procId}'. {e}") - pass self.kill() @@ -288,26 +287,26 @@ class Download: return False def delete_temp(self): - if self.tempKeep is True or not self.tempPath: + if self.temp_keep is True or not self.temp_path: return if "finished" != self.info.status and self.is_live: LOG.warning( - f"Keeping live temp folder '{self.tempPath}', as the reported status is not finished '{self.info.status}'." + f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'." ) return - if not os.path.exists(self.tempPath): + if not os.path.exists(self.temp_path): return - if self.tempPath == self.temp_dir: + if self.temp_path == self.temp_dir: LOG.warning( - f"Attempted to delete video temp folder: {self.tempPath}, but it is the same as main temp folder." + f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder." ) return - LOG.info(f"Deleting Temp folder '{self.tempPath}'.") - shutil.rmtree(self.tempPath, ignore_errors=True) + LOG.info(f"Deleting Temp folder '{self.temp_path}'.") + shutil.rmtree(self.temp_path, ignore_errors=True) async def progress_update(self): """ @@ -343,7 +342,6 @@ class Download: self.info.file_size = os.path.getsize(status.get("filename")) except FileNotFoundError: self.info.file_size = 0 - pass self.info.status = status.get("status", self.info.status) self.info.msg = status.get("msg") @@ -385,7 +383,7 @@ class Download: except Exception as e: self.info.extras["is_video"] = True self.info.extras["is_audio"] = True - LOG.exception(f"Failed to ffprobe: {status.get}. {e}") LOG.exception(e) + LOG.error(f"Failed to ffprobe: {status.get}. {e}") asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index fd0cd2e0..a14fd8f9 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -1,23 +1,24 @@ import asyncio -import datetime import json import logging import os import time +from datetime import UTC, datetime from email.utils import formatdate from sqlite3 import Connection import yt_dlp +from aiohttp import web from .AsyncPool import AsyncPool from .config import Config from .DataStore import DataStore from .Download import Download from .Emitter import Emitter +from .EventsSubscriber import Events from .ItemDTO import ItemDTO from .Singleton import Singleton -from .EventsSubscriber import Events -from .Utils import ExtractInfo, calcDownloadPath, get_opts, isDownloaded, mergeConfig +from .Utils import calc_download_path, extract_info, get_opts, is_downloaded, merge_config LOG = logging.getLogger("DownloadQueue") @@ -42,6 +43,8 @@ class DownloadQueue(metaclass=Singleton): pool: AsyncPool | None = None """Pool of workers to download the files.""" + _active_downloads: dict[str, Download] = {} + _instance = None """Instance of the DownloadQueue.""" @@ -65,18 +68,39 @@ class DownloadQueue(metaclass=Singleton): Returns: DownloadQueue: The instance of the DownloadQueue + """ if not DownloadQueue._instance: DownloadQueue._instance = DownloadQueue() return DownloadQueue._instance + def attach(self, app: web.Application): + """ + Attach the download queue to the application. + + Args: + app (web.Application): The application to attach the download queue to. + + """ + app.on_startup.append(lambda _: self.initialize()) + # app.on_shutdown.append(self.on_shutdown) + + # async def close_pool(_: web.Application): + # try: + # await self.pool.on_shutdown(_) + # except Exception as e: + # LOG.error(f"Failed to cleanup download pool. {e!s}") + + # app.on_cleanup.append(close_pool) + async def test(self) -> bool: """ Test the datastore connection to the database. Returns: bool: True if the test is successful, False otherwise. + """ await self.done.test() return True @@ -88,18 +112,20 @@ class DownloadQueue(metaclass=Singleton): LOG.info( f"Using '{self.config.max_workers}' worker/s for downloading. Can be configured via `YTP_MAX_WORKERS` environment variable." ) - asyncio.create_task(self.__download_pool(), name="download_pool") + asyncio.create_task(self._download_pool(), name="download_pool") - def pause(self) -> bool: + def pause(self, shutdown: bool = False) -> bool: """ Pause the download queue. Returns: bool: True if the download is paused, False otherwise + """ if self.paused.is_set(): self.paused.clear() - LOG.warning(f"Download paused at. {datetime.datetime.now().isoformat()}") + if not shutdown: + LOG.warning(f"Download paused at. {datetime.now(tz=UTC).isoformat()}") return True return False @@ -110,29 +136,40 @@ class DownloadQueue(metaclass=Singleton): Returns: bool: True if the download is resumed, False otherwise + """ if not self.paused.is_set(): self.paused.set() - LOG.warning(f"Downloading resumed at. {datetime.datetime.now().isoformat()}") + LOG.warning(f"Downloading resumed at. {datetime.now(tz=UTC).isoformat()}") return True return False - def isPaused(self) -> bool: + def is_paused(self) -> bool: """ Check if the download queue is paused. Returns: bool: True if the download queue is paused, False otherwise + """ - return False if self.paused.is_set() else True + return not self.paused.is_set() + + async def on_shutdown(self, _: web.Application): + LOG.debug("Canceling all active downloads.") + if self._active_downloads: + self.pause() + try: + await self.cancel(list(self._active_downloads.keys())) + except Exception as e: + LOG.error(f"Failed to cancel downloads. {e!s}") async def __add_entry( self, entry: dict, preset: str, folder: str, - config: dict = {}, + config: dict | None = None, cookies: str = "", template: str = "", already=None, @@ -151,7 +188,11 @@ class DownloadQueue(metaclass=Singleton): Returns: dict: The status of the operation. + """ + if not config: + config = {} + if not entry: return {"status": "error", "msg": "Invalid/empty data was given."} @@ -161,13 +202,14 @@ class DownloadQueue(metaclass=Singleton): live_in: str | None = None eventType = entry.get("_type") or "video" + if "playlist" == eventType: entries = entry.get("entries", []) playlist_index_digits = len(str(len(entries))) results = [] for i, etr in enumerate(entries, start=1): etr["playlist"] = entry.get("id") - etr["playlist_index"] = "{{0:0{0:d}d}}".format(playlist_index_digits).format(i) + etr["playlist_index"] = f"{{0:0{playlist_index_digits:d}d}}".format(i) for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry.get(property) @@ -194,7 +236,8 @@ class DownloadQueue(metaclass=Singleton): } return {"status": "ok"} - elif ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry: + + if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry: # check if the video is live stream. if "live_status" in entry and "is_upcoming" == entry.get("live_status"): if "release_timestamp" in entry and entry.get("release_timestamp"): @@ -202,11 +245,9 @@ class DownloadQueue(metaclass=Singleton): else: error = "Live stream not yet started. And no date is set." else: - error = entry.get("msg", None) + error = entry.get("msg") - LOG.debug( - f"Entry id '{entry.get('id', None)}' url '{entry.get('webpage_url', None)} - {entry.get('url', None)}'." - ) + LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.") if self.done.exists(key=entry["id"], url=str(entry.get("webpage_url") or entry.get("url"))): item = self.done.get(key=entry["id"], url=entry.get("webpage_url") or entry["url"]) @@ -226,10 +267,10 @@ class DownloadQueue(metaclass=Singleton): options.update({"is_manifestless": is_manifestless}) live_status: list = ["is_live", "is_upcoming"] - is_live = bool(entry.get("is_live", None) or live_in or entry.get("live_status", None) in live_status) + is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) try: - download_dir = calcDownloadPath(basePath=self.config.download_path, folder=folder) + download_dir = calc_download_path(base_path=self.config.download_path, folder=folder) except Exception as e: LOG.exception(e) return {"status": "error", "msg": str(e)} @@ -237,7 +278,7 @@ class DownloadQueue(metaclass=Singleton): extras: dict = {} fields: tuple = ("uploader", "channel", "thumbnail") for field in fields: - if entry.get(field, None): + if entry.get(field): extras[field] = entry.get(field) dl = ItemDTO( @@ -245,7 +286,6 @@ class DownloadQueue(metaclass=Singleton): title=str(entry.get("title")), url=str(entry.get("webpage_url") or entry.get("url")), preset=preset, - thumbnail=entry.get("thumbnail", None), folder=folder, download_dir=download_dir, temp_dir=self.config.temp_path, @@ -267,7 +307,7 @@ class DownloadQueue(metaclass=Singleton): dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug)) - if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status", None): + if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"): dlInfo.info.status = "not_live" itemDownload = self.done.put(dlInfo) NotifyEvent = Events.COMPLETED @@ -286,7 +326,8 @@ class DownloadQueue(metaclass=Singleton): ) return {"status": "ok"} - elif eventType.startswith("url"): + + if eventType.startswith("url"): return await self.add( url=str(entry.get("url")), preset=preset, @@ -304,7 +345,7 @@ class DownloadQueue(metaclass=Singleton): url: str, preset: str, folder: str, - config: dict = {}, + config: dict | None = None, cookies: str = "", template: str = "", already=None, @@ -312,7 +353,7 @@ class DownloadQueue(metaclass=Singleton): config = config if config else {} folder = str(folder) if folder else "" - filePath = calcDownloadPath(basePath=self.config.download_path, folder=folder) + filePath = calc_download_path(base_path=self.config.download_path, folder=folder) LOG.info( f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {cookies}' 'YTConfig: {config}'." @@ -322,8 +363,8 @@ class DownloadQueue(metaclass=Singleton): try: config = json.loads(config) except Exception as e: - LOG.error(f"Unable to load '{config=}'. {str(e)}") - return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {str(e)}"} + LOG.error(f"Unable to load '{config=}'. {e!s}") + return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {e!s}"} already = set() if already is None else already @@ -334,7 +375,7 @@ class DownloadQueue(metaclass=Singleton): already.add(url) try: - downloaded, id_dict = self.isDownloaded(url) + downloaded, id_dict = self._is_downloaded(url) if downloaded is True and id_dict: message = f"This url with ID '{id_dict.get('id')}' has been downloaded already and recorded in archive." LOG.info(message) @@ -346,8 +387,8 @@ class DownloadQueue(metaclass=Singleton): entry = await asyncio.wait_for( fut=asyncio.get_running_loop().run_in_executor( None, - ExtractInfo, - get_opts(preset, mergeConfig(self.config.ytdl_options, config)), + extract_info, + get_opts(preset, merge_config(self.config.ytdl_options, config)), url, bool(self.config.ytdl_debug), ), @@ -361,13 +402,13 @@ class DownloadQueue(metaclass=Singleton): f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'." ) except yt_dlp.utils.ExistingVideoReached as exc: - LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{str(exc)}'.") + LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.") return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} except yt_dlp.utils.YoutubeDLError as exc: - LOG.error(f"YoutubeDLError: Unable to extract info. '{str(exc)}'.") + LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.") return {"status": "error", "msg": str(exc)} except asyncio.exceptions.TimeoutError as exc: - LOG.error(f"TimeoutError: Unable to extract info. '{str(exc)}'.") + LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.") return { "status": "error", "msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.", @@ -384,6 +425,16 @@ class DownloadQueue(metaclass=Singleton): ) async def cancel(self, ids: list[str]) -> dict[str, str]: + """ + Cancel the download. + + Args: + ids (list): The list of ids to cancel. + + Returns: + dict: The status of the operation. + + """ status: dict[str, str] = {"status": "ok"} for id in ids: @@ -392,32 +443,43 @@ class DownloadQueue(metaclass=Singleton): except KeyError as e: status[id] = str(e) status["status"] = "error" - LOG.warning(f"Requested cancel for non-existent download {id=}. {str(e)}") + LOG.warning(f"Requested cancel for non-existent download {id=}. {e!s}") continue - itemMessage = f"{id=} {item.info.id=} {item.info.title=}" + item_ref = f"{id=} {item.info.id=} {item.info.title=}" if item.running(): - LOG.debug(f"Canceling {itemMessage}") + LOG.debug(f"Canceling {item_ref}") item.cancel() - LOG.info(f"Cancelled {itemMessage}") + LOG.info(f"Cancelled {item_ref}") await item.close() else: await item.close() - LOG.debug(f"Deleting from queue {itemMessage}") + LOG.debug(f"Deleting from queue {item_ref}") self.queue.delete(id) asyncio.create_task(self.emitter.cancelled(dl=item.info.serialize()), name=f"notifier-c-{id}") item.info.status = "cancelled" item.info.error = "Cancelled by user." self.done.put(item) asyncio.create_task(self.emitter.completed(dl=item.info.serialize()), name=f"notifier-d-{id}") - LOG.info(f"Deleted from queue {itemMessage}") + LOG.info(f"Deleted from queue {item_ref}") status[id] = "ok" return status async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]: + """ + Clear the download history. + + Args: + ids (list): The list of ids to clear. + remove_file (bool): True to remove the file, False otherwise. Default is False. + + Returns: + dict: The status of the operation. + + """ status: dict[str, str] = {"status": "ok"} for id in ids: @@ -426,7 +488,7 @@ class DownloadQueue(metaclass=Singleton): except KeyError as e: status[id] = str(e) status["status"] = "error" - LOG.warning(f"Requested delete for non-existent download {id=}. {str(e)}") + LOG.warning(f"Requested delete for non-existent download {id=}. {e!s}") continue itemRef: str = f"{id=} {item.info.id=} {item.info.title=}" @@ -439,10 +501,10 @@ class DownloadQueue(metaclass=Singleton): filename = f"{item.info.folder}/{item.info.filename}" try: - realFile: str = calcDownloadPath( - basePath=self.config.download_path, + realFile: str = calc_download_path( + base_path=self.config.download_path, folder=filename, - createPath=False, + create_path=False, ) if realFile and os.path.exists(realFile): os.remove(realFile) @@ -450,7 +512,7 @@ class DownloadQueue(metaclass=Singleton): else: LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.") except Exception as e: - LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {str(e)}") + LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {e!s}") self.done.delete(id) asyncio.create_task(self.emitter.cleared(dl=item.info.serialize()), name=f"notifier-c-{id}") @@ -469,6 +531,7 @@ class DownloadQueue(metaclass=Singleton): Returns: dict: The download queue and the download history. + """ items = {"queue": {}, "done": {}} @@ -488,11 +551,18 @@ class DownloadQueue(metaclass=Singleton): return items - async def __download_pool(self): + async def _download_pool(self) -> None: + """ + Create a pool of workers to download the files. + + Returns: + None + + """ self.pool = AsyncPool( loop=asyncio.get_running_loop(), num_workers=self.config.max_workers, - worker_co=self.__downloadFile, + worker_co=self._download_file, name="download_pool", logger=logging.getLogger("WorkerPool"), ) @@ -510,19 +580,19 @@ class DownloadQueue(metaclass=Singleton): LOG.info(f"Waiting for worker to be free. {self.pool.get_workers_status()}") await asyncio.sleep(1) - while not self.queue.hasDownloads(): + while not self.queue.has_downloads(): LOG.info(f"Waiting for item to download. '{self.pool.get_available_workers()}' free workers.") if self.event: await self.event.wait() self.event.clear() LOG.debug("Cleared wait event.") - if self.paused and isinstance(self.paused, asyncio.Event) and self.isPaused(): + if self.paused and isinstance(self.paused, asyncio.Event) and self.is_paused(): LOG.info("Download pool is paused.") await self.paused.wait() LOG.info("Download pool resumed downloading.") - entry = self.queue.getNextDownload() + entry = self.queue.get_next_download() await asyncio.sleep(0.2) if entry is None: @@ -535,13 +605,25 @@ class DownloadQueue(metaclass=Singleton): LOG.debug(f"Pushed {entry=} to executor.") await asyncio.sleep(1) - async def __downloadFile(self, id: str, entry: Download): - filePath = calcDownloadPath(basePath=self.config.download_path, folder=entry.info.folder) + async def _download_file(self, id: str, entry: Download) -> None: + """ + Download the file. + + Args: + id (str): The id of the download. + entry (Download): The download entry. + + Returns: + None + + """ + filePath = calc_download_path(base_path=self.config.download_path, folder=entry.info.folder) LOG.info( f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {filePath}'." ) try: + self._active_downloads[entry.info._id] = entry await entry.start(self.emitter) if "finished" != entry.info.status: @@ -554,6 +636,9 @@ class DownloadQueue(metaclass=Singleton): entry.info.status = "error" finally: + if entry.info._id in self._active_downloads: + self._active_downloads.pop(entry.info._id, None) + await entry.close() if self.queue.exists(key=id): @@ -573,8 +658,18 @@ class DownloadQueue(metaclass=Singleton): if self.event: self.event.set() - def isDownloaded(self, url: str) -> tuple[bool, dict | None]: + def _is_downloaded(self, url: str) -> tuple[bool, dict | None]: + """ + Check if the url has been downloaded already. + + Args: + url (str): The url to check. + + Returns: + tuple: A tuple with the status of the operation and the id of the downloaded item. + + """ if not url or not self.config.keep_archive: return False, None - return isDownloaded(self.config.ytdl_options.get("download_archive", None), url) + return is_downloaded(self.config.ytdl_options.get("download_archive", None), url) diff --git a/app/library/Emitter.py b/app/library/Emitter.py index f0000c2a..8348ab70 100644 --- a/app/library/Emitter.py +++ b/app/library/Emitter.py @@ -1,9 +1,9 @@ import asyncio import logging -from typing import Awaitable +from collections.abc import Awaitable -from .Singleton import Singleton from .EventsSubscriber import Events +from .Singleton import Singleton LOG = logging.getLogger("Emitter") @@ -28,6 +28,7 @@ class Emitter(metaclass=Singleton): Returns: Emitter: The instance of the Emitter + """ if not Emitter._instance: Emitter._instance = Emitter() @@ -43,6 +44,7 @@ class Emitter(metaclass=Singleton): Returns: Emitter: The instance of the Emitter + """ if not isinstance(emitter, list): emitter = [emitter] @@ -67,19 +69,19 @@ class Emitter(metaclass=Singleton): async def cleared(self, dl: dict | None = None, local: bool = False, **kwargs): await self.emit(Events.CLEARED, data=dl, local=local, **kwargs) - async def error(self, message: str, data: dict = {}, local: bool = False, **kwargs): + async def error(self, message: str, data: dict|None = None, local: bool = False, **kwargs): msg = {"type": "error", "message": message, "data": {}} if data: msg.update({"data": data}) await self.emit(Events.ERROR, data=msg, local=local, **kwargs) - async def info(self, message: str, data: dict = {}, local: bool = False, **kwargs): + async def info(self, message: str, data: dict|None = None, local: bool = False, **kwargs): msg = {"type": "info", "message": message, "data": {}} if data: msg.update({"data": data}) await self.emit(Events.LOG_INFO, data=msg, local=local, **kwargs) - async def success(self, message: str, data: dict = {}, local: bool = False, **kwargs): + async def success(self, message: str, data: dict|None = None, local: bool = False, **kwargs): msg = {"type": "success", "message": message, "data": {}} if data: msg.update({"data": data}) @@ -97,6 +99,7 @@ class Emitter(metaclass=Singleton): Returns: None + """ tasks = [] @@ -122,8 +125,8 @@ class Emitter(metaclass=Singleton): await asyncio.wait_for(asyncio.gather(*tasks), timeout=60) except asyncio.CancelledError: LOG.error(f"Cancelled sending event '{event}'.") - except asyncio.TimeoutError: + except TimeoutError: LOG.error(f"Timed out sending event '{event}'.") except Exception as e: LOG.exception(e) - LOG.error(f"Failed to send event '{event}'. '{str(e)}'.") + LOG.error(f"Failed to send event '{event}'. '{e!s}'.") diff --git a/app/library/EventsSubscriber.py b/app/library/EventsSubscriber.py index 19c9ae9e..2fb5a683 100644 --- a/app/library/EventsSubscriber.py +++ b/app/library/EventsSubscriber.py @@ -1,7 +1,6 @@ import asyncio import logging -from typing import Awaitable - +from collections.abc import Awaitable from dataclasses import dataclass from .Singleton import Singleton @@ -32,12 +31,10 @@ class Events: CLI_OUTPUT = "cli_output" UPDATE = "update" TEST = "test" - UPDATED = "updated" ADD_URL = "add_url" CLI_POST = "cli_post" PAUSED = "paused" - TEST = "test" TASKS_ADD = "task_add" TASK_DISPATCHED = "task_dispatched" @@ -72,6 +69,7 @@ class EventsSubscriber(metaclass=Singleton): Returns: EventsSubscriber: The instance of the EventsSubscriber + """ if not EventsSubscriber._instance: EventsSubscriber._instance = EventsSubscriber() @@ -88,6 +86,7 @@ class EventsSubscriber(metaclass=Singleton): Returns: EventsSubscriber: The instance of the EventsSubscriber + """ if isinstance(event, str): event = [event] @@ -110,8 +109,8 @@ class EventsSubscriber(metaclass=Singleton): Returns: EventsSubscriber: The instance of the EventsSubscriber - """ + """ if isinstance(event, str): event = [event] @@ -148,8 +147,8 @@ class EventsSubscriber(metaclass=Singleton): results.append(asyncio.get_event_loop().run_until_complete(callback(event, data))) except Exception as e: - LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.") LOG.exception(e) + LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{e!s}'.") return results @@ -164,10 +163,10 @@ class EventsSubscriber(metaclass=Singleton): Returns: Awaitable: The task that was created to run the event. - """ + """ if event not in self._listeners: - return + return None tasks = [] for id, callback in self._listeners[event].items(): @@ -181,7 +180,7 @@ class EventsSubscriber(metaclass=Singleton): tasks.append(asyncio.create_task(callback(event, data))) except Exception as e: - LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.") LOG.exception(e) + LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{e!s}'.") return asyncio.gather(*tasks) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index f13351fb..f11e087b 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -8,9 +8,9 @@ import os import random import time import uuid -from datetime import datetime +from collections.abc import Awaitable +from datetime import UTC, datetime from pathlib import Path -from typing import Awaitable import httpx import magic @@ -18,7 +18,7 @@ from aiohttp import web from aiohttp.web import Request, RequestHandler, Response from .cache import Cache -from .common import common +from .common import Common from .config import Config from .DownloadQueue import DownloadQueue from .Emitter import Emitter @@ -31,21 +31,24 @@ from .Playlist import Playlist from .Segments import Segments from .Subtitle import Subtitle from .Tasks import Task, Tasks -from .Utils import StreamingError, arg_converter, calcDownloadPath, getVideoInfo, validate_url, validate_uuid +from .Utils import StreamingError, arg_converter, calc_download_path, get_video_info, validate_url, validate_uuid LOG = logging.getLogger("http_api") MIME = magic.Magic(mime=True) -class HttpAPI(common): - staticHolder: dict = {} - extToMime: dict = { +class HttpAPI(Common): + _static_holder: dict = {} + """Holds loaded static assets.""" + + _ext_to_mime: dict = { ".html": "text/html", ".css": "text/css", ".js": "application/javascript", ".json": "application/json", ".ico": "image/x-icon", } + """Map ext to mimetype""" def __init__( self, @@ -65,6 +68,7 @@ class HttpAPI(common): super().__init__(queue=self.queue, encoder=self.encoder, config=self.config) + @staticmethod def route(method: str, path: str) -> Awaitable: """ Decorator to mark a method as an HTTP route handler. @@ -75,6 +79,7 @@ class HttpAPI(common): Returns: Awaitable: The decorated function. + """ def decorator(func): @@ -88,6 +93,9 @@ class HttpAPI(common): return decorator + async def on_shutdown(self, _: web.Application): + LOG.debug("Shutting down http API server.") + def attach(self, app: web.Application) -> "HttpAPI": """ Attach the routes to the application. @@ -97,6 +105,7 @@ class HttpAPI(common): Returns: HttpAPI: The instance of the HttpAPI. + """ if self.config.auth_username and self.config.auth_password: app.middlewares.append(HttpAPI.basic_auth(self.config.auth_username, self.config.auth_password)) @@ -117,9 +126,10 @@ class HttpAPI(common): except Exception as e: LOG.exception(e) + app.on_shutdown.append(self.on_shutdown) return self - async def staticFile(self, req: Request) -> Response: + async def _static_file(self, req: Request) -> Response: """ Preload static files from the ui/exported folder. @@ -128,13 +138,14 @@ class HttpAPI(common): Returns: Response: The response object. + """ path = req.path - if req.path not in self.staticHolder: + if req.path not in self._static_holder: return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code) - item: dict = self.staticHolder[req.path] + item: dict = self._static_holder[req.path] return web.Response( body=item.get("content"), @@ -142,12 +153,12 @@ class HttpAPI(common): "Pragma": "public", "Cache-Control": "public, max-age=31536000", "Content-Type": item.get("content_type"), - "X-Via": "memory" if not item.get("file", None) else "disk", + "X-Via": "memory" if not item.get("file") else "disk", }, status=web.HTTPOk.status_code, ) - def preloadStatic(self, app: web.Application) -> "HttpAPI": + def _preload_static(self, app: web.Application) -> "HttpAPI": """ Preload static files from the ui/exported folder. @@ -156,11 +167,12 @@ class HttpAPI(common): Returns: HttpAPI: The instance of the HttpAPI. - """ + """ staticDir = os.path.join(self.rootPath, "ui", "exported") if not os.path.exists(staticDir): - raise ValueError(f"Could not find the frontend UI static assets. '{staticDir}'.") + msg = f"Could not find the frontend UI static assets. '{staticDir}'." + raise ValueError(msg) preloaded = 0 @@ -172,21 +184,23 @@ class HttpAPI(common): file = os.path.join(root, file) urlPath = f"/{file.replace(f'{staticDir}/', '')}" - content = open(file, "rb").read() - contentType = self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)) + with open(file, "rb") as f: + content = f.read() - self.staticHolder[urlPath] = {"content": content, "content_type": contentType} + contentType = self._ext_to_mime.get(os.path.splitext(file)[1], MIME.from_file(file)) + + self._static_holder[urlPath] = {"content": content, "content_type": contentType} LOG.debug(f"Preloading '{urlPath}'.") - app.router.add_get(urlPath, self.staticFile) + app.router.add_get(urlPath, self._static_file) preloaded += 1 if urlPath.endswith("/index.html") and urlPath != "/index.html": parentSlash = urlPath.replace("/index.html", "/") parentNoSlash = urlPath.replace("/index.html", "") - self.staticHolder[parentSlash] = {"content": content, "content_type": contentType} - self.staticHolder[parentNoSlash] = {"content": content, "content_type": contentType} - app.router.add_get(parentSlash, self.staticFile) - app.router.add_get(parentNoSlash, self.staticFile) + self._static_holder[parentSlash] = {"content": content, "content_type": contentType} + self._static_holder[parentNoSlash] = {"content": content, "content_type": contentType} + app.router.add_get(parentSlash, self._static_file) + app.router.add_get(parentNoSlash, self._static_file) preloaded += 2 if preloaded < 1: @@ -210,8 +224,8 @@ class HttpAPI(common): Returns: HttpAPI: The instance of the HttpAPI. - """ + """ for attr_name in dir(self): method = getattr(self, attr_name) if hasattr(method, "_http_method") and hasattr(method, "_http_path"): @@ -222,15 +236,17 @@ class HttpAPI(common): self.routes.route(method._http_method, f"/{http_path}")(method) self.routes.static("/api/download/", self.config.download_path) - self.preloadStatic(app) + self._preload_static(app) try: app.add_routes(self.routes) except ValueError as e: if "ui/exported" in str(e): - raise RuntimeError(f"Could not find the frontend UI static assets. '{e}'.") from e - raise e + msg = f"Could not find the frontend UI static assets. '{e}'." + raise RuntimeError(msg) from e + raise + @staticmethod def basic_auth(username: str, password: str) -> Awaitable: """ Middleware to handle basic authentication. @@ -241,6 +257,7 @@ class HttpAPI(common): Returns: Awaitable: The middleware handler. + """ @web.middleware @@ -291,6 +308,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code) @@ -304,6 +322,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ await self.queue.test() return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code) @@ -318,6 +337,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ post = await request.json() args: str | None = post.get("args") @@ -345,6 +365,7 @@ class HttpAPI(common): Returns: Response: The response object + """ url = request.query.get("url") if not url: @@ -372,7 +393,7 @@ class HttpAPI(common): "proxy": self.config.ytdl_options.get("proxy", None), } - data = getVideoInfo(url=url, ytdlp_opts=opts, no_archive=True) + data = get_video_info(url=url, ytdlp_opts=opts, no_archive=True) self.cache.set(key=self.cache.hash(url), value=data, ttl=300) data["_cached"] = { "key": self.cache.hash(url), @@ -403,6 +424,7 @@ class HttpAPI(common): Returns: Response: The response object + """ url: str | None = request.query.get("url") if not url: @@ -425,6 +447,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ data = await request.json() @@ -456,9 +479,10 @@ class HttpAPI(common): Returns: Response: The response object. + """ return web.json_response( - data=Tasks.get_instance().getTasks(), status=web.HTTPOk.status_code, dumps=self.encoder.encode + data=Tasks.get_instance().get_tasks(), status=web.HTTPOk.status_code, dumps=self.encoder.encode ) @route("PUT", "api/tasks") @@ -471,6 +495,7 @@ class HttpAPI(common): Returns: Response: The response object + """ data = await request.json() @@ -500,7 +525,7 @@ class HttpAPI(common): item["id"] = str(uuid.uuid4()) if not item.get("timer", None) or str(item.get("timer")).strip() == "": - item["timer"] = f"{random.randint(1,59)} */1 * * *" + item["timer"] = f"{random.randint(1,59)} */1 * * *" # noqa: S311 if not item.get("cookies", None): item["cookies"] = "" @@ -509,20 +534,20 @@ class HttpAPI(common): item["config"] = {} if not item.get("template", None): - item["template"] = str() + item["template"] = "" try: ins.validate(item) except ValueError as e: return web.json_response( - {"error": f"Failed to validate task '{item.get('name')}'. '{str(e)}'"}, + {"error": f"Failed to validate task '{item.get('name')}'. '{e!s}'"}, status=web.HTTPBadRequest.status_code, ) tasks.append(Task(**item)) try: - tasks = ins.save(tasks=tasks).load().getTasks() + tasks = ins.save(tasks=tasks).load().get_tasks() except Exception as e: LOG.exception(e) return web.json_response( @@ -542,6 +567,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ data = await request.json() ids = data.get("ids") @@ -569,12 +595,13 @@ class HttpAPI(common): Returns: Response: The response object. + """ id: str = request.match_info.get("id") if not id: return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) - item = self.queue.done.getById(id) + item = self.queue.done.get_by_id(id) if not item: return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) @@ -615,6 +642,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ data: dict = {"queue": [], "history": []} @@ -635,6 +663,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ if self.queue.pool is None: return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) @@ -672,6 +701,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ if self.queue.pool is None: return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) @@ -690,6 +720,7 @@ class HttpAPI(common): Returns: Response: The response object + """ id: str = request.match_info.get("id") if not id: @@ -712,6 +743,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ id: str = request.match_info.get("id") if not id: @@ -734,6 +766,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ file: str = request.match_info.get("file") @@ -767,6 +800,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ file: str = request.match_info.get("file") mode: str = request.match_info.get("mode") @@ -817,6 +851,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ file: str = request.match_info.get("file") segment: int = request.match_info.get("segment") @@ -829,7 +864,7 @@ class HttpAPI(common): if request.if_modified_since: lastMod = time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple() ) if os.path.exists(file_path) and request.if_modified_since.timestamp() == os.path.getmtime(file_path): return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) @@ -842,9 +877,9 @@ class HttpAPI(common): segmenter = Segments( index=int(segment), - duration=float("{:.6f}".format(float(sd if sd else M3u8.duration))), - vconvert=True if vc == 1 else False, - aconvert=True if ac == 1 else False, + duration=float(f"{float(sd if sd else M3u8.duration):.6f}"), + vconvert=vc == 1, + aconvert=ac == 1, ) return web.Response( @@ -856,10 +891,10 @@ class HttpAPI(common): "Pragma": "public", "Cache-Control": f"public, max-age={time.time() + 31536000}", "Last-Modified": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple() ), "Expires": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple() ), }, status=web.HTTPOk.status_code, @@ -875,6 +910,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ file: str = request.match_info.get("file") file_path: str = os.path.normpath(os.path.join(self.config.download_path, file)) @@ -883,7 +919,7 @@ class HttpAPI(common): if request.if_modified_since: lastMod = time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple() ) if os.path.exists(file_path) and request.if_modified_since.timestamp() == os.path.getmtime(file_path): return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) @@ -900,10 +936,10 @@ class HttpAPI(common): "Pragma": "public", "Cache-Control": f"public, max-age={time.time() + 31536000}", "Last-Modified": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple() ), "Expires": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple() ), }, status=web.HTTPOk.status_code, @@ -919,14 +955,15 @@ class HttpAPI(common): Returns: Response: The response object. + """ - if "/index.html" not in self.staticHolder: + if "/index.html" not in self._static_holder: LOG.error("Static frontend files not found.") return web.json_response( data={"error": "File not found.", "file": "/index.html"}, status=web.HTTPNotFound.status_code ) - data = self.staticHolder["/index.html"] + data = self._static_holder["/index.html"] return web.Response( body=data.get("content"), content_type=data.get("content_type"), @@ -944,6 +981,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ url = request.query.get("url") if not url: @@ -974,7 +1012,8 @@ class HttpAPI(common): "Access-Control-Allow-Origin": "*", "Cache-Control": f"public, max-age={time.time() + 31536000}", "Expires": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000).timetuple() + "%a, %d %b %Y %H:%M:%S GMT", + datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple(), ), }, ) @@ -994,11 +1033,15 @@ class HttpAPI(common): Returns: Response: The response object. + """ file: str = request.match_info.get("file") + if not file: + return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) + try: - realFile: str = calcDownloadPath(basePath=self.config.download_path, folder=file, createPath=False) - if not os.path.exists(realFile): + realFile: str = calc_download_path(base_path=self.config.download_path, folder=file, create_path=False) + if not os.path.exists(realFile) or not os.path.isfile(realFile): return web.json_response( data={"error": f"File '{file}' does not exist."}, status=web.HTTPNotFound.status_code ) @@ -1019,6 +1062,7 @@ class HttpAPI(common): Returns: Response: The response object + """ cookie_file = self.config.ytdl_options.get("cookiefile", None) if not cookie_file: @@ -1066,7 +1110,7 @@ class HttpAPI(common): LOG.error(f"Failed to request '{url}'. '{e}'.") LOG.exception(e) return web.json_response( - data={"message": f"Failed to request website. {str(e)}"}, + data={"message": f"Failed to request website. {e!s}"}, status=web.HTTPInternalServerError.status_code, ) @@ -1080,11 +1124,12 @@ class HttpAPI(common): Returns: Response: The response object. + """ return web.json_response( data={ - "notifications": Notification.get_instance().getTargets(), - "allowedTypes": list(NotificationEvents.getEvents().values()), + "notifications": Notification.get_instance().get_targets(), + "allowedTypes": list(NotificationEvents.get_events().values()), }, status=web.HTTPOk.status_code, dumps=self.encoder.encode, @@ -1100,6 +1145,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ post = await request.json() if not isinstance(post, list): @@ -1125,11 +1171,11 @@ class HttpAPI(common): Notification.validate(item) except ValueError as e: return web.json_response( - {"error": f"Invalid notification target settings. {str(e)}", "data": item}, + {"error": f"Invalid notification target settings. {e!s}", "data": item}, status=web.HTTPBadRequest.status_code, ) - targets.append(ins.makeTarget(item)) + targets.append(ins.make_target(item)) try: if len(targets) < 1: @@ -1141,7 +1187,7 @@ class HttpAPI(common): LOG.exception(e) return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code) - data = {"notifications": targets, "allowedTypes": list(NotificationEvents.getEvents().values())} + data = {"notifications": targets, "allowedTypes": list(NotificationEvents.get_events().values())} return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) @@ -1155,6 +1201,7 @@ class HttpAPI(common): Returns: Response: The response object. + """ data = {"type": "test", "message": "This is a test notification."} await self.emitter.emit(Events.TEST, data) diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index f2907240..435204ca 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -6,24 +6,25 @@ import os import pty import shlex import time -from datetime import datetime - -import socketio -from aiohttp import web +from datetime import UTC, datetime from typing import Any -from .common import common +import anyio +import socketio +from aiohttp import web + +from .common import Common from .config import Config from .DownloadQueue import DownloadQueue from .Emitter import Emitter from .encoder import Encoder -from .Utils import isDownloaded, arg_converter -from .EventsSubscriber import EventsSubscriber, Events, Event +from .EventsSubscriber import Event, Events, EventsSubscriber +from .Utils import arg_converter, is_downloaded LOG = logging.getLogger("socket_api") -class HttpSocket(common): +class HttpSocket(Common): """ This class is used to handle WebSocket events. """ @@ -54,6 +55,7 @@ class HttpSocket(common): super().__init__(queue=queue, encoder=encoder, config=config) + @staticmethod def ws_event(func): # type: ignore """ Decorator to mark a method as a socket event. @@ -66,6 +68,9 @@ class HttpSocket(common): wrapper._ws_event = func.__name__ # type: ignore return wrapper + async def on_shutdown(self, _: web.Application): + LOG.debug("Shutting down socket server.") + def attach(self, app: web.Application): self.sio.attach(app, socketio_path=self.config.url_socketio) @@ -82,7 +87,10 @@ class HttpSocket(common): EventsSubscriber.get_instance().subscribe(Events.ADD_URL, "socket_add_url", handle_event) - @ws_event # type: ignore + # register the shutdown event. + app.on_shutdown.append(self.on_shutdown) + + @ws_event async def cli_post(self, sid: str, data): if not data: await self.emitter.emit(Events.CLI_CLOSE, {"exitcode": 0}, to=sid) @@ -91,7 +99,7 @@ class HttpSocket(common): try: LOG.info(f"Cli command from client '{sid}'. '{data}'") - args = ["yt-dlp"] + shlex.split(data) + args = ["yt-dlp", *shlex.split(data)] _env = os.environ.copy() _env.update( { @@ -119,7 +127,7 @@ class HttpSocket(common): try: os.close(slave_fd) except Exception as e: - LOG.error(f"Error closing PTY. '{str(e)}'.") + LOG.error(f"Error closing PTY. '{e!s}'.") async def read_pty(sid: str): loop = asyncio.get_running_loop() @@ -130,8 +138,7 @@ class HttpSocket(common): except OSError as e: if e.errno == errno.EIO: break - else: - raise + raise if not chunk: # No more output @@ -154,7 +161,7 @@ class HttpSocket(common): try: os.close(master_fd) except Exception as e: - LOG.error(f"Error closing PTY. '{str(e)}'.") + LOG.error(f"Error closing PTY. '{e!s}'.") # Start reading output from PTY read_task = asyncio.create_task(read_pty(sid=sid)) @@ -172,7 +179,7 @@ class HttpSocket(common): await self.emitter.emit(Events.CLI_OUTPUT, {"type": "stderr", "line": str(e)}, to=sid) await self.emitter.emit(Events.CLI_CLOSE, {"exitcode": -1}, to=sid) - @ws_event # type: ignore + @ws_event async def add_url(self, sid: str, data: dict): url: str | None = data.get("url") @@ -183,13 +190,13 @@ class HttpSocket(common): try: item = self.format_item(data) except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + await self.emitter.error(str(e), to=sid) + return status = await self.add(**item) - await self.emitter.emit(event=Events.STATUS, data=status, to=sid) - @ws_event # type: ignore + @ws_event async def item_cancel(self, sid: str, id: str): if not id: await self.emitter.error("Invalid request.", to=sid) @@ -201,7 +208,7 @@ class HttpSocket(common): await self.emitter.emit(event=Events.ITEM_CANCEL, data=status) - @ws_event # type: ignore + @ws_event async def item_delete(self, sid: str, data: dict): if not data: await self.emitter.error("Invalid request.", to=sid) @@ -218,8 +225,8 @@ class HttpSocket(common): await self.emitter.emit(event=Events.ITEM_DELETE, data=status) - @ws_event # type: ignore - async def archive_item(self, sid: str, data: dict): + @ws_event + async def archive_item(self, _: str, data: dict): if not isinstance(data, dict) or "url" not in data or not self.config.keep_archive: return @@ -231,36 +238,36 @@ class HttpSocket(common): if not file: return - exists, idDict = isDownloaded(file, data["url"]) + exists, idDict = is_downloaded(file, data["url"]) if exists or "archive_id" not in idDict or idDict["archive_id"] is None: return - with open(file, "a") as f: - f.write(f"{idDict['archive_id']}\n") + async with await anyio.open_file(file, "a") as f: + await f.write(f"{idDict['archive_id']}\n") manual_archive = self.config.manual_archive if manual_archive: previouslyArchived = False if os.path.exists(manual_archive): - with open(manual_archive, "r") as f: - for line in f.readlines(): + async with await anyio.open_file(manual_archive) as f: + async for line in f: if idDict["archive_id"] in line: previouslyArchived = True break if not previouslyArchived: - with open(manual_archive, "a") as f: - f.write(f"{idDict['archive_id']} - at: {datetime.now().isoformat()}\n") + async with await anyio.open_file(manual_archive, "a") as f: + await f.write(f"{idDict['archive_id']} - at: {datetime.now(UTC).isoformat()}\n") LOG.info(f"Archiving url '{data['url']}' with id '{idDict['archive_id']}'.") - @ws_event # type: ignore + @ws_event async def connect(self, sid: str, _=None): data = { **self.queue.get(), "config": self.config.frontend(), "presets": self.config.presets, - "paused": self.queue.isPaused(), + "paused": self.queue.is_paused(), } # get download folder listing. @@ -269,13 +276,13 @@ class HttpSocket(common): await self.emitter.emit(event=Events.INITIAL_DATA, data=data, to=sid) - @ws_event # type: ignore - async def pause(self, sid: str, _=None): + @ws_event + async def pause(self, *_, **__): self.queue.pause() await self.emitter.emit(event=Events.PAUSED, data={"paused": True, "at": time.time()}) - @ws_event # type: ignore - async def resume(self, sid: str, _=None): + @ws_event + async def resume(self, *_, **__): self.queue.resume() await self.emitter.emit(event=Events.PAUSED, data={"paused": False, "at": time.time()}) @@ -293,9 +300,10 @@ class HttpSocket(common): try: await self.emitter.emit(event=Events.YTDLP_CONVERT, data=arg_converter(args), to=sid) - return except Exception as e: err = str(e).strip() err = err.split("\n")[-1] if "\n" in err else err LOG.error(f"Failed to convert args. '{err}'.") await self.emitter.error(f"Failed to convert options. '{err}'.", to=sid) + + return diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index 29045fe2..a3020545 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -30,7 +30,7 @@ class ItemDTO: config: dict = field(default_factory=dict) template: str | None = None template_chapter: str | None = None - timestamp: float = time.time_ns() + timestamp: float = field(default_factory=lambda: time.time_ns()) is_live: bool | None = None datetime: str = field(default_factory=lambda: str(formatdate(time.time()))) live_in: str | None = None @@ -92,3 +92,6 @@ class ItemDTO: def get(self, key: str, default: Any = None) -> Any: return self.__dict__.get(key, default) + + def get_id(self) -> str: + return self._id diff --git a/app/library/M3u8.py b/app/library/M3u8.py index bfb90843..e72a2a5c 100644 --- a/app/library/M3u8.py +++ b/app/library/M3u8.py @@ -1,33 +1,27 @@ import math import os from urllib.parse import quote -from .Utils import calcDownloadPath, StreamingError + from .ffprobe import ffprobe +from .Utils import StreamingError, calc_download_path class M3u8: duration: float = 6.000000 - ok_vcodecs: tuple = ( - "h264", - "x264", - "avc", - ) - ok_acodecs: tuple = ( - "aac", - "m4a", - "mp3", - ) + ok_vcodecs: tuple = ("h264", "x264", "avc") + ok_acodecs: tuple = ("aac", "m4a", "mp3") - def __init__(self, url: str, segment_duration: float = None): + def __init__(self, url: str, segment_duration: float | None = None): self.url = url self.duration = float(segment_duration) if segment_duration is not None else self.duration async def make_stream(self, download_path: str, file: str) -> str: - realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) + realFile: str = calc_download_path(base_path=download_path, folder=file, create_path=False) if not os.path.exists(realFile): - raise StreamingError(f"File '{realFile}' does not exist.") + error = f"File '{realFile}' does not exist." + raise StreamingError(error) try: ff = await ffprobe(realFile) @@ -35,7 +29,8 @@ class M3u8: pass if "duration" not in ff.metadata: - raise StreamingError(f"Unable to get '{realFile}' play duration.") + error = f"Unable to get '{realFile}' play duration." + raise StreamingError(error) duration: float = float(ff.metadata.get("duration")) @@ -47,22 +42,20 @@ class M3u8: m3u8.append("#EXT-X-MEDIA-SEQUENCE:0") m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD") - segmentSize: float = "{:.6f}".format(self.duration) + segmentSize: float = f"{self.duration:.6f}" splits: int = math.ceil(duration / self.duration) segmentParams: dict = {} for stream in ff.streams(): - if stream.is_video(): - if stream.codec_name not in self.ok_vcodecs: - segmentParams["vc"] = 1 - if stream.is_audio(): - if stream.codec_name not in self.ok_acodecs: - segmentParams["ac"] = 1 + if stream.is_video() and stream.codec_name not in self.ok_vcodecs: + segmentParams["vc"] = 1 + if stream.is_audio() and stream.codec_name not in self.ok_acodecs: + segmentParams["ac"] = 1 for i in range(splits): if (i + 1) == splits: - segmentSize = "{:.6f}".format(duration - (i * self.duration)) + segmentSize = f"{duration - (i * self.duration):.6f}" segmentParams.update({"sd": segmentSize}) m3u8.append(f"#EXTINF:{segmentSize},") @@ -78,10 +71,11 @@ class M3u8: return "\n".join(m3u8) async def make_subtitle(self, download_path: str, file: str, duration: float) -> str: - realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) + realFile: str = calc_download_path(base_path=download_path, folder=file, create_path=False) if not os.path.exists(realFile): - raise StreamingError(f"File '{realFile}' does not exist.") + error = f"File '{realFile}' does not exist." + raise StreamingError(error) m3u8 = [] diff --git a/app/library/Notifications.py b/app/library/Notifications.py index c10cee8d..d8922c7b 100644 --- a/app/library/Notifications.py +++ b/app/library/Notifications.py @@ -1,26 +1,26 @@ import asyncio -from datetime import datetime, timezone import json import logging import os -from typing import List, Any - from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + import httpx from .config import Config -from .ItemDTO import ItemDTO -from .Singleton import Singleton -from .Utils import validate_uuid, ag -from .version import APP_VERSION from .encoder import Encoder from .EventsSubscriber import Events +from .ItemDTO import ItemDTO +from .Singleton import Singleton +from .Utils import ag, validate_uuid +from .version import APP_VERSION LOG = logging.getLogger("notifications") @dataclass(kw_only=True) -class targetRequestHeader: +class TargetRequestHeader: """Request header details for a notification target.""" key: str @@ -37,13 +37,13 @@ class targetRequestHeader: @dataclass(kw_only=True) -class targetRequest: +class TargetRequest: """Request details for a notification target.""" type: str method: str url: str - headers: list[targetRequestHeader] = field(default_factory=list) + headers: list[TargetRequestHeader] = field(default_factory=list) def serialize(self) -> dict: return { @@ -66,8 +66,8 @@ class Target: id: str name: str - on: List[str] - request: targetRequest + on: list[str] + request: TargetRequest def serialize(self) -> dict: return { @@ -95,17 +95,12 @@ class NotificationEvents: TEST = Events.TEST @staticmethod - def getEvents() -> dict[str, str]: - data: dict[str, str] = {} - for k, v in vars(NotificationEvents).items(): - if not k.startswith("__") and not callable(v): - data[k] = v - - return data + def get_events() -> dict[str, str]: + return {k: v for k, v in vars(NotificationEvents).items() if not k.startswith("__") and not callable(v)} @staticmethod - def isValid(event: str) -> bool: - return event in NotificationEvents.getEvents().values() + def is_valid(event: str) -> bool: + return event in NotificationEvents.get_events().values() class Notification(metaclass=Singleton): @@ -147,7 +142,7 @@ class Notification(metaclass=Singleton): return Notification._instance - def getTargets(self) -> list[Target]: + def get_targets(self) -> list[Target]: """Get the list of notification targets.""" return self._targets @@ -165,16 +160,15 @@ class Notification(metaclass=Singleton): Returns: Notification: The Notification instance. - """ + """ LOG.info(f"Saving notification targets to '{self._file}'.") try: with open(self._file, "w") as f: json.dump([t.serialize() for t in targets], fp=f, indent=4) except Exception as e: LOG.exception(e) - LOG.error(f"Error saving notification targets to '{self._file}'. '{e}'") - pass + LOG.error(f"Error saving notification targets to '{self._file}'. '{e!s}'") return self @@ -192,21 +186,20 @@ class Notification(metaclass=Singleton): LOG.info(f"Loading notification targets from '{self._file}'.") try: - with open(self._file, "r") as f: + with open(self._file) as f: targets = json.load(f) except Exception as e: - LOG.error(f"Error loading notification targets from '{self._file}'. '{e}'") - pass + LOG.error(f"Error loading notification targets from '{self._file}'. '{e!s}'") for target in targets: try: try: Notification.validate(target) except ValueError as e: - LOG.error(f"Invalid notification target '{target}'. '{e}'") + LOG.error(f"Invalid notification target '{target}'. '{e!s}'") continue - target = self.makeTarget(target) + target = self.make_target(target) self._targets.append(target) @@ -214,8 +207,7 @@ class Notification(metaclass=Singleton): f"Will send '{target.on if len(target.on) > 0 else 'all'}' as {target.request.type} notification events to '{target.name}'." ) except Exception as e: - LOG.error(f"Error loading notification target '{target}'. '{e}'") - pass + LOG.error(f"Error loading notification target '{target}'. '{e!s}'") return self @@ -229,46 +221,57 @@ class Notification(metaclass=Singleton): Returns: bool: True if the target is valid, False otherwise. - """ + """ if not isinstance(target, dict): target = target.serialize() if "id" not in target or validate_uuid(target["id"], version=4) is False: - raise ValueError("Invalid notification target. No ID found.") + msg = "Invalid notification target. No ID found." + raise ValueError(msg) if "name" not in target: - raise ValueError("Invalid notification target. No name found.") + msg = "Invalid notification target. No name found." + raise ValueError(msg) if "request" not in target: - raise ValueError("Invalid notification target. No request details found.") + msg = "Invalid notification target. No request details found." + raise ValueError(msg) if "url" not in target["request"]: - raise ValueError("Invalid notification target. No URL found.") + msg = "Invalid notification target. No URL found." + raise ValueError(msg) if "method" in target["request"] and target["request"]["method"].upper() not in ["POST", "PUT"]: - raise ValueError("Invalid notification target. Invalid method found.") + msg = "Invalid notification target. Invalid method found." + raise ValueError(msg) if "type" in target["request"] and target["request"]["type"].lower() not in ["json", "form"]: - raise ValueError("Invalid notification target. Invalid type found.") + msg = "Invalid notification target. Invalid type found." + raise ValueError(msg) if "on" in target: if not isinstance(target["on"], list): - raise ValueError("Invalid notification target. Invalid 'on' event list found.") + msg = "Invalid notification target. Invalid 'on' event list found." + raise ValueError(msg) for e in target["on"]: - if e not in NotificationEvents.getEvents().values(): - raise ValueError(f"Invalid notification target. Invalid event '{e}' found.") + if e not in NotificationEvents.get_events().values(): + msg = f"Invalid notification target. Invalid event '{e}' found." + raise ValueError(msg) if "headers" in target["request"]: if not isinstance(target["request"]["headers"], list): - raise ValueError("Invalid notification target. Invalid headers list found.") + msg = "Invalid notification target. Invalid headers list found." + raise ValueError(msg) for h in target["request"]["headers"]: if "key" not in h: - raise ValueError("Invalid notification target. No header key found.") + msg = "Invalid notification target. No header key found." + raise ValueError(msg) if "value" not in h: - raise ValueError("Invalid notification target. No header value found.") + msg = "Invalid notification target. No header value found." + raise ValueError(msg) return True @@ -315,7 +318,7 @@ class Notification(metaclass=Singleton): reqBody["json" if "json" == target.request.type.lower() else "data"] = { "event": event, - "created_at": datetime.now(tz=timezone.utc).isoformat(), + "created_at": datetime.now(tz=UTC).isoformat(), "payload": item.__dict__ if isinstance(item, ItemDTO) else item, } @@ -337,7 +340,7 @@ class Notification(metaclass=Singleton): LOG.error(f"Error sending Notification event '{event}' id '{itemId}' to '{target.name}'. '{e}'.") return {"url": target.request.url, "status": 500, "text": str(e)} - def makeTarget(self, target: dict) -> Target: + def make_target(self, target: dict) -> Target: """ Make a notification target from a dictionary. @@ -346,27 +349,28 @@ class Notification(metaclass=Singleton): Returns: Target: The notification target. + """ return Target( id=target.get("id"), name=target.get("name"), on=target.get("on", []), - request=targetRequest( + request=TargetRequest( type=target.get("request", {}).get("type", "json"), method=target.get("request", {}).get("method", "POST"), url=target.get("request", {}).get("url"), headers=[ - targetRequestHeader(key=h.get("key"), value=h.get("value")) + TargetRequestHeader(key=h.get("key"), value=h.get("value")) for h in target.get("request", {}).get("headers", []) ], ), ) - def emit(self, event, data, **kwargs): + def emit(self, event, data, **kwargs): # noqa: ARG002 if len(self._targets) < 1: return False - if not NotificationEvents.isValid(event): + if not NotificationEvents.is_valid(event): return False return self.send(event, data) diff --git a/app/library/PackageInstaller.py b/app/library/PackageInstaller.py index 2c71ece4..3ceb01c9 100644 --- a/app/library/PackageInstaller.py +++ b/app/library/PackageInstaller.py @@ -1,8 +1,9 @@ +import importlib import logging import os import subprocess import sys -import importlib + from library.config import Config LOG = logging.getLogger("package_installer") @@ -24,8 +25,8 @@ class PackageInstaller: return [] if os.access(file_path, os.R_OK): - with open(file_path, "r") as f: - return [pkg.strip() for pkg in f.readlines() if pkg.strip()] + with open(file_path) as f: + return [pkg.strip() for pkg in f if pkg.strip()] else: LOG.error(f"Could not read pip packages from '{file_path}'.") return [] @@ -37,10 +38,10 @@ class PackageInstaller: LOG.info(f"'{pkg}' is already installed. Skipping upgrades. as requested.") return LOG.info(f"'{pkg}' is already installed. Checking for upgrades...") - subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", pkg], check=True) + subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", pkg], check=True) # noqa: S603 except ImportError: LOG.info(f"'{pkg}' is not installed. Installing...") - subprocess.run([sys.executable, "-m", "pip", "install", pkg], check=True) + subprocess.run([sys.executable, "-m", "pip", "install", pkg], check=True) # noqa: S603 def check(self): """ @@ -57,5 +58,5 @@ class PackageInstaller: try: self.action(package) except Exception as e: - LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {str(e)}") + LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}") LOG.exception(e) diff --git a/app/library/Playlist.py b/app/library/Playlist.py index 11337602..a83074ca 100644 --- a/app/library/Playlist.py +++ b/app/library/Playlist.py @@ -2,10 +2,12 @@ import glob import re from pathlib import Path from urllib.parse import quote -from aiohttp.web import Response + +from aiohttp.web import HTTPFound, Response + from .ffprobe import ffprobe from .Subtitle import Subtitle -from .Utils import calcDownloadPath, checkId, StreamingError +from .Utils import StreamingError, calc_download_path, check_id class Playlist: @@ -15,15 +17,16 @@ class Playlist: self.url = url async def make(self, download_path: str, file: str) -> str | Response: - rFile = Path(calcDownloadPath(basePath=download_path, folder=file, createPath=False)) + rFile = Path(calc_download_path(base_path=download_path, folder=file, create_path=False)) if not rFile.exists(): - possibleFile = checkId(download_path, rFile) + possibleFile = check_id(file=rFile) if not possibleFile: - raise StreamingError(f"File '{rFile}' does not exist.") + msg = f"File '{rFile}' does not exist." + raise StreamingError(msg) return Response( - status=302, + status=HTTPFound.status_code, headers={ "Location": f"{self.url}api/player/playlist/{quote(str(possibleFile).replace(download_path, '').strip('/'))}.m3u8" }, @@ -35,7 +38,8 @@ class Playlist: pass if "duration" not in ff.metadata: - raise StreamingError(f"Unable to get '{rFile}' duration.") + msg = f"Unable to get '{rFile}' duration." + raise StreamingError(msg) duration: float = float(ff.metadata.get("duration")) @@ -45,8 +49,8 @@ class Playlist: subs = "" index = 0 - for item in self.getSideCarFiles(rFile): - if item.suffix not in Subtitle.allowedExtensions: + for item in self.get_sidecar_files(rFile): + if item.suffix not in Subtitle.allowed_extensions: continue index += 1 @@ -68,7 +72,7 @@ class Playlist: return "\n".join(playlist) - def getSideCarFiles(self, file: Path) -> list[Path]: + def get_sidecar_files(self, file: Path) -> list[Path]: """ Get sidecar files for the given file. diff --git a/app/library/Segments.py b/app/library/Segments.py index 8c96f3e4..b45d894d 100644 --- a/app/library/Segments.py +++ b/app/library/Segments.py @@ -4,9 +4,9 @@ import logging import os import tempfile -from .ffprobe import ffprobe from .config import Config -from .Utils import calcDownloadPath, StreamingError +from .ffprobe import ffprobe +from .Utils import StreamingError, calc_download_path LOG = logging.getLogger("player.segments") @@ -25,10 +25,11 @@ class Segments: self.aconvert = True async def stream(self, path: str, file: str) -> bytes: - realFile: str = calcDownloadPath(basePath=path, folder=file, createPath=False) + realFile: str = calc_download_path(base_path=path, folder=file, create_path=False) if not os.path.exists(realFile): - raise StreamingError(f"File {realFile} does not exist.") + msg = f"File {realFile} does not exist." + raise StreamingError(msg) try: ff = await ffprobe(realFile) @@ -36,15 +37,15 @@ class Segments: pass tmpDir: str = tempfile.gettempdir() - tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.md5(realFile.encode("utf-8")).hexdigest()}') + tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.sha256(realFile.encode("utf-8")).hexdigest()}') if not os.path.exists(tmpFile): os.symlink(realFile, tmpFile) if self.index == 0: - startTime: float = "{:.6f}".format(0) + startTime: float = f"{0:.6f}" else: - startTime: float = "{:.6f}".format((self.duration * self.index)) + startTime: float = f"{self.duration * self.index:.6f}" fargs = [] fargs.append("-xerror") @@ -55,7 +56,7 @@ class Segments: fargs.append("-ss") fargs.append(str(startTime if startTime else "0.00000")) fargs.append("-t") - fargs.append(str("{:.6f}".format(self.duration))) + fargs.append(str(f"{self.duration:.6f}")) fargs.append("-copyts") @@ -107,7 +108,8 @@ class Segments: data, err = await proc.communicate() if 0 != proc.returncode: - LOG.error(f'Failed to stream {realFile} segment {self.index}. {err.decode("utf-8")}.') - raise StreamingError(f"Failed to stream {realFile} segment {self.index}.") + LOG.error(f"Failed to stream '{realFile}' segment '{self.index}'. {err.decode('utf-8')}.") + msg = f"Failed to stream '{realFile}' segment '{self.index}'." + raise StreamingError(msg) return data diff --git a/app/library/Singleton.py b/app/library/Singleton.py index d786bbf4..96e983b0 100644 --- a/app/library/Singleton.py +++ b/app/library/Singleton.py @@ -1,5 +1,5 @@ -from typing import Any, Dict import threading +from typing import Any class Singleton(type): @@ -7,7 +7,7 @@ class Singleton(type): A metaclass that creates a Singleton base class when called. """ - _instances: Dict[type, Any] = {} + _instances: dict[type, Any] = {} def __call__(cls, *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: @@ -16,12 +16,12 @@ class Singleton(type): return cls._instances[cls] -class threadSafe(type): +class ThreadSafe(type): """ A metaclass that creates a Singleton base class when called. """ - _instances: Dict[type, Any] = {} + _instances: dict[type, Any] = {} _lock = threading.Lock() def __call__(cls, *args: Any, **kwargs: Any) -> Any: diff --git a/app/library/Subtitle.py b/app/library/Subtitle.py index 589fd9c6..31742d45 100644 --- a/app/library/Subtitle.py +++ b/app/library/Subtitle.py @@ -1,9 +1,12 @@ import logging -from .Utils import calcDownloadPath import pathlib + +import anyio import pysubs2 -from pysubs2.time import ms_to_times from pysubs2.formats.substation import SubstationFormat +from pysubs2.time import ms_to_times + +from .Utils import calc_download_path LOG = logging.getLogger("player.subtitle") @@ -19,34 +22,34 @@ SubstationFormat.ms_to_timestamp = ms_to_timestamp class Subtitle: - allowedExtensions: tuple[str] = ( + allowed_extensions: tuple[str] = ( ".srt", ".vtt", ".ass", ) async def make(self, path: str, file: str) -> str: - realFile: str = calcDownloadPath(basePath=path, folder=file, createPath=False) + realFile: str = calc_download_path(base_path=path, folder=file, create_path=False) rFile = pathlib.Path(realFile) if not rFile.exists(): - raise Exception(f"File '{file}' does not exist.") + msg = f"File '{file}' does not exist." + raise Exception(msg) - if rFile.suffix not in self.allowedExtensions: - raise Exception(f"File '{file}' subtitle type is not supported.") + if rFile.suffix not in self.allowed_extensions: + msg = f"File '{file}' subtitle type is not supported." + raise Exception(msg) if rFile.suffix == ".vtt": - subData = "" - with open(realFile, "r") as f: - subData = f.read() - - return subData + async with await anyio.open_file(realFile) as f: + return await f.read() subs = pysubs2.load(path=str(rFile)) if len(subs.events) < 1: - raise Exception(f"No subtitle events were found in '{rFile}'.") + msg = f"No subtitle events were found in '{rFile}'." + raise Exception(msg) if len(subs.events) < 2: return subs.to_string("vtt") diff --git a/app/library/Tasks.py b/app/library/Tasks.py index 37bfe4c2..35b52ddc 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -1,11 +1,11 @@ import asyncio -from datetime import datetime import json import logging import os import time from dataclasses import dataclass, field -from typing import Any, List +from datetime import UTC, datetime +from typing import Any import httpx from aiocron import Cron, crontab @@ -55,7 +55,7 @@ class Tasks(metaclass=Singleton): This class is used to manage the tasks. """ - _jobs: List[Job] = [] + _jobs: list[Job] = [] """The jobs for the tasks.""" _instance = None @@ -100,12 +100,24 @@ class Tasks(metaclass=Singleton): Returns: Tasks: The instance of the Tasks - """ + """ if not Tasks._instance: Tasks._instance = Tasks() return Tasks._instance + async def on_shutdown(self, _: web.Application): + """ + Shutdown the socket server. + + Args: + _: The aiohttp application. + + """ + LOG.debug("Shutting down tasks runner.") + self.clear(shutdown=True) + LOG.debug("Tasks runner has been shut down.") + def attach(self, _: web.Application): """ Attach the tasks to the aiohttp application. @@ -115,16 +127,11 @@ class Tasks(metaclass=Singleton): Returns: None + """ self.load() - def shutdown(self): - """ - Shutdown the tasks service. - """ - self.clear() - - def getTasks(self) -> List[Task]: + def get_tasks(self) -> list[Task]: """Return the tasks.""" return [job.task for job in self._jobs] @@ -134,6 +141,7 @@ class Tasks(metaclass=Singleton): Returns: Tasks: The current instance. + """ self.clear() @@ -142,7 +150,7 @@ class Tasks(metaclass=Singleton): LOG.info(f"Loading tasks from '{self._file}'.") try: - with open(self._file, "r") as f: + with open(self._file) as f: tasks = json.load(f) except Exception as e: LOG.error(f"Failed to parse tasks from '{self._file}'. '{e}'.") @@ -156,7 +164,7 @@ class Tasks(metaclass=Singleton): try: task = Task(**task) except Exception as e: - LOG.error(f"Failed to parse task at list position '{i}'. '{str(e)}'.") + LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.") continue try: @@ -171,16 +179,17 @@ class Tasks(metaclass=Singleton): LOG.info(f"Task '{i}: {task.name}' queued to be executed every '{task.timer}'.") except Exception as e: LOG.exception(e) - LOG.error(f"Failed to queue task '{i}: {task['name']}'. '{str(e)}'.") + LOG.error(f"Failed to queue task '{i}: {task['name']}'. '{e!s}'.") return self - def clear(self) -> "Tasks": + def clear(self, shutdown: bool = False) -> "Tasks": """ Clear all tasks. Returns: Tasks: The current instance. + """ if len(self._jobs) < 1: return self @@ -190,8 +199,9 @@ class Tasks(metaclass=Singleton): LOG.info(f"Stopping job '{task.id}: {task.name}'.") task.job.stop() except Exception as e: - LOG.exception(e) - LOG.error(f"Failed to stop job '{task.id}: {task.name}'. '{str(e)}'.") + if not shutdown: + LOG.exception(e) + LOG.error(f"Failed to stop job '{task.id}: {task.name}'. '{e!s}'.") self._jobs.clear() @@ -202,56 +212,63 @@ class Tasks(metaclass=Singleton): Validate the task. Args: - tasks (Task|dict): The task to validate. + task (Task|dict): The task to validate. Returns: bool: True if the task is valid, False otherwise. - """ + """ if not isinstance(task, dict): if not isinstance(task, Task): - raise ValueError("Invalid task type.") + msg = "Invalid task type." + raise ValueError(msg) # noqa: TRY004 task = task.serialize() if not task.get("name"): - raise ValueError("No name found.") + msg = "No name found." + raise ValueError(msg) if not task.get("timer"): - raise ValueError("No timer found.") + msg = "No timer found." + raise ValueError(msg) if not task.get("url"): - raise ValueError("No URL found.") + msg = "No URL found." + raise ValueError(msg) if not isinstance(task.get("cookies"), str): - raise ValueError("Invalid cookies type.") + msg = "Invalid cookies type." + raise ValueError(msg) # noqa: TRY004 if not isinstance(task.get("config"), dict): - raise ValueError("Invalid config type.") + msg = "Invalid config type." + raise ValueError(msg) # noqa: TRY004 if not isinstance(task.get("template"), str): - raise ValueError("Invalid template type.") + msg = "Invalid template type." + raise ValueError(msg) # noqa: TRY004 return True - def save(self, tasks: List[Task | dict]) -> "Tasks": + def save(self, tasks: list[Task | dict]) -> "Tasks": """ Save the tasks. Args: - tasks (List[Task]): The tasks to save. + tasks (list[Task]): The tasks to save. Returns: Tasks: The current instance. - """ + """ for i, task in enumerate(tasks): try: if not isinstance(task, Task): task = Task(**task) tasks[i] = task except Exception as e: - LOG.error(f"Failed to save task '{i}' unable to parse task. '{str(e)}'.") + LOG.error(f"Failed to save task '{i}' unable to parse task. '{e!s}'.") continue try: @@ -266,7 +283,7 @@ class Tasks(metaclass=Singleton): LOG.info(f"Tasks saved to '{self._file}'.") except Exception as e: - LOG.error(f"Failed to save tasks to '{self._file}'. '{str(e)}'.") + LOG.error(f"Failed to save tasks to '{self._file}'. '{e!s}'.") return self @@ -279,9 +296,10 @@ class Tasks(metaclass=Singleton): Returns: None + """ try: - timeNow = datetime.now().isoformat() + timeNow = datetime.now(UTC).isoformat() started = time.time() if not task.url: LOG.error(f"Failed to dispatch task '{task.id}: {task.name}'. No URL found.") @@ -297,7 +315,7 @@ class Tasks(metaclass=Singleton): try: config = json.loads(config) except Exception as e: - LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {str(e)}") + LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {e!s}") return LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.") @@ -324,13 +342,13 @@ class Tasks(metaclass=Singleton): await asyncio.wait_for(asyncio.gather(*tasks), timeout=None) - timeNow = datetime.now().isoformat() + timeNow = datetime.now(UTC).isoformat() ended = time.time() LOG.info(f"Task '{task.id}: {task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") await self._emitter.success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds.") except Exception as e: - timeNow = datetime.now().isoformat() - LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{str(e)}'.") - await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{str(e)}'.") + timeNow = datetime.now(UTC).isoformat() + LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{e!s}'.") + await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{e!s}'.") diff --git a/app/library/Utils.py b/app/library/Utils.py index ab82ca81..951650ea 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -8,7 +8,7 @@ import re import shlex import socket import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from functools import lru_cache from typing import Any @@ -26,14 +26,11 @@ IGNORED_KEYS: tuple[str] = ( "download_archive", ) YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None -OS_ALT_SEP: list[str] = list(sep for sep in [os.sep, os.path.altsep] if sep is not None and sep != "/") class StreamingError(Exception): """Raised when an error occurs during streaming.""" - pass - def get_opts(preset: str, ytdl_opts: dict) -> dict: """ @@ -45,6 +42,7 @@ def get_opts(preset: str, ytdl_opts: dict) -> dict: Returns: ytdl extra options + """ opts = copy.deepcopy(ytdl_opts) @@ -80,7 +78,7 @@ def get_opts(preset: str, ytdl_opts: dict) -> dict: return opts -def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> Any | dict[str, Any] | None: +def get_video_info(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> Any | dict[str, Any] | None: """ Extracts video information from the given URL. @@ -91,6 +89,7 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> Returns: dict: Video information. + """ params: dict = { "quiet": True, @@ -110,28 +109,34 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> return yt_dlp.YoutubeDL().extract_info(url, download=False) -def calcDownloadPath(basePath: str, folder: str | None = None, createPath: bool = True) -> str: - """Calculates download path and prevents folder traversal. +def calc_download_path(base_path: str, folder: str | None = None, create_path: bool = True) -> str: + """ + Calculates download path and prevents folder traversal. Returns: Download path with base folder factored in. + """ if not folder: - return basePath + return base_path - realBasePath = os.path.realpath(basePath) - download_path = os.path.realpath(os.path.join(basePath, folder)) + if folder.startswith("/"): + folder = folder[1:] + + realBasePath = os.path.realpath(base_path) + download_path = os.path.realpath(os.path.join(base_path, folder)) if not download_path.startswith(realBasePath): - raise Exception(f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".') + msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".' + raise Exception(msg) - if not os.path.isdir(download_path) and createPath: + if not os.path.isdir(download_path) and create_path: os.makedirs(download_path, exist_ok=True) return download_path -def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict: +def extract_info(config: dict, url: str, debug: bool = False) -> dict: params: dict = { "color": "no_color", "extract_flat": True, @@ -163,14 +168,14 @@ def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict: return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) -def mergeDict(source: dict, destination: dict) -> dict: +def merge_dict(source: dict, destination: dict) -> dict: """Merge data from source into destination""" destination_copy = copy.deepcopy(destination) for key, value in source.items(): destination_key_value = destination_copy.get(key) if isinstance(value, dict) and isinstance(destination_key_value, dict): - destination_copy[key] = mergeDict(source=value, destination=destination_copy.setdefault(key, {})) + destination_copy[key] = merge_dict(source=value, destination=destination_copy.setdefault(key, {})) elif isinstance(value, list) and isinstance(destination_key_value, list): destination_copy[key] = destination_key_value + value else: @@ -179,7 +184,7 @@ def mergeDict(source: dict, destination: dict) -> dict: return destination_copy -def mergeConfig(config: dict, new_config: dict) -> dict: +def merge_config(config: dict, new_config: dict) -> dict: """ Merge user provided config into default config @@ -189,13 +194,14 @@ def mergeConfig(config: dict, new_config: dict) -> dict: Returns: dict: Merged config + """ for key in IGNORED_KEYS: if key in new_config: LOG.error(f"Key '{key}' is not allowed to be manually set.") del new_config[key] - conf = mergeDict(new_config, config) + conf = merge_dict(new_config, config) if "impersonate" in conf: conf["impersonate"] = ImpersonateTarget.from_str(conf["impersonate"]) @@ -203,8 +209,8 @@ def mergeConfig(config: dict, new_config: dict) -> dict: return conf -def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]: - global YTDLP_INFO_CLS +def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]: + global YTDLP_INFO_CLS # noqa: PLW0603 idDict = { "id": None, @@ -247,35 +253,26 @@ def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, st break if not idDict["archive_id"]: - return ( - False, - idDict, - ) + return (False, idDict) - with open(archive_file, "r") as f: - for line in f.readlines(): + with open(archive_file) as f: + for line in f: if idDict["archive_id"] in line: - return ( - True, - idDict, - ) + return (True, idDict) - return ( - False, - idDict, - ) + return (False, idDict) -def jsonCookie(cookies: dict[dict[str, any]]) -> str | None: - """Converts JSON cookies to Netscape cookies +def json_cookie(cookies: dict[dict[str, any]]) -> str | None: + """ + Converts JSON cookies to Netscape cookies Returns None if no cookies are found, otherwise returns a string of cookies in Netscape format. """ - netscapeCookies = "# Netscape HTTP Cookie File\n# https://curl.haxx.se/docs/http-cookies.html\n# This file was generated by libcurl! Edit at your own risk.\n\n" hasCookies: bool = False - for domain in cookies: + for domain in cookies: # noqa: PLC0206 if not isinstance(cookies[domain], dict): continue @@ -290,7 +287,7 @@ def jsonCookie(cookies: dict[dict[str, any]]) -> str | None: cookieDict = cookies[domain][subDomain][cookie] if 0 == int(cookieDict["expirationDate"]): - cookieDict["expirationDate"] = datetime.now(timezone.utc).timestamp() + (86400 * 1000) + cookieDict["expirationDate"] = datetime.now(UTC).timestamp() + (86400 * 1000) hasCookies = True netscapeCookies += ( @@ -325,13 +322,14 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded. bool: True if the file was loaded successfully. str: Error message if the file could not be loaded. + """ try: with open(file) as json_data: opts = json.load(json_data) if check_type: - assert isinstance(opts, check_type) + assert isinstance(opts, check_type) # noqa: S101 return ( opts, @@ -346,7 +344,7 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: opts = json5_load(json_data) if check_type: - assert isinstance(opts, check_type) + assert isinstance(opts, check_type) # noqa: S101 return ( opts, @@ -367,7 +365,7 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: ) -def checkId(basePath: str, file: pathlib.Path) -> bool | str: +def check_id(file: pathlib.Path) -> bool | str: """ Check if we are able to get an id from the file name. if so check if any video file with the same id exists. @@ -377,7 +375,6 @@ def checkId(basePath: str, file: pathlib.Path) -> bool | str: :return: False if no id found, otherwise the id. """ - match = re.search(r"(?<=\[)(?:youtube-)?(?P[a-zA-Z0-9\-_]{11})(?=\])", file.stem, re.IGNORECASE) if not match: return False @@ -388,16 +385,7 @@ def checkId(basePath: str, file: pathlib.Path) -> bool | str: if id not in f.stem: continue - if f.suffix not in ( - ".mp4", - ".mkv", - ".webm", - ".m4v", - ".m4a", - ".mp3", - ".aac", - ".ogg", - ): + if f.suffix not in (".mp4", ".mkv", ".webm", ".m4v", ".m4a", ".mp3", ".aac", ".ogg"): continue return f.absolute() @@ -422,7 +410,6 @@ def ag(array: dict | list, path: list[str | int] | str | int, default: Any = Non :param separator: Separator for nested paths in strings. :return: The found value or the default if not found. """ - if path is None or path == "": return array @@ -482,24 +469,29 @@ def validate_url(url: str) -> bool: Returns: bool: True if the URL is valid and allowed. + """ if not url: - raise ValueError("URL is required.") + msg = "URL is required." + raise ValueError(msg) try: from yarl import URL parsed_url = URL(url) except ValueError: - raise ValueError("Invalid URL.") + msg = "Invalid URL." + raise ValueError(msg) # noqa: B904 # Check allowed schemes if parsed_url.scheme not in ["http", "https"]: - raise ValueError("Invalid scheme usage. Only HTTP or HTTPS allowed.") + msg = "Invalid scheme usage. Only HTTP or HTTPS allowed." + raise ValueError(msg) hostname = parsed_url.host if not hostname or is_private_address(hostname): - raise ValueError("Access to internal urls or private networks is not allowed.") + msg = "Access to internal urls or private networks is not allowed." + raise ValueError(msg) return True @@ -509,16 +501,17 @@ def arg_converter(args: str) -> dict: Convert yt-dlp options to a dictionary. Args: - opts (str): yt-dlp options string. + args (str): yt-dlp options string. Returns: dict: yt-dlp options dictionary. + """ import yt_dlp.options create_parser = yt_dlp.options.create_parser - def defaultOpts(args: str): + def _default_opts(args: str): patched_parser = create_parser() try: yt_dlp.options.create_parser = lambda: patched_parser @@ -526,7 +519,7 @@ def arg_converter(args: str) -> dict: finally: yt_dlp.options.create_parser = create_parser - default_opts = defaultOpts([]).ydl_opts + default_opts = _default_opts([]).ydl_opts opts = yt_dlp.parse_options(shlex.split(args)).ydl_opts @@ -543,9 +536,11 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool: Args: uuid_str (str): UUID to validate. + version (int): The UUID version Returns: bool: True if the UUID is valid. + """ try: uuid.UUID(uuid_str, version=version) diff --git a/app/library/cache.py b/app/library/cache.py index b2e33f02..c35830bc 100644 --- a/app/library/cache.py +++ b/app/library/cache.py @@ -1,12 +1,12 @@ import hashlib import threading import time -from typing import Any, Dict, Optional, Tuple +from typing import Any -from .Singleton import threadSafe +from .Singleton import ThreadSafe -class Cache(metaclass=threadSafe): +class Cache(metaclass=ThreadSafe): def __init__(self) -> None: """ Initialize the Cache. @@ -14,11 +14,11 @@ class Cache(metaclass=threadSafe): # Prevent reinitialization in singleton context. if hasattr(self, "_initialized") and self._initialized: return - self._cache: Dict[str, Tuple[Any, Optional[float]]] = {} + self._cache: dict[str, tuple[Any, float | None]] = {} self._lock = threading.Lock() self._initialized = True - def set(self, key: str, value: Any, ttl: Optional[float] = None) -> None: + def set(self, key: str, value: Any, ttl: float | None = None) -> None: """ Synchronously set a value in the cache with an optional time-to-live. If ttl is None, the entry never expires. @@ -27,7 +27,7 @@ class Cache(metaclass=threadSafe): with self._lock: self._cache[key] = (value, expire_at) - def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]: + def get(self, key: str, default: Any | None = None) -> Any | None: """ Synchronously retrieve a value from the cache if it exists and hasn't expired. """ @@ -81,13 +81,13 @@ class Cache(metaclass=threadSafe): return hashlib.sha256(key.encode("utf-8")).hexdigest() # Asynchronous counterparts for non-blocking interfaces - async def aset(self, key: str, value: Any, ttl: Optional[float] = None) -> None: + async def aset(self, key: str, value: Any, ttl: float | None = None) -> None: """ Asynchronously set a value in the cache. """ self.set(key, value, ttl) - async def aget(self, key: str, default: Optional[Any] = None) -> Optional[Any]: + async def aget(self, key: str, default: Any | None = None) -> Any | None: """ Asynchronously retrieve a value from the cache. """ diff --git a/app/library/common.py b/app/library/common.py index d02955d3..884bf1ce 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -1,14 +1,14 @@ import json import logging +from .config import Config from .DownloadQueue import DownloadQueue from .encoder import Encoder -from .config import Config LOG = logging.getLogger("common") -class common: +class Common: """ This class is used to share common methods between the socket and the API gateways. """ @@ -43,6 +43,7 @@ class common: Returns: dict[str, str]: The status of the download. { "status": "text" } + """ if cookies and isinstance(cookies, dict): cookies = self.encoder.encode(cookies) @@ -69,11 +70,13 @@ class common: Returns: dict: The formatted item + """ url: str = item.get("url") if not url: - raise ValueError("url param is required.") + msg = "url param is required." + raise ValueError(msg) preset: str = str(item.get("preset", self.config.default_preset)) folder: str = str(item.get("folder")) if item.get("folder") else "" @@ -85,9 +88,10 @@ class common: try: config = json.loads(config) except Exception as e: - raise ValueError(f"Failed to parse json yt-dlp config for '{url}'. {str(e)}") + msg = f"Failed to parse json yt-dlp config for '{url}'. {e!s}" + raise ValueError(msg) from e - item = { + return { "url": url, "preset": preset, "folder": folder, @@ -95,5 +99,3 @@ class common: "config": config if isinstance(config, dict) else {}, "template": template, } - - return item diff --git a/app/library/config.py b/app/library/config.py index 1b2e7d94..45e8b6e7 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -12,7 +12,7 @@ import coloredlogs from dotenv import load_dotenv from yt_dlp.version import __version__ as YTDLP_VERSION -from .Utils import IGNORED_KEYS, load_file, mergeDict +from .Utils import IGNORED_KEYS, load_file, merge_dict from .version import APP_VERSION @@ -113,9 +113,6 @@ class Config: ytdl_options: dict = {} "The options to use for yt-dlp." - tasks: list = [] - "The tasks to run." - new_version_available: bool = False "A new version of the application is available." @@ -215,9 +212,10 @@ class Config: def __init__(self): """Virtually private constructor.""" if Config.__instance is not None: - raise Exception("This class is a singleton. Use Config.get_instance() instead.") - else: - Config.__instance = self + msg = "This class is a singleton. Use Config.get_instance() instead." + raise Exception(msg) + + Config.__instance = self baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute()) @@ -233,7 +231,7 @@ class Config: logging.info(f"Loading environment variables from '{envFile}'.") load_dotenv(envFile) - for k, v in self._getAttributes().items(): + for k, v in self._get_attributes().items(): if k.startswith("_") or k in self._manual_vars: continue @@ -243,7 +241,7 @@ class Config: continue lookUpKey: str = f"YTP_{k}".upper() - setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v) + setattr(self, k, os.environ.get(lookUpKey, v)) for k, v in self.__dict__.items(): if k.startswith("_") or k in self._immutable or k in self._manual_vars: @@ -262,7 +260,8 @@ class Config: if k in self._boolean_vars: if str(v).lower() not in (True, False, "true", "false", "on", "off", "1", "0"): - raise ValueError(f"Config variable '{k}' is set to a non-boolean value '{v}'.") + msg = f"Config variable '{k}' is set to a non-boolean value '{v}'." + raise ValueError(msg) setattr(self, k, str(v).lower() in (True, "true", "on", "1")) @@ -271,7 +270,8 @@ class Config: numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): - raise ValueError(f"Invalid log level '{self.log_level}' specified.") + msg = f"Invalid log level '{self.log_level}' specified." + raise TypeError(msg) coloredlogs.install( level=numeric_level, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S" @@ -304,26 +304,14 @@ class Config: LOG.error(f"Key '{key}' is not allowed to be loaded via 'ytdlp.json' file.") del opts[key] - self.ytdl_options = mergeDict(self.ytdl_options, opts) + self.ytdl_options = merge_dict(self.ytdl_options, opts) else: LOG.error(f"Invalid yt-dlp custom options file '{optsFile}'.") else: LOG.info(f"No yt-dlp custom options found at '{optsFile}'.") - tasksFile = os.path.join(self.config_path, "tasks.json") - if os.path.exists(tasksFile) and os.path.getsize(tasksFile) > 0: - LOG.info(f"Loading tasks from '{tasksFile}'.") - try: - (tasks, status, error) = load_file(tasksFile, list) - if not status: - LOG.error(f"Could not load tasks file from '{tasksFile}'. '{error}'.") - sys.exit(1) - self.tasks.extend(tasks) - except Exception: - pass - # Load default presets. - with open(os.path.join(os.path.dirname(__file__), "presets.json"), "r") as f: + with open(os.path.join(os.path.dirname(__file__), "presets.json")) as f: self.presets.extend(json.load(f)) # Load user defined presets. @@ -378,11 +366,11 @@ class Config: self.started = time.time() - def _getAttributes(self) -> dict: + def _get_attributes(self) -> dict: attrs: dict = {} vClass: str = self.__class__ - for attribute in vClass.__dict__.keys(): + for attribute in vClass.__dict__: if attribute.startswith("_"): continue @@ -398,8 +386,8 @@ class Config: Returns: dict: A dictionary with the frontend configuration - """ + """ data = {k: getattr(self, k) for k in self._frontend_vars} hasCookies = self.ytdl_options.get("cookiefile", None) data["has_cookies"] = hasCookies is not None and os.path.exists(hasCookies) diff --git a/app/library/ffprobe.py b/app/library/ffprobe.py index e2f57718..b38da7a7 100644 --- a/app/library/ffprobe.py +++ b/app/library/ffprobe.py @@ -8,6 +8,8 @@ import json import operator import os +import anyio + # parameter-less decorator def async_lru_cache_decorator(async_function): @@ -85,10 +87,7 @@ class FFStream: if self.__dict__.get("codec_type", None) != "video": return False - if self.__dict__.get("codec_name", None) in ["png", "mjpeg", "gif", "bmp", "tiff", "webp"]: - return False - - return True + return self.__dict__.get("codec_name", None) not in ["png", "mjpeg", "gif", "bmp", "tiff", "webp"] def is_subtitle(self): """ @@ -115,8 +114,9 @@ class FFStream: if width and height: try: size = (int(width), int(height)) - except ValueError: - raise FFProbeError("None integer size {}:{}".format(width, height)) + except ValueError as e: + msg = f"None integer size {width}:{height}" + raise FFProbeError(msg) from e else: return None @@ -136,8 +136,9 @@ class FFStream: if self.is_video() or self.is_audio(): try: frame_count = int(self.__dict__.get("nb_frames", "")) - except ValueError: - raise FFProbeError("None integer frame count") + except ValueError as e: + msg = "None integer frame count" + raise FFProbeError(msg) from e else: frame_count = 0 @@ -151,8 +152,9 @@ class FFStream: if self.is_video() or self.is_audio(): try: duration = float(self.__dict__.get("duration", "")) - except ValueError: - raise FFProbeError("None numeric duration") + except ValueError as e: + msg = "None numeric duration" + raise FFProbeError(msg) from e else: duration = 0.0 @@ -188,8 +190,9 @@ class FFStream: """ try: return int(self.__dict__.get("bit_rate", "")) - except ValueError: - raise FFProbeError("None integer bit_rate") + except ValueError as e: + msg = "None integer bit_rate" + raise FFProbeError(msg) from e class FFProbeResult: @@ -212,19 +215,19 @@ class FFProbeResult: return getattr(self, key) if hasattr(self, key) else default def streams(self) -> list[FFStream]: - "List of all streams." + """List of all streams.""" return self.video + self.audio + self.subtitle + self.attachment def has_video(self): - "Is there a video stream?" + """Is there a video stream?""" return len(self.video) > 0 def has_audio(self): - "Is there an audio stream?" + """Is there an audio stream?""" return len(self.audio) > 0 def has_subtitle(self): - "Is there a subtitle stream?" + """Is there a subtitle stream?""" return len(self.subtitle) > 0 def __repr__(self): @@ -259,15 +262,18 @@ async def ffprobe(file: str) -> FFProbeResult: Returns: dict: A dictionary containing the parsed data. + """ try: - with open(os.devnull, "w") as tempf: + async with await anyio.open_file(os.devnull, "w") as tempf: await asyncio.create_subprocess_exec("ffprobe", "-h", stdout=tempf, stderr=tempf) - except FileNotFoundError: - raise IOError("ffprobe not found.") + except FileNotFoundError as e: + msg = "ffprobe not found." + raise OSError(msg) from e if not os.path.isfile(file): - raise IOError(f"No such media file '{file}'.") + msg = f"No such media file '{file}'." + raise OSError(msg) args = ["-v", "quiet", "-of", "json", "-show_streams", "-show_format", file] @@ -284,13 +290,14 @@ async def ffprobe(file: str) -> FFProbeResult: if 0 == exitCode: parsed: dict = json.loads(data.decode("utf-8")) else: - raise FFProbeError(f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'") + msg = f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'" + raise FFProbeError(msg) result = FFProbeResult() result.metadata = parsed.get("format", {}) - for stream in parsed.get("streams", []): - stream = FFStream(stream) + for raw_stream in parsed.get("streams", []): + stream = FFStream(raw_stream) if stream.is_audio(): result.audio.append(stream) elif stream.is_video(): diff --git a/app/main.py b/app/main.py index 7af485e5..ee857722 100644 --- a/app/main.py +++ b/app/main.py @@ -13,7 +13,6 @@ from library.config import Config from library.DownloadQueue import DownloadQueue from library.Emitter import Emitter from library.EventsSubscriber import EventsSubscriber -from library.HttpAPI import LOG as http_logger from library.HttpAPI import HttpAPI from library.HttpSocket import HttpSocket from library.Notifications import Notification @@ -25,110 +24,97 @@ MIME = magic.Magic(mime=True) class Main: - config: Config - app: web.Application - http: HttpAPI - socket: HttpSocket - def __init__(self): - self.config = Config.get_instance() + self._config = Config.get_instance() self.rootPath = str(Path(__file__).parent.absolute()) - self.app = web.Application() + self._app = web.Application() - self.checkFolders() + self._check_folders() try: - PackageInstaller(self.config).check() + PackageInstaller(self._config).check() except Exception as e: - LOG.error(f"Failed to check for packages. Error message '{str(e)}'.") LOG.exception(e) + LOG.error(f"Failed to check for packages. Error message '{e!s}'.") - caribou.upgrade(self.config.db_file, os.path.join(self.rootPath, "migrations")) + caribou.upgrade(self._config.db_file, os.path.join(self.rootPath, "migrations")) - connection = sqlite3.connect(database=self.config.db_file, isolation_level=None) + connection = sqlite3.connect(database=self._config.db_file, isolation_level=None) connection.row_factory = sqlite3.Row connection.execute("PRAGMA journal_mode=wal") + + async def _close_connection(_): + LOG.debug("Closing database connection.") + connection.close() + LOG.debug("Database connection closed.") + try: - if "600" != oct(os.stat(self.config.db_file).st_mode)[-3:]: - os.chmod(self.config.db_file, 0o600) + if "600" != oct(os.stat(self._config.db_file).st_mode)[-3:]: + os.chmod(self._config.db_file, 0o600) except Exception: pass - queue = DownloadQueue(connection=connection) - - self.http = HttpAPI(queue=queue) - self.socket = HttpSocket(queue=queue) + self._queue = DownloadQueue(connection=connection) + self._http = HttpAPI(queue=self._queue) + self._socket = HttpSocket(queue=self._queue) Emitter.get_instance().add_emitter([Notification().emit], local=False).add_emitter( [EventsSubscriber().emit], local=True ) - self.app.on_startup.append(lambda _: queue.initialize()) + self._app.on_cleanup.append(_close_connection) - def checkFolders(self): - """ - Check if the required folders exist and create them if they do not. - """ - try: - LOG.debug(f"Checking download folder at '{self.config.download_path}'.") - if not os.path.exists(self.config.download_path): - LOG.info(f"Creating download folder at '{self.config.download_path}'.") - os.makedirs(self.config.download_path, exist_ok=True) - except OSError as e: - LOG.error(f"Could not create download folder at '{self.config.download_path}'.") - raise e + def _check_folders(self): + """Check if the required folders exist and create them if they do not.""" + folders = (self._config.download_path, self._config.temp_path, self._config.config_path) + + for folder in folders: + try: + LOG.debug(f"Checking folder at '{folder}'.") + if not os.path.exists(folder): + LOG.info(f"Creating folder at '{folder}'.") + os.makedirs(folder, exist_ok=True) + except OSError: + LOG.error(f"Could not create folder at '{folder}'.") + raise try: - LOG.debug(f"Checking temp folder at '{self.config.temp_path}'.") - if not os.path.exists(self.config.temp_path): - LOG.info(f"Creating temp folder at '{self.config.temp_path}'.") - os.makedirs(self.config.temp_path, exist_ok=True) - except OSError as e: - LOG.error(f"Could not create temp folder at '{self.config.temp_path}'.") - raise e - - try: - LOG.debug(f"Checking config folder at '{self.config.config_path}'.") - if not os.path.exists(self.config.config_path): - LOG.info(f"Creating config folder at '{self.config.config_path}'.") - os.makedirs(self.config.config_path, exist_ok=True) - except OSError as e: - LOG.error(f"Could not create config folder at '{self.config.config_path}'.") - raise e - - try: - LOG.debug(f"Checking database file at '{self.config.db_file}'.") - if not os.path.exists(self.config.db_file): - LOG.info(f"Creating database file at '{self.config.db_file}'.") - with open(self.config.db_file, "w") as _: + LOG.debug(f"Checking database file at '{self._config.db_file}'.") + if not os.path.exists(self._config.db_file): + LOG.info(f"Creating database file at '{self._config.db_file}'.") + with open(self._config.db_file, "w") as _: pass - except OSError as e: - LOG.error(f"Could not create database file at '{self.config.db_file}'.") - raise e + except OSError: + LOG.error(f"Could not create database file at '{self._config.db_file}'.") + raise def start(self): """ Start the application. """ - self.socket.attach(self.app) - self.http.attach(self.app) - Tasks.get_instance().attach(self.app) + self._socket.attach(self._app) + self._http.attach(self._app) + self._queue.attach(self._app) + Tasks.get_instance().attach(self._app) def started(_): LOG.info("=" * 40) - LOG.info(f"YTPTube v{self.config.version} - started on http://{self.config.host}:{self.config.port}") + LOG.info(f"YTPTube v{self._config.version} - started on http://{self._config.host}:{self._config.port}") LOG.info("=" * 40) - if self.config.access_log: - http_logger.addFilter(lambda record: "GET /api/ping" not in record.getMessage()) + HTTP_LOGGER = None + if self._config.access_log: + from library.HttpAPI import LOG as HTTP_LOGGER + + HTTP_LOGGER.addFilter(lambda record: "GET /api/ping" not in record.getMessage()) web.run_app( - self.app, - host=self.config.host, - port=self.config.port, + self._app, + host=self._config.host, + port=self._config.port, reuse_port=True, loop=asyncio.get_event_loop(), - access_log=http_logger if self.config.access_log else None, + access_log=HTTP_LOGGER, print=started, ) diff --git a/entrypoint.sh b/entrypoint.sh index 94a38918..2706d0ee 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,4 +19,17 @@ if [ ! -w "${YTP_CONFIG_PATH}" ]; then exit 1 fi +if [ ! -w "${YTP_DOWNLOAD_PATH}" ]; then + CH_USER=$(stat -c "%u" "${YTP_DOWNLOAD_PATH}") + CH_GRP=$(stat -c "%g" "${YTP_DOWNLOAD_PATH}") + echo_err "ERROR: Unable to write to '${YTP_DOWNLOAD_PATH}' downloads directory. Current user id '${UID}' while directory owner is '${CH_USER}'." + echo_err "[Running under docker]" + echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\"" + echo_err "Run the following command to change the directory ownership" + echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config" + echo_err "[Running under podman]" + echo_err "change docker-compose.yaml user: to user:\"0:0\"" + exit 1 +fi + exec "${@}" diff --git a/pyproject.toml b/pyproject.toml index 8013192d..7307cfa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ exclude = [ "node_modules", "site-packages", "venv", + ".venv", ] # Same as Black. @@ -39,8 +40,75 @@ target-version = "py311" # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or # McCabe complexity (`C901`) by default. -select = ["E4", "E7", "E9", "F", "G", "W"] -ignore = ["PTH", "G0", "G1", "G201"] +#select = ["E4", "E7", "E9", "F", "G", "W"] +#ignore = ["PTH", "G0", "G1", "G201"] +select = [ + "ALL", # include all the rules, including new ones +] +ignore = [ + #### modules + "ANN", # flake8-annotations + "COM", # flake8-commas + "C90", # mccabe complexity + "DJ", # django + "EXE", # flake8-executable + "T10", # debugger + "TID", # flake8-tidy-imports + + #### specific rules + "D100", # ignore missing docs + "D101", + "D102", + "D103", + "D104", + "D105", + "D106", + "D107", + "D200", + "D205", + "D212", + "D400", + "D401", + "D415", + "E402", # false positives for local imports + "E501", # line too long + "TRY003", # external messages in exceptions are too verbose + "TD002", + "TD003", + "FIX002", # too verbose descriptions of todos, + "N806", # variable names should be snake_case + "PTH", # prefer using absolute imports + "PGH003", + "D404", + "PLR0913", + "INP001", + "G004", + "SIM105", + "SIM300", + "BLE001", + "TRY400", + "S104", + "PLR2004", + "S110", + "N812", + "S108", + "RUF006", + "RUF012", + "RUF013", + "TRY002", + "TRY401", + "A001", + "A002", + "SLF001", + "FBT001", + "FBT002", + "TRY300", + "PLR0911", + "PLR0912", + "PLR0915", + "PLW2901", + "ERA001" +] # Allow fix for all enabled rules (when `--fix`) is provided. fixable = ["ALL"] diff --git a/ui/layouts/default.vue b/ui/layouts/default.vue index ff6f047e..e08c4d54 100644 --- a/ui/layouts/default.vue +++ b/ui/layouts/default.vue @@ -13,11 +13,40 @@