From df86cc0001ca2b4665856d77d1ac036d166b2db4 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Thu, 19 Dec 2024 22:58:33 +0300 Subject: [PATCH] updated some type checks --- .editorconfig | 1 + app/library/DownloadQueue.py | 233 ++++++++++++++++++----------------- app/library/HttpAPI.py | 2 +- app/library/config.py | 204 ++++++++++++++++-------------- app/main.py | 47 ++++--- pyproject.toml | 78 ++++++++++++ 6 files changed, 333 insertions(+), 232 deletions(-) diff --git a/.editorconfig b/.editorconfig index 9e851b92..77836166 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,3 +15,4 @@ trim_trailing_whitespace = false [*.py] indent_size = 4 +max_line_length = 120 diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index c92a89f9..f64939d7 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -10,13 +10,13 @@ from .config import Config from .Download import Download from .ItemDTO import ItemDTO from .DataStore import DataStore -from .Utils import ag, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig +from .Utils import calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig from .AsyncPool import AsyncPool from .Emitter import Emitter -LOG = logging.getLogger('DownloadQueue') -TYPE_DONE: str = 'done' -TYPE_QUEUE: str = 'queue' +LOG = logging.getLogger("DownloadQueue") +TYPE_DONE: str = "done" +TYPE_QUEUE: str = "queue" class DownloadQueue: @@ -41,11 +41,11 @@ class DownloadQueue: async def initialize(self): self.event = asyncio.Event() - LOG.info(f'Using {self.config.max_workers} workers for downloading.') + LOG.info(f"Using {self.config.max_workers} workers for downloading.") asyncio.create_task( self.__download_pool() if self.config.max_workers > 1 else self.__download(), - name='download_pool' if self.config.max_workers > 1 else 'download_worker' + name="download_pool" if self.config.max_workers > 1 else "download_worker", ) async def __add_entry( @@ -54,29 +54,29 @@ class DownloadQueue: preset: str, folder: str, ytdlp_config: dict = {}, - ytdlp_cookies: str = '', - output_template: str = '', - already=None + ytdlp_cookies: str = "", + output_template: str = "", + already=None, ): if not entry: - return {'status': 'error', 'msg': 'Invalid/empty data was given.'} + return {"status": "error", "msg": "Invalid/empty data was given."} options: dict = {} error: str = None live_in: str = None - eventType = entry.get('_type') or 'video' - if eventType == 'playlist': - entries = entry['entries'] + eventType = entry.get("_type") or "video" + if eventType == "playlist": + entries = entry["entries"] playlist_index_digits = len(str(len(entries))) results = [] for i, etr in enumerate(entries, start=1): - etr['playlist'] = entry.get('id') - etr['playlist_index'] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(i) - for property in ('id', 'title', 'uploader', 'uploader_id'): + etr["playlist"] = entry.get("id") + etr["playlist_index"] = "{{0:0{0:d}d}}".format(playlist_index_digits).format(i) + for property in ("id", "title", "uploader", "uploader_id"): if property in entry: - etr[f'playlist_{property}'] = entry.get(property) + etr[f"playlist_{property}"] = entry.get(property) results.append( await self.__add_entry( @@ -86,60 +86,63 @@ class DownloadQueue: ytdlp_config=ytdlp_config, ytdlp_cookies=ytdlp_cookies, output_template=output_template, - already=already + already=already, ) ) - if any(res['status'] == 'error' for res in results): - return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} + if any(res["status"] == "error" for res in results): + return { + "status": "error", + "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res), + } - return {'status': 'ok'} - elif (eventType == 'video' or eventType.startswith('url')) and 'id' in entry and 'title' in entry: + return {"status": "ok"} + elif (eventType == "video" or eventType.startswith("url")) and "id" in entry and "title" in entry: # check if the video is live stream. - if 'live_status' in entry and entry.get('live_status') == 'is_upcoming': - if 'release_timestamp' in entry and entry.get('release_timestamp'): - live_in = formatdate(entry.get('release_timestamp')) + if "live_status" in entry and entry.get("live_status") == "is_upcoming": + if "release_timestamp" in entry and entry.get("release_timestamp"): + live_in = formatdate(entry.get("release_timestamp")) else: - error = 'Live stream not yet started. And no date is set.' + error = "Live stream not yet started. And no date is set." else: - error = entry.get('msg', None) + error = entry.get("msg", None) LOG.debug( f"Entry id '{entry.get('id', None)}' url '{entry.get('webpage_url', None)} - {entry.get('url', None)}'." ) - if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry.get('url')): - item = self.done.get(key=entry['id'], url=entry.get('webpage_url') or entry['url']) + if self.done.exists(key=entry["id"], url=entry.get("webpage_url") or entry.get("url")): + item = self.done.get(key=entry["id"], url=entry.get("webpage_url") or entry["url"]) LOG.warning(f"Item '{item.info.id}' - '{item.info.title}' already downloaded. Removing from history.") await self.clear([item.info._id]) try: - item = self.queue.get(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url')) + item = self.queue.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url")) if item is not None: err_message = f"Item ID '{item.info.id}' - '{item.info.title}' already in download queue." LOG.info(err_message) - return {'status': 'error', 'msg': err_message} + return {"status": "error", "msg": err_message} except KeyError: pass - is_manifestless = entry.get('is_manifestless', False) - options.update({'is_manifestless': is_manifestless}) + is_manifestless = entry.get("is_manifestless", False) + options.update({"is_manifestless": is_manifestless}) - live_status: list = ['is_live', 'is_upcoming'] - is_live = entry.get('is_live', None) or live_in or entry.get('live_status', None) in live_status + live_status: list = ["is_live", "is_upcoming"] + is_live = entry.get("is_live", None) or live_in or entry.get("live_status", None) in live_status try: download_dir = calcDownloadPath(basePath=self.config.download_path, folder=folder) except Exception as e: LOG.exception(e) - return {'status': 'error', 'msg': str(e)} + return {"status": "error", "msg": str(e)} dl = ItemDTO( - id=entry.get('id'), - title=entry.get('title'), - url=entry.get('webpage_url') or entry.get('url'), + id=entry.get("id"), + title=entry.get("title"), + url=entry.get("webpage_url") or entry.get("url"), preset=preset, - thumbnail=entry.get('thumbnail', None), + thumbnail=entry.get("thumbnail", None), folder=folder, download_dir=download_dir, temp_dir=self.config.temp_path, @@ -160,42 +163,37 @@ class DownloadQueue: dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug)) - if dlInfo.info.live_in or 'is_upcoming' == entry.get('live_status', None): - dlInfo.info.status = 'not_live' + if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status", None): + dlInfo.info.status = "not_live" itemDownload = self.done.put(dlInfo) - NotifyEvent = 'completed' + NotifyEvent = "completed" elif self.config.allow_manifestless is False and is_manifestless is True: - dlInfo.info.status = 'error' - dlInfo.info.error = 'Video is in post-live manifestless mode.' + dlInfo.info.status = "error" + dlInfo.info.error = "Video is in post-live manifestless mode." itemDownload = self.done.put(dlInfo) - NotifyEvent = 'completed' + NotifyEvent = "completed" else: - NotifyEvent = 'added' + NotifyEvent = "added" itemDownload = self.queue.put(dlInfo) self.event.set() asyncio.create_task( - self.emitter.emit(NotifyEvent, itemDownload.info), - name=f'notifier-{NotifyEvent}-{itemDownload.info.id}') + self.emitter.emit(NotifyEvent, itemDownload.info), name=f"notifier-{NotifyEvent}-{itemDownload.info.id}" + ) - return { - 'status': 'ok' - } - elif eventType.startswith('url'): + return {"status": "ok"} + elif eventType.startswith("url"): return await self.add( - url=entry.get('url'), + url=entry.get("url"), preset=preset, folder=folder, ytdlp_config=ytdlp_config, ytdlp_cookies=ytdlp_cookies, output_template=output_template, - already=already + already=already, ) - return { - 'status': 'error', - 'msg': f'Unsupported resource "{eventType}"' - } + return {"status": "error", "msg": f'Unsupported resource "{eventType}"'} async def add( self, @@ -203,26 +201,28 @@ class DownloadQueue: preset: str, folder: str, ytdlp_config: dict = {}, - ytdlp_cookies: str = '', - output_template: str = '', - already=None + ytdlp_cookies: str = "", + output_template: str = "", + already=None, ): ytdlp_config = ytdlp_config if ytdlp_config else {} - LOG.info(f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'.") + LOG.info( + f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'." + ) if isinstance(ytdlp_config, str): try: ytdlp_config = json.loads(ytdlp_config) except Exception as e: LOG.error(f"Unable to load '{ytdlp_config=}'. {str(e)}") - return {'status': 'error', 'msg': f"Failed to parse json yt-dlp config. {str(e)}"} + return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {str(e)}"} already = set() if already is None else already if url in already: LOG.warning(f"Recursion detected with url '{url}' skipping.") - return {'status': 'ok'} + return {"status": "ok"} already.add(url) @@ -231,10 +231,10 @@ class DownloadQueue: if downloaded is True: message = f"This url with ID '{id_dict.get('id')}' has been downloaded already and recorded in archive." LOG.info(message) - return {'status': 'error', 'msg': message} + return {"status": "error", "msg": message} started = time.perf_counter() - LOG.debug(f'extract_info: checking {url=}') + LOG.debug(f"extract_info: checking {url=}") entry = await asyncio.wait_for( fut=asyncio.get_running_loop().run_in_executor( @@ -242,24 +242,29 @@ class DownloadQueue: ExtractInfo, mergeConfig(self.config.ytdl_options, ytdlp_config), url, - bool(self.config.ytdl_debug) + bool(self.config.ytdl_debug), ), - timeout=self.config.extract_info_timeout) + timeout=self.config.extract_info_timeout, + ) if not entry: - return {'status': 'error', 'msg': 'Unable to extract info check logs.'} + return {"status": "error", "msg": "Unable to extract info check logs."} LOG.debug( - f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'.") + f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'." + ) except yt_dlp.utils.ExistingVideoReached as exc: - LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}') - return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'} + LOG.error(f"Video has been downloaded already and recorded in archive.log file. {str(exc)}") + return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} except yt_dlp.utils.YoutubeDLError as exc: - LOG.error(f'YoutubeDLError: Unable to extract info. {str(exc)}') - return {'status': 'error', 'msg': str(exc)} + LOG.error(f"YoutubeDLError: Unable to extract info. {str(exc)}") + return {"status": "error", "msg": str(exc)} except asyncio.exceptions.TimeoutError as exc: - LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}') - return {'status': 'error', 'msg': f'TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.'} + LOG.error(f"TimeoutError: Unable to extract info. {str(exc)}") + return { + "status": "error", + "msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.", + } return await self.__add_entry( entry=entry, @@ -279,27 +284,27 @@ class DownloadQueue: item = self.queue.get(key=id) except KeyError as e: status[id] = str(e) - status['status'] = 'error' - LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}') + status["status"] = "error" + LOG.warning(f"Requested cancel for non-existent download {id=}. {str(e)}") continue itemMessage = f"{id=} {item.info.id=} {item.info.title=}" if item.running(): - LOG.debug(f'Canceling {itemMessage}') + LOG.debug(f"Canceling {itemMessage}") item.cancel() - LOG.info(f'Cancelled {itemMessage}') + LOG.info(f"Cancelled {itemMessage}") await item.close() else: await item.close() - LOG.debug(f'Deleting from queue {itemMessage}') + LOG.debug(f"Deleting from queue {itemMessage}") self.queue.delete(id) - asyncio.create_task(self.emitter.canceled(id=id, dl=item), name=f'notifier-c-{id}') + asyncio.create_task(self.emitter.canceled(id=id, dl=item), name=f"notifier-c-{id}") self.done.put(item) - asyncio.create_task(self.emitter.completed(dl=item), name=f'notifier-d-{id}') - LOG.info(f'Deleted from queue {itemMessage}') + asyncio.create_task(self.emitter.completed(dl=item), name=f"notifier-d-{id}") + LOG.info(f"Deleted from queue {itemMessage}") - status[id] = 'ok' + status[id] = "ok" return status @@ -311,35 +316,35 @@ class DownloadQueue: item = self.done.get(key=id) except KeyError as e: status[id] = str(e) - status['status'] = 'error' - LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}') + status["status"] = "error" + LOG.warning(f"Requested delete for non-existent download {id=}. {str(e)}") continue itemMessage = f"{id=} {item.info.id=} {item.info.title=}" - LOG.debug(f'Deleting completed download {itemMessage}') + LOG.debug(f"Deleting completed download {itemMessage}") self.done.delete(id) - asyncio.create_task(self.emitter.cleared(id, dl=item), name=f'notifier-c-{id}') - LOG.info(f'Deleted completed download {itemMessage}') - status[id] = 'ok' + asyncio.create_task(self.emitter.cleared(id, dl=item), name=f"notifier-c-{id}") + LOG.info(f"Deleted completed download {itemMessage}") + status[id] = "ok" return status def get(self) -> dict[str, list[dict[str, ItemDTO]]]: - items = {'queue': {}, 'done': {}} + items = {"queue": {}, "done": {}} for k, v in self.queue.saved_items(): - items['queue'][k] = v + items["queue"][k] = v for k, v in self.done.saved_items(): - items['done'][k] = v + items["done"][k] = v for k, v in self.queue.items(): - if not k in items['queue']: - items['queue'][k] = v.info + if k not in items["queue"]: + items["queue"][k] = v.info for k, v in self.done.items(): - if not k in items['done']: - items['done'][k] = v.info + if k not in items["done"]: + items["done"][k] = v.info return items @@ -348,8 +353,8 @@ class DownloadQueue: loop=asyncio.get_running_loop(), num_workers=self.config.max_workers, worker_co=self.__downloadFile, - name='download_pool', - logger=logging.getLogger('WorkerPool'), + name="download_pool", + logger=logging.getLogger("WorkerPool"), ) self.pool.start() @@ -362,14 +367,14 @@ class DownloadQueue: break if time.time() - lastLog > 120: lastLog = time.time() - LOG.info(f'Waiting for worker to be free. {self.pool.get_workers_status()}') + LOG.info(f"Waiting for worker to be free. {self.pool.get_workers_status()}") await asyncio.sleep(1) while not self.queue.hasDownloads(): LOG.info(f"Waiting for item to download. '{self.pool.get_available_workers()}' free workers.") await self.event.wait() self.event.clear() - LOG.debug(f"Cleared wait event.") + LOG.debug("Cleared wait event.") entry = self.queue.getNextDownload() await asyncio.sleep(0.2) @@ -387,7 +392,7 @@ class DownloadQueue: async def __download(self): while True: while self.queue.empty(): - LOG.info('Waiting for item to download.') + LOG.info("Waiting for item to download.") await self.event.wait() self.event.clear() @@ -396,34 +401,34 @@ class DownloadQueue: async def __downloadFile(self, id: str, entry: Download): LOG.info( - f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'folder: {entry.info.folder}'.") + f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'folder: {entry.info.folder}'." + ) try: await entry.start(self.emitter) - if entry.info.status != 'finished': + if entry.info.status != "finished": if entry.tmpfilename and os.path.isfile(entry.tmpfilename): try: os.remove(entry.tmpfilename) entry.tmpfilename = None - except: + except Exception: pass - entry.info.status = 'error' + entry.info.status = "error" finally: await entry.close() if self.queue.exists(key=id): - self.queue.delete(key=id) if entry.is_canceled() is True: - asyncio.create_task(self.emitter.canceled(id, dl=entry.info), name=f'notifier-c-{id}') - entry.info.status = 'canceled' - entry.info.error = 'Canceled by user.' + asyncio.create_task(self.emitter.canceled(id, dl=entry.info), name=f"notifier-c-{id}") + entry.info.status = "canceled" + entry.info.error = "Canceled by user." self.done.put(value=entry) - asyncio.create_task(self.emitter.completed(dl=entry.info), name=f'notifier-d-{id}') + asyncio.create_task(self.emitter.completed(dl=entry.info), name=f"notifier-d-{id}") self.event.set() @@ -431,4 +436,4 @@ class DownloadQueue: if not url or not self.config.keep_archive: return False, None - return isDownloaded(self.config.ytdl_options.get('download_archive', None), url) + return isDownloaded(self.config.ytdl_options.get("download_archive", None), url) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index d4a1f48e..28e06061 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -448,7 +448,7 @@ class HttpAPI(common): 'Access-Control-Max-Age': "300", }) - @route('GET', 'player/segments/{segment:\d+}/{file:.*}.ts') + @route('GET', 'player/segments/{segment:d+}/{file:.*}.ts') async def segments(self, request: Request) -> Response: file: str = request.match_info.get('file') segment: int = request.match_info.get('segment') diff --git a/app/library/config.py b/app/library/config.py index bcc87e82..eade2eb7 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -13,31 +13,31 @@ from pathlib import Path class Config: __instance = None - config_path: str = '.' - download_path: str = '.' - temp_path: str = '/tmp' + config_path: str = "." + download_path: str = "." + temp_path: str = "/tmp" temp_keep: bool = False - db_file: str = '{config_path}/ytptube.db' - manual_archive: str = '{config_path}/archive.manual.log' + db_file: str = "{config_path}/ytptube.db" + manual_archive: str = "{config_path}/archive.manual.log" - url_host: str = '' - url_prefix: str = '' - url_socketio: str = '{url_prefix}socket.io' + url_host: str = "" + url_prefix: str = "" + url_socketio: str = "{url_prefix}socket.io" - output_template: str = '%(title)s.%(ext)s' - output_template_chapter: str = '%(title)s - %(section_number)s %(section_title)s.%(ext)s' + output_template: str = "%(title)s.%(ext)s" + output_template_chapter: str = "%(title)s - %(section_number)s %(section_title)s.%(ext)s" ytdl_options: dict | str = {} ytdl_debug: bool = False - host: str = '0.0.0.0' + host: str = "0.0.0.0" port: int = 8081 keep_archive: bool = True - base_path: str = '' + base_path: str = "" - log_level: str = 'info' + log_level: str = "info" allow_manifestless: bool = False @@ -57,8 +57,8 @@ class Config: started: int = 0 - streamer_vcodec = 'libx264' - streamer_acodec = 'aac' + streamer_vcodec = "libx264" + streamer_acodec = "aac" auth_username: str = None auth_password: str = None @@ -66,73 +66,96 @@ class Config: ytdlp_version: str = YTDLP_VERSION tasks: list = [] presets: list = [ - {'name': 'default', 'format': 'default', 'postprocessors': [], - 'args': {}}, - {'name': 'Video - 1080p h264/acc', 'format': 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]', - 'args': {'format_sort': ['vcodec:h264'], }, }, - {'name': 'Video - 720p h264/acc', 'format': 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]', - 'args': {'format_sort': ['vcodec:h264'], }, }, + {"name": "default", "format": "default", "postprocessors": [], "args": {}}, { - 'name': 'Audio - Best audio [MP3]', - 'format': 'bestaudio/best', - 'postprocessors': [ + "name": "Video - 1080p h264/acc", + "format": "bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]", + "args": { + "format_sort": ["vcodec:h264"], + }, + }, + { + "name": "Video - 720p h264/acc", + "format": "bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]", + "args": { + "format_sort": ["vcodec:h264"], + }, + }, + { + "name": "Audio - Best audio [MP3]", + "format": "bestaudio/best", + "postprocessors": [ { "key": "FFmpegExtractAudio", "preferredcodec": "mp3", "preferredquality": "5", - "nopostoverwrites": False + "nopostoverwrites": False, }, - { - "key": "FFmpegMetadata", - "add_chapters": True, - "add_metadata": True, - "add_infojson": "if_exists" - }, - { - "key": "EmbedThumbnail", - "already_have_thumbnail": False - }, - { - "key": "FFmpegConcat", - "only_multi_video": True, - "when": "playlist" - } + {"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": True, "add_infojson": "if_exists"}, + {"key": "EmbedThumbnail", "already_have_thumbnail": False}, + {"key": "FFmpegConcat", "only_multi_video": True, "when": "playlist"}, ], - 'args': { - "outtmpl": { - "pl_thumbnail": "" - }, + "args": { + "outtmpl": {"pl_thumbnail": ""}, "ignoreerrors": "only_download", "retries": 10, "fragment_retries": 10, "writethumbnail": True, "extract_flat": "discard_in_playlist", "final_ext": "mp3", - } - } + }, + }, ] - _manual_vars: tuple = ('temp_path', 'config_path', 'download_path',) + _manual_vars: tuple = ( + "temp_path", + "config_path", + "download_path", + ) _immutable: tuple = ( - 'version', '__instance', 'ytdl_options', 'tasks', - 'new_version_available', 'ytdlp_version', 'started', 'presets', + "version", + "__instance", + "ytdl_options", + "tasks", + "new_version_available", + "ytdlp_version", + "started", + "presets", ) - _int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout', 'debugpy_port',) - _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',) + _int_vars: tuple = ( + "port", + "max_workers", + "socket_timeout", + "extract_info_timeout", + "debugpy_port", + ) + _boolean_vars: tuple = ( + "keep_archive", + "ytdl_debug", + "debug", + "temp_keep", + "allow_manifestless", + ) _frontend_vars: tuple = ( - 'download_path', 'keep_archive', 'output_template', - 'ytdlp_version', 'version', 'url_host', 'started', 'url_prefix', + "download_path", + "keep_archive", + "output_template", + "ytdlp_version", + "version", + "url_host", + "started", + "url_prefix", ) - @ staticmethod + @staticmethod def get_instance(): - """ Static access method. """ + """Static access method.""" return Config() if not Config.__instance else Config.__instance def __init__(self): - """ Virtually private constructor. """ + """Virtually private constructor.""" if Config.__instance is not None: raise Exception("This class is a singleton. Use Config.get_instance() instead.") else: @@ -140,20 +163,20 @@ class Config: baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute()) - self.temp_path = os.environ.get('YTP_TEMP_PATH', None) or os.path.join(baseDefaultPath, 'var', 'tmp') - self.config_path = os.environ.get('YTP_CONFIG_PATH', None) or os.path.join(baseDefaultPath, 'var', 'config') - self.download_path = os.environ.get( - 'YTP_DOWNLOAD_PATH', None) or os.path.join( - baseDefaultPath, 'var', 'downloads') + self.temp_path = os.environ.get("YTP_TEMP_PATH", None) or os.path.join(baseDefaultPath, "var", "tmp") + self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or os.path.join(baseDefaultPath, "var", "config") + self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or os.path.join( + baseDefaultPath, "var", "downloads" + ) - envFile: str = os.path.join(self.config_path, '.env') + envFile: str = os.path.join(self.config_path, ".env") if os.path.exists(envFile): logging.info(f"Loading environment variables from '{envFile}'.") load_dotenv(envFile) for k, v in self._getAttributes().items(): - if k.startswith('_') or k in self._manual_vars: + if k.startswith("_") or k in self._manual_vars: continue # If the variable declared as immutable, set it to the default value. @@ -161,15 +184,15 @@ class Config: setattr(self, k, v) continue - lookUpKey: str = f'YTP_{k}'.upper() + lookUpKey: str = f"YTP_{k}".upper() setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v) for k, v in self.__dict__.items(): - if k.startswith('_') or k in self._immutable or k in self._manual_vars: + if k.startswith("_") or k in self._immutable or k in self._manual_vars: continue - if isinstance(v, str) and '{' in v and '}' in v: - for key in re.findall(r'\{.*?\}', v): + if isinstance(v, str) and "{" in v and "}" in v: + for key in re.findall(r"\{.*?\}", v): localKey: str = key[1:-1] if localKey not in self.__dict__: logging.error(f"Config variable '{k}' had non-existing config reference '{key}'.") @@ -180,32 +203,31 @@ class Config: setattr(self, k, v) if k in self._boolean_vars: - if str(v).lower() not in (True, False, 'true', 'false', 'on', 'off', '1', '0'): + if str(v).lower() not in (True, False, "true", "false", "on", "off", "1", "0"): raise ValueError(f"Config variable '{k}' is set to a non-boolean value '{v}'.") - setattr(self, k, str(v).lower() in (True, 'true', 'on', '1')) + setattr(self, k, str(v).lower() in (True, "true", "on", "1")) if k in self._int_vars: setattr(self, k, int(v)) - if not self.url_prefix.endswith('/'): - self.url_prefix += '/' + if not self.url_prefix.endswith("/"): + self.url_prefix += "/" numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError(f"Invalid log level '{self.log_level}' specified.") coloredlogs.install( - level=numeric_level, - fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", - datefmt='%H:%M:%S' + level=numeric_level, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S" ) - LOG = logging.getLogger('config') + LOG = logging.getLogger("config") if self.debug: try: import debugpy + debugpy.listen(("0.0.0.0", self.debugpy_port)) LOG.info(f"starting debugpy server on '0.0.0.0:{self.debugpy_port}'.") except ImportError: @@ -213,7 +235,7 @@ class Config: except Exception as e: LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}") - optsFile: str = os.path.join(self.config_path, 'ytdlp.json') + optsFile: str = os.path.join(self.config_path, "ytdlp.json") if os.path.exists(optsFile) and os.path.getsize(optsFile) > 4: LOG.info(f"Loading yt-dlp custom options from '{optsFile}'.") @@ -226,7 +248,7 @@ class Config: else: LOG.info(f"No yt-dlp custom options found at '{optsFile}'.") - tasksFile = os.path.join(self.config_path, 'tasks.json') + tasksFile = os.path.join(self.config_path, "tasks.json") if os.path.exists(tasksFile) and os.path.getsize(tasksFile) > 0: LOG.info(f"Loading tasks from '{tasksFile}'.") try: @@ -235,10 +257,10 @@ class Config: LOG.error(f"Could not load tasks file from '{tasksFile}'. '{error}'.") sys.exit(1) self.tasks.extend(tasks) - except Exception as e: + except Exception: pass - presetsFile = os.path.join(self.config_path, 'presets.json') + presetsFile = os.path.join(self.config_path, "presets.json") if os.path.exists(presetsFile) and os.path.getsize(presetsFile) > 0: LOG.info(f"Loading extra presets from '{presetsFile}'.") try: @@ -248,32 +270,32 @@ class Config: sys.exit(1) for preset in presets: - if 'name' not in preset: + if "name" not in preset: LOG.error(f"Missing 'name' key in preset '{preset}'.") continue - if 'format' not in preset: + if "format" not in preset: LOG.error(f"Missing 'format' key in preset '{preset}'.") continue - if 'args' not in preset: - preset['args'] = {} + if "args" not in preset: + preset["args"] = {} - if 'postprocessors' not in preset: - preset['postprocessors'] = [] + if "postprocessors" not in preset: + preset["postprocessors"] = [] self.presets.append(preset) - except Exception as e: + except Exception: pass - self.ytdl_options['socket_timeout'] = self.socket_timeout + self.ytdl_options["socket_timeout"] = self.socket_timeout if self.keep_archive: - LOG.info(f"keep archive option is enabled.") - self.ytdl_options['download_archive'] = os.path.join(self.config_path, 'archive.log') + LOG.info("keep archive option is enabled.") + self.ytdl_options["download_archive"] = os.path.join(self.config_path, "archive.log") if self.temp_keep: - LOG.info(f'Keep temp files option is enabled.') + LOG.info("Keep temp files option is enabled.") if self.auth_password and self.auth_username: LOG.warning(f"Basic authentication enabled with username '{self.auth_username}'.") @@ -285,7 +307,7 @@ class Config: vClass: str = self.__class__ for attribute in vClass.__dict__.keys(): - if attribute.startswith('_'): + if attribute.startswith("_"): continue value = getattr(vClass, attribute) diff --git a/app/main.py b/app/main.py index 16971048..a38cd3cc 100644 --- a/app/main.py +++ b/app/main.py @@ -1,33 +1,34 @@ #!/usr/bin/env python3 import asyncio +import logging import os import random -import logging -import caribou import sqlite3 -import magic from datetime import datetime -from aiohttp import web from pathlib import Path -from library.Emitter import Emitter -from library.config import Config -from library.encoder import Encoder -from library.DownloadQueue import DownloadQueue -from library.Webhooks import Webhooks -from library.HttpSocket import HttpSocket -from library.HttpAPI import HttpAPI + +import caribou +import magic from aiocron import crontab +from aiohttp import web +from library.config import Config +from library.DownloadQueue import DownloadQueue +from library.Emitter import Emitter +from library.encoder import Encoder +from library.HttpAPI import HttpAPI +from library.HttpSocket import HttpSocket +from library.Webhooks import Webhooks LOG = logging.getLogger("app") MIME = magic.Magic(mime=True) -class main: - config: Config = None - app: web.Application = None - http: HttpAPI = None - socket: HttpSocket = None +class Main: + config: Config + app: web.Application + http: HttpAPI + socket: HttpSocket def __init__(self): self.config = Config.get_instance() @@ -61,9 +62,7 @@ class main: LOG.info(f"Creating download folder at '{self.config.download_path}'.") os.makedirs(self.config.download_path, exist_ok=True) except OSError as e: - LOG.error( - f"Could not create download folder at '{self.config.download_path}'." - ) + LOG.error(f"Could not create download folder at '{self.config.download_path}'.") raise e try: @@ -111,9 +110,7 @@ class main: LOG.info(f"Completed 'Task: {taskName}' at '{timeNow}'.") except Exception as e: timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - LOG.error( - f"Failed 'Task: {taskName}' at '{timeNow}'. Error message '{str(e)}'." - ) + LOG.error(f"Failed 'Task: {taskName}' at '{timeNow}'. Error message '{str(e)}'.") def load_tasks(self): for task in self.config.tasks: @@ -131,9 +128,7 @@ class main: loop=asyncio.get_event_loop(), ) - LOG.info( - f"Added 'Task: {task.get('name', task.get('url'))}' to be executed every '{cron_timer}'." - ) + LOG.info(f"Added 'Task: {task.get('name', task.get('url'))}' to be executed every '{cron_timer}'.") def start(self): self.socket.attach(self.app) @@ -155,4 +150,4 @@ class main: if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) - main().start() + Main().start() diff --git a/pyproject.toml b/pyproject.toml index dbf4699b..8013192d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,80 @@ +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 120 +indent-width = 4 + +# Assume Python 3.11 +target-version = "py311" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F", "G", "W"] +ignore = ["PTH", "G0", "G1", "G201"] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" + [tool.autopep8] --max-line-length = 120