From 7a519bf16c2d7fccb3b22d2893958c315b53d2a5 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 14 May 2024 12:41:36 +0300 Subject: [PATCH] support jsonc in tasks/webhooks/ytdlp json files. --- Pipfile | 1 + Pipfile.lock | 135 +++++++++++++++++++++++++++++++++++++++++++++++- app/Config.py | 13 +++-- app/Utils.py | 37 +++++++++++++ app/Webhooks.py | 11 ++-- app/main.py | 19 ++++--- 6 files changed, 197 insertions(+), 19 deletions(-) diff --git a/Pipfile b/Pipfile index 1c4587f6..b22a3f2d 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ python-magic = ">=0.4.27" debugpy = ">=1.8.1" httpx = "*" async-timeout = "*" +pyjson5 = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index d42c216d..5685a357 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "dc6e4544c55ad5440a21d9bedd84e11bdcd375c8a3c1ddf7708e8f99e2f7167f" + "sha256": "d5812b5e6776f99dd75ccf98252f83238c48638b157d84695dd5cea36596c67c" }, "pipfile-spec": 6, "requires": { @@ -675,6 +675,139 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==3.20.0" }, + "pyjson5": { + "hashes": [ + "sha256:011957e1e7143c82ee5701dd39292d3864f6989247d1423462b1ef405390435e", + "sha256:02d60cd0f98d39347416d9172f38cd9601dfcf9976536688deb82c387ac22db1", + "sha256:0570cd200f5b52938769705c4f3092f1fcbb850f62278d30db8b8c6ae5ef0617", + "sha256:0abc60630bb1601ae9005e251d30cef363cb4d6cef2d2fb7a160329569170e38", + "sha256:0b9718821bfc36c61dd2ae8456fafbe3e9eb46df13cb2ac1ade38c5163ff9c92", + "sha256:0c069dfbc4939ce5d8b6d72b7755bb287988aab819435349e6f52da4e62fac9c", + "sha256:0d8063c83603e4eda99dc9396113521f0d277b027ccb3182e8e47ea284db8a70", + "sha256:0d828b98667ebca9fbd172f221746ecba105081b77f5ac6bbbf055f2faa38e14", + "sha256:122a980366db9c56b35bead60debf832d614ebe47a65d34bc292b2211e42e547", + "sha256:127f145467b61ef8a9a1d01c5c33a433d14dbca70c70d0b0800f4a19581303ff", + "sha256:144ea7654b64226cd24c6cc856827180b2e04ddc4404f9513ba41c6f284aa4c7", + "sha256:14671bbb72df85b895570f092e1b0aa34437a8df3e609e70a00459943aa1b75c", + "sha256:149f6ca0e346fce26ccb154e6633d3dbe10f197aae2b306cf3e3f09a57b6e4f7", + "sha256:14cad404a8dff0ea57370b98350b8fedb7611ee8023090b8a991673b4542abf2", + "sha256:17b258ba83c2d888568a098cab8f7df1ceffded589386eb5d70960ab83cffa9d", + "sha256:1c2ae4f322d2c859abd45efa893d06737b12575ca32fb068d2ab1ff7e1dacf7c", + "sha256:1d0bd69aa1b82410593341eb43c546c567bee5acb151666857c9df98e2cdfc09", + "sha256:1e3cf693ac82c7ee1da149e172bbe0cf59d11b06c31331f254daf8825d566033", + "sha256:1ea61368eb930a39441a7e9dd34ecb9af026a7b35bd03570910558abcd296215", + "sha256:20df8c5dbbe0d653f403da88b7520be44fc8d74697bbdd1ead688458b5691a02", + "sha256:211a9e260cf7d122e4d8881ae8724067cf7dfa48317760adb59b96bcf8e0a407", + "sha256:259f1797cd25e13396b6cb08dc0625c5de26a4214a8279e5304d6c91e5c8eb50", + "sha256:270600deefe532138d00cec1e4029c716fc86eaa52dabfb12494dca1cb30e611", + "sha256:2add15588939d8b1b7c59fd3f5480cce76231e0b014a2edebf3443ba92789d38", + "sha256:2cec7163effe96abe40ff098ffd2b16f5b322628bdac34c7aa5d316588524c42", + "sha256:314537a41af768367ab19a50d697a6ea85cb1c39bac4a2b93680cab930002e31", + "sha256:3145da3885e063b360bd9cc0bff6c9b0c83e3a2e822f83b4f7f40b26b41e1598", + "sha256:34500d5373dff030ab5183a69b3eb545a3a89385a3949142ea61ca95e653cbf8", + "sha256:36bdfa9334d00be616a63fd15295e9885d0e91dfa75cda5b6a8d1a2b406d6252", + "sha256:3d2829447d9d6a51a5976a3eb0a2a0069122ab741378c60367883e60160cb6de", + "sha256:410c15018434271408bb9410ac7db5195beccc33d8d1fbc8fb4a39858427e0df", + "sha256:415805f111faa16b050827beda8d763d4391becc24f86438848c7cb23ce63c55", + "sha256:420d9f970d678d8d1fab8aefdd483747c75e64681766e771494910b8030f41fa", + "sha256:4724d6bc1e369b29fb9fab7b10530cc11ccba60dc403cef3a83c213bc88d541c", + "sha256:48d02c97e34e64eefca4794dc3717373637aec4bd97b6e094cbed926b1893097", + "sha256:493fa3a81ce879e7ae47d1aa6a2b465a207087857b693404e523f069f531d01d", + "sha256:4f1a9eebe9d793e5b6ed00691744255400f57666004f454b09e2e651657e15fe", + "sha256:4fedb2ab5cfc821f9b9ed44dc1eae5d22a39a040950bda77c8b26e04df327629", + "sha256:50edc8c1f8c903f6e5df8547ae4195e3456ba2cdb45fbbad14022937a27ffa7c", + "sha256:521e7691fe38bd56bc4764e7c9c54b4783910f602904d8ca1a6571a9e82a3f82", + "sha256:55df02651c98dc8ad7fa30fd4fc5a4789a83ed0498312154c35ee46bdd027ccd", + "sha256:567437862f410a5912eee4cf13dd01a8c28ce9c9bf95590b9b9a4cb20e9daaed", + "sha256:57231901f4c96cc5bc630daef3fc3eadc95b8babe019b9f26b4915296a755bb5", + "sha256:59166de551e7321dbd3aa552ef647d652873701faadb021eae20f55ba6705829", + "sha256:5940c9c2feb9dbcda0cb87d2fc7edf330de231469c882f1e584d96814b6efd0d", + "sha256:5c7643308f995d746af8210b9a418c287885a0f8c0104b5e5160c190e56fbd0c", + "sha256:5f541197328ac3433863e2e67e4d34fccf1590bb88cc7f2c3fc2a81b8cde2681", + "sha256:62cd10320a1e80b1ce386ccaed9073cffd729e0e5b7a8793f2083291770b0db3", + "sha256:64381957edf773f4c67cc32c6e44fc791034efde5d654626171829d1219e4247", + "sha256:6b19a7025546406ca91184125aadc89a961189a9f5d4a76c0534a04f814c8882", + "sha256:6c8af6410aa6525783ceef0008f4f86480a4f04c4b921dd4295a2f2ba62f682d", + "sha256:6e2c27e3dd5caef56910c6da6a0c97cfc015a1dbdc6c2f2bd63a3ad86a16b0bd", + "sha256:6e358035e6fd975933f00a01c2ed3cc1e344f9a06b6edc445ad4b3bca7509de4", + "sha256:7366857ff307ef1bae3f1096651a2b2f86ef87b8ff4ea7264c6d7db9176adaab", + "sha256:759ef75276235d9491ecf31c4c7ba057cdcd45707c04d2bdd0c0a18bd9f36f71", + "sha256:76154b0e0245b9dbb97be34055d5be38cb6a55508785594b53e0f4d57b0267eb", + "sha256:7d1fe850aa1763e8f9945a7120f032f5f707d0372d52c8e0fecda2a79640820e", + "sha256:7d6e8f92e2f0b84e6ede805aa68bbee408f639c29f64fd14960a242bb518dcc6", + "sha256:7dffc9dcbdf09663f6aefcf03a48cfb029437ee60c0d4372e2218f30929d3035", + "sha256:7f9f8072be8d2cd3ba40346efeaaaba089a6e9be04ef23a3d690aaa897c53f71", + "sha256:7fe091a563a1fe8f1c495d96d26fd40f9d19b9c3674dbf89dd3c3df8bf46cfe5", + "sha256:80a96a264b7eb7fcbeaf646f44e18c22f98d1f72cc9e9ca284f785612df457d8", + "sha256:868619dc4d271ea9ec5f4041901238167526be369e3598867ca0d9170827692e", + "sha256:86ac9e2b5dc086a979a115661f7886208352a45f3276453db914adbda54adbb7", + "sha256:87063a381b2db5f5af794ba6a3ec16c9a9f8fc5e974e41c299f761e7af138ec3", + "sha256:8afb7a6f3e020b9a2996bb62f98067ebf793e86599f349910a7129fbfaebdc79", + "sha256:8b869fead9daa2ef5972f30abd76fdb8acfe3d6f73c0e3783fe13e1775d4aa05", + "sha256:8eddf775c2846e24f5f17c0ef2dc20f7de628148d03394a46ac1fd866a59ab78", + "sha256:9293b3768e27a69ef764daa10168c199887ffbe820ef86ea9c2ff155bdd27fba", + "sha256:9371ff9d5c6a92676935171eafa9cc6a541702352f747f77000b343d3101b0c0", + "sha256:94b2d2a26ecdd9ecef704a5a0c86a272e85d1df6bb30f44fb7f825d6975f1fb3", + "sha256:94e20d28d2cfba1b2be59c63ad8ae221e709a84158dc7f138d3b7258319f59b2", + "sha256:985f51077649fd0c84fcc7057d48b862c843f30f77c9f5c27102363c653c9e5e", + "sha256:990bf3aa3df840dfb98e3663d645d6c897d00ccf1e6cc2a594a547e8c239cc52", + "sha256:a093e24dd14a97f06d1596158d413b3b556cdf2b0d7a1af45b546d1d90224de7", + "sha256:a29645584914f7b1840e916777d402264ea3cbe3e04ff47ea00a799de17552d6", + "sha256:a3b0e45a2f1a84e78d8ecd70aecce0f84b360af37b18b2123646c3b310ea10a7", + "sha256:a79e914cba2a45725c2b10801bbdc7975e2b0926c4406f3cbd67d3976c2ded9c", + "sha256:a873c3a0d38f4f901e2d721ea3875ecf8d704a6d6c642cf31f5e17d37b60ca38", + "sha256:a8e4974cd32697f3a2c6738b597eaf66f2d23eb34dcdd79b8b6bb92549b56395", + "sha256:aae2ad989da263216f556f3c5a3ea3eaf8c45894c9fea641c483576adb27494f", + "sha256:ab2d7b402b8f27a866431dc1b476c4f9f0ccbb0811425c846985f05f5de9236b", + "sha256:ae55e95b6c59e9ad86662b6be213a6ea3f1e0c7f3d5627d88ca3dbe293a0a23a", + "sha256:b21596c5d805c5e8fae5ba2c80317f2a6441c62ea91d1bd6515c43c23911a16a", + "sha256:b2f96647371dcab50060e5d6908f152ad33c10e534f9695f81e3f59733748f0f", + "sha256:bdcc8073544acbb625dfcd64af8d6fdddefa4dd8f6898eb0bea1d50bfc7db071", + "sha256:beaa44d409e16e31341713890baa4e612909250247201af49ddc57815c52a2e7", + "sha256:bff1c85756a80fba14285bcaef576419b4b159543fdb8c20714331c1d873e9e1", + "sha256:c0836a8181e4e857a91c1ab55adfee2dc4e60fff5e67f9e45f885a69586702f8", + "sha256:c5ee52cc58080472bd5668a2e114b779a56e200cdae52870e61a72c95c0760d4", + "sha256:c8c19dd552551b256ec70beed516d47953cbf153cc7b04ec7189b9000211f372", + "sha256:caebf75a83015299e45006819caac156ac2ef1e057158c6c1453c0ebf73de1d9", + "sha256:cb1fec1b2a03a219c9fb3cccb255140bc11daa86ce844ffe7354db7b2bc9223f", + "sha256:cbdcd7cb1745ce82b744fdadcfce167217c4c815e76e05897f806f99397fa092", + "sha256:cd44211d3430fc32abad56a991fe5279243cbdae303a1c00ce418e0562f315cb", + "sha256:cdf1f8140eccab24bd917a4111cc2783b1b430e8b9a6443b2ec4dce7267cfa4e", + "sha256:ce1c193b0b35c40aa8e2097c88cb92674fa4b37016724cd8e2dc2a12784fad4f", + "sha256:cf1f437b4258661d0cf884675487220fd035f67e50a648732e02b22b1e417d60", + "sha256:d1649e043e1277aae474e72f8fa3431cbf83605059e733043e718f77f59aef29", + "sha256:d29b5cefef1d2a2044fb800d7d7af2705b7acafac1bfd2630840a9a080d1c70d", + "sha256:d373e93e3384670b71fa42f4453cc1f979e15b34a21c0b746c5a6d14b6ebbb12", + "sha256:d404ec8a8d8301d803ccf037b3f0fb5fc8051aaa8a9a737cd6d4c490911d316a", + "sha256:d539ed7039ca0677a9eee156bd7551e158fd4c8e67b500fba4e9b42c2178dbde", + "sha256:d6cb856116850100d46df398cd68fc05d61ae5525eb0164b1aa9310e14c957d6", + "sha256:d8eb8edf6607a6648316f8a5a76bbd1bcb6626990dd9bd6c4b1dee91ec73650e", + "sha256:d9661f68bcf033a80da5292a87ab1abcbd6462ec308f83cc9a96d5df4255a500", + "sha256:d9facfbdf1952d3c96016ce17159a3ce97b296454bc900e94c5ea1f0ae85f00c", + "sha256:db57d37d1c2cc07e50dc8a37c1fd4903a38feb1b236e8f9094249d621deb39e5", + "sha256:de97c9e246bce7231dab34248a66218b48be5af5d5ae69c78421a095b0e0ab26", + "sha256:df2429ec7da1495122b63df437c04d3d5af8cf23654848d20637fa41c4aee1b5", + "sha256:e23d76671b1ea35d31144ea94f911d7096808e63834ee412105e9c281d50802a", + "sha256:e4b0a14f9226aa425bec207b9ea5ec149b6b9ff63d40399001b9ad4a1f9920df", + "sha256:e5f192cf3570c0588511afac7f0aa155f66dbf0e61ae39c8db63f67e0a3c9788", + "sha256:e62998daf49ccfa106459724ab3e57a87afc6b4e17e3422db633bf837918ee01", + "sha256:e6dedeb9aa7a4d6703c1b5ffd3ec97f4ee6549f4f6550f644f43a74f43fcc89b", + "sha256:e6f791cb35ec48a5822a888225b54b36e53e10ae7ec193f85d26c188702861a9", + "sha256:e94bb47d5fbacf7859f16117ab572b063705fdc6d3caf0afd92e02bbe1a0adfb", + "sha256:ed4c50a117e8ff0733e2c2b761adf183150ee9bf9294f06c01983d76e89f5f2c", + "sha256:f069e4c6336b19afdc594d04c81eb900af5d834a6cd8b395015389974b044972", + "sha256:f3a120f1ac4dfe9d7fdfadd3cd2e0f02260c2e1c1418767946fa6ce6a786dcd0", + "sha256:f3fb709498a53e0f9fb8d18612ae88b11bd472cce80431e312f1a6ad65bce673", + "sha256:f46d276c64b787c895323c82acb0f0151c4e0b275cf1ef001e2c7633a29f3068", + "sha256:f48861b7aaafee14c2c1769e28070cae1aeb011e54cdc75a7dae8ed732115c72", + "sha256:f6453b85fd4839660beff29165cdee0e0c486d32d8e7be002daffbf40c64c445", + "sha256:f8b738a46b7a4c6910a8cd34c3fec752c26afb5156939241b080311df1ace484", + "sha256:fb6a4bfdfb47904f1a4fdc135ab000e36bf79a0bc70aa9ec98275493ae19fd55" + ], + "index": "pypi", + "markers": "python_version ~= '3.5'", + "version": "==1.6.6" + }, "python-dateutil": { "hashes": [ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", diff --git a/app/Config.py b/app/Config.py index 591d0e80..95653bda 100644 --- a/app/Config.py +++ b/app/Config.py @@ -8,6 +8,7 @@ import coloredlogs from version import APP_VERSION from dotenv import load_dotenv from yt_dlp.version import __version__ as YTDLP_VERSION +from Utils import load_file class Config: @@ -151,14 +152,12 @@ class Config: if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0: LOG.info(f'Loading yt-dlp custom options from "{optsFile}"') - try: - with open(optsFile) as json_data: - opts = json.load(json_data) - assert isinstance(opts, dict) - self.ytdl_options.update(opts) - except (json.decoder.JSONDecodeError, AssertionError) as e: - LOG.error(f'JSON error in "{optsFile}": {e}') + (opts, status, error) = load_file(optsFile, dict) + if not status: + LOG.error(f'Error loading "{optsFile}": {error}') sys.exit(1) + + self.ytdl_options.update(opts) else: LOG.info(f'No custom yt-dlp options found in "{self.config_path}"') diff --git a/app/Utils.py b/app/Utils.py index 5e8f9901..1452a333 100644 --- a/app/Utils.py +++ b/app/Utils.py @@ -351,3 +351,40 @@ class Notifier: await asyncio.wait_for(asyncio.gather(*tasks), timeout=60) except asyncio.TimeoutError: LOG.error(f"Timed out sending event {event} to webhooks.") + + +def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: + """ + Load a JSON or JSON5 file and return the contents as a dictionary + + Args: + file (str): File path + check_type (type): Type to check the loaded file against. + + Returns tuple: + dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded. + bool: True if the file was loaded successfully. + str: Error message if the file could not be loaded. + """ + try: + with open(file) as json_data: + opts = json.load(json_data) + + if check_type: + assert isinstance(opts, check_type) + + return (opts, True, '',) + except Exception: + with open(file) as json_data: + from pyjson5 import load as json5_load + try: + opts = json5_load(json_data) + + if check_type: + assert isinstance(opts, check_type) + + return (opts, True, '',) + except AssertionError: + return ({}, False, f"Failed to assert that the contents '{type(opts)}' are of type '{check_type}'.",) + except Exception as e: + return ({}, False, f'{e}',) diff --git a/app/Webhooks.py b/app/Webhooks.py index 86c0cde2..c604b524 100644 --- a/app/Webhooks.py +++ b/app/Webhooks.py @@ -1,5 +1,4 @@ import asyncio -import json import logging import os from ItemDTO import ItemDTO @@ -9,7 +8,7 @@ LOG = logging.getLogger('Webhooks') class Webhooks: - targets: list[dict] = {} + targets: list[dict] = [] def __init__(self, file: str): if os.path.exists(file): @@ -21,8 +20,12 @@ class Webhooks: raise Exception(f'file is empty.') LOG.info(f'Loading webhooks from {file}') - with open(file, 'r') as f: - self.targets = json.load(f) + from Utils import load_file + (target, status, error) = load_file(file, list) + if not status: + raise Exception(f'{error}') + + self.targets = target except Exception as e: LOG.error(f'Error loading webhooks from {file}: {e}') pass diff --git a/app/main.py b/app/main.py index e6fcc3e5..005b6f77 100644 --- a/app/main.py +++ b/app/main.py @@ -4,12 +4,11 @@ import asyncio from datetime import datetime import json import os -import pathlib import random import time from Config import Config from DownloadQueue import DownloadQueue -from Utils import ObjectSerializer, Notifier +from Utils import ObjectSerializer, Notifier, load_file from aiohttp import web, client from aiohttp.web import Request, Response from Webhooks import Webhooks @@ -129,8 +128,12 @@ class Main: if os.path.exists(os.path.join(self.config.config_path, 'tasks.json')): try: - with open(os.path.join(self.config.config_path, 'tasks.json'), 'r') as f: - data['tasks'] = json.load(f) + (tasks, status, error) = load_file(os.path.join(self.config.config_path, 'tasks.json'), list) + if status is False: + LOG.error(f"Could not load tasks file. Error message '{error}'.") + else: + data['tasks'] = tasks + except Exception as e: pass @@ -166,10 +169,12 @@ class Main: return try: - with open(tasks_file, 'r') as f: - tasks = json.load(f) + (tasks, status, error) = load_file(tasks_file, list) + if status is False: + raise Exception(error) + except Exception as e: - LOG.error(f'Could not load tasks file [{tasks_file}]. Error message [{str(e)}]. Skipping Tasks.') + LOG.error(f"Could not load tasks file '{tasks_file}'. Error message '{str(e)}'. Skipping Tasks.") return for task in tasks: