Merge pull request #92 from arabcoders/dev

support jsonc in tasks/webhooks/ytdlp json files.
2024-05-14 12:49:48 +03:00 · 2024-05-14 12:49:48 +03:00 · a7633c07a5
commit a7633c07a5
parent 047bbfbe06 7a519bf16c
6 changed files with 197 additions and 19 deletions
--- a/1
+++ b/1
@ -15,6 +15,7 @@ python-magic = ">=0.4.27"
 debugpy = ">=1.8.1"
 httpx = "*"
 async-timeout = "*"
+pyjson5 = "*"

 [dev-packages]

--- a/Pipfile.lock
+++ b/Pipfile.lock
@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "dc6e4544c55ad5440a21d9bedd84e11bdcd375c8a3c1ddf7708e8f99e2f7167f"
+            "sha256": "d5812b5e6776f99dd75ccf98252f83238c48638b157d84695dd5cea36596c67c"
        },
        "pipfile-spec": 6,
        "requires": {
@ -675,6 +675,139 @@
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
            "version": "==3.20.0"
        },
+        "pyjson5": {
+            "hashes": [
+                "sha256:011957e1e7143c82ee5701dd39292d3864f6989247d1423462b1ef405390435e",
+                "sha256:02d60cd0f98d39347416d9172f38cd9601dfcf9976536688deb82c387ac22db1",
+                "sha256:0570cd200f5b52938769705c4f3092f1fcbb850f62278d30db8b8c6ae5ef0617",
+                "sha256:0abc60630bb1601ae9005e251d30cef363cb4d6cef2d2fb7a160329569170e38",
+                "sha256:0b9718821bfc36c61dd2ae8456fafbe3e9eb46df13cb2ac1ade38c5163ff9c92",
+                "sha256:0c069dfbc4939ce5d8b6d72b7755bb287988aab819435349e6f52da4e62fac9c",
+                "sha256:0d8063c83603e4eda99dc9396113521f0d277b027ccb3182e8e47ea284db8a70",
+                "sha256:0d828b98667ebca9fbd172f221746ecba105081b77f5ac6bbbf055f2faa38e14",
+                "sha256:122a980366db9c56b35bead60debf832d614ebe47a65d34bc292b2211e42e547",
+                "sha256:127f145467b61ef8a9a1d01c5c33a433d14dbca70c70d0b0800f4a19581303ff",
+                "sha256:144ea7654b64226cd24c6cc856827180b2e04ddc4404f9513ba41c6f284aa4c7",
+                "sha256:14671bbb72df85b895570f092e1b0aa34437a8df3e609e70a00459943aa1b75c",
+                "sha256:149f6ca0e346fce26ccb154e6633d3dbe10f197aae2b306cf3e3f09a57b6e4f7",
+                "sha256:14cad404a8dff0ea57370b98350b8fedb7611ee8023090b8a991673b4542abf2",
+                "sha256:17b258ba83c2d888568a098cab8f7df1ceffded589386eb5d70960ab83cffa9d",
+                "sha256:1c2ae4f322d2c859abd45efa893d06737b12575ca32fb068d2ab1ff7e1dacf7c",
+                "sha256:1d0bd69aa1b82410593341eb43c546c567bee5acb151666857c9df98e2cdfc09",
+                "sha256:1e3cf693ac82c7ee1da149e172bbe0cf59d11b06c31331f254daf8825d566033",
+                "sha256:1ea61368eb930a39441a7e9dd34ecb9af026a7b35bd03570910558abcd296215",
+                "sha256:20df8c5dbbe0d653f403da88b7520be44fc8d74697bbdd1ead688458b5691a02",
+                "sha256:211a9e260cf7d122e4d8881ae8724067cf7dfa48317760adb59b96bcf8e0a407",
+                "sha256:259f1797cd25e13396b6cb08dc0625c5de26a4214a8279e5304d6c91e5c8eb50",
+                "sha256:270600deefe532138d00cec1e4029c716fc86eaa52dabfb12494dca1cb30e611",
+                "sha256:2add15588939d8b1b7c59fd3f5480cce76231e0b014a2edebf3443ba92789d38",
+                "sha256:2cec7163effe96abe40ff098ffd2b16f5b322628bdac34c7aa5d316588524c42",
+                "sha256:314537a41af768367ab19a50d697a6ea85cb1c39bac4a2b93680cab930002e31",
+                "sha256:3145da3885e063b360bd9cc0bff6c9b0c83e3a2e822f83b4f7f40b26b41e1598",
+                "sha256:34500d5373dff030ab5183a69b3eb545a3a89385a3949142ea61ca95e653cbf8",
+                "sha256:36bdfa9334d00be616a63fd15295e9885d0e91dfa75cda5b6a8d1a2b406d6252",
+                "sha256:3d2829447d9d6a51a5976a3eb0a2a0069122ab741378c60367883e60160cb6de",
+                "sha256:410c15018434271408bb9410ac7db5195beccc33d8d1fbc8fb4a39858427e0df",
+                "sha256:415805f111faa16b050827beda8d763d4391becc24f86438848c7cb23ce63c55",
+                "sha256:420d9f970d678d8d1fab8aefdd483747c75e64681766e771494910b8030f41fa",
+                "sha256:4724d6bc1e369b29fb9fab7b10530cc11ccba60dc403cef3a83c213bc88d541c",
+                "sha256:48d02c97e34e64eefca4794dc3717373637aec4bd97b6e094cbed926b1893097",
+                "sha256:493fa3a81ce879e7ae47d1aa6a2b465a207087857b693404e523f069f531d01d",
+                "sha256:4f1a9eebe9d793e5b6ed00691744255400f57666004f454b09e2e651657e15fe",
+                "sha256:4fedb2ab5cfc821f9b9ed44dc1eae5d22a39a040950bda77c8b26e04df327629",
+                "sha256:50edc8c1f8c903f6e5df8547ae4195e3456ba2cdb45fbbad14022937a27ffa7c",
+                "sha256:521e7691fe38bd56bc4764e7c9c54b4783910f602904d8ca1a6571a9e82a3f82",
+                "sha256:55df02651c98dc8ad7fa30fd4fc5a4789a83ed0498312154c35ee46bdd027ccd",
+                "sha256:567437862f410a5912eee4cf13dd01a8c28ce9c9bf95590b9b9a4cb20e9daaed",
+                "sha256:57231901f4c96cc5bc630daef3fc3eadc95b8babe019b9f26b4915296a755bb5",
+                "sha256:59166de551e7321dbd3aa552ef647d652873701faadb021eae20f55ba6705829",
+                "sha256:5940c9c2feb9dbcda0cb87d2fc7edf330de231469c882f1e584d96814b6efd0d",
+                "sha256:5c7643308f995d746af8210b9a418c287885a0f8c0104b5e5160c190e56fbd0c",
+                "sha256:5f541197328ac3433863e2e67e4d34fccf1590bb88cc7f2c3fc2a81b8cde2681",
+                "sha256:62cd10320a1e80b1ce386ccaed9073cffd729e0e5b7a8793f2083291770b0db3",
+                "sha256:64381957edf773f4c67cc32c6e44fc791034efde5d654626171829d1219e4247",
+                "sha256:6b19a7025546406ca91184125aadc89a961189a9f5d4a76c0534a04f814c8882",
+                "sha256:6c8af6410aa6525783ceef0008f4f86480a4f04c4b921dd4295a2f2ba62f682d",
+                "sha256:6e2c27e3dd5caef56910c6da6a0c97cfc015a1dbdc6c2f2bd63a3ad86a16b0bd",
+                "sha256:6e358035e6fd975933f00a01c2ed3cc1e344f9a06b6edc445ad4b3bca7509de4",
+                "sha256:7366857ff307ef1bae3f1096651a2b2f86ef87b8ff4ea7264c6d7db9176adaab",
+                "sha256:759ef75276235d9491ecf31c4c7ba057cdcd45707c04d2bdd0c0a18bd9f36f71",
+                "sha256:76154b0e0245b9dbb97be34055d5be38cb6a55508785594b53e0f4d57b0267eb",
+                "sha256:7d1fe850aa1763e8f9945a7120f032f5f707d0372d52c8e0fecda2a79640820e",
+                "sha256:7d6e8f92e2f0b84e6ede805aa68bbee408f639c29f64fd14960a242bb518dcc6",
+                "sha256:7dffc9dcbdf09663f6aefcf03a48cfb029437ee60c0d4372e2218f30929d3035",
+                "sha256:7f9f8072be8d2cd3ba40346efeaaaba089a6e9be04ef23a3d690aaa897c53f71",
+                "sha256:7fe091a563a1fe8f1c495d96d26fd40f9d19b9c3674dbf89dd3c3df8bf46cfe5",
+                "sha256:80a96a264b7eb7fcbeaf646f44e18c22f98d1f72cc9e9ca284f785612df457d8",
+                "sha256:868619dc4d271ea9ec5f4041901238167526be369e3598867ca0d9170827692e",
+                "sha256:86ac9e2b5dc086a979a115661f7886208352a45f3276453db914adbda54adbb7",
+                "sha256:87063a381b2db5f5af794ba6a3ec16c9a9f8fc5e974e41c299f761e7af138ec3",
+                "sha256:8afb7a6f3e020b9a2996bb62f98067ebf793e86599f349910a7129fbfaebdc79",
+                "sha256:8b869fead9daa2ef5972f30abd76fdb8acfe3d6f73c0e3783fe13e1775d4aa05",
+                "sha256:8eddf775c2846e24f5f17c0ef2dc20f7de628148d03394a46ac1fd866a59ab78",
+                "sha256:9293b3768e27a69ef764daa10168c199887ffbe820ef86ea9c2ff155bdd27fba",
+                "sha256:9371ff9d5c6a92676935171eafa9cc6a541702352f747f77000b343d3101b0c0",
+                "sha256:94b2d2a26ecdd9ecef704a5a0c86a272e85d1df6bb30f44fb7f825d6975f1fb3",
+                "sha256:94e20d28d2cfba1b2be59c63ad8ae221e709a84158dc7f138d3b7258319f59b2",
+                "sha256:985f51077649fd0c84fcc7057d48b862c843f30f77c9f5c27102363c653c9e5e",
+                "sha256:990bf3aa3df840dfb98e3663d645d6c897d00ccf1e6cc2a594a547e8c239cc52",
+                "sha256:a093e24dd14a97f06d1596158d413b3b556cdf2b0d7a1af45b546d1d90224de7",
+                "sha256:a29645584914f7b1840e916777d402264ea3cbe3e04ff47ea00a799de17552d6",
+                "sha256:a3b0e45a2f1a84e78d8ecd70aecce0f84b360af37b18b2123646c3b310ea10a7",
+                "sha256:a79e914cba2a45725c2b10801bbdc7975e2b0926c4406f3cbd67d3976c2ded9c",
+                "sha256:a873c3a0d38f4f901e2d721ea3875ecf8d704a6d6c642cf31f5e17d37b60ca38",
+                "sha256:a8e4974cd32697f3a2c6738b597eaf66f2d23eb34dcdd79b8b6bb92549b56395",
+                "sha256:aae2ad989da263216f556f3c5a3ea3eaf8c45894c9fea641c483576adb27494f",
+                "sha256:ab2d7b402b8f27a866431dc1b476c4f9f0ccbb0811425c846985f05f5de9236b",
+                "sha256:ae55e95b6c59e9ad86662b6be213a6ea3f1e0c7f3d5627d88ca3dbe293a0a23a",
+                "sha256:b21596c5d805c5e8fae5ba2c80317f2a6441c62ea91d1bd6515c43c23911a16a",
+                "sha256:b2f96647371dcab50060e5d6908f152ad33c10e534f9695f81e3f59733748f0f",
+                "sha256:bdcc8073544acbb625dfcd64af8d6fdddefa4dd8f6898eb0bea1d50bfc7db071",
+                "sha256:beaa44d409e16e31341713890baa4e612909250247201af49ddc57815c52a2e7",
+                "sha256:bff1c85756a80fba14285bcaef576419b4b159543fdb8c20714331c1d873e9e1",
+                "sha256:c0836a8181e4e857a91c1ab55adfee2dc4e60fff5e67f9e45f885a69586702f8",
+                "sha256:c5ee52cc58080472bd5668a2e114b779a56e200cdae52870e61a72c95c0760d4",
+                "sha256:c8c19dd552551b256ec70beed516d47953cbf153cc7b04ec7189b9000211f372",
+                "sha256:caebf75a83015299e45006819caac156ac2ef1e057158c6c1453c0ebf73de1d9",
+                "sha256:cb1fec1b2a03a219c9fb3cccb255140bc11daa86ce844ffe7354db7b2bc9223f",
+                "sha256:cbdcd7cb1745ce82b744fdadcfce167217c4c815e76e05897f806f99397fa092",
+                "sha256:cd44211d3430fc32abad56a991fe5279243cbdae303a1c00ce418e0562f315cb",
+                "sha256:cdf1f8140eccab24bd917a4111cc2783b1b430e8b9a6443b2ec4dce7267cfa4e",
+                "sha256:ce1c193b0b35c40aa8e2097c88cb92674fa4b37016724cd8e2dc2a12784fad4f",
+                "sha256:cf1f437b4258661d0cf884675487220fd035f67e50a648732e02b22b1e417d60",
+                "sha256:d1649e043e1277aae474e72f8fa3431cbf83605059e733043e718f77f59aef29",
+                "sha256:d29b5cefef1d2a2044fb800d7d7af2705b7acafac1bfd2630840a9a080d1c70d",
+                "sha256:d373e93e3384670b71fa42f4453cc1f979e15b34a21c0b746c5a6d14b6ebbb12",
+                "sha256:d404ec8a8d8301d803ccf037b3f0fb5fc8051aaa8a9a737cd6d4c490911d316a",
+                "sha256:d539ed7039ca0677a9eee156bd7551e158fd4c8e67b500fba4e9b42c2178dbde",
+                "sha256:d6cb856116850100d46df398cd68fc05d61ae5525eb0164b1aa9310e14c957d6",
+                "sha256:d8eb8edf6607a6648316f8a5a76bbd1bcb6626990dd9bd6c4b1dee91ec73650e",
+                "sha256:d9661f68bcf033a80da5292a87ab1abcbd6462ec308f83cc9a96d5df4255a500",
+                "sha256:d9facfbdf1952d3c96016ce17159a3ce97b296454bc900e94c5ea1f0ae85f00c",
+                "sha256:db57d37d1c2cc07e50dc8a37c1fd4903a38feb1b236e8f9094249d621deb39e5",
+                "sha256:de97c9e246bce7231dab34248a66218b48be5af5d5ae69c78421a095b0e0ab26",
+                "sha256:df2429ec7da1495122b63df437c04d3d5af8cf23654848d20637fa41c4aee1b5",
+                "sha256:e23d76671b1ea35d31144ea94f911d7096808e63834ee412105e9c281d50802a",
+                "sha256:e4b0a14f9226aa425bec207b9ea5ec149b6b9ff63d40399001b9ad4a1f9920df",
+                "sha256:e5f192cf3570c0588511afac7f0aa155f66dbf0e61ae39c8db63f67e0a3c9788",
+                "sha256:e62998daf49ccfa106459724ab3e57a87afc6b4e17e3422db633bf837918ee01",
+                "sha256:e6dedeb9aa7a4d6703c1b5ffd3ec97f4ee6549f4f6550f644f43a74f43fcc89b",
+                "sha256:e6f791cb35ec48a5822a888225b54b36e53e10ae7ec193f85d26c188702861a9",
+                "sha256:e94bb47d5fbacf7859f16117ab572b063705fdc6d3caf0afd92e02bbe1a0adfb",
+                "sha256:ed4c50a117e8ff0733e2c2b761adf183150ee9bf9294f06c01983d76e89f5f2c",
+                "sha256:f069e4c6336b19afdc594d04c81eb900af5d834a6cd8b395015389974b044972",
+                "sha256:f3a120f1ac4dfe9d7fdfadd3cd2e0f02260c2e1c1418767946fa6ce6a786dcd0",
+                "sha256:f3fb709498a53e0f9fb8d18612ae88b11bd472cce80431e312f1a6ad65bce673",
+                "sha256:f46d276c64b787c895323c82acb0f0151c4e0b275cf1ef001e2c7633a29f3068",
+                "sha256:f48861b7aaafee14c2c1769e28070cae1aeb011e54cdc75a7dae8ed732115c72",
+                "sha256:f6453b85fd4839660beff29165cdee0e0c486d32d8e7be002daffbf40c64c445",
+                "sha256:f8b738a46b7a4c6910a8cd34c3fec752c26afb5156939241b080311df1ace484",
+                "sha256:fb6a4bfdfb47904f1a4fdc135ab000e36bf79a0bc70aa9ec98275493ae19fd55"
+            ],
+            "index": "pypi",
+            "markers": "python_version ~= '3.5'",
+            "version": "==1.6.6"
+        },
        "python-dateutil": {
            "hashes": [
                "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
--- a/app/Config.py
+++ b/app/Config.py
@ -8,6 +8,7 @@ import coloredlogs
 from version import APP_VERSION
 from dotenv import load_dotenv
 from yt_dlp.version import __version__ as YTDLP_VERSION
+from Utils import load_file


 class Config:
@ -151,14 +152,12 @@ class Config:
        if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0:
            LOG.info(f'Loading yt-dlp custom options from "{optsFile}"')

-            try:
-                with open(optsFile) as json_data:
-                    opts = json.load(json_data)
-                assert isinstance(opts, dict)
-                self.ytdl_options.update(opts)
-            except (json.decoder.JSONDecodeError, AssertionError) as e:
-                LOG.error(f'JSON error in "{optsFile}": {e}')
+            (opts, status, error) = load_file(optsFile, dict)
+            if not status:
+                LOG.error(f'Error loading "{optsFile}": {error}')
                sys.exit(1)
+
+            self.ytdl_options.update(opts)
        else:
            LOG.info(f'No custom yt-dlp options found in "{self.config_path}"')

--- a/app/Utils.py
+++ b/app/Utils.py
@ -351,3 +351,40 @@ class Notifier:
            await asyncio.wait_for(asyncio.gather(*tasks), timeout=60)
        except asyncio.TimeoutError:
            LOG.error(f"Timed out sending event {event} to webhooks.")
+
+
+def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
+    """
+    Load a JSON or JSON5 file and return the contents as a dictionary
+
+    Args:
+        file (str): File path
+        check_type (type): Type to check the loaded file against.
+
+    Returns tuple:
+        dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded.
+        bool: True if the file was loaded successfully.
+        str: Error message if the file could not be loaded.
+    """
+    try:
+        with open(file) as json_data:
+            opts = json.load(json_data)
+
+        if check_type:
+            assert isinstance(opts, check_type)
+
+        return (opts, True, '',)
+    except Exception:
+        with open(file) as json_data:
+            from pyjson5 import load as json5_load
+            try:
+                opts = json5_load(json_data)
+
+                if check_type:
+                    assert isinstance(opts, check_type)
+
+                return (opts, True, '',)
+            except AssertionError:
+                return ({}, False, f"Failed to assert that the contents '{type(opts)}' are of type '{check_type}'.",)
+            except Exception as e:
+                return ({}, False, f'{e}',)
--- a/app/Webhooks.py
+++ b/app/Webhooks.py
@ -1,5 +1,4 @@
 import asyncio
-import json
 import logging
 import os
 from ItemDTO import ItemDTO
@ -9,7 +8,7 @@ LOG = logging.getLogger('Webhooks')


 class Webhooks:
-    targets: list[dict] = {}
+    targets: list[dict] = []

    def __init__(self, file: str):
        if os.path.exists(file):
@ -21,8 +20,12 @@ class Webhooks:
                raise Exception(f'file is empty.')

            LOG.info(f'Loading webhooks from {file}')
-            with open(file, 'r') as f:
-                self.targets = json.load(f)
+            from Utils import load_file
+            (target, status, error) = load_file(file, list)
+            if not status:
+                raise Exception(f'{error}')
+
+            self.targets = target
        except Exception as e:
            LOG.error(f'Error loading webhooks from {file}: {e}')
            pass
--- a/app/main.py
+++ b/app/main.py
@ -4,12 +4,11 @@ import asyncio
 from datetime import datetime
 import json
 import os
-import pathlib
 import random
 import time
 from Config import Config
 from DownloadQueue import DownloadQueue
-from Utils import ObjectSerializer, Notifier
+from Utils import ObjectSerializer, Notifier, load_file
 from aiohttp import web, client
 from aiohttp.web import Request, Response
 from Webhooks import Webhooks
@ -129,8 +128,12 @@ class Main:

        if os.path.exists(os.path.join(self.config.config_path, 'tasks.json')):
            try:
-                with open(os.path.join(self.config.config_path, 'tasks.json'), 'r') as f:
-                    data['tasks'] = json.load(f)
+                (tasks, status, error) = load_file(os.path.join(self.config.config_path, 'tasks.json'), list)
+                if status is False:
+                    LOG.error(f"Could not load tasks file. Error message '{error}'.")
+                else:
+                    data['tasks'] = tasks
+
            except Exception as e:
                pass

@ -166,10 +169,12 @@ class Main:
            return

        try:
-            with open(tasks_file, 'r') as f:
-                tasks = json.load(f)
+            (tasks, status, error) = load_file(tasks_file, list)
+            if status is False:
+                raise Exception(error)
+
        except Exception as e:
-            LOG.error(f'Could not load tasks file [{tasks_file}]. Error message [{str(e)}]. Skipping Tasks.')
+            LOG.error(f"Could not load tasks file '{tasks_file}'. Error message '{str(e)}'. Skipping Tasks.")
            return

        for task in tasks: