Merge pull request #92 from arabcoders/dev

support jsonc in tasks/webhooks/ytdlp json files.
This commit is contained in:
Abdulmohsen 2024-05-14 12:49:48 +03:00 committed by GitHub
commit a7633c07a5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 197 additions and 19 deletions

View file

@ -15,6 +15,7 @@ python-magic = ">=0.4.27"
debugpy = ">=1.8.1" debugpy = ">=1.8.1"
httpx = "*" httpx = "*"
async-timeout = "*" async-timeout = "*"
pyjson5 = "*"
[dev-packages] [dev-packages]

135
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "dc6e4544c55ad5440a21d9bedd84e11bdcd375c8a3c1ddf7708e8f99e2f7167f" "sha256": "d5812b5e6776f99dd75ccf98252f83238c48638b157d84695dd5cea36596c67c"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -675,6 +675,139 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.20.0" "version": "==3.20.0"
}, },
"pyjson5": {
"hashes": [
"sha256:011957e1e7143c82ee5701dd39292d3864f6989247d1423462b1ef405390435e",
"sha256:02d60cd0f98d39347416d9172f38cd9601dfcf9976536688deb82c387ac22db1",
"sha256:0570cd200f5b52938769705c4f3092f1fcbb850f62278d30db8b8c6ae5ef0617",
"sha256:0abc60630bb1601ae9005e251d30cef363cb4d6cef2d2fb7a160329569170e38",
"sha256:0b9718821bfc36c61dd2ae8456fafbe3e9eb46df13cb2ac1ade38c5163ff9c92",
"sha256:0c069dfbc4939ce5d8b6d72b7755bb287988aab819435349e6f52da4e62fac9c",
"sha256:0d8063c83603e4eda99dc9396113521f0d277b027ccb3182e8e47ea284db8a70",
"sha256:0d828b98667ebca9fbd172f221746ecba105081b77f5ac6bbbf055f2faa38e14",
"sha256:122a980366db9c56b35bead60debf832d614ebe47a65d34bc292b2211e42e547",
"sha256:127f145467b61ef8a9a1d01c5c33a433d14dbca70c70d0b0800f4a19581303ff",
"sha256:144ea7654b64226cd24c6cc856827180b2e04ddc4404f9513ba41c6f284aa4c7",
"sha256:14671bbb72df85b895570f092e1b0aa34437a8df3e609e70a00459943aa1b75c",
"sha256:149f6ca0e346fce26ccb154e6633d3dbe10f197aae2b306cf3e3f09a57b6e4f7",
"sha256:14cad404a8dff0ea57370b98350b8fedb7611ee8023090b8a991673b4542abf2",
"sha256:17b258ba83c2d888568a098cab8f7df1ceffded589386eb5d70960ab83cffa9d",
"sha256:1c2ae4f322d2c859abd45efa893d06737b12575ca32fb068d2ab1ff7e1dacf7c",
"sha256:1d0bd69aa1b82410593341eb43c546c567bee5acb151666857c9df98e2cdfc09",
"sha256:1e3cf693ac82c7ee1da149e172bbe0cf59d11b06c31331f254daf8825d566033",
"sha256:1ea61368eb930a39441a7e9dd34ecb9af026a7b35bd03570910558abcd296215",
"sha256:20df8c5dbbe0d653f403da88b7520be44fc8d74697bbdd1ead688458b5691a02",
"sha256:211a9e260cf7d122e4d8881ae8724067cf7dfa48317760adb59b96bcf8e0a407",
"sha256:259f1797cd25e13396b6cb08dc0625c5de26a4214a8279e5304d6c91e5c8eb50",
"sha256:270600deefe532138d00cec1e4029c716fc86eaa52dabfb12494dca1cb30e611",
"sha256:2add15588939d8b1b7c59fd3f5480cce76231e0b014a2edebf3443ba92789d38",
"sha256:2cec7163effe96abe40ff098ffd2b16f5b322628bdac34c7aa5d316588524c42",
"sha256:314537a41af768367ab19a50d697a6ea85cb1c39bac4a2b93680cab930002e31",
"sha256:3145da3885e063b360bd9cc0bff6c9b0c83e3a2e822f83b4f7f40b26b41e1598",
"sha256:34500d5373dff030ab5183a69b3eb545a3a89385a3949142ea61ca95e653cbf8",
"sha256:36bdfa9334d00be616a63fd15295e9885d0e91dfa75cda5b6a8d1a2b406d6252",
"sha256:3d2829447d9d6a51a5976a3eb0a2a0069122ab741378c60367883e60160cb6de",
"sha256:410c15018434271408bb9410ac7db5195beccc33d8d1fbc8fb4a39858427e0df",
"sha256:415805f111faa16b050827beda8d763d4391becc24f86438848c7cb23ce63c55",
"sha256:420d9f970d678d8d1fab8aefdd483747c75e64681766e771494910b8030f41fa",
"sha256:4724d6bc1e369b29fb9fab7b10530cc11ccba60dc403cef3a83c213bc88d541c",
"sha256:48d02c97e34e64eefca4794dc3717373637aec4bd97b6e094cbed926b1893097",
"sha256:493fa3a81ce879e7ae47d1aa6a2b465a207087857b693404e523f069f531d01d",
"sha256:4f1a9eebe9d793e5b6ed00691744255400f57666004f454b09e2e651657e15fe",
"sha256:4fedb2ab5cfc821f9b9ed44dc1eae5d22a39a040950bda77c8b26e04df327629",
"sha256:50edc8c1f8c903f6e5df8547ae4195e3456ba2cdb45fbbad14022937a27ffa7c",
"sha256:521e7691fe38bd56bc4764e7c9c54b4783910f602904d8ca1a6571a9e82a3f82",
"sha256:55df02651c98dc8ad7fa30fd4fc5a4789a83ed0498312154c35ee46bdd027ccd",
"sha256:567437862f410a5912eee4cf13dd01a8c28ce9c9bf95590b9b9a4cb20e9daaed",
"sha256:57231901f4c96cc5bc630daef3fc3eadc95b8babe019b9f26b4915296a755bb5",
"sha256:59166de551e7321dbd3aa552ef647d652873701faadb021eae20f55ba6705829",
"sha256:5940c9c2feb9dbcda0cb87d2fc7edf330de231469c882f1e584d96814b6efd0d",
"sha256:5c7643308f995d746af8210b9a418c287885a0f8c0104b5e5160c190e56fbd0c",
"sha256:5f541197328ac3433863e2e67e4d34fccf1590bb88cc7f2c3fc2a81b8cde2681",
"sha256:62cd10320a1e80b1ce386ccaed9073cffd729e0e5b7a8793f2083291770b0db3",
"sha256:64381957edf773f4c67cc32c6e44fc791034efde5d654626171829d1219e4247",
"sha256:6b19a7025546406ca91184125aadc89a961189a9f5d4a76c0534a04f814c8882",
"sha256:6c8af6410aa6525783ceef0008f4f86480a4f04c4b921dd4295a2f2ba62f682d",
"sha256:6e2c27e3dd5caef56910c6da6a0c97cfc015a1dbdc6c2f2bd63a3ad86a16b0bd",
"sha256:6e358035e6fd975933f00a01c2ed3cc1e344f9a06b6edc445ad4b3bca7509de4",
"sha256:7366857ff307ef1bae3f1096651a2b2f86ef87b8ff4ea7264c6d7db9176adaab",
"sha256:759ef75276235d9491ecf31c4c7ba057cdcd45707c04d2bdd0c0a18bd9f36f71",
"sha256:76154b0e0245b9dbb97be34055d5be38cb6a55508785594b53e0f4d57b0267eb",
"sha256:7d1fe850aa1763e8f9945a7120f032f5f707d0372d52c8e0fecda2a79640820e",
"sha256:7d6e8f92e2f0b84e6ede805aa68bbee408f639c29f64fd14960a242bb518dcc6",
"sha256:7dffc9dcbdf09663f6aefcf03a48cfb029437ee60c0d4372e2218f30929d3035",
"sha256:7f9f8072be8d2cd3ba40346efeaaaba089a6e9be04ef23a3d690aaa897c53f71",
"sha256:7fe091a563a1fe8f1c495d96d26fd40f9d19b9c3674dbf89dd3c3df8bf46cfe5",
"sha256:80a96a264b7eb7fcbeaf646f44e18c22f98d1f72cc9e9ca284f785612df457d8",
"sha256:868619dc4d271ea9ec5f4041901238167526be369e3598867ca0d9170827692e",
"sha256:86ac9e2b5dc086a979a115661f7886208352a45f3276453db914adbda54adbb7",
"sha256:87063a381b2db5f5af794ba6a3ec16c9a9f8fc5e974e41c299f761e7af138ec3",
"sha256:8afb7a6f3e020b9a2996bb62f98067ebf793e86599f349910a7129fbfaebdc79",
"sha256:8b869fead9daa2ef5972f30abd76fdb8acfe3d6f73c0e3783fe13e1775d4aa05",
"sha256:8eddf775c2846e24f5f17c0ef2dc20f7de628148d03394a46ac1fd866a59ab78",
"sha256:9293b3768e27a69ef764daa10168c199887ffbe820ef86ea9c2ff155bdd27fba",
"sha256:9371ff9d5c6a92676935171eafa9cc6a541702352f747f77000b343d3101b0c0",
"sha256:94b2d2a26ecdd9ecef704a5a0c86a272e85d1df6bb30f44fb7f825d6975f1fb3",
"sha256:94e20d28d2cfba1b2be59c63ad8ae221e709a84158dc7f138d3b7258319f59b2",
"sha256:985f51077649fd0c84fcc7057d48b862c843f30f77c9f5c27102363c653c9e5e",
"sha256:990bf3aa3df840dfb98e3663d645d6c897d00ccf1e6cc2a594a547e8c239cc52",
"sha256:a093e24dd14a97f06d1596158d413b3b556cdf2b0d7a1af45b546d1d90224de7",
"sha256:a29645584914f7b1840e916777d402264ea3cbe3e04ff47ea00a799de17552d6",
"sha256:a3b0e45a2f1a84e78d8ecd70aecce0f84b360af37b18b2123646c3b310ea10a7",
"sha256:a79e914cba2a45725c2b10801bbdc7975e2b0926c4406f3cbd67d3976c2ded9c",
"sha256:a873c3a0d38f4f901e2d721ea3875ecf8d704a6d6c642cf31f5e17d37b60ca38",
"sha256:a8e4974cd32697f3a2c6738b597eaf66f2d23eb34dcdd79b8b6bb92549b56395",
"sha256:aae2ad989da263216f556f3c5a3ea3eaf8c45894c9fea641c483576adb27494f",
"sha256:ab2d7b402b8f27a866431dc1b476c4f9f0ccbb0811425c846985f05f5de9236b",
"sha256:ae55e95b6c59e9ad86662b6be213a6ea3f1e0c7f3d5627d88ca3dbe293a0a23a",
"sha256:b21596c5d805c5e8fae5ba2c80317f2a6441c62ea91d1bd6515c43c23911a16a",
"sha256:b2f96647371dcab50060e5d6908f152ad33c10e534f9695f81e3f59733748f0f",
"sha256:bdcc8073544acbb625dfcd64af8d6fdddefa4dd8f6898eb0bea1d50bfc7db071",
"sha256:beaa44d409e16e31341713890baa4e612909250247201af49ddc57815c52a2e7",
"sha256:bff1c85756a80fba14285bcaef576419b4b159543fdb8c20714331c1d873e9e1",
"sha256:c0836a8181e4e857a91c1ab55adfee2dc4e60fff5e67f9e45f885a69586702f8",
"sha256:c5ee52cc58080472bd5668a2e114b779a56e200cdae52870e61a72c95c0760d4",
"sha256:c8c19dd552551b256ec70beed516d47953cbf153cc7b04ec7189b9000211f372",
"sha256:caebf75a83015299e45006819caac156ac2ef1e057158c6c1453c0ebf73de1d9",
"sha256:cb1fec1b2a03a219c9fb3cccb255140bc11daa86ce844ffe7354db7b2bc9223f",
"sha256:cbdcd7cb1745ce82b744fdadcfce167217c4c815e76e05897f806f99397fa092",
"sha256:cd44211d3430fc32abad56a991fe5279243cbdae303a1c00ce418e0562f315cb",
"sha256:cdf1f8140eccab24bd917a4111cc2783b1b430e8b9a6443b2ec4dce7267cfa4e",
"sha256:ce1c193b0b35c40aa8e2097c88cb92674fa4b37016724cd8e2dc2a12784fad4f",
"sha256:cf1f437b4258661d0cf884675487220fd035f67e50a648732e02b22b1e417d60",
"sha256:d1649e043e1277aae474e72f8fa3431cbf83605059e733043e718f77f59aef29",
"sha256:d29b5cefef1d2a2044fb800d7d7af2705b7acafac1bfd2630840a9a080d1c70d",
"sha256:d373e93e3384670b71fa42f4453cc1f979e15b34a21c0b746c5a6d14b6ebbb12",
"sha256:d404ec8a8d8301d803ccf037b3f0fb5fc8051aaa8a9a737cd6d4c490911d316a",
"sha256:d539ed7039ca0677a9eee156bd7551e158fd4c8e67b500fba4e9b42c2178dbde",
"sha256:d6cb856116850100d46df398cd68fc05d61ae5525eb0164b1aa9310e14c957d6",
"sha256:d8eb8edf6607a6648316f8a5a76bbd1bcb6626990dd9bd6c4b1dee91ec73650e",
"sha256:d9661f68bcf033a80da5292a87ab1abcbd6462ec308f83cc9a96d5df4255a500",
"sha256:d9facfbdf1952d3c96016ce17159a3ce97b296454bc900e94c5ea1f0ae85f00c",
"sha256:db57d37d1c2cc07e50dc8a37c1fd4903a38feb1b236e8f9094249d621deb39e5",
"sha256:de97c9e246bce7231dab34248a66218b48be5af5d5ae69c78421a095b0e0ab26",
"sha256:df2429ec7da1495122b63df437c04d3d5af8cf23654848d20637fa41c4aee1b5",
"sha256:e23d76671b1ea35d31144ea94f911d7096808e63834ee412105e9c281d50802a",
"sha256:e4b0a14f9226aa425bec207b9ea5ec149b6b9ff63d40399001b9ad4a1f9920df",
"sha256:e5f192cf3570c0588511afac7f0aa155f66dbf0e61ae39c8db63f67e0a3c9788",
"sha256:e62998daf49ccfa106459724ab3e57a87afc6b4e17e3422db633bf837918ee01",
"sha256:e6dedeb9aa7a4d6703c1b5ffd3ec97f4ee6549f4f6550f644f43a74f43fcc89b",
"sha256:e6f791cb35ec48a5822a888225b54b36e53e10ae7ec193f85d26c188702861a9",
"sha256:e94bb47d5fbacf7859f16117ab572b063705fdc6d3caf0afd92e02bbe1a0adfb",
"sha256:ed4c50a117e8ff0733e2c2b761adf183150ee9bf9294f06c01983d76e89f5f2c",
"sha256:f069e4c6336b19afdc594d04c81eb900af5d834a6cd8b395015389974b044972",
"sha256:f3a120f1ac4dfe9d7fdfadd3cd2e0f02260c2e1c1418767946fa6ce6a786dcd0",
"sha256:f3fb709498a53e0f9fb8d18612ae88b11bd472cce80431e312f1a6ad65bce673",
"sha256:f46d276c64b787c895323c82acb0f0151c4e0b275cf1ef001e2c7633a29f3068",
"sha256:f48861b7aaafee14c2c1769e28070cae1aeb011e54cdc75a7dae8ed732115c72",
"sha256:f6453b85fd4839660beff29165cdee0e0c486d32d8e7be002daffbf40c64c445",
"sha256:f8b738a46b7a4c6910a8cd34c3fec752c26afb5156939241b080311df1ace484",
"sha256:fb6a4bfdfb47904f1a4fdc135ab000e36bf79a0bc70aa9ec98275493ae19fd55"
],
"index": "pypi",
"markers": "python_version ~= '3.5'",
"version": "==1.6.6"
},
"python-dateutil": { "python-dateutil": {
"hashes": [ "hashes": [
"sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",

View file

@ -8,6 +8,7 @@ import coloredlogs
from version import APP_VERSION from version import APP_VERSION
from dotenv import load_dotenv from dotenv import load_dotenv
from yt_dlp.version import __version__ as YTDLP_VERSION from yt_dlp.version import __version__ as YTDLP_VERSION
from Utils import load_file
class Config: class Config:
@ -151,14 +152,12 @@ class Config:
if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0: if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0:
LOG.info(f'Loading yt-dlp custom options from "{optsFile}"') LOG.info(f'Loading yt-dlp custom options from "{optsFile}"')
try: (opts, status, error) = load_file(optsFile, dict)
with open(optsFile) as json_data: if not status:
opts = json.load(json_data) LOG.error(f'Error loading "{optsFile}": {error}')
assert isinstance(opts, dict)
self.ytdl_options.update(opts)
except (json.decoder.JSONDecodeError, AssertionError) as e:
LOG.error(f'JSON error in "{optsFile}": {e}')
sys.exit(1) sys.exit(1)
self.ytdl_options.update(opts)
else: else:
LOG.info(f'No custom yt-dlp options found in "{self.config_path}"') LOG.info(f'No custom yt-dlp options found in "{self.config_path}"')

View file

@ -351,3 +351,40 @@ class Notifier:
await asyncio.wait_for(asyncio.gather(*tasks), timeout=60) await asyncio.wait_for(asyncio.gather(*tasks), timeout=60)
except asyncio.TimeoutError: except asyncio.TimeoutError:
LOG.error(f"Timed out sending event {event} to webhooks.") LOG.error(f"Timed out sending event {event} to webhooks.")
def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
"""
Load a JSON or JSON5 file and return the contents as a dictionary
Args:
file (str): File path
check_type (type): Type to check the loaded file against.
Returns tuple:
dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded.
bool: True if the file was loaded successfully.
str: Error message if the file could not be loaded.
"""
try:
with open(file) as json_data:
opts = json.load(json_data)
if check_type:
assert isinstance(opts, check_type)
return (opts, True, '',)
except Exception:
with open(file) as json_data:
from pyjson5 import load as json5_load
try:
opts = json5_load(json_data)
if check_type:
assert isinstance(opts, check_type)
return (opts, True, '',)
except AssertionError:
return ({}, False, f"Failed to assert that the contents '{type(opts)}' are of type '{check_type}'.",)
except Exception as e:
return ({}, False, f'{e}',)

View file

@ -1,5 +1,4 @@
import asyncio import asyncio
import json
import logging import logging
import os import os
from ItemDTO import ItemDTO from ItemDTO import ItemDTO
@ -9,7 +8,7 @@ LOG = logging.getLogger('Webhooks')
class Webhooks: class Webhooks:
targets: list[dict] = {} targets: list[dict] = []
def __init__(self, file: str): def __init__(self, file: str):
if os.path.exists(file): if os.path.exists(file):
@ -21,8 +20,12 @@ class Webhooks:
raise Exception(f'file is empty.') raise Exception(f'file is empty.')
LOG.info(f'Loading webhooks from {file}') LOG.info(f'Loading webhooks from {file}')
with open(file, 'r') as f: from Utils import load_file
self.targets = json.load(f) (target, status, error) = load_file(file, list)
if not status:
raise Exception(f'{error}')
self.targets = target
except Exception as e: except Exception as e:
LOG.error(f'Error loading webhooks from {file}: {e}') LOG.error(f'Error loading webhooks from {file}: {e}')
pass pass

View file

@ -4,12 +4,11 @@ import asyncio
from datetime import datetime from datetime import datetime
import json import json
import os import os
import pathlib
import random import random
import time import time
from Config import Config from Config import Config
from DownloadQueue import DownloadQueue from DownloadQueue import DownloadQueue
from Utils import ObjectSerializer, Notifier from Utils import ObjectSerializer, Notifier, load_file
from aiohttp import web, client from aiohttp import web, client
from aiohttp.web import Request, Response from aiohttp.web import Request, Response
from Webhooks import Webhooks from Webhooks import Webhooks
@ -129,8 +128,12 @@ class Main:
if os.path.exists(os.path.join(self.config.config_path, 'tasks.json')): if os.path.exists(os.path.join(self.config.config_path, 'tasks.json')):
try: try:
with open(os.path.join(self.config.config_path, 'tasks.json'), 'r') as f: (tasks, status, error) = load_file(os.path.join(self.config.config_path, 'tasks.json'), list)
data['tasks'] = json.load(f) if status is False:
LOG.error(f"Could not load tasks file. Error message '{error}'.")
else:
data['tasks'] = tasks
except Exception as e: except Exception as e:
pass pass
@ -166,10 +169,12 @@ class Main:
return return
try: try:
with open(tasks_file, 'r') as f: (tasks, status, error) = load_file(tasks_file, list)
tasks = json.load(f) if status is False:
raise Exception(error)
except Exception as e: except Exception as e:
LOG.error(f'Could not load tasks file [{tasks_file}]. Error message [{str(e)}]. Skipping Tasks.') LOG.error(f"Could not load tasks file '{tasks_file}'. Error message '{str(e)}'. Skipping Tasks.")
return return
for task in tasks: for task in tasks: