From 74169b99396bbd34f56ae6701edddcb77f25fdad Mon Sep 17 00:00:00 2001 From: jbannon Date: Thu, 31 Mar 2022 01:27:56 +0000 Subject: [PATCH] config validator working --- config.yaml | 10 +-- tests/unit/entries/test_entry_formatter.py | 10 +-- ytdl_subscribe/entries/entry.py | 41 +----------- ytdl_subscribe/enums.py | 2 +- ytdl_subscribe/main.py | 64 ++++++++++--------- ytdl_subscribe/parse.py | 16 +++-- .../subscriptions => utils}/__init__.py | 0 ytdl_subscribe/utils/formatter_validator.py | 41 ++++++++++++ ytdl_subscribe/validators/config/__init__.py | 0 ytdl_subscribe/validators/config/config.py | 37 +++++++++++ .../preset.py} | 29 +++++---- .../{subscriptions => config}/sources.py | 31 +++++---- ytdl_subscribe/validators/exceptions.py | 4 ++ .../validators/native/object_validator.py | 20 ++++-- ytdl_subscribe/validators/native/validator.py | 4 +- 15 files changed, 187 insertions(+), 122 deletions(-) rename ytdl_subscribe/{validators/subscriptions => utils}/__init__.py (100%) create mode 100644 ytdl_subscribe/utils/formatter_validator.py create mode 100644 ytdl_subscribe/validators/config/__init__.py create mode 100644 ytdl_subscribe/validators/config/config.py rename ytdl_subscribe/validators/{subscriptions/subscription_preset.py => config/preset.py} (53%) rename ytdl_subscribe/validators/{subscriptions => config}/sources.py (65%) create mode 100644 ytdl_subscribe/validators/exceptions.py diff --git a/config.yaml b/config.yaml index 3cb55c26..4a686373 100644 --- a/config.yaml +++ b/config.yaml @@ -1,12 +1,12 @@ -config: - working_directory: 'tmp' +# ytdl-sub needs a temporary place to download files. Choose that path here +working_directory: 'tmp' presets: soundcloud_with_id3_tags: soundcloud: - download_strategy: "albums_then_tracks" + download_strategy: "albums_and_singles" skip_premiere_tracks: True - ytdl_opts: + ytdl_options: format: 'bestaudio[ext=mp3]' post_process: file_name: "[{album_year}] {sanitized_album}/{track_number_padded} - {sanitized_title}.{ext}" @@ -23,7 +23,7 @@ presets: music_videos: youtube: download_strategy: "playlist" - ytdl_opts: + ytdl_options: format: 'best' ignoreerrors: True post_process: diff --git a/tests/unit/entries/test_entry_formatter.py b/tests/unit/entries/test_entry_formatter.py index 7be76c29..0057de5d 100644 --- a/tests/unit/entries/test_entry_formatter.py +++ b/tests/unit/entries/test_entry_formatter.py @@ -1,6 +1,6 @@ import pytest -from ytdl_subscribe.entries.entry import EntryFormatter +from ytdl_subscribe.utils.formatter_validator import FormatterValidator @pytest.fixture @@ -44,11 +44,11 @@ def error_message_unequal_regex_matches_generator( class TestEntryFormatter(object): def test_parse(self): format_string = "Here is my {var_one} and {var_two} 💩" - assert EntryFormatter(format_string).parse() == ["var_one", "var_two"] + assert FormatterValidator(format_string).parse() == ["var_one", "var_two"] def test_parse_no_variables(self): format_string = "No vars 💩" - assert EntryFormatter(format_string).parse() == [] + assert FormatterValidator(format_string).parse() == [] @pytest.mark.parametrize( "format_string", @@ -65,7 +65,7 @@ class TestEntryFormatter(object): expected_error_msg = error_message_unequal_brackets_generator(format_string) with pytest.raises(ValueError, match=expected_error_msg): - assert EntryFormatter(format_string).parse() + assert FormatterValidator(format_string).parse() @pytest.mark.parametrize( "format_string", @@ -86,4 +86,4 @@ class TestEntryFormatter(object): ) with pytest.raises(ValueError, match=expected_error_msg): - assert EntryFormatter(format_string).parse() + assert FormatterValidator(format_string).parse() diff --git a/ytdl_subscribe/entries/entry.py b/ytdl_subscribe/entries/entry.py index b5c8e0e8..cc030429 100644 --- a/ytdl_subscribe/entries/entry.py +++ b/ytdl_subscribe/entries/entry.py @@ -1,48 +1,11 @@ -import re from pathlib import Path from typing import Any from typing import Dict -from typing import List from typing import Optional from sanitize_filename import sanitize - -class EntryFormatter: - """ - Ensures user-created formatter strings are valid - """ - - FIELDS_VALIDATOR = re.compile(r"{([a-z_]+?)}") - - def __init__(self, format_string: str): - self.format_string = format_string - self._error_prefix = f"Format string '{self.format_string}' is invalid:" - - def parse(self) -> List[str]: - """ - Returns - ------- - List of fields to format - """ - open_bracket_count = self.format_string.count("{") - close_bracket_count = self.format_string.count("}") - - if open_bracket_count != close_bracket_count: - raise ValueError( - f"{self._error_prefix} Brackets are reserved for {{variable_names}} " - f"and should contain a single open and close bracket." - ) - - parsed_fields = re.findall(EntryFormatter.FIELDS_VALIDATOR, self.format_string) - - if len(parsed_fields) != open_bracket_count: - raise ValueError( - f"{self._error_prefix} {{variable_names}} should only contain " - f"lowercase letters and underscores with a single open and close bracket." - ) - - return sorted(parsed_fields) +from ytdl_subscribe.utils.formatter_validator import FormatterValidator class Entry: @@ -141,7 +104,7 @@ class Entry: if overrides: entry_dict = dict(entry_dict, **overrides) - field_names = EntryFormatter(format_string).parse() + field_names = FormatterValidator(format_string).parse() for field_name in field_names: if field_name not in entry_dict: diff --git a/ytdl_subscribe/enums.py b/ytdl_subscribe/enums.py index 78e996b5..6ef0bed9 100644 --- a/ytdl_subscribe/enums.py +++ b/ytdl_subscribe/enums.py @@ -19,4 +19,4 @@ class SubscriptionSourceName(object): class YAMLSection(object): CONFIG_KEY = "config" PRESET_KEY = "presets" - SUBSCRIPTIONS_KEY = "subscriptions" + SUBSCRIPTIONS_KEY = "config" diff --git a/ytdl_subscribe/main.py b/ytdl_subscribe/main.py index 151f1d50..6034d61a 100644 --- a/ytdl_subscribe/main.py +++ b/ytdl_subscribe/main.py @@ -1,8 +1,14 @@ import argparse +from typing import List from ytdl_subscribe.parse import parse_presets from ytdl_subscribe.parse import parse_subscriptions from ytdl_subscribe.parse import parse_subscriptions_file +from ytdl_subscribe.parse import read_config_file + +################################################################################################### +# GLOBAL PARSER +from ytdl_subscribe.validators.config.config import Config parser = argparse.ArgumentParser( description="YoutubeDL-Subscribe: Download and organize your favorite media hassle-free." @@ -12,50 +18,46 @@ parser.add_argument( "--config", metavar="CONFIGPATH", type=str, - help="path to the config yaml, uses subscriptions.yaml if not provided", - default="subscriptions.yaml", + help="path to the config yaml, uses config.yaml if not provided", + default="config.yaml", ) +################################################################################################### +# SUBSCRIPTION PARSER subparsers = parser.add_subparsers(dest="subparser") -subscription_parser = subparsers.add_parser("s") +subscription_parser = subparsers.add_parser("sub") subscription_parser.add_argument( - "subscriptions", - metavar="SUB", + "subscription_paths", + metavar="SUBPATH", nargs="*", - help="specific subscriptions to download, downloads all if not provided", + help="path to subscription files, uses config.yaml if not provided", + default="config.yaml", ) +################################################################################################### +# INTERACTIVE DOWNLOAD PARSER download_parser = subparsers.add_parser("dl") download_parser.add_argument( - "-p", "--preset", metavar="PRESET", help="which preset to apply" -) -download_parser.add_argument("-o", "--output", metavar="OUT", help="output path") -download_parser.add_argument( - "-r", - "--overrides", - metavar="NAME:VALUE", - nargs="+", - help="variable overrides formmatted as name:value", + "url", + help="url to the desired content to download, will prompt for further arguments needed", ) if __name__ == "__main__": args = parser.parse_args() - print(args) + config: Config = Config.from_file_path(args.config) + if args.subparser == "sub": + subscription_paths: List[str] = args.subscription_paths - # subscriptions = parse_subscriptions(args.config) - - yaml_dict = parse_subscriptions_file(args.config, set_config_variables=True) - presets = parse_presets(yaml_dict) - - # Subscription download - if args.subparser == "s": - subscriptions = parse_subscriptions( - yaml_dict, presets, subscriptions=args.subscriptions - ) - for s in subscriptions: - s.extract_info() + # for subscription_path in subscription_paths: + # + # + # subscriptions = parse_subscriptions( + # yaml_dict, presets, subscriptions=args.subscriptions + # ) + # for s in subscriptions: + # s.extract_info() # One-off download if args.subparser == "dl": - preset = args.preset - output_path = args.output_path - overrides = args.overrides + print("Interactive download is not supported yet. Stay tuned!") + + exit(0) diff --git a/ytdl_subscribe/parse.py b/ytdl_subscribe/parse.py index 8b819eed..10ec22e0 100644 --- a/ytdl_subscribe/parse.py +++ b/ytdl_subscribe/parse.py @@ -8,12 +8,20 @@ from ytdl_subscribe.enums import YAMLSection from ytdl_subscribe.subscriptions.soundcloud import SoundcloudSubscription from ytdl_subscribe.subscriptions.subscription import Subscription from ytdl_subscribe.subscriptions.youtube import YoutubeSubscription +from ytdl_subscribe.validators.config.config import Config def _set_config_variables(config): Subscription.WORKING_DIRECTORY = config.get("working_directory", "") +def read_config_file(config_path: str) -> Config: + with open(config_path, "r", encoding="utf-8") as file: + config_dict = yaml.safe_load(file) + + return Config(name="config", value=config_dict).validate() + + def parse_subscriptions_file(subscription_yaml_path, set_config_variables=True): """ Reads and parses a subscription yaml from its path. @@ -62,14 +70,14 @@ def parse_presets(yaml_dict): def parse_subscriptions(yaml_dict: dict, presets: dict, subscriptions: Optional[list]): """ - Parses subscriptions from a subscription yaml dict + Parses config from a subscription yaml dict Parameters ---------- yaml_dict: dict presets: dict subscriptions: list[str] or None - If present, only parse these subscriptions + If present, only parse these config Returns ------- @@ -77,7 +85,7 @@ def parse_subscriptions(yaml_dict: dict, presets: dict, subscriptions: Optional[ """ parsed_subscriptions = [] - # Parse all subscriptions even if all are not used for validation's sake + # Parse all config even if all are not used for validation's sake for name, subscription in yaml_dict[YAMLSection.SUBSCRIPTIONS_KEY].items(): preset = {} if subscription.get("preset") in presets: @@ -104,7 +112,7 @@ def parse_subscriptions(yaml_dict: dict, presets: dict, subscriptions: Optional[ ) ) - # Filter subscriptions if present + # Filter config if present if subscriptions: parsed_subscriptions = [ sub for sub in parsed_subscriptions if sub.name in subscriptions diff --git a/ytdl_subscribe/validators/subscriptions/__init__.py b/ytdl_subscribe/utils/__init__.py similarity index 100% rename from ytdl_subscribe/validators/subscriptions/__init__.py rename to ytdl_subscribe/utils/__init__.py diff --git a/ytdl_subscribe/utils/formatter_validator.py b/ytdl_subscribe/utils/formatter_validator.py new file mode 100644 index 00000000..b7e4f890 --- /dev/null +++ b/ytdl_subscribe/utils/formatter_validator.py @@ -0,0 +1,41 @@ +import re +from typing import List + + +class FormatterValidator: + """ + Ensures user-created formatter strings are valid + """ + + FIELDS_VALIDATOR = re.compile(r"{([a-z_]+?)}") + + def __init__(self, format_string: str): + self.format_string = format_string + self._error_prefix = f"Format string '{self.format_string}' is invalid:" + + def parse(self) -> List[str]: + """ + Returns + ------- + List of fields to format + """ + open_bracket_count = self.format_string.count("{") + close_bracket_count = self.format_string.count("}") + + if open_bracket_count != close_bracket_count: + raise ValueError( + f"{self._error_prefix} Brackets are reserved for {{variable_names}} " + f"and should contain a single open and close bracket." + ) + + parsed_fields = re.findall( + FormatterValidator.FIELDS_VALIDATOR, self.format_string + ) + + if len(parsed_fields) != open_bracket_count: + raise ValueError( + f"{self._error_prefix} {{variable_names}} should only contain " + f"lowercase letters and underscores with a single open and close bracket." + ) + + return sorted(parsed_fields) diff --git a/ytdl_subscribe/validators/config/__init__.py b/ytdl_subscribe/validators/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ytdl_subscribe/validators/config/config.py b/ytdl_subscribe/validators/config/config.py new file mode 100644 index 00000000..7621215e --- /dev/null +++ b/ytdl_subscribe/validators/config/config.py @@ -0,0 +1,37 @@ +from typing import Any +from typing import Dict +from typing import Optional + +import yaml + +from ytdl_subscribe.validators.config.preset import Preset +from ytdl_subscribe.validators.native.object_validator import ObjectValidator + + +class Config(ObjectValidator): + required_fields = {"working_directory", "presets"} + + def __init__(self, name: str, value: Any): + super().__init__(name, value) + self.working_directory: Optional[str] = None + self.presets: Optional[Dict[str, Preset]] = None + + def validate(self) -> "Config": + super().validate() + + self.working_directory = self.get("working_directory") + self.presets = {} + for preset_name, preset_obj in self.get("presets").items(): + self.presets[preset_name] = Preset( + name=f"{self.name}.presets.{preset_name}", value=preset_obj + ).validate() + + return self + + @classmethod + def from_file_path(cls, config_path) -> "Config": + # TODO: Create separate yaml file loader class + with open(config_path, "r", encoding="utf-8") as file: + config_dict = yaml.safe_load(file) + + return Config(name="config", value=config_dict).validate() diff --git a/ytdl_subscribe/validators/subscriptions/subscription_preset.py b/ytdl_subscribe/validators/config/preset.py similarity index 53% rename from ytdl_subscribe/validators/subscriptions/subscription_preset.py rename to ytdl_subscribe/validators/config/preset.py index ba429ea7..466d856e 100644 --- a/ytdl_subscribe/validators/subscriptions/subscription_preset.py +++ b/ytdl_subscribe/validators/config/preset.py @@ -2,45 +2,46 @@ from typing import Any from typing import Optional from ytdl_subscribe.enums import SubscriptionSourceName +from ytdl_subscribe.validators.config.sources import PresetSource +from ytdl_subscribe.validators.config.sources import SoundcloudSource +from ytdl_subscribe.validators.config.sources import YoutubeSource +from ytdl_subscribe.validators.exceptions import ValidationException from ytdl_subscribe.validators.native.object_validator import ObjectValidator -from ytdl_subscribe.validators.subscriptions.sources import SoundcloudSource -from ytdl_subscribe.validators.subscriptions.sources import SubscriptionSource -from ytdl_subscribe.validators.subscriptions.sources import YoutubeSource -class SubscriptionPreset(ObjectValidator): +class Preset(ObjectValidator): required_fields = {"post_process"} optional_fields = {"ytdl_options", *SubscriptionSourceName.all()} allow_extra_fields = False - def __init__(self, key: str, value: Any): - super().__init__(key=key, value=value) - self.subscription_source: Optional[SubscriptionSource] = None + def __init__(self, name: str, value: Any): + super().__init__(name=name, value=value) + self.subscription_source: Optional[PresetSource] = None def _validate_subscription_source(self): # Make sure we only have a single subscription source for object_key, object_value in self.object_items: if object_key in SubscriptionSourceName.all() and self.subscription_source: - raise ValueError( - f"'{self.key}' can only have one of the following sources: {SubscriptionSourceName.pretty_all()}" + raise ValidationException( + f"'{self.name}' can only have one of the following sources: {SubscriptionSourceName.pretty_all()}" ) if object_key == SubscriptionSourceName.SOUNDCLOUD: self.subscription_source = SoundcloudSource( - key=object_key, value=object_value + name=f"{self.name}{object_key}", value=object_value ).validate() elif object_key == SubscriptionSourceName.YOUTUBE: self.subscription_source = YoutubeSource( - key=object_key, value=object_value + name=f"{self.name}{object_key}", value=object_value ).validate() # If subscription source was not set, error if not self.subscription_source: - raise ValueError( - f"'{self.key} must have one of the following sources: {SubscriptionSourceName.pretty_all()}" + raise ValidationException( + f"'{self.name} must have one of the following sources: {SubscriptionSourceName.pretty_all()}" ) - def validate(self) -> "SubscriptionPreset": + def validate(self) -> "Preset": super().validate() self._validate_subscription_source() return self diff --git a/ytdl_subscribe/validators/subscriptions/sources.py b/ytdl_subscribe/validators/config/sources.py similarity index 65% rename from ytdl_subscribe/validators/subscriptions/sources.py rename to ytdl_subscribe/validators/config/sources.py index e670429b..6054df61 100644 --- a/ytdl_subscribe/validators/subscriptions/sources.py +++ b/ytdl_subscribe/validators/config/sources.py @@ -2,34 +2,35 @@ from typing import Any from typing import Optional from typing import Set +from ytdl_subscribe.validators.exceptions import ValidationException from ytdl_subscribe.validators.native.object_validator import ObjectValidator -class SubscriptionSource(ObjectValidator): +class PresetSource(ObjectValidator): required_fields = {"download_strategy"} download_strategies: Set[str] = {} - def __init__(self, key: str, value: Any): - super().__init__(key=key, value=value) + def __init__(self, name: str, value: Any): + super().__init__(name=name, value=value) self.download_strategy: Optional[str] = None - def validate(self) -> "SubscriptionSource": + def validate(self) -> "PresetSource": super().validate() if self.value["download_strategy"] not in self.download_strategies: - raise ValueError( + raise ValidationException( f"'download_strategy' must be one of the following: {', '.join(self.download_strategies)}" ) return self -class YoutubeSource(SubscriptionSource): +class YoutubeSource(PresetSource): optional_fields = {"playlist_id"} download_strategies = {"playlist"} - def __init__(self, key: str, value: Any): - super().__init__(key=key, value=value) + def __init__(self, name: str, value: Any): + super().__init__(name=name, value=value) self.playlist_id: Optional[str] = None def validate(self) -> "YoutubeSource": @@ -38,28 +39,30 @@ class YoutubeSource(SubscriptionSource): if self.download_strategy == "playlist": self.playlist_id = self.value.get("playlist_id") if not self.playlist_id or not isinstance(self.playlist_id, str): - raise ValueError( + raise ValidationException( "Youtube download_strategy 'playlist' requires the field 'playlist_id' to be a string" ) return self -class SoundcloudSource(SubscriptionSource): - optional_fields = {"username"} +class SoundcloudSource(PresetSource): + optional_fields = {"username", "skip_premiere_tracks"} download_strategies = {"albums_and_singles"} - def __init__(self, key: str, value: Any): - super().__init__(key=key, value=value) + def __init__(self, name: str, value: Any): + super().__init__(name=name, value=value) self.username: Optional[str] = None + self.skip_premiere_tracks: Optional[bool] = None def validate(self) -> "SoundcloudSource": super().validate() + self.skip_premiere_tracks = self.get("skip_premiere_tracks", True) if self.download_strategy == "albums_and_singles": self.username = self.value.get("username") if not self.username or not isinstance(self.username, str): - raise ValueError( + raise ValidationException( "Soundcloud download_strategy 'albums_and_singles' requires the field 'username'" ) diff --git a/ytdl_subscribe/validators/exceptions.py b/ytdl_subscribe/validators/exceptions.py new file mode 100644 index 00000000..f00ba7b2 --- /dev/null +++ b/ytdl_subscribe/validators/exceptions.py @@ -0,0 +1,4 @@ +class ValidationException(ValueError): + """Any user-caused configuration error should result in this error""" + + pass diff --git a/ytdl_subscribe/validators/native/object_validator.py b/ytdl_subscribe/validators/native/object_validator.py index 59dcaf88..4864b412 100644 --- a/ytdl_subscribe/validators/native/object_validator.py +++ b/ytdl_subscribe/validators/native/object_validator.py @@ -1,5 +1,8 @@ +from typing import Any +from typing import Optional from typing import Set +from ytdl_subscribe.validators.exceptions import ValidationException from ytdl_subscribe.validators.native.validator import Validator @@ -21,36 +24,39 @@ class ObjectValidator(Validator): def object_items(self): return self.value.items() + def get(self, object_key: str, default: Optional[Any] = None) -> Any: + return self.value.get(object_key, default) + def validate(self) -> "ObjectValidator": super().validate() # Ensure the value is an object if not isinstance(self.value, dict): - error_msg = f"'{self.key}' must be an object" + error_msg = f"'{self.name}' must be an object" if self.required_fields: error_msg = f"{error_msg} with the required field: {', '.join(self.required_fields)}" - raise ValueError(error_msg) + raise ValidationException(error_msg) # Ensure the object contains all the required fields for required_key in self.required_fields: if required_key not in self.value: error_msg = ( - f"'{self.key}' is missing the required field '{required_key}'" + f"'{self.name}' is missing the required field '{required_key}'" ) - raise ValueError(error_msg) + raise ValidationException(error_msg) # If extra fields are not allowed, ensure we only have required or optional keys if not self.allow_extra_fields: for object_key in self.object_keys: if ( object_key not in self.required_fields - or object_key not in self.optional_fields + and object_key not in self.optional_fields ): error_msg = ( - f"'{self.key}' contains the field '{object_key} which is not allowed. " + f"'{self.name}' contains the field '{object_key}' which is not allowed. " f"Allowed fields: {', '.join(self.allowed_fields)}" ) - raise ValueError(error_msg) + raise ValidationException(error_msg) return self diff --git a/ytdl_subscribe/validators/native/validator.py b/ytdl_subscribe/validators/native/validator.py index ac975185..c210beb3 100644 --- a/ytdl_subscribe/validators/native/validator.py +++ b/ytdl_subscribe/validators/native/validator.py @@ -2,8 +2,8 @@ from typing import Any class Validator: - def __init__(self, key: str, value: Any): - self.key = key + def __init__(self, name: str, value: Any): + self.name = name self.value = value def validate(self) -> "Validator":