protected variables for easier debugging

This commit is contained in:
jbannon 2022-04-05 00:26:45 +00:00
parent ac378ab962
commit 922ac3b7cf
21 changed files with 152 additions and 138 deletions

View file

@ -1,5 +1,6 @@
# ytdl-sub needs a temporary place to download files. Choose that path here
working_directory: 'tmp'
configuration:
working_directory: 'tmp'
presets:
soundcloud_with_id3_tags:

View file

@ -65,7 +65,7 @@ class TestEntry(object):
def test_entry_formatter_fails_missing_field(self, mock_entry):
format_string = f"prefix {{bah_humbug}} suffix"
available_fields = ", ".join(sorted(mock_entry.to_dict().keys()))
available_fields = ", ".join(sorted(mock_entry.to_dict()._keys()))
expected_error_msg = f"Format variable 'bah_humbug' does not exist for Entry. Available fields: {available_fields}"
with pytest.raises(ValueError, match=expected_error_msg):

View file

@ -44,18 +44,15 @@ class Downloader:
def __init__(
self,
output_directory: str,
working_directory: Optional[str] = None,
ytdl_options: Optional[Dict] = None,
):
self.output_path = output_directory
self.working_directory = working_directory
if self.working_directory is None:
self.working_directory = tempfile.TemporaryDirectory().name
self.output_directory = output_directory
if self.output_directory is None:
self.output_directory = tempfile.TemporaryDirectory().name
self.ytdl_options = Downloader._configure_ytdl_options(
ytdl_options=ytdl_options,
working_directory=self.working_directory,
working_directory=self.output_directory,
)
@contextmanager

View file

@ -26,7 +26,7 @@ class YoutubeDownloader(Downloader):
"""
ytdl_metadata_override = {
"download_archive": str(
Path(self.working_directory) / "ytdl-download-archive.txt"
Path(self.output_directory) / "ytdl-download-archive.txt"
),
"writeinfojson": True,
}
@ -44,12 +44,12 @@ class YoutubeDownloader(Downloader):
entries: List[YoutubeVideo] = []
# Load the entries from info.json, ignore the playlist entry
for file_name in os.listdir(self.working_directory):
for file_name in os.listdir(self.output_directory):
if file_name.endswith(".info.json") and not file_name.startswith(
playlist_id
):
with open(
Path(self.working_directory) / file_name, "r", encoding="utf-8"
Path(self.output_directory) / file_name, "r", encoding="utf-8"
) as file:
entries.append(YoutubeVideo(**json.load(file)))

View file

@ -4,7 +4,7 @@ from typing import List
###################################################################################################
# GLOBAL PARSER
from ytdl_subscribe.validators.config.config_validator import ConfigValidator
from ytdl_subscribe.validators.config.config_validator import ConfigFileValidator
from ytdl_subscribe.validators.config.subscription_validator import (
SubscriptionValidator,
)
@ -42,7 +42,7 @@ download_parser.add_argument(
if __name__ == "__main__":
args = parser.parse_args()
config: ConfigValidator = ConfigValidator.from_file_path(args.config)
config: ConfigFileValidator = ConfigFileValidator.from_file_path(args.config)
if args.subparser == "sub":
subscription_paths: List[str] = args.subscription_paths
subscriptions: List[SubscriptionValidator] = []

View file

@ -30,8 +30,7 @@ class SoundcloudAlbumsAndSinglesSubscription(Subscription):
tracks: List[SoundcloudTrack] = []
soundcloud_downloader = SoundcloudDownloader(
output_directory=self.output_options.output_directory.value,
working_directory=self.config_options.working_directory.value,
output_directory=self.config_options.working_directory.value,
ytdl_options=self.ytdl_options.dict,
)

View file

@ -9,7 +9,7 @@ import music_tag
from PIL import Image
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.validators.config.config_validator import ConfigValidator
from ytdl_subscribe.validators.config.config_validator import ConfigOptionsValidator
from ytdl_subscribe.validators.config.metadata_options.metadata_options_validator import (
MetadataOptionsValidator,
)
@ -34,7 +34,7 @@ class Subscription(object):
def __init__(
self,
name: str,
config_options: ConfigValidator,
config_options: ConfigOptionsValidator,
source_options: SourceValidator,
output_options: OutputOptionsValidator,
metadata_options: MetadataOptionsValidator,
@ -46,7 +46,7 @@ class Subscription(object):
----------
name: str
Name of the subscription
config_options: ConfigValidator
config_options: ConfigOptionsValidator
source_options: SourceValidator
output_options: OutputOptionsValidator
metadata_options: MetadataOptionsValidator

View file

@ -22,8 +22,7 @@ class YoutubeSubscription(Subscription):
def extract_info(self):
youtube_downloader = YoutubeDownloader(
output_directory=self.output_options.output_directory.value,
working_directory=self.config_options.working_directory.value,
output_directory=self.config_options.working_directory.value,
ytdl_options=self.ytdl_options.dict,
)

View file

@ -25,13 +25,13 @@ class StrictDictValidator(DictValidator):
# Ensure all required keys are present
for required_key in self._required_keys:
if required_key not in self.value:
if required_key not in self._dict:
raise self._validation_exception(
f"missing the required field '{required_key}'"
)
# Ensure an empty dict was not passed as the value
if not self.dict:
if not self._dict:
raise self._validation_exception(
f"at least one of the following fields must be defined: "
f"{', '.join(self._optional_keys)}'"
@ -39,10 +39,10 @@ class StrictDictValidator(DictValidator):
# Ensure all keys are either required or optional keys if no extra field are allowed
if not self._allow_extra_keys:
for object_key in self.keys:
for object_key in self._keys:
if object_key not in self._allowed_keys:
error_msg = (
f"'{self.name}' contains the field '{object_key}' which is not allowed. "
f"'{self._name}' contains the field '{object_key}' which is not allowed. "
f"Allowed fields: {', '.join(self._allowed_keys)}"
)
raise ValidationException(error_msg)

View file

@ -1,5 +1,6 @@
import re
from keyword import iskeyword
from typing import Dict
from typing import List
from ytdl_subscribe.validators.base.validators import DictValidator
@ -13,7 +14,7 @@ class StringFormatterValidator(StringValidator):
_expected_value_type_name = "format string"
FIELDS_VALIDATOR = re.compile(r"{([a-z_]+?)}")
__fields_validator = re.compile(r"{([a-z_]+?)}")
def __validate_and_get_format_variables(self) -> List[str]:
"""
@ -37,7 +38,7 @@ class StringFormatterValidator(StringValidator):
)
format_variables: List[str] = list(
re.findall(StringFormatterValidator.FIELDS_VALIDATOR, self.format_string)
re.findall(StringFormatterValidator.__fields_validator, self.format_string)
)
if len(format_variables) != open_bracket_count:
@ -76,5 +77,9 @@ class DictFormatterValidator(DictValidator):
def __init__(self, name, value):
super().__init__(name, value)
for key in self.keys:
_ = self.validate_key(key=key, validator=StringFormatterValidator)
for key in self._keys:
_ = self._validate_key(key=key, validator=StringFormatterValidator)
@property
def dict(self) -> Dict[str, str]:
return self._dict

View file

@ -21,7 +21,7 @@ class Validator:
_expected_value_type_name: Optional[str] = None
def __init__(self, name: str, value: Any):
self.name = name
self._name = name
self._value = value
if not isinstance(self._value, self._expected_value_type):
@ -32,15 +32,6 @@ class Validator:
error_message=f"should be of type {expected_value_type_name}."
)
@property
def value(self) -> object:
"""
Returns
-------
Value of the validator
"""
return self._value
def _validation_exception(self, error_message: str) -> ValidationException:
"""
Parameters
@ -52,7 +43,7 @@ class Validator:
-------
Validation exception with a consistent prefix.
"""
prefix = f"Validation error in {self.name}: "
prefix = f"Validation error in {self._name}: "
return ValidationException(f"{prefix}{error_message}")
@ -79,7 +70,7 @@ class StringValidator(Validator):
Validates string fields.
"""
_expected_value_type: Type = str
_expected_value_type = str
_expected_value_type_name = "string"
@property
@ -105,7 +96,7 @@ class DictValidator(Validator):
_expected_value_type_name = "object"
@property
def dict(self) -> dict:
def _dict(self) -> dict:
"""
Returns
-------
@ -114,15 +105,15 @@ class DictValidator(Validator):
return self._value
@property
def keys(self) -> List[str]:
def _keys(self) -> List[str]:
"""
Returns
-------
Sorted list of dictionary keys
"""
return sorted(list(self.dict.keys()))
return sorted(list(self._dict.keys()))
def validate_key(
def _validate_key(
self,
key: str,
validator: Type[T],
@ -142,24 +133,24 @@ class DictValidator(Validator):
-------
An instance of the specified validator
"""
value = self.dict.get(key, default)
value = self._dict.get(key, default)
if value is None:
raise self._validation_exception(
f"{key} is missing when it should be present."
)
return validator(
name=f"{self.name}.{key}",
name=f"{self._name}.{key}",
value=value,
)
def validate_key_if_present(
def _validate_key_if_present(
self,
key: str,
validator: Type[T],
default: Optional[Any] = None,
) -> Optional[T]:
if key not in self.dict:
if key not in self._dict:
return None
return self.validate_key(key=key, validator=validator, default=default)
return self._validate_key(key=key, validator=validator, default=default)

View file

@ -7,21 +7,34 @@ from ytdl_subscribe.validators.base.validators import DictValidator
from ytdl_subscribe.validators.base.validators import StringValidator
class ConfigValidator(StrictDictValidator):
_required_keys = {"working_directory", "presets"}
class ConfigOptionsValidator(StrictDictValidator):
_required_keys = {"working_directory"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.working_directory = self.validate_key("working_directory", StringValidator)
self.presets = self.validate_key("presets", DictValidator)
self.working_directory = self._validate_key(
key="working_directory", validator=StringValidator
)
class ConfigFileValidator(StrictDictValidator):
_required_keys = {"configuration", "presets"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.config_options = self._validate_key(
"configuration", ConfigOptionsValidator
)
self.presets = self._validate_key("presets", DictValidator)
@classmethod
def from_dict(cls, config_dict) -> "ConfigValidator":
return ConfigValidator(name="config", value=config_dict)
def from_dict(cls, config_dict) -> "ConfigFileValidator":
return ConfigFileValidator(name="", value=config_dict)
@classmethod
def from_file_path(cls, config_path) -> "ConfigValidator":
def from_file_path(cls, config_path) -> "ConfigFileValidator":
# TODO: Create separate yaml file loader class
with open(config_path, "r", encoding="utf-8") as file:
config_dict = yaml.safe_load(file)
return ConfigValidator.from_dict(config_dict)
return ConfigFileValidator.from_dict(config_dict)

View file

@ -17,11 +17,11 @@ class Id3Validator(StrictDictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.id3_version = self.validate_key(
self.id3_version = self._validate_key(
key="id3_version", validator=Id3VersionValidator
).value
self.tags = self.validate_key(key="tags", validator=DictFormatterValidator)
self.tags = self._validate_key(key="tags", validator=DictFormatterValidator)
self.multi_value_separator = self.validate_key_if_present(
self.multi_value_separator = self._validate_key_if_present(
key="multi_value_separator", validator=StringValidator
)

View file

@ -9,5 +9,5 @@ class MetadataOptionsValidator(StrictDictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.id3 = self.validate_key_if_present(key="id3", validator=Id3Validator)
self.nfo = self.validate_key_if_present(key="nfo", validator=NFOValidator)
self.id3 = self._validate_key_if_present(key="id3", validator=Id3Validator)
self.nfo = self._validate_key_if_present(key="nfo", validator=NFOValidator)

View file

@ -13,10 +13,10 @@ class NFOValidator(StrictDictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.nfo_name = self.validate_key(
self.nfo_name = self._validate_key(
key="nfo_name", validator=StringFormatterValidator
)
self.nfo_root = self.validate_key(
self.nfo_root = self._validate_key(
key="nfo_root", validator=StringFormatterValidator
)
self.tags = self.validate_key(key="tags", validator=DictFormatterValidator)
self.tags = self._validate_key(key="tags", validator=DictFormatterValidator)

View file

@ -18,16 +18,16 @@ class OutputOptionsValidator(StrictDictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.output_directory: StringFormatterValidator = self.validate_key(
self.output_directory: StringFormatterValidator = self._validate_key(
key="output_directory", validator=StringFormatterValidator
)
self.file_name: StringFormatterValidator = self.validate_key(
self.file_name: StringFormatterValidator = self._validate_key(
key="file_name", validator=StringFormatterValidator
)
self.convert_thumbnail = self.validate_key_if_present(
self.convert_thumbnail = self._validate_key_if_present(
key="convert_thumbnail", validator=ConvertThumbnailValidator
)
self.thumbnail_name = self.validate_key_if_present(
self.thumbnail_name = self._validate_key_if_present(
key="thumbnail_name", validator=StringFormatterValidator
)

View file

@ -30,15 +30,17 @@ from ytdl_subscribe.validators.exceptions import ValidationException
class YTDLOptionsValidator(DictValidator):
pass
@property
def dict(self) -> Dict:
return self._dict
class OverridesValidator(DictFormatterValidator):
@property
def dict(self) -> dict:
def dict(self) -> Dict[str, str]:
"""For overrides, create sanitized versions of each entry for convenience"""
output_dict = copy.deepcopy(super().dict)
for key in list(output_dict.keys()):
output_dict = copy.deepcopy(self._dict)
for key in self._keys:
output_dict[f"sanitized_{key}"] = sanitize_filename.sanitize(
output_dict[key]
)
@ -46,69 +48,69 @@ class OverridesValidator(DictFormatterValidator):
return output_dict
class PresetValidator(StrictDictValidator):
_subscription_source_validator_mapping: Dict[str, Type[SourceValidator]] = {
"soundcloud": SoundcloudSourceValidator,
"youtube": YoutubeSourceValidator,
}
PRESET_SOURCE_VALIDATOR_MAPPING: Dict[str, Type[SourceValidator]] = {
"soundcloud": SoundcloudSourceValidator,
"youtube": YoutubeSourceValidator,
}
_required_keys = {"output_options"}
_optional_keys = {
"metadata_options",
"ytdl_options",
"overrides",
*_subscription_source_validator_mapping.keys(),
}
PRESET_REQUIRED_KEYS = {"output_options"}
PRESET_OPTIONAL_KEYS = {
"metadata_options",
"ytdl_options",
"overrides",
*PRESET_SOURCE_VALIDATOR_MAPPING.keys(),
}
class PresetValidator(StrictDictValidator):
_required_keys = PRESET_REQUIRED_KEYS
_optional_keys = PRESET_OPTIONAL_KEYS
@property
def available_sources(self) -> List[str]:
return sorted(list(self._subscription_source_validator_mapping.keys()))
def __available_sources(self) -> List[str]:
return sorted(list(PRESET_SOURCE_VALIDATOR_MAPPING.keys()))
def __validate_and_get_subscription_source(self) -> Tuple[str, SourceValidator]:
def __validate_and_get_subscription_source(self) -> SourceValidator:
subscription_source: Optional[SourceValidator] = None
subscription_source_name: Optional[str] = None
for key in self.keys:
if key in self.available_sources and subscription_source:
for key in self._keys:
if key in self.__available_sources and subscription_source:
raise ValidationException(
f"'{self.name}' can only have one of the following sources: "
f"{', '.join(self.available_sources)}"
f"'{self._name}' can only have one of the following sources: "
f"{', '.join(self.__available_sources)}"
)
if key in self._subscription_source_validator_mapping:
subscription_source_name = key
subscription_source = self.validate_key(
key=key, validator=self._subscription_source_validator_mapping[key]
if key in PRESET_SOURCE_VALIDATOR_MAPPING:
subscription_source = self._validate_key(
key=key, validator=PRESET_SOURCE_VALIDATOR_MAPPING[key]
)
# If subscription source was not set, error
if not subscription_source:
raise ValidationException(
f"'{self.name} must have one of the following sources: "
f"{', '.join(self.available_sources)}"
f"'{self._name} must have one of the following sources: "
f"{', '.join(self.__available_sources)}"
)
return subscription_source_name, subscription_source
return subscription_source
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
(
self.subscription_source_name,
self.subscription_source,
) = self.__validate_and_get_subscription_source()
self.output_options = self.validate_key(
self.subscription_source = self.__validate_and_get_subscription_source()
self.output_options = self._validate_key(
key="output_options",
validator=OutputOptionsValidator,
)
self.metadata_options = self.validate_key(
self.metadata_options = self._validate_key(
key="metadata_options", validator=MetadataOptionsValidator
)
self.ytdl_options = self.validate_key(
self.ytdl_options = self._validate_key(
key="ytdl_options", validator=YTDLOptionsValidator, default={}
)
self.overrides = self.validate_key(
self.overrides = self._validate_key(
key="overrides", validator=OverridesValidator, default={}
)

View file

@ -11,18 +11,18 @@ class SoundcloudAlbumsAndSinglesDownloadValidator(DownloadStrategyValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.username = self.validate_key(key="username", validator=StringValidator)
self.username = self._validate_key(key="username", validator=StringValidator)
class SoundcloudSourceValidator(SourceValidator):
_optional_keys = {"skip_premiere_tracks"}
download_strategy_validator_mapping = {
_download_strategy_validator_mapping = {
"albums_and_singles": SoundcloudAlbumsAndSinglesDownloadValidator
}
def __init__(self, name: str, value: dict):
super().__init__(name=name, value=value)
self.skip_premiere_tracks = self.validate_key(
self.skip_premiere_tracks = self._validate_key(
"skip_premiere_tracks", BoolValidator, default=True
)

View file

@ -18,32 +18,34 @@ class SourceValidator(StrictDictValidator):
# Extra fields will be strict-validated using other StictDictValidators
_allow_extra_keys = True
download_strategy_validator_mapping: Dict[str, Type[DownloadStrategyValidator]] = {}
_download_strategy_validator_mapping: Dict[
str, Type[DownloadStrategyValidator]
] = {}
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
self.download_strategy_name = self.validate_key(
download_strategy_name = self._validate_key(
key="download_strategy",
validator=StringValidator,
).value
if self.download_strategy_name not in self.possible_download_strategies:
if download_strategy_name not in self._possible_download_strategies:
raise self._validation_exception(
f"download_strategy must be one of the following: {', '.join(self.possible_download_strategies)}"
f"download_strategy must be one of the following: {', '.join(self._possible_download_strategies)}"
)
# Remove all non-download strategy keys before passing the dict to the validator
download_strategy_dict = copy.deepcopy(self.dict)
download_strategy_dict = copy.deepcopy(self._dict)
for key_to_delete in self._allowed_keys:
del download_strategy_dict[key_to_delete]
download_strategy_class = self.download_strategy_validator_mapping[
self.download_strategy_name
download_strategy_class = self._download_strategy_validator_mapping[
download_strategy_name
]
self.download_strategy = download_strategy_class(
name=self.name, value=download_strategy_dict
name=self._name, value=download_strategy_dict
)
@property
def possible_download_strategies(self):
return sorted(list(self.download_strategy_validator_mapping.keys()))
def _possible_download_strategies(self):
return sorted(list(self._download_strategy_validator_mapping.keys()))

View file

@ -12,11 +12,13 @@ class YoutubePlaylistDownloadValidator(DownloadStrategyValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.playlist_id = self.validate_key("playlist_id", StringValidator)
self.playlist_id = self._validate_key("playlist_id", StringValidator)
class YoutubeSourceValidator(SourceValidator):
download_strategy_validator_mapping = {"playlist": YoutubePlaylistDownloadValidator}
_download_strategy_validator_mapping = {
"playlist": YoutubePlaylistDownloadValidator
}
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)

View file

@ -11,7 +11,12 @@ from ytdl_subscribe.subscriptions.subscription import Subscription
from ytdl_subscribe.subscriptions.youtube import YoutubeSubscription
from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.base.validators import StringValidator
from ytdl_subscribe.validators.config.config_validator import ConfigValidator
from ytdl_subscribe.validators.config.config_validator import ConfigFileValidator
from ytdl_subscribe.validators.config.preset_validator import PRESET_OPTIONAL_KEYS
from ytdl_subscribe.validators.config.preset_validator import PRESET_REQUIRED_KEYS
from ytdl_subscribe.validators.config.preset_validator import (
PRESET_SOURCE_VALIDATOR_MAPPING,
)
from ytdl_subscribe.validators.config.preset_validator import OverridesValidator
from ytdl_subscribe.validators.config.preset_validator import PresetValidator
from ytdl_subscribe.validators.config.sources.soundcloud_validators import (
@ -28,27 +33,25 @@ class SubscriptionValidator(StrictDictValidator):
"""
_required_keys = {"preset"}
_optional_keys = PresetValidator._required_keys.union(
PresetValidator._optional_keys
)
_optional_keys = PRESET_REQUIRED_KEYS.union(PRESET_OPTIONAL_KEYS)
def __init__(self, config: ConfigValidator, name: str, value: Any):
def __init__(self, config: ConfigFileValidator, name: str, value: Any):
super().__init__(name, value)
self.config = config
# Ensure the overrides defined here are valid
_ = self.validate_key(
_ = self._validate_key(
key="overrides",
validator=OverridesValidator,
default={},
)
preset_name = self.validate_key(
preset_name = self._validate_key(
key="preset",
validator=StringValidator,
).value
available_presets = self.config.presets.keys
available_presets = self.config.presets._keys
if preset_name not in available_presets:
raise self._validation_exception(
f"'preset '{preset_name}' does not exist in the provided config. "
@ -57,14 +60,14 @@ class SubscriptionValidator(StrictDictValidator):
# A little hacky, we will override the preset with the contents of this subscription, then validate it
preset_dict = mergedeep.merge(
self.config.presets.dict[preset_name],
self.dict,
self.config.presets._dict[preset_name],
self._dict,
strategy=mergedeep.Strategy.REPLACE,
)
del preset_dict["preset"]
self.preset = PresetValidator(
name=f"{self.name}.{preset_name}",
name=f"{self._name}.{preset_name}",
value=preset_dict,
)
@ -83,8 +86,8 @@ class SubscriptionValidator(StrictDictValidator):
raise ValueError("subscription source class not found")
return subscription_class(
name=self.name,
config_options=self.config,
name=self._name,
config_options=self.config.config_options,
source_options=self.preset.subscription_source,
output_options=self.preset.output_options,
metadata_options=self.preset.metadata_options,
@ -94,7 +97,7 @@ class SubscriptionValidator(StrictDictValidator):
@classmethod
def from_file_path(
cls, config: ConfigValidator, subscription_path: str
cls, config: ConfigFileValidator, subscription_path: str
) -> List["SubscriptionValidator"]:
# TODO: Create separate yaml file loader class
with open(subscription_path, "r", encoding="utf-8") as file: