danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more yolo refactors

Browse source
This commit is contained in:
jbannon 2022-04-22 17:33:33 +00:00
parent 139d82e126
commit 87338853bc
12 changed files with 151 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
ytdl_subscribe/config/config_file.py Normal file
View file

@ -0,0 +1,42 @@
from typing import Any
import yaml
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import LiteralDictValidator
from ytdl_subscribe.validators.validators import StringValidator
class ConfigOptions(StrictDictValidator):
"""Validation for global config options"""
_required_keys = {"working_directory"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.working_directory = self._validate_key(
key="working_directory", validator=StringValidator
)
class ConfigFile(StrictDictValidator):
_required_keys = {"configuration", "presets"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.config_options = self._validate_key("configuration", ConfigOptions)
# Make sure presets is a dictionary. Will be validated in `PresetValidator`
self.presets = self._validate_key("presets", LiteralDictValidator)
@classmethod
def from_dict(cls, config_dict) -> "ConfigFile":
return ConfigFile(name="", value=config_dict)
@classmethod
def from_file_path(cls, config_path) -> "ConfigFile":
# TODO: Create separate yaml file loader class
with open(config_path, "r", encoding="utf-8") as file:
config_dict = yaml.safe_load(file)
return ConfigFile.from_dict(config_dict)

31
ytdl_subscribe/config/config_file_validator.py
View file

@ -1,31 +0,0 @@
from typing import Any
import yaml
from ytdl_subscribe.config.config_options_validator import ConfigOptionsValidator
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import LiteralDictValidator
class ConfigPresetsValidator(LiteralDictValidator):
"""Shallow validator checking for the presets dict in the config"""
class ConfigFileValidator(StrictDictValidator):
_required_keys = {"configuration", "presets"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.config_options = self._validate_key("configuration", ConfigOptionsValidator)
self.presets = self._validate_key("presets", ConfigPresetsValidator)
@classmethod
def from_dict(cls, config_dict) -> "ConfigFileValidator":
return ConfigFileValidator(name="", value=config_dict)
@classmethod
def from_file_path(cls, config_path) -> "ConfigFileValidator":
# TODO: Create separate yaml file loader class
with open(config_path, "r", encoding="utf-8") as file:
config_dict = yaml.safe_load(file)
return ConfigFileValidator.from_dict(config_dict)

17
ytdl_subscribe/config/config_options_validator.py
View file

@ -1,17 +0,0 @@
from typing import Any
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import StringValidator
class ConfigOptionsValidator(StrictDictValidator):
"""Validation for the config options"""
_required_keys = {"working_directory"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.working_directory = self._validate_key(
key="working_directory", validator=StringValidator
)

44
ytdl_subscribe/config/output_options_validator.py
View file

@ -1,44 +0,0 @@
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator
from ytdl_subscribe.validators.validators import BoolValidator
class OutputOptionsValidator(StrictDictValidator):
"""Where to output the final files and thumbnails"""
_required_keys = {"output_directory", "file_name"}
_optional_keys = {
"thumbnail_name",
"maintain_download_archive",
"maintain_stale_file_deletion",
}
def __init__(self, name, value):
super().__init__(name, value)
# Output directory should resolve without any entry variables.
# This is to check the directory for any download-archives before any downloads begin
self.output_directory: OverridesStringFormatterValidator = self._validate_key(
key="output_directory", validator=OverridesStringFormatterValidator
)
# file name and thumbnails however can use entry variables
self.file_name: StringFormatterValidator = self._validate_key(
key="file_name", validator=StringFormatterValidator
)
self.thumbnail_name = self._validate_key_if_present(
key="thumbnail_name", validator=StringFormatterValidator
)
self.maintain_download_archive = self._validate_key_if_present(
key="maintain_download_archive", validator=BoolValidator, default=False
)
self.maintain_stale_file_deletion = self._validate_key_if_present(
key="maintain_stale_file_deletion", validator=BoolValidator, default=False
)
if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value:
raise self._validation_exception(
"maintain_stale_file_deletion requires maintain_download_archive set to True"
)

36
ytdl_subscribe/config/overrides_validator.py
View file

@ -1,36 +0,0 @@
from typing import Optional
from yt_dlp.utils import sanitize_filename
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator
class OverridesValidator(DictFormatterValidator):
"""Ensures `overrides` is a dict"""
def __init__(self, name, value):
super().__init__(name, value)
for key in self._keys:
sanitized_key_name = f"sanitized_{key}"
# First, sanitize the format string
self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string)
# Then, convert it into a StringFormatterValidator
self._value[sanitized_key_name] = StringFormatterValidator(
name="__should_never_fail__",
value=self._value[sanitized_key_name],
)
def apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> str:
"""
Returns the format_string after .format has been called on it using entry (if provided) and
override values
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
return formatter.apply_formatter(variable_dict)

91
ytdl_subscribe/config/preset_validator.py → ytdl_subscribe/config/preset.py
View file

@ -4,18 +4,22 @@ from typing import List
from typing import Optional
from typing import Type
from yt_dlp.utils import sanitize_filename
from ytdl_subscribe.config.download_strategy_validators import DownloadStrategyValidator
from ytdl_subscribe.config.download_strategy_validators import SoundcloudDownloadStrategyValidator
from ytdl_subscribe.config.download_strategy_validators import YoutubeDownloadStrategyValidator
from ytdl_subscribe.config.output_options_validator import OutputOptionsValidator
from ytdl_subscribe.config.overrides_validator import OverridesValidator
from ytdl_subscribe.config.ytdl_options_validator import YTDLOptionsValidator
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_subscribe.utils.exceptions import ValidationException
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator
from ytdl_subscribe.validators.validators import BoolValidator
from ytdl_subscribe.validators.validators import DictValidator
from ytdl_subscribe.validators.validators import LiteralDictValidator
from ytdl_subscribe.validators.validators import Validator
PRESET_SOURCE_VALIDATOR_MAPPING: Dict[str, Type[DownloadStrategyValidator]] = {
@ -32,6 +36,79 @@ PRESET_OPTIONAL_KEYS = {
}
class YTDLOptions(LiteralDictValidator):
"""Ensures `ytdl_options` is a dict"""
class Overrides(DictFormatterValidator):
"""Ensures `overrides` is a dict"""
def __init__(self, name, value):
super().__init__(name, value)
for key in self._keys:
sanitized_key_name = f"sanitized_{key}"
# First, sanitize the format string
self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string)
# Then, convert it into a StringFormatterValidator
self._value[sanitized_key_name] = StringFormatterValidator(
name="__should_never_fail__",
value=self._value[sanitized_key_name],
)
def apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> str:
"""
Returns the format_string after .format has been called on it using entry (if provided) and
override values
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
return formatter.apply_formatter(variable_dict)
class OutputOptions(StrictDictValidator):
"""Where to output the final files and thumbnails"""
_required_keys = {"output_directory", "file_name"}
_optional_keys = {
"thumbnail_name",
"maintain_download_archive",
"maintain_stale_file_deletion",
}
def __init__(self, name, value):
super().__init__(name, value)
# Output directory should resolve without any entry variables.
# This is to check the directory for any download-archives before any downloads begin
self.output_directory: OverridesStringFormatterValidator = self._validate_key(
key="output_directory", validator=OverridesStringFormatterValidator
)
# file name and thumbnails however can use entry variables
self.file_name: StringFormatterValidator = self._validate_key(
key="file_name", validator=StringFormatterValidator
)
self.thumbnail_name = self._validate_key_if_present(
key="thumbnail_name", validator=StringFormatterValidator
)
self.maintain_download_archive = self._validate_key_if_present(
key="maintain_download_archive", validator=BoolValidator, default=False
)
self.maintain_stale_file_deletion = self._validate_key_if_present(
key="maintain_stale_file_deletion", validator=BoolValidator, default=False
)
if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value:
raise self._validation_exception(
"maintain_stale_file_deletion requires maintain_download_archive set to True"
)
class PresetValidator(StrictDictValidator):
_required_keys = PRESET_REQUIRED_KEYS
_optional_keys = PRESET_OPTIONAL_KEYS
@ -113,18 +190,16 @@ class PresetValidator(StrictDictValidator):
self.output_options = self._validate_key(
key="output_options",
validator=OutputOptionsValidator,
validator=OutputOptions,
)
# TODO: REPLACE METADATA OPTIONS WITH PLUGINS
self.ytdl_options = self._validate_key(
key="ytdl_options", validator=YTDLOptionsValidator, default={}
key="ytdl_options", validator=YTDLOptions, default={}
)
self.overrides = self._validate_key(
key="overrides", validator=OverridesValidator, default={}
)
self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
# After all options are initialized, perform a recursive post-validate that requires
# values from multiple validators

18
ytdl_subscribe/config/subscription_validator.py → ytdl_subscribe/config/subscription.py
View file

@ -8,11 +8,11 @@ from typing import Type
import yaml
from mergedeep import mergedeep
from ytdl_subscribe.config.config_file_validator import ConfigFileValidator
from ytdl_subscribe.config.overrides_validator import OverridesValidator
from ytdl_subscribe.config.preset_validator import PRESET_OPTIONAL_KEYS
from ytdl_subscribe.config.preset_validator import PRESET_REQUIRED_KEYS
from ytdl_subscribe.config.preset_validator import PresetValidator
from ytdl_subscribe.config.config_file import ConfigFile
from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS
from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset import PresetValidator
from ytdl_subscribe.downloaders.soundcloud_downloader import (
SoundcloudAlbumsAndSinglesSourceValidator,
)
@ -36,14 +36,14 @@ class SubscriptionValidator(StrictDictValidator):
_required_keys = {"preset"}
_optional_keys = PRESET_REQUIRED_KEYS.union(PRESET_OPTIONAL_KEYS)
def __init__(self, config: ConfigFileValidator, name: str, value: Any):
def __init__(self, config: ConfigFile, name: str, value: Any):
super().__init__(name, value)
self.config = config
# Ensure the overrides defined here are valid
_ = self._validate_key(
key="overrides",
validator=OverridesValidator,
validator=Overrides,
default={},
)
@ -90,13 +90,13 @@ class SubscriptionValidator(StrictDictValidator):
@classmethod
def from_dict(
cls, config: ConfigFileValidator, subscription_name, subscription_dict: Dict
cls, config: ConfigFile, subscription_name, subscription_dict: Dict
) -> "SubscriptionValidator":
return SubscriptionValidator(config=config, name=subscription_name, value=subscription_dict)
@classmethod
def from_file_path(
cls, config: ConfigFileValidator, subscription_path: str
cls, config: ConfigFile, subscription_path: str
) -> List["SubscriptionValidator"]:
# TODO: Create separate yaml file loader class
with open(subscription_path, "r", encoding="utf-8") as file:

5
ytdl_subscribe/config/ytdl_options_validator.py
View file

@ -1,5 +0,0 @@
from ytdl_subscribe.validators.validators import LiteralDictValidator
class YTDLOptionsValidator(LiteralDictValidator):
"""Ensures `ytdl_options` is a dict"""

12
ytdl_subscribe/main.py
View file

@ -5,14 +5,12 @@ from typing import List
from ytdl_subscribe.cli.download_args_parser import DownloadArgsParser
from ytdl_subscribe.cli.main_args_parser import parser
from ytdl_subscribe.config.config_file_validator import ConfigFileValidator
from ytdl_subscribe.config.subscription_validator import SubscriptionValidator
from ytdl_subscribe.config.config_file import ConfigFile
from ytdl_subscribe.config.subscription import SubscriptionValidator
from ytdl_subscribe.utils.exceptions import ValidationException
def _download_subscriptions_from_yaml_files(
config: ConfigFileValidator, args: argparse.Namespace
) -> None:
def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.Namespace) -> None:
"""
Downloads all subscriptions from one or many subscription yaml files.
@ -31,7 +29,7 @@ def _download_subscriptions_from_yaml_files(
subscription.to_subscription().download()
def _download_subscription_from_cli(config: ConfigFileValidator, extra_args: List[str]) -> None:
def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None:
"""
Downloads a one-off subscription using the CLI
@ -53,7 +51,7 @@ def _main():
"""Entrypoint for ytdl-subscribe"""
args, extra_args = parser.parse_known_args()
config: ConfigFileValidator = ConfigFileValidator.from_file_path(args.config)
config: ConfigFile = ConfigFile.from_file_path(args.config)
if args.subparser == "sub":
_download_subscriptions_from_yaml_files(config=config, args=args)
print("Subscription download complete!")

4
ytdl_subscribe/plugins/convert_thumbnail.py
View file

@ -19,7 +19,7 @@ class ConvertThumbnailValidator(PluginValidator):
def __init__(self, name, value):
super().__init__(name, value)
self.to = self._validate_key(key="to", validator=ThumbnailTypes)
self.convert_to = self._validate_key(key="to", validator=ThumbnailTypes)
class ConvertThumbnail(Plugin[ConvertThumbnailValidator]):
@ -34,7 +34,7 @@ class ConvertThumbnail(Plugin[ConvertThumbnailValidator]):
image = Image.open(entry.download_thumbnail_path).convert("RGB")
# Pillow likes the formal 'jpeg' name and not 'jpg'
thumbnail_format = self.plugin_options.to.value
thumbnail_format = self.plugin_options.convert_to.value
if thumbnail_format == "jpg":
thumbnail_format = "jpeg"

4
ytdl_subscribe/plugins/plugin.py
View file

@ -3,7 +3,7 @@ from typing import Generic
from typing import TypeVar
from typing import final
from ytdl_subscribe.config.overrides_validator import OverridesValidator
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -30,7 +30,7 @@ class Plugin(Generic[PluginValidatorT], ABC):
self,
plugin_options: PluginValidatorT,
output_directory: str,
overrides: OverridesValidator,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
self.plugin_options = plugin_options

16
ytdl_subscribe/subscriptions/subscription.py
View file

@ -11,10 +11,10 @@ from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_subscribe.config.config_options_validator import ConfigOptionsValidator
from ytdl_subscribe.config.output_options_validator import OutputOptionsValidator
from ytdl_subscribe.config.overrides_validator import OverridesValidator
from ytdl_subscribe.config.preset_validator import PresetValidator
from ytdl_subscribe.config.config_file import ConfigOptions
from ytdl_subscribe.config.preset import OutputOptions
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset import PresetValidator
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.entry import Entry
@ -43,7 +43,7 @@ class Subscription(Generic[SourceT, EntryT], ABC):
def __init__(
self,
name: str,
config_options: ConfigOptionsValidator,
config_options: ConfigOptions,
preset_options: PresetValidator,
):
"""
@ -51,7 +51,7 @@ class Subscription(Generic[SourceT, EntryT], ABC):
----------
name: str
Name of the subscription
config_options: ConfigOptionsValidator
config_options: ConfigOptions
preset_options: PresetValidator
"""
self.name = name
@ -77,12 +77,12 @@ class Subscription(Generic[SourceT, EntryT], ABC):
return self.__preset_options.subscription_source
@property
def output_options(self) -> OutputOptionsValidator:
def output_options(self) -> OutputOptions:
"""Returns the output options defined for this subscription"""
return self.__preset_options.output_options
@property
def overrides(self) -> OverridesValidator:
def overrides(self) -> Overrides:
"""Returns the overrides defined for this subscription"""
return self.__preset_options.overrides