danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

insane number of changes

Browse source
This commit is contained in:
jbannon 2022-04-22 20:34:44 +00:00
parent 6157648183
commit f2ec156138
14 changed files with 299 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
ytdl_subscribe/config/download_strategy_validators.py
View file

@ -1,68 +0,0 @@
import copy
from abc import ABC
from typing import Any
from typing import Dict
from typing import Type
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.downloaders.soundcloud_downloader import (
SoundcloudAlbumsAndSinglesDownloadOptions,
)
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloaderOptions
from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloaderOptions
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import StringValidator
class DownloadStrategyValidator(StrictDictValidator, ABC):
"""
Validates the download strategy of a source. Does not validate the source options.
"""
# All media sources must define a download strategy
_required_keys = {"download_strategy"}
# Extra fields will be strict-validated using other StictDictValidators
_allow_extra_keys = True
_download_strategy_to_source_mapping: Dict[str, Type[DownloaderValidator]] = {}
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
download_strategy_name = self._validate_key(
key="download_strategy",
validator=StringValidator,
).value
if download_strategy_name not in self._possible_download_strategies:
raise self._validation_exception(
f"download_strategy must be one of the following: "
f"{', '.join(self._possible_download_strategies)}"
)
# Remove the 'download_strategy' key before passing the other keys to the actual
# source validator
source_validator_dict = copy.deepcopy(self._dict)
del source_validator_dict["download_strategy"]
source_validator_class = self._download_strategy_to_source_mapping[download_strategy_name]
self.source_validator = source_validator_class(name=self._name, value=source_validator_dict)
@property
def _possible_download_strategies(self):
return sorted(list(self._download_strategy_to_source_mapping.keys()))
class SoundcloudDownloadStrategyValidator(DownloadStrategyValidator):
_download_strategy_to_source_mapping = {
"albums_and_singles": SoundcloudAlbumsAndSinglesDownloadOptions
}
class YoutubeDownloadStrategyValidator(DownloadStrategyValidator):
_download_strategy_to_source_mapping = {
"channel": YoutubeChannelDownloaderOptions,
"playlist": YoutubePlaylistDownloaderOptions,
"video": YoutubeVideoDownloaderOptions,
}

118
ytdl_subscribe/config/preset.py
View file

@ -1,16 +1,20 @@
from abc import ABC
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from yt_dlp.utils import sanitize_filename
from ytdl_subscribe.config.download_strategy_validators import DownloadStrategyValidator
from ytdl_subscribe.config.download_strategy_validators import SoundcloudDownloadStrategyValidator
from ytdl_subscribe.config.download_strategy_validators import YoutubeDownloadStrategyValidator
from ytdl_subscribe.config.preset_class_mappings import DownloadStrategyMapping
from ytdl_subscribe.config.preset_class_mappings import PluginMapping
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_subscribe.utils.exceptions import ValidationException
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
@ -20,19 +24,15 @@ from ytdl_subscribe.validators.string_formatter_validators import StringFormatte
from ytdl_subscribe.validators.validators import BoolValidator
from ytdl_subscribe.validators.validators import DictValidator
from ytdl_subscribe.validators.validators import LiteralDictValidator
from ytdl_subscribe.validators.validators import StringValidator
from ytdl_subscribe.validators.validators import Validator
PRESET_SOURCE_VALIDATOR_MAPPING: Dict[str, Type[DownloadStrategyValidator]] = {
"soundcloud": SoundcloudDownloadStrategyValidator,
"youtube": YoutubeDownloadStrategyValidator,
}
PRESET_REQUIRED_KEYS = {"output_options"}
PRESET_OPTIONAL_KEYS = {
"metadata_options",
"ytdl_options",
"overrides",
*PRESET_SOURCE_VALIDATOR_MAPPING.keys(),
*DownloadStrategyMapping.sources(),
*PluginMapping.plugins(),
}
@ -109,38 +109,95 @@ class OutputOptions(StrictDictValidator):
)
class DownloadStrategyValidator(StrictDictValidator, ABC):
"""
Ensures a download strategy exists for a source. Does not validate any more than that.
The respective Downloader's option validator will do that.
"""
# All media sources must define a download strategy
_required_keys = {"download_strategy"}
# Extra fields will be strict-validated using other StictDictValidators
_allow_extra_keys = True
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
self.name = self._validate_key(
key="download_strategy",
validator=StringValidator,
).value
class PresetValidator(StrictDictValidator):
_required_keys = PRESET_REQUIRED_KEYS
_optional_keys = PRESET_OPTIONAL_KEYS
@property
def __available_sources(self) -> List[str]:
return sorted(list(PRESET_SOURCE_VALIDATOR_MAPPING.keys()))
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
downloader_strategy = self._validate_key(
key=downloader_source, validator=DownloadStrategyValidator
).name
def __validate_and_get_subscription_source(self) -> DownloaderValidator:
download_strategy_validator: Optional[DownloadStrategyValidator] = None
return DownloadStrategyMapping.get(
source=downloader_source, download_strategy=downloader_strategy
)
def __validate_and_get_downloader_options(
self, downloader_source: str, downloader: Type[Downloader]
) -> DownloaderValidator:
# Remove the download_strategy key before validating it against the downloader options
# TODO: make this cleaner
del self._dict[downloader_source]["download_strategy"]
return self._validate_key(
key=downloader_source, validator=downloader.downloader_options_type
)
def __validate_and_get_downloader_and_options(
self,
) -> Tuple[Type[Downloader], DownloaderValidator]:
downloader: Optional[Type[Downloader]] = None
download_options: Optional[DownloaderValidator] = None
downloader_sources = DownloadStrategyMapping.sources()
for key in self._keys:
# skip if the key is not a download source
if key not in downloader_sources:
continue
# Ensure there are not multiple sources, i.e. youtube and soundcloud
if key in self.__available_sources and download_strategy_validator:
if downloader:
raise ValidationException(
f"'{self._name}' can only have one of the following sources: "
f"{', '.join(self.__available_sources)}"
f"{', '.join(downloader_sources)}"
)
if key in PRESET_SOURCE_VALIDATOR_MAPPING:
download_strategy_validator = self._validate_key(
key=key, validator=PRESET_SOURCE_VALIDATOR_MAPPING[key]
)
# If subscription source was not set, error
if not download_strategy_validator:
raise ValidationException(
f"'{self._name} must have one of the following sources: "
f"{', '.join(self.__available_sources)}"
downloader = self.__validate_and_get_downloader(downloader_source=key)
download_options = self.__validate_and_get_downloader_options(
downloader_source=key, downloader=downloader
)
return download_strategy_validator.source_validator
# If downloader was not set, error since it is required
if not downloader:
raise ValidationException(
f"'{self._name} must have one of the following sources: "
f"{', '.join(downloader_sources)}"
)
return downloader, download_options
def __validate_and_get_plugins(self) -> List[Tuple[Type[Plugin], PluginOptions]]:
plugins: List[Tuple[Type[Plugin], PluginOptions]] = []
for key in self._keys:
if key not in PluginMapping.plugins():
continue
plugin = PluginMapping.get(plugin=key)
plugin_options = self._validate_key(key=key, validator=plugin.plugin_options_type)
plugins.append((plugin, plugin_options))
return plugins
def __validate_override_string_formatter_validator(
self, formatter_validator: OverridesStringFormatterValidator
@ -186,20 +243,19 @@ class PresetValidator(StrictDictValidator):
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
self.subscription_source = self.__validate_and_get_subscription_source()
self.downloader, self.downloader_options = self.__validate_and_get_downloader_and_options()
self.output_options = self._validate_key(
key="output_options",
validator=OutputOptions,
)
# TODO: REPLACE METADATA OPTIONS WITH PLUGINS
self.ytdl_options = self._validate_key(
key="ytdl_options", validator=YTDLOptions, default={}
)
self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
self.plugins = self.__validate_and_get_plugins()
# After all options are initialized, perform a recursive post-validate that requires
# values from multiple validators

107
ytdl_subscribe/config/preset_class_mappings.py Normal file
View file

@ -0,0 +1,107 @@
from typing import Dict
from typing import List
from typing import Type
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.soundcloud_downloader import SoundcloudAlbumsAndSinglesDownloader
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeChannelDownloader
from ytdl_subscribe.downloaders.youtube_downloader import YoutubePlaylistDownloader
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeVideoDownloader
from ytdl_subscribe.plugins.convert_thumbnail import ConvertThumbnailPlugin
from ytdl_subscribe.plugins.music_tags import MusicTagsPlugin
from ytdl_subscribe.plugins.nfo_tags import NfoTagsPlugin
from ytdl_subscribe.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_subscribe.plugins.plugin import Plugin
class DownloadStrategyMapping:
"""
Maps downloader strategies defined in the preset to its respective downloader class
"""
_MAPPING: Dict[str, Dict[str, Type[Downloader]]] = {
"youtube": {
"video": YoutubeVideoDownloader,
"playlist": YoutubePlaylistDownloader,
"channel": YoutubeChannelDownloader,
},
"soundcloud": {
"albums and singles": SoundcloudAlbumsAndSinglesDownloader,
},
}
@classmethod
def sources(cls) -> List[str]:
"""
Returns
-------
Available download sources
"""
return sorted(list(cls._MAPPING.keys()))
@classmethod
def _validate_is_source(cls, source: str) -> None:
"""
Ensure the source exists
"""
if source not in cls.sources():
raise ValueError(f"Tried to use source '{source}' that does not exist")
@classmethod
def source_download_strategies(cls, source: str) -> List[str]:
"""
Parameters
----------
source
Name of the source
Returns
-------
Available download strategies for the given source
"""
cls._validate_is_source(source)
return sorted(list(cls._MAPPING[source].keys()))
@classmethod
def _validate_is_download_strategy(cls, source: str, download_strategy: str) -> None:
"""
Ensure the download strategy for the given source exists
"""
if download_strategy not in cls.source_download_strategies(source):
raise ValueError(
f"Tried to use download strategy '{download_strategy}' with source '{source}', "
f"which does not exist"
)
@classmethod
def get(cls, source: str, download_strategy: str) -> Type[Downloader]:
cls._validate_is_download_strategy(source, download_strategy)
return cls._MAPPING[source][download_strategy]
class PluginMapping:
"""
Maps plugins defined in the preset to its respective plugin class
"""
_MAPPING: Dict[str, Type[Plugin]] = {
"convert_thumbnail": ConvertThumbnailPlugin,
"music_tags": MusicTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
}
@classmethod
def plugins(cls) -> List[str]:
"""
Returns
-------
Available download sources
"""
return sorted(list(cls._MAPPING.keys()))
@classmethod
def get(cls, plugin: str) -> Type[Plugin]:
if plugin not in cls.plugins():
raise ValueError(f"Tried to use plugin '{plugin}' that does not exist")
return cls._MAPPING[plugin]

0
ytdl_subscribe/config/preset_downloader.py
View file

24
ytdl_subscribe/config/subscription.py
View file

@ -58,25 +58,11 @@ class SubscriptionValidator(StrictDictValidator):
)
def to_subscription(self) -> Subscription:
# TODO: Fix this abomination
return None
# subscription_class: Optional[Type[Subscription]] = None
# if isinstance(self.preset.subscription_source, SoundcloudAlbumsAndSinglesDownloadOptions):
# subscription_class = SoundcloudAlbumsAndSinglesSubscription
# elif isinstance(self.preset.subscription_source, YoutubePlaylistDownloaderValidator):
# subscription_class = YoutubePlaylistSubscription
# elif isinstance(self.preset.subscription_source, YoutubeVideoDownloaderValidator):
# subscription_class = YoutubeVideoSubscription
# elif isinstance(self.preset.subscription_source, YoutubeChannelDownloaderValidator):
# subscription_class = YoutubeChannelSubscription
# if subscription_class is None:
# raise ValueError("subscription source class not found")
#
# return subscription_class(
# name=self._name,
# config_options=self.config.config_options,
# preset_options=self.preset,
# )
return Subscription(
name=self._name,
config_options=self.config.config_options,
preset_options=self.preset,
)
@classmethod
def from_dict(

6
ytdl_subscribe/downloaders/downloader.py
View file

@ -6,6 +6,7 @@ from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
import yt_dlp as ytdl
@ -24,12 +25,15 @@ DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT]):
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
"""
Class that interacts with ytdl to perform the download of metadata and content,
and should translate that to list of Entry objects.
"""
downloader_options_type: Type[DownloaderOptionsT] = NotImplemented
downloader_entry_type: Type[DownloaderEntryT] = NotImplemented
@classmethod
def ytdl_option_overrides(cls) -> Dict:
"""Global overrides that even overwrite user input"""

6
ytdl_subscribe/downloaders/soundcloud_downloader.py
View file

@ -35,8 +35,8 @@ SoundcloudDownloaderOptionsT = TypeVar(
class SoundcloudDownloader(
Generic[SoundcloudDownloaderOptionsT],
Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
Generic[SoundcloudDownloaderOptionsT],
ABC,
):
"""
@ -44,6 +44,8 @@ class SoundcloudDownloader(
SoundcloudTrack / SoundcloudAlbumTrack objects
"""
downloader_entry_type = SoundcloudTrack
@classmethod
def ytdl_option_defaults(cls) -> Dict:
"""Returns default format to be best mp3"""
@ -97,6 +99,8 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):
class SoundcloudAlbumsAndSinglesDownloader(
SoundcloudDownloader[SoundcloudAlbumsAndSinglesDownloadOptions]
):
downloader_options_type = SoundcloudDownloaderOptions
def download(self) -> List[SoundcloudTrack]:
"""
Soundcloud subscription to download albums and tracks as singles.

10
ytdl_subscribe/downloaders/youtube_downloader.py
View file

@ -30,13 +30,15 @@ YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDo
class YoutubeDownloader(
Generic[YoutubeDownloaderOptionsT], Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], ABC
Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], Generic[YoutubeDownloaderOptionsT], ABC
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo objects
"""
downloader_entry_type = YoutubeVideo
@classmethod
def playlist_url(cls, playlist_id: str) -> str:
"""Returns full playlist url"""
@ -131,6 +133,8 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]):
downloader_options_type = YoutubeVideoDownloaderOptions
def download(self) -> List[YoutubeVideo]:
video = self.download_video(video_id=self.download_options.video_id.value)
return [video]
@ -149,6 +153,8 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions]):
downloader_options_type = YoutubePlaylistDownloaderOptions
def download(self) -> List[YoutubeVideo]:
return self.download_playlist(playlist_id=self.download_options.playlist_id.value)
@ -168,6 +174,8 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DownloadDateRang
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions]):
downloader_options_type = YoutubeChannelDownloaderOptions
def download(self) -> List[YoutubeVideo]:
ytdl_options_overrides = {}
source_date_range = self.download_options.get_date_range()

8
ytdl_subscribe/plugins/convert_thumbnail.py
View file

@ -4,7 +4,7 @@ from PIL.Image import Image
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginValidator
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.validators.string_select_validator import StringSelectValidator
@ -14,7 +14,7 @@ class ThumbnailTypes(StringSelectValidator):
_select_values = {"jpg"}
class ConvertThumbnailValidator(PluginValidator):
class ConvertThumbnailOptions(PluginOptions):
_required_keys = {"to"}
def __init__(self, name, value):
@ -22,7 +22,9 @@ class ConvertThumbnailValidator(PluginValidator):
self.convert_to = self._validate_key(key="to", validator=ThumbnailTypes)
class ConvertThumbnail(Plugin[ConvertThumbnailValidator]):
class ConvertThumbnailPlugin(Plugin[ConvertThumbnailOptions]):
plugin_options_type = ConvertThumbnailOptions
def post_process_entry(self, entry: Entry):
"""
Convert the source thumbnail to the desired type. Leave the original file name and

8
ytdl_subscribe/plugins/music_tags.py
View file

@ -2,12 +2,12 @@ import music_tag
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginValidator
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator
from ytdl_subscribe.validators.validators import StringValidator
class MusicTagsValidator(PluginValidator):
class MusicTagsOptions(PluginOptions):
_required_keys = {"tags"}
_optional_keys = {"multi_value_separator"}
@ -21,7 +21,9 @@ class MusicTagsValidator(PluginValidator):
)
class MusicTags(Plugin[MusicTagsValidator]):
class MusicTagsPlugin(Plugin[MusicTagsOptions]):
plugin_options_type = MusicTagsOptions
def post_process_entry(self, entry: Entry):
"""
Tags the entry's audio file using values defined in the metadata options

8
ytdl_subscribe/plugins/nfo_tags.py
View file

@ -4,12 +4,12 @@ import dicttoxml
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginValidator
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator
class NfoTagsValidator(PluginValidator):
class NfoTagsOptions(PluginOptions):
_required_keys = {"nfo_name", "nfo_root", "tags"}
def __init__(self, name, value):
@ -20,7 +20,9 @@ class NfoTagsValidator(PluginValidator):
self.tags = self._validate_key(key="tags", validator=DictFormatterValidator)
class NfoTags(Plugin[NfoTagsValidator]):
class NfoTagsPlugin(Plugin[NfoTagsOptions]):
plugin_options_type = NfoTagsOptions
def post_process_entry(self, entry: Entry):
"""
Creates an entry's NFO file using values defined in the metadata options

8
ytdl_subscribe/plugins/output_directory_nfo_tags.py
View file

@ -3,12 +3,12 @@ from pathlib import Path
import dicttoxml
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginValidator
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator
class OutputDirectoryNfoTagsValidator(PluginValidator):
class OutputDirectoryNfoTagsOptions(PluginOptions):
"""
Validates config values for the output directory nfo tags plugin.
"""
@ -28,7 +28,9 @@ class OutputDirectoryNfoTagsValidator(PluginValidator):
self.tags = self._validate_key(key="tags", validator=OverridesDictFormatterValidator)
class NfoTags(Plugin[OutputDirectoryNfoTagsValidator]):
class OutputDirectoryNfoTagsPlugin(Plugin[OutputDirectoryNfoTagsOptions]):
plugin_options_type = OutputDirectoryNfoTagsOptions
def post_process_subscription(self):
"""
Creates an NFO file in the root of the output directory

11
ytdl_subscribe/plugins/plugin.py
View file

@ -1,5 +1,6 @@
from abc import ABC
from typing import Generic
from typing import Type
from typing import TypeVar
from typing import final
@ -9,24 +10,26 @@ from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class PluginValidator(StrictDictValidator):
class PluginOptions(StrictDictValidator):
"""
Class that defines the parameters to a plugin
"""
PluginValidatorT = TypeVar("PluginValidatorT", bound=PluginValidator)
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
class Plugin(Generic[PluginValidatorT], ABC):
class Plugin(Generic[PluginOptionsT], ABC):
"""
Class to define the new plugin functionality
"""
plugin_options_type: Type[PluginOptionsT] = NotImplemented
@final
def __init__(
self,
plugin_options: PluginValidatorT,
plugin_options: PluginOptionsT,
output_directory: str,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,

115
ytdl_subscribe/subscriptions/subscription.py
View file

@ -1,32 +1,29 @@
import contextlib
import os
import shutil
from abc import ABC
from pathlib import Path
from shutil import copyfile
from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from typing import TypeVar
from ytdl_subscribe.config.config_file import ConfigOptions
from ytdl_subscribe.config.preset import OutputOptions
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset import PresetValidator
from ytdl_subscribe.config.preset import YTDLOptions
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource
from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
SourceT = TypeVar("SourceT", bound=DownloaderValidator)
EntryT = TypeVar("EntryT", bound=Entry)
DownloaderT = TypeVar("DownloaderT", bound=Downloader)
class Subscription(Generic[SourceT, EntryT], ABC):
class Subscription:
"""
Subscription classes are the 'controllers' that perform...
@ -65,16 +62,20 @@ class Subscription(Generic[SourceT, EntryT], ABC):
)
@property
def entry_type(self) -> Type[EntryT]:
"""
:return: The Entry type this subscription uses to represent downloaded media
"""
return EntryT.__class__
def downloader_class(self) -> Type[Downloader]:
return self.__preset_options.downloader
@property
def source_options(self) -> SourceT:
"""Returns the source options defined for this subscription"""
return self.__preset_options.subscription_source
def downloader_options(self) -> DownloaderValidator:
return self.__preset_options.downloader_options
@property
def plugins(self) -> List[Tuple[Type[Plugin], PluginOptions]]:
return self.__preset_options.plugins
@property
def ytdl_options(self) -> YTDLOptions:
return self.__preset_options.ytdl_options
@property
def output_options(self) -> OutputOptions:
@ -98,21 +99,6 @@ class Subscription(Generic[SourceT, EntryT], ABC):
"""
return self.overrides.apply_formatter(formatter=self.output_options.output_directory)
def get_downloader(
self, downloader_type: Type[DownloaderT], source_ytdl_options: Optional[Dict] = None
) -> DownloaderT:
"""Returns the downloader that will be used to download media for this subscription"""
# if source_ytdl_options are present, override the ytdl_options with them
ytdl_options = self.__preset_options.ytdl_options.dict
if source_ytdl_options:
ytdl_options = dict(ytdl_options, **source_ytdl_options)
return downloader_type(
working_directory=self.working_directory,
ytdl_options=ytdl_options,
download_archive_file_name=self._enhanced_download_archive.archive_file_name,
)
def _copy_file_to_output_directory(self, source_file_path: str, output_file_name: str):
destination_file_path = Path(self.output_directory) / Path(output_file_name)
os.makedirs(os.path.dirname(destination_file_path), exist_ok=True)
@ -157,42 +143,49 @@ class Subscription(Generic[SourceT, EntryT], ABC):
yield
date_range = None
if isinstance(self.source_options, DownloadDateRangeSource):
date_range = self.source_options.get_date_range()
# TODO: add date range in output options
# date_range = None
# if isinstance(self.source_options, DownloadDateRangeSource):
# date_range = self.source_options.get_date_range()
#
# if date_range and self.output_options.maintain_stale_file_deletion:
# self._enhanced_download_archive.remove_stale_files(date_range=date_range)
#
# self._enhanced_download_archive.save_download_archive()
if date_range and self.output_options.maintain_stale_file_deletion:
self._enhanced_download_archive.remove_stale_files(date_range=date_range)
def _initialize_plugins(self) -> List[Plugin]:
plugins: List[Plugin] = []
for plugin_type, plugin_options in self.plugins:
plugin = plugin_type(
plugin_options=plugin_options,
output_directory=self.output_directory,
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
)
self._enhanced_download_archive.save_download_archive()
plugins.append(plugin)
return plugins
def download(self):
"""
Performs the subscription download.
"""
plugins = self._initialize_plugins()
with self._prepare_working_directory(), self._maintain_archive_file():
entries = self._extract_info()
downloader = self.downloader_class(
working_directory=self.working_directory,
download_options=self.downloader_options,
ytdl_options=self.ytdl_options.dict,
download_archive_file_name=self._enhanced_download_archive.archive_file_name,
)
entries = downloader.download()
for plugin in plugins:
for entry in entries:
plugin.post_process_entry(entry)
plugin.post_process_subscription()
for entry in entries:
self.post_process_entry(entry)
def _extract_info(self) -> List[EntryT]:
"""
Extracts only the info of the source, does not download it
"""
raise NotImplementedError("Each source needs to implement how it extracts info")
def post_process_entry(self, entry: Entry) -> None:
"""
After downloading an entry to the working directory, perform all post-processing, which
includes:
* Adding metadata to the entry file itself (music tags)
* Moving the entry file + thumbnail to the output directory with its formatted name
* Creating new metadata files (NFO) to reside alongside the entry files
:param entry: The entry to post-process
"""
# before move
self._copy_entry_files_to_output_directory(entry=entry)
# after move
self._copy_entry_files_to_output_directory(entry=entry)