diff --git a/docs/config.rst b/docs/config.rst index 0927bee5..ada53acb 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -114,7 +114,7 @@ for available source variables to use. albums_and_singles __________________ .. autoclass:: ytdl_sub.downloaders.soundcloud.albums_and_singles.SoundcloudAlbumsAndSinglesDownloadOptions() - :members: + :members: url, skip_premiere_tracks :member-order: bysource :inherited-members: diff --git a/examples/kodi_tv_shows_config.yaml b/examples/kodi_tv_shows_config.yaml index d91c25d1..020de14e 100644 --- a/examples/kodi_tv_shows_config.yaml +++ b/examples/kodi_tv_shows_config.yaml @@ -85,31 +85,3 @@ presets: overrides: youtube_tv_shows_directory: "/path/to/youtube_tv_shows" episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {title_sanitized}" - - ############################################################################### - # RECENT ARCHIVE - # This preset shows how you can download just the last 14 days-worth of videos - # on each download invocation. - yt_channel_as_tv__recent: - # `preset` can be set to any other preset in the config, and will inherit all its defined fields. - # This helps reduce copy-paste in the config.yaml - preset: "yt_channel_as_tv" - - # We will add the `after` field onto `yt_channel_as_tv`'s YouTube channel download strategy. - # This is saying 'only download videos uploaded in the last 2 weeks' - youtube: - after: "today-2weeks" - - ############################################################################### - # ROLLING ARCHIVE - # This example shows how to only keep the last 14-days worth of videos, and - # delete the rest. - yt_channel_as_tv__only_recent: - # Reuse `yt_channel_as_tv__recent` to only download the last 2 weeks' of videos - preset: "yt_channel_as_tv__recent" - - # This is saying "only keep files if they were uploaded in the last 2 weeks". - # All other files that this subscription had previously downloaded will be - # deleted. - output_options: - keep_files_after: "today-2weeks" \ No newline at end of file diff --git a/examples/kodi_tv_shows_subscriptions.yaml b/examples/kodi_tv_shows_subscriptions.yaml index b8502693..fa456225 100644 --- a/examples/kodi_tv_shows_subscriptions.yaml +++ b/examples/kodi_tv_shows_subscriptions.yaml @@ -3,7 +3,7 @@ # in each example. ############################################################################### -# LEVEL 1 - FULL ARCHIVE +# FULL ARCHIVE # # Subscription names are defined by you. We will call this one # john_smith_archive because it will download every single video in @@ -25,27 +25,3 @@ john_smith_archive: # '_sanitized' version that is safe for file systems. overrides: tv_show_name: "John /\ Smith" - -############################################################################### -# LEVEL 2 - RECENT ARCHIVE -# -# Use the "yt_channel_as_tv__recent" preset to only download the last two -# weeks' worth of YouTube videos. -john_smith_recent_archive: - preset: "yt_channel_as_tv__recent" - youtube: - channel_url: "https://youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - overrides: - tv_show_name: "John /\ Smith" - -############################################################################### -# LEVEL 3 - ROLLING ARCHIVE -# -# Use the "yt_channel_as_tv__recent_only" preset to only download the last two -# weeks' worth of YouTube videos, and delete any existing older videos. -john_smith_rolling_archive: - preset: "yt_channel_as_tv__only_recent" - youtube: - channel_url: "https://youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - overrides: - tv_show_name: "John /\ Smith" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d11f30ae..1f0df34f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ disable = [ "R0903", # too-few-public-methods "R0801", # similar lines "R0913", # Too many arguments + "R0901", # too-many-ancestors "W0511", # TODO ] diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 93681817..af8880e0 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -186,7 +186,6 @@ class Preset(StrictDictValidator): def __validate_and_get_plugins(self) -> PresetPlugins: preset_plugins = PresetPlugins() - source_variables = copy.deepcopy(self._source_variables) for key in self._keys: if key not in PluginMapping.plugins(): @@ -197,20 +196,30 @@ class Preset(StrictDictValidator): preset_plugins.add(plugin_type=plugin, plugin_options=plugin_options) - for plugin, plugin_options in sorted( - preset_plugins.zipped(), - key=lambda _plugin_and_options: _plugin_and_options[0].priority.modify_entry, + return preset_plugins + + def __validate_added_variables(self): + source_variables = copy.deepcopy(self._source_variables) + + # Validate added download option variables here since plugins could subsequently use them + self.downloader_options.validate_with_variables( + source_variables=source_variables, + override_variables=self.overrides.dict_with_format_strings, + ) + source_variables.extend(self.downloader_options.added_source_variables()) + + for _, plugin_options in sorted( + self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry ): # Validate current plugin using source + added plugin variables plugin_options.validate_with_variables( - source_variables=source_variables, override_variables=self.overrides.keys + source_variables=source_variables, + override_variables=self.overrides.dict_with_format_strings, ) # Extend existing source variables with ones created from this plugin source_variables.extend(plugin_options.added_source_variables()) - return preset_plugins - def __validate_override_string_formatter_validator( self, formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], @@ -224,7 +233,11 @@ class Preset(StrictDictValidator): # If the formatter supports source variables, set the formatter variables to include # both source and override variables if not isinstance(formatter_validator, OverridesStringFormatterValidator): - source_variables = {source_var: "dummy_string" for source_var in self._source_variables} + source_variables = { + source_var: "dummy_string" + for source_var in self._source_variables + + self.downloader_options.added_source_variables() + } variable_dict = dict(source_variables, **variable_dict) # For all plugins, add in any extra added source variables @@ -330,6 +343,7 @@ class Preset(StrictDictValidator): self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) self.plugins: PresetPlugins = self.__validate_and_get_plugins() + self.__validate_added_variables() # After all options are initialized, perform a recursive post-validate that requires # values from multiple validators diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index d1ddb35a..2a67b011 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,4 +1,6 @@ +from abc import ABC from typing import Dict +from typing import List from typing import Optional from yt_dlp.utils import sanitize_filename @@ -13,6 +15,43 @@ from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import LiteralDictValidator +# pylint: disable=no-self-use +# pylint: disable=unused-argument +class AddsVariablesMixin(ABC): + """ + Mixin for parts of the Preset that adds source variables + """ + + def added_source_variables(self) -> List[str]: + """ + If the plugin adds source variables, list them here. + + Returns + ------- + List of added source variables this plugin creates + """ + return [] + + def validate_with_variables( + self, source_variables: List[str], override_variables: Dict[str, str] + ) -> None: + """ + Optional validation after init with the session's source and override variables. + + Parameters + ---------- + source_variables + Available source variables when running the plugin + override_variables + Available override variables when running the plugin + """ + return None + + +# pylint: enable=no-self-use +# pylint: enable=unused-argument + + class YTDLOptions(LiteralDictValidator): """ Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index cdb85691..e4c67b1f 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -21,6 +21,7 @@ import yt_dlp as ytdl from yt_dlp.utils import ExistingVideoReached from yt_dlp.utils import RejectedVideoReached +from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.base_entry import BaseEntry @@ -36,7 +37,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr download_logger = Logger.get(name="downloader") -class DownloaderValidator(StrictDictValidator, ABC): +class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): """ Placeholder class to define downloader options """ @@ -203,37 +204,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] f"yt-dlp failed to download an entry with these arguments: {error_dict}" ) - def _filter_entry_dicts( - self, - entry_dicts: List[Dict], - extractor: Optional[str] = None, - sort_by: Optional[str] = None, - ) -> List[Dict]: - """ - Parameters - ---------- - entry_dicts - entry dicts to filter - extractor - Optional. Extractor that the entry dicts must have. If None, defaults to the - entry type's extractor - sort_by - Optional. Sort the entry dicts on this key - - Returns - ------- - filtered entry dicts - """ - if extractor is None: - extractor = self.downloader_entry_type.entry_extractor - - output = [ - entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor - ] - if sort_by: - output = sorted(output, key=lambda entry_dict: entry_dict[sort_by]) - return output - def _get_entry_dicts_from_info_json_files(self) -> List[Dict]: """ Returns @@ -265,7 +235,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] info_json_listener = LogEntriesDownloadedListener( working_directory=self.working_directory, - info_json_extractor=self.downloader_entry_type.entry_extractor, log_prefix=log_prefix, ) diff --git a/src/ytdl_sub/downloaders/generic/__init__.py b/src/ytdl_sub/downloaders/generic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/downloaders/generic/collection.py b/src/ytdl_sub/downloaders/generic/collection.py new file mode 100644 index 00000000..64157d0d --- /dev/null +++ b/src/ytdl_sub/downloaders/generic/collection.py @@ -0,0 +1,286 @@ +import contextlib +import os.path +from typing import Dict +from typing import Generator +from typing import List +from typing import Set + +from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.downloaders.downloader import Downloader +from ytdl_sub.downloaders.downloader import DownloaderOptionsT +from ytdl_sub.downloaders.downloader import DownloaderValidator +from ytdl_sub.downloaders.downloader import download_logger +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder +from ytdl_sub.entries.base_entry import BaseEntry +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.entry_parent import EntryParent +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator +from ytdl_sub.validators.validators import ListValidator +from ytdl_sub.validators.validators import StringValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + + +def _entry_key(entry: BaseEntry) -> str: + return entry.extractor + entry.uid + + +def _get_parent_entry_variables(parent: EntryParent) -> Dict[str, str | int]: + """ + Adds source variables to the child entry derived from the parent entry. + """ + if not parent.child_entries: + return {} + + return { + "playlist_max_upload_year": max( + child_entry.to_type(Entry).upload_year for child_entry in parent.child_entries + ) + } + + +class CollectionUrlValidator(StrictDictValidator): + _required_keys = {"url"} + _optional_keys = {"variables"} + + def __init__(self, name, value): + super().__init__(name, value) + + # TODO: url validate using yt-dlp IE + self._url = self._validate_key(key="url", validator=StringValidator) + variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator) + self._variables = variables.dict_with_format_strings if variables else {} + + @property + def url(self) -> str: + """ + Returns + ------- + URL to download from + """ + return self._url.value + + @property + def variables(self) -> Dict[str, str]: + """ + Variables to add to each entry + """ + return self._variables + + +class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): + _inner_list_type = CollectionUrlValidator + _expected_value_type_name = "collection url list" + + def __init__(self, name, value): + super().__init__(name, value) + + added_variables: Dict[str, str] = self.list[0].variables + + for idx, collection_url_validator in enumerate(self.list[1:]): + collection_variables = collection_url_validator.variables + + # see if this collection contains new added vars (it should not) + for var in collection_variables.keys(): + if var not in added_variables: + raise self._validation_exception( + f"Collection url {idx} contains the variable '{var}' that the first " + f"collection url does not. The first collection url must define all added " + f"variables." + ) + + # see if this collection is missing any added vars (if so, inherit from the top) + for var in added_variables.keys(): + if var not in collection_variables: + collection_variables[var] = added_variables[var] + + +class CollectionDownloadOptions(DownloaderValidator): + """ + Downloads from multiple URLs. If an entry is returned from more than one URL, it will + resolve to the bottom-most URL settings. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + generic: + # required + download_strategy: "collection" + urls: + - url: "soundcloud.com/tracks" + variables: + season: "1" + album: "{title}" + - url: "soundcloud.com/albums" + variables: + season: "1" + album: "{playlist_title}" + """ + + _required_keys = {"urls"} + + def __init__(self, name, value): + super().__init__(name, value) + self._urls = self._validate_key(key="urls", validator=CollectionUrlListValidator) + + @property + def collection_urls(self) -> CollectionUrlListValidator: + """ + Required. The Soundcloud user's url, i.e. ``soundcloud.com/the_username`` + """ + return self._urls + + def added_source_variables(self) -> List[str]: + """ + Returns + ------- + List of variables added. The first collection url always contains all the variables. + """ + return list(self._urls.list[0].variables.keys()) + + def validate_with_variables( + self, source_variables: List[str], override_variables: Dict[str, str] + ) -> None: + """ + Ensures new variables added are not existing variables + """ + for source_var_name in self.added_source_variables(): + if source_var_name in source_variables: + raise self._validation_exception( + f"'{source_var_name}' cannot be used as a variable name because it " + f"is an existing source variable" + ) + + base_variables = dict( + override_variables, **{source_var: "dummy_string" for source_var in source_variables} + ) + + # Apply formatting to each new source variable, ensure it resolves + for collection_url in self.collection_urls.list: + for source_var_name, source_var_formatter_str in collection_url.variables.items(): + _ = StringFormatterValidator( + name=f"{self._name}.{source_var_name}", value=source_var_formatter_str + ).apply_formatter(base_variables) + + +class CollectionDownloader(Downloader[CollectionDownloadOptions, Entry]): + downloader_options_type = CollectionDownloadOptions + downloader_entry_type = Entry + + def __init__( + self, + download_options: DownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, + ytdl_options_builder: YTDLOptionsBuilder, + overrides: Overrides, + ): + super().__init__( + download_options=download_options, + enhanced_download_archive=enhanced_download_archive, + ytdl_options_builder=ytdl_options_builder, + overrides=overrides, + ) + + self.parents: List[EntryParent] = [] + self.downloaded_entries: Set[str] = set() + + @contextlib.contextmanager + def _separate_download_archives(self): + """ + Separate download archive writing between collection urls. This is so break_on_existing + does not break when downloading from subset urls. + """ + archive_path = self._ytdl_options_builder.to_dict().get("download_archive", "") + backup_archive_path = f"{archive_path}.backup" + + # If archive path exists, maintain download archive is enable + if archive_file_exists := archive_path and os.path.isfile(archive_path): + archive_file_exists = True + + # If a backup exists, it's the one prior to any downloading, use that. + if os.path.isfile(backup_archive_path): + FileHandler.copy(src_file_path=backup_archive_path, dst_file_path=archive_path) + # If not, create the backup + else: + FileHandler.copy(src_file_path=archive_path, dst_file_path=backup_archive_path) + + yield + + # If an archive path did not exist at first, but now exists, delete it + if not archive_file_exists and os.path.isfile(archive_path): + FileHandler.delete(file_path=archive_path) + # If the archive file did exist, restore the backup + elif archive_file_exists: + FileHandler.copy(src_file_path=backup_archive_path, dst_file_path=archive_path) + + def _download_entry(self, entry: Entry) -> Entry: + download_logger.info("Downloading entry %s", entry.title) + download_entry_dict = self.extract_info_with_retry( + is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded, + url=entry.webpage_url, + ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run}, + ) + + # Workaround for the ytdlp issue + # pylint: disable=protected-access + entry._kwargs["requested_subtitles"] = download_entry_dict.get("requested_subtitles") + # pylint: enable=protected-access + + return entry + + def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]: + """Download in reverse order, that way we download older entries ones first""" + for entry_child in reversed(parent.entry_children()): + if _entry_key(entry_child) in self.downloaded_entries: + continue + + yield self._download_entry(entry_child.to_type(Entry)) + self.downloaded_entries.add(_entry_key(entry_child)) + + for parent_child in reversed(parent.parent_children()): + for entry_child in self._download_parent_entry(parent=parent_child): + yield entry_child + + def download_url_metadata(self, collection_url: CollectionUrlValidator) -> List[EntryParent]: + """ + Downloads only info.json files and forms EntryParent trees + """ + with self._separate_download_archives(): + entry_dicts = self.extract_info_via_info_json( + only_info_json=True, + url=collection_url.url, + log_prefix_on_info_json_dl="Downloading metadata for", + ) + + return EntryParent.from_entry_dicts( + entry_dicts=entry_dicts, working_directory=self.working_directory + ) + + def download_url( + self, collection_url: CollectionUrlValidator, parents: List[EntryParent] + ) -> Generator[Entry, None, None]: + """ + Downloads the leaf entries from EntryParent trees + """ + with self._separate_download_archives(): + for parent in parents: + for entry_child in self._download_parent_entry(parent=parent): + entry_child.add_variables( + dict(_get_parent_entry_variables(parent), **collection_url.variables) + ) + yield entry_child + + def download(self) -> Generator[Entry, None, None]: + """ + Soundcloud subscription to download albums and tracks as singles. + """ + # download the bottom-most urls first since they are top-priority + for collection_url in reversed(self.download_options.collection_urls.list): + parents = self.download_url_metadata(collection_url=collection_url) + for entry in self.download_url(collection_url=collection_url, parents=parents): + yield entry diff --git a/src/ytdl_sub/downloaders/soundcloud/abc.py b/src/ytdl_sub/downloaders/soundcloud/abc.py deleted file mode 100644 index cba3e289..00000000 --- a/src/ytdl_sub/downloaders/soundcloud/abc.py +++ /dev/null @@ -1,65 +0,0 @@ -from abc import ABC -from typing import Generic -from typing import TypeVar - -from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.entries.soundcloud import SoundcloudTrack -from ytdl_sub.validators.validators import BoolValidator - - -class SoundcloudDownloaderOptions(DownloaderValidator, ABC): - """ - Abstract source validator for all soundcloud sources. - """ - - _optional_keys = {"skip_premiere_tracks"} - - def __init__(self, name: str, value: dict): - super().__init__(name=name, value=value) - self._skip_premiere_tracks = self._validate_key( - "skip_premiere_tracks", BoolValidator, default=True - ) - - @property - def skip_premiere_tracks(self) -> bool: - """ - Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True. - """ - return self._skip_premiere_tracks.value - - -SoundcloudDownloaderOptionsT = TypeVar( - "SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions -) - - -class SoundcloudDownloader( - Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack], - Generic[SoundcloudDownloaderOptionsT], - ABC, -): - """ - Class that handles downloading soundcloud entries via ytdl and converting them into - SoundcloudTrack / SoundcloudAlbumTrack objects - """ - - downloader_entry_type = SoundcloudTrack - - @classmethod - def artist_albums_url(cls, artist_url: str) -> str: - """ - Returns - ------- - Full artist album url - """ - return f"{artist_url}/albums" - - @classmethod - def artist_tracks_url(cls, artist_url: str) -> str: - """ - Returns - ------- - Full artist tracks url - """ - return f"{artist_url}/tracks" diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index 6b590daa..f40fbfcf 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -2,16 +2,16 @@ from typing import Dict from typing import Generator from typing import List -from ytdl_sub.downloaders.downloader import download_logger -from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloader -from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloaderOptions -from ytdl_sub.entries.entry_parent import EntryParent -from ytdl_sub.entries.soundcloud import SoundcloudAlbumTrack -from ytdl_sub.entries.soundcloud import SoundcloudTrack +from ytdl_sub.downloaders.downloader import Downloader +from ytdl_sub.downloaders.downloader import DownloaderValidator +from ytdl_sub.downloaders.generic.collection import CollectionDownloader +from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions +from ytdl_sub.entries.entry import Entry from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator +from ytdl_sub.validators.validators import BoolValidator -class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions): +class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator): """ Downloads a soundcloud user's entire discography. Groups together album tracks and considers any track not in an album as a single. Also includes any collaboration tracks. @@ -32,12 +32,53 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions): """ _required_keys = {"url"} + _optional_keys = {"skip_premiere_tracks"} def __init__(self, name, value): super().__init__(name, value) self._url = self._validate_key( key="url", validator=SoundcloudUsernameUrlValidator ).username_url + self._skip_premiere_tracks = self._validate_key( + "skip_premiere_tracks", BoolValidator, default=True + ) + + self.collection_validator = CollectionDownloadOptions( + name=self._name, + value={ + "urls": [ + { + "url": f"{self._url}/tracks", + "variables": { + "track_number": "1", + "track_number_padded": "01", + "track_count": "1", + "album": "{title}", + "album_sanitized": "{title_sanitized}", + "album_year": "{upload_year}", + }, + }, + { + "url": f"{self._url}/albums", + "variables": { + "track_number": "{playlist_index}", + "track_number_padded": "{playlist_index_padded}", + "track_count": "{playlist_count}", + "album": "{playlist}", + "album_sanitized": "{playlist_sanitized}", + "album_year": "{playlist_max_upload_year}", + }, + }, + ] + }, + ) + + @property + def skip_premiere_tracks(self) -> bool: + """ + Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True. + """ + return self._skip_premiere_tracks.value @property def url(self) -> str: @@ -46,11 +87,25 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions): """ return self._url + def added_source_variables(self) -> List[str]: + """Validate using collection_validator""" + return self.collection_validator.added_source_variables() + + def validate_with_variables( + self, source_variables: List[str], override_variables: Dict[str, str] + ) -> None: + """Validate using collection_validator""" + return self.collection_validator.validate_with_variables( + source_variables, override_variables + ) + class SoundcloudAlbumsAndSinglesDownloader( - SoundcloudDownloader[SoundcloudAlbumsAndSinglesDownloadOptions] + Downloader[SoundcloudAlbumsAndSinglesDownloadOptions, Entry] ): downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions + downloader_entry_type = Entry + supports_subtitles = False supports_chapters = False @@ -72,104 +127,29 @@ class SoundcloudAlbumsAndSinglesDownloader( }, ) - def _get_singles( - self, entry_dicts: List[Dict], albums: List[EntryParent] - ) -> List[SoundcloudTrack]: - tracks: List[SoundcloudTrack] = [] + def _should_skip(self, entry: Entry) -> bool: + if not self.download_options.skip_premiere_tracks: + return False - # Get all tracks that are not part of an album - for entry_dict in self._filter_entry_dicts(entry_dicts): - if not any(entry_dict in album for album in albums): - tracks.append( - SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory) - ) + for url in [entry.kwargs_get("url", ""), entry.webpage_url]: + if "/preview/" in url: + return True - return tracks + return False - def _get_albums(self) -> List[EntryParent]: - # Dry-run to get the info json files - artist_albums_url = self.artist_albums_url(artist_url=self.download_options.url) - entry_dicts = self.extract_info_via_info_json( - only_info_json=True, - log_prefix_on_info_json_dl="Downloading metadata for", - url=artist_albums_url, - ) - - albums: List[EntryParent] = [] - for entry_dict in self._filter_entry_dicts(entry_dicts, extractor="soundcloud:set"): - albums.append( - EntryParent( - entry_dict=entry_dict, working_directory=self.working_directory - ).read_children_from_entry_dicts( - entry_dicts=entry_dicts, child_class=SoundcloudAlbumTrack - ) - ) - - return albums - - def _get_album_tracks( - self, albums: List[EntryParent] - ) -> Generator[SoundcloudAlbumTrack, None, None]: - for album in albums: - if album.child_count > 0: - download_logger.info("Downloading album %s", album.title) - - for track in album.child_entries: - if self.download_options.skip_premiere_tracks and track.is_premiere(): - continue - - download_logger.info( - "Downloading album track %d/%d %s", - track.track_number, - track.track_count, - track.title, - ) - if not self.is_dry_run: - _ = self.extract_info_with_retry( - is_downloaded_fn=track.is_downloaded, - url=album.webpage_url, - ytdl_options_overrides={ - "playlist_items": str(track.kwargs("playlist_index")), - "writeinfojson": False, - }, - ) - - yield track - - def _get_single_tracks( - self, albums: List[EntryParent] - ) -> Generator[SoundcloudTrack, None, None]: - artist_tracks_url = self.artist_tracks_url(artist_url=self.download_options.url) - tracks_entry_dicts = self.extract_info_via_info_json( - only_info_json=True, - log_prefix_on_info_json_dl="Downloading metadata for", - url=artist_tracks_url, - ) - - # Then, get all singles - tracks = self._get_singles(entry_dicts=tracks_entry_dicts, albums=albums) - for track in tracks: - # Filter any premiere tracks if specified - if self.download_options.skip_premiere_tracks and track.is_premiere(): - continue - - download_logger.info("Downloading single track %s", track.title) - if not self.is_dry_run: - _ = self.extract_info_with_retry( - is_downloaded_fn=track.is_downloaded, - url=track.webpage_url, - ytdl_options_overrides={"writeinfojson": False}, - ) - - yield track - - def download(self) -> Generator[SoundcloudTrack, None, None]: + def download(self) -> Generator[Entry, None, None]: """ Soundcloud subscription to download albums and tracks as singles. """ - albums = self._get_albums() - for album_track in self._get_album_tracks(albums=albums): - yield album_track + downloader = CollectionDownloader( + download_options=self.download_options.collection_validator, + enhanced_download_archive=self._enhanced_download_archive, + ytdl_options_builder=self._ytdl_options_builder, + overrides=self.overrides, + ) - for single_track in self._get_single_tracks(albums=albums): - yield single_track + for entry in downloader.download(): + if self._should_skip(entry): + continue + + yield entry diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index d185a71a..4e581e7b 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -7,14 +7,14 @@ from typing import Optional from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.downloader import DownloaderOptionsT from ytdl_sub.downloaders.downloader import download_logger +from ytdl_sub.downloaders.generic.collection import CollectionDownloader +from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.youtube import YoutubeVideo -from ytdl_sub.utils.datetime import to_date_range_hack from ytdl_sub.utils.thumbnail import convert_url_thumbnail -from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -41,8 +41,6 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): _required_keys = {"channel_url"} _optional_keys = { - "before", - "after", "channel_avatar_path", "channel_banner_path", } @@ -58,8 +56,11 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): self._channel_banner_path = self._validate_key_if_present( "channel_banner_path", OverridesStringFormatterValidator ) - self._before = self._validate_key_if_present("before", StringDatetimeValidator) - self._after = self._validate_key_if_present("after", StringDatetimeValidator) + + self.collection_validator = CollectionDownloadOptions( + name=self._name, + value={"urls": [{"url": self._channel_url}]}, + ) @property def channel_url(self) -> str: @@ -84,22 +85,6 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): """ return self._channel_banner_path - @property - def before(self) -> Optional[StringDatetimeValidator]: - """ - DEPRECATED: use the `date_range` plugin instead. Will be removed in version 0.5.0 - Optional. Only download videos before this datetime. - """ - return self._before - - @property - def after(self) -> Optional[StringDatetimeValidator]: - """ - DEPRECATED: use the `date_range` plugin instead. Will be removed in version 0.5.0 - Optional. Only download videos after this datetime. - """ - return self._after - class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]): downloader_options_type = YoutubeChannelDownloaderOptions @@ -162,31 +147,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions """ Downloads all videos from a channel """ - ytdl_options_overrides = {} - - # If a date range is specified when download a YT channel, add it into the ytdl options - source_date_range = to_date_range_hack( - before=self.download_options.before, after=self.download_options.after + downloader = CollectionDownloader( + download_options=self.download_options.collection_validator, + enhanced_download_archive=self._enhanced_download_archive, + ytdl_options_builder=self._ytdl_options_builder, + overrides=self.overrides, ) - if source_date_range: - ytdl_options_overrides["daterange"] = source_date_range + collection_url = self.download_options.collection_validator.collection_urls.list[0] - # dry-run the entire channel download first, this will get the - # videos that will be downloaded. Afterwards, download each video one-by-one - entry_dicts = self.extract_info_via_info_json( - ytdl_options_overrides=ytdl_options_overrides, - only_info_json=True, - log_prefix_on_info_json_dl="Downloading metadata for", - url=self.download_options.channel_url, - ) - - self.channel = EntryParent.from_entry_dicts_with_children( - entry_dicts=entry_dicts, - working_directory=self.working_directory, - child_class=YoutubeVideo, - extractor="youtube:tab", - ) + parents = downloader.download_url_metadata(collection_url=collection_url) + assert len(parents) == 1, "Channel should be the only entry parent" + self.channel = parents[0] + # TODO: Handle this better self.overrides.add_override_variables( variables_to_add={ "source_uploader": self.channel.kwargs_get("uploader", "__failed_to_scrape__"), @@ -195,27 +168,11 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions } ) - # Iterate in descending order to process older videos first. In case an error occurs and a - # the channel must be redownloaded, it will fetch most recent metadata first, and break - # on the older video that's been processed and is in the download archive. - for idx, video in enumerate(reversed(self.channel.child_entries), start=1): - download_logger.info("Downloading %d/%d %s", idx, self.channel.child_count, video.title) - - # Re-download the contents even if it's a dry-run as a single video. At this time, - # channels do not download subtitles or subtitle metadata - as_single_video_dict = self.extract_info_with_retry( - is_downloaded_fn=None if self.is_dry_run else video.is_downloaded, - ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run}, - url=video.webpage_url, - ) - - # Workaround for the ytdlp issue + for entry in downloader.download_url(collection_url=collection_url, parents=parents): # pylint: disable=protected-access - video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles") + yield YoutubeVideo(entry_dict=entry._kwargs, working_directory=self.working_directory) # pylint: enable=protected-access - yield video - def _download_thumbnail( self, thumbnail_url: str, diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index 25d08eba..b81a66ce 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -2,10 +2,10 @@ from typing import Dict from typing import Generator from typing import List -from ytdl_sub.downloaders.downloader import download_logger +from ytdl_sub.downloaders.generic.collection import CollectionDownloader +from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions -from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.youtube import YoutubePlaylistVideo from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator @@ -34,6 +34,11 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): "playlist_url", YoutubePlaylistUrlValidator ).playlist_url + self.collection_validator = CollectionDownloadOptions( + name=self._name, + value={"urls": [{"url": self._playlist_url}]}, + ) + @property def playlist_url(self) -> str: """ @@ -85,22 +90,20 @@ class YoutubePlaylistDownloader( def download(self) -> Generator[YoutubePlaylistVideo, None, None]: """ Downloads all videos in a Youtube playlist. - - Dry-run the entire playlist download first. This will get the videos that will be - downloaded. Afterwards, download each video one-by-one """ - entry_dicts = self.extract_info_via_info_json( - only_info_json=True, - log_prefix_on_info_json_dl="Downloading metadata for", - url=self.download_options.playlist_url, + downloader = CollectionDownloader( + download_options=self.download_options.collection_validator, + enhanced_download_archive=self._enhanced_download_archive, + ytdl_options_builder=self._ytdl_options_builder, + overrides=self.overrides, ) + collection_url = self.download_options.collection_validator.collection_urls.list[0] - playlist: EntryParent = EntryParent.from_entry_dicts_with_children( - entry_dicts=entry_dicts, - working_directory=self.working_directory, - child_class=YoutubePlaylistVideo, - extractor="youtube:tab", - ) + parents = downloader.download_url_metadata(collection_url=collection_url) + assert len(parents) == 1, "Playlist should be the only entry parent" + playlist = parents[0] + + # TODO: Handle this better self.overrides.add_override_variables( variables_to_add={ "source_title": playlist.title, @@ -109,23 +112,9 @@ class YoutubePlaylistDownloader( } ) - # Iterate in reverse order to process older videos first. In case an error occurs and a - # the playlist must be redownloaded, it will fetch most recent metadata first, and break - # on the older video that's been processed and is in the download archive. - for idx, video in enumerate(reversed(playlist.child_entries), start=1): - download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) - - # Re-download the contents even if it's a dry-run as a single video. At this time, - # playlists do not download subtitles or subtitle metadata - as_single_video_dict = self.extract_info_with_retry( - is_downloaded_fn=None if self.is_dry_run else video.is_downloaded, - ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run}, - url=video.webpage_url, - ) - - # Workaround for the ytdlp issue + for entry in downloader.download_url(collection_url=collection_url, parents=parents): # pylint: disable=protected-access - video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles") + yield YoutubePlaylistVideo( + entry_dict=entry._kwargs, working_directory=self.working_directory + ) # pylint: enable=protected-access - - yield video diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 22d5fa5b..e86cf2f6 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -88,9 +88,6 @@ class BaseEntry(BaseEntryVariables, ABC): Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc). """ - # The ytdl extractor type that the entry represents - entry_extractor: str - def __init__(self, entry_dict: Dict, working_directory: str): """ Initialize the entry using ytdl metadata @@ -126,7 +123,7 @@ class BaseEntry(BaseEntryVariables, ABC): """ Dict get on kwargs """ - if not self.kwargs_contains(key): + if not self.kwargs_contains(key) or self.kwargs(key) is None: return default return self.kwargs(key) @@ -192,19 +189,3 @@ class BaseEntry(BaseEntryVariables, ABC): source_var: getattr(self, source_var) for source_var in self.source_variables() } return dict(source_variable_dict, **self._added_variables()) - - # TODO: super typing - @classmethod - def from_entry_dicts( - cls, entry_dicts: List[Dict], working_directory: str, extractor: Optional[str] - ): - """ - Load the entry from a list of dicts. There should only be one entry with the extractor - type - """ - extractor = extractor or cls.extractor - output = [ - entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor - ] - assert len(output) == 1, f"Expected a single entry with extractor of type '{extractor}'" - return cls(entry_dict=output[0], working_directory=working_directory) diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index c2af6341..a9adebc9 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -6,70 +6,49 @@ from typing import TypeVar from ytdl_sub.entries.base_entry import BaseEntry -TChildEntry = TypeVar("TChildEntry", bound=BaseEntry) +TBaseEntry = TypeVar("TBaseEntry", bound=BaseEntry) class EntryParent(BaseEntry): def __init__(self, entry_dict: Dict, working_directory: str): super().__init__(entry_dict=entry_dict, working_directory=working_directory) - self._child_entries: List[TChildEntry] = [] + self.child_entries: List["EntryParent"] = [] - # pylint: disable=no-self-use - def _get_children_entry_variables_to_add( - self, child_entries: List[TChildEntry] - ) -> Dict[str, str | int]: + def is_entry(self) -> bool: """ - Adds source variables to the child entry derived from the parent entry. - """ - if not child_entries: - return {} - - return {"playlist_max_upload_year": max(entry.upload_year for entry in child_entries)} - - # pylint: enable=no-self-use - - def read_children_from_entry_dicts( - self, entry_dicts: List[Dict], child_class: Type[TChildEntry] - ) -> "EntryParent": - """ - Parameters - ---------- - entry_dicts - Entry dicts to look for children from - child_class - The class to convert the entry dict to - Returns ------- - List of children + True if the entry contains a media file. False otherwise. """ - child_entries: List[TChildEntry] = [] + return self.kwargs_contains("ext") + + def parent_children(self) -> List["EntryParent"]: + """This parent's children that are also parents""" + return [child for child in self.child_entries if child.child_count() > 0] + + def entry_children(self) -> List["EntryParent"]: + """This parent's children that are entries""" + return [child for child in self.child_entries if child.is_entry()] + + def read_children_from_entry_dicts(self, entry_dicts: List[Dict]) -> "EntryParent": + """ + Populates a tree of EntryParents that belong to this instance + """ + child_entries: List["EntryParent"] = [] for entry_dict in entry_dicts: if entry_dict.get("playlist_id") == self.uid: child_entries.append( - child_class(entry_dict=entry_dict, working_directory=self.working_directory()) + self.__class__( + entry_dict=entry_dict, + working_directory=self.working_directory(), + ) ) + child_entries[-1].read_children_from_entry_dicts(entry_dicts) - child_variables_to_add = self._get_children_entry_variables_to_add(child_entries) - for child_entry in child_entries: - child_entry.add_variables(variables_to_add=child_variables_to_add) - - self._child_entries = sorted( - child_entries, key=lambda entry: entry.kwargs("playlist_index") - ) + self.child_entries = sorted(child_entries, key=lambda entry: entry.kwargs("playlist_index")) return self - @property - def child_entries(self) -> List[TChildEntry]: - """ - Returns - ------- - List of child entities - """ - return self._child_entries - - @property def child_count(self) -> int: """ Returns @@ -96,37 +75,25 @@ class EntryParent(BaseEntry): return thumbnail["url"] return None - def __contains__(self, item): + @classmethod + def from_entry_dicts( + cls, entry_dicts: List[Dict], working_directory: str + ) -> List["EntryParent"]: + """ + Reads all entry dicts and builds a tree of EntryParents + """ + return [ + EntryParent( + entry_dict=entry_dict, working_directory=working_directory + ).read_children_from_entry_dicts(entry_dicts) + for entry_dict in entry_dicts + if "playlist_id" not in entry_dict + ] + + def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry: """ Returns ------- - True if the the item (entry_dict) has the same id as one of the tracks. False otherwise. + Converted EntryParent to Entry-like class """ - uid: Optional[str] = None - if isinstance(item, BaseEntry): - uid = item.uid - elif isinstance(item, dict): - uid = item.get("id") - - if uid is not None: - return any(uid == child_entry.uid for child_entry in self._child_entries) - return False - - @classmethod - def from_entry_dicts_with_children( - cls, - entry_dicts: List[Dict], - working_directory: str, - child_class: Type[TChildEntry], - extractor: Optional[str], - ) -> "EntryParent": - """ - Load the parent entry and its children - """ - entry_parent: EntryParent = cls.from_entry_dicts( - entry_dicts=entry_dicts, working_directory=working_directory, extractor=extractor - ) - entry_parent.read_children_from_entry_dicts( - entry_dicts=entry_dicts, child_class=child_class - ) - return entry_parent + return entry_type(entry_dict=self._kwargs, working_directory=self._working_directory) diff --git a/src/ytdl_sub/entries/soundcloud.py b/src/ytdl_sub/entries/soundcloud.py index 0d775c6c..1ab9f74b 100644 --- a/src/ytdl_sub/entries/soundcloud.py +++ b/src/ytdl_sub/entries/soundcloud.py @@ -1,45 +1,11 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.variables.soundcloud_variables import SoundcloudVariables +# TODO: Delete since not used + class SoundcloudTrack(SoundcloudVariables, Entry): """ Entry object to represent a Soundcloud track yt-dlp that is a single, which implies it does not belong to an album. """ - - entry_extractor = "soundcloud" - - def is_premiere(self) -> bool: - """ - Returns whether the entry is a premier track. Check this by seeing if the track's url is - a preview. - """ - return "/preview/" in self.kwargs("url") - - -class SoundcloudAlbumTrack(SoundcloudTrack): - """ - Entry object to represent a Soundcloud track yt-dlp that belongs to an album. - """ - - @property - def track_number(self) -> int: - """Returns the entry's track number""" - return self.kwargs("playlist_index") - - @property - def track_count(self) -> int: - """Returns the entry's total tracks in album""" - return self.kwargs("playlist_count") - - @property - def album(self) -> str: - """Returns the entry's album name, fetched from its internal album""" - return self.kwargs("playlist") - - @property - def album_year(self) -> int: - """Returns the entry's album year, fetched from its internal album""" - # added from parent entry - return self._additional_variables["playlist_max_upload_year"] diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py index 155caa51..658208f6 100644 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ b/src/ytdl_sub/entries/variables/entry_variables.py @@ -1,3 +1,5 @@ +from yt_dlp.utils import sanitize_filename + from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import BaseEntryVariables @@ -14,6 +16,72 @@ _days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] class EntryVariables(BaseEntryVariables): + @property + def playlist(self: BaseEntry) -> str: + """ + Returns + ------- + str + Name of its parent playlist/channel if it exists, otherwise returns its title. + """ + return self.kwargs_get("playlist", self.title) + + @property + def playlist_sanitized(self) -> str: + """ + Returns + ------- + str + The playlist name, sanitized + """ + return sanitize_filename(self.playlist) + + @property + def playlist_index(self: BaseEntry) -> int: + """ + Returns + ------- + int + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, an index of 1 implies it's the most recent + uploaded entry. It is recommended to not use this unless you know the channel/playlist + will never add new content, i.e. a music album. + """ + return self.kwargs_get("playlist_index", 1) + + @property + def playlist_index_padded(self) -> str: + """ + Returns + ------- + str + playlist_index padded two digits + """ + return _pad(self.playlist_index, width=2) + + @property + def playlist_count(self: BaseEntry) -> int: + """ + Returns + ------- + int + Playlist count if it exists, otherwise returns ``1``. + """ + return self.kwargs_get("playlist_count", 1) + + @property + def playlist_max_upload_year(self) -> int: + """ + Returns + ------- + int + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + """ + # override in EntryParent + return self.upload_year + @property def ext(self: BaseEntry) -> str: """ diff --git a/src/ytdl_sub/entries/youtube.py b/src/ytdl_sub/entries/youtube.py index 6f5dece2..1bfa8513 100644 --- a/src/ytdl_sub/entries/youtube.py +++ b/src/ytdl_sub/entries/youtube.py @@ -10,8 +10,6 @@ class YoutubeVideo(YoutubeVideoVariables, Entry): Entry object to represent a Youtube video. Reserved for shared Youtube entry logic. """ - entry_extractor = "youtube" - @property def ext(self) -> str: """ diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index 040e2ceb..8b4b3480 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -7,7 +7,6 @@ from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator -from ytdl_sub.validators.validators import StringValidator class MusicTagsOptions(PluginOptions): @@ -34,17 +33,12 @@ class MusicTagsOptions(PluginOptions): """ _required_keys = {"tags"} - _optional_keys = {"multi_value_separator"} def __init__(self, name, value): super().__init__(name, value) self._tags = self._validate_key(key="tags", validator=DictFormatterValidator) - self.multi_value_separator = self._validate_key_if_present( - key="multi_value_separator", validator=StringValidator - ) - @property def tags(self) -> DictFormatterValidator: """ diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 8c555641..7ee6d8a0 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -8,6 +8,7 @@ from typing import Type from typing import TypeVar from typing import final +from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata @@ -39,40 +40,11 @@ class PluginPriority: return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT -class PluginOptions(StrictDictValidator): +class PluginOptions(StrictDictValidator, AddsVariablesMixin, ABC): """ Class that defines the parameters to a plugin """ - # pylint: disable=no-self-use - def added_source_variables(self) -> List[str]: - """ - If the plugin adds source variables, list them here. - - Returns - ------- - List of added source variables this plugin creates - """ - return [] - - # pylint: disable=unused-argument - def validate_with_variables( - self, source_variables: List[str], override_variables: List[str] - ) -> None: - """ - Optional validation after init with the session's source and override variables. - - Parameters - ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin - """ - return None - - # pylint: enable=no-self-use,unused-argument - PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions) diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 34070fbd..b818e9da 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -165,7 +165,7 @@ class RegexOptions(PluginOptions): return self._skip_if_match_fails def validate_with_variables( - self, source_variables: List[str], override_variables: List[str] + self, source_variables: List[str], override_variables: Dict[str, str] ) -> None: """ Ensures each source variable capture group is valid diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index f70a4f7a..de98f888 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -1,4 +1,3 @@ -from pathlib import Path from typing import Dict from typing import List from typing import Optional @@ -21,14 +20,6 @@ SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"} logger = Logger.get(name="subtitles") -def _is_entry_subtitle_file(path: Path, entry: Entry) -> bool: - if path.is_file() and path.name.startswith(entry.uid): - for ext in SUBTITLE_EXTENSIONS: - if path.name.endswith(f".{ext}"): - return True - return False - - class SubtitlesTypeValidator(StringSelectValidator): _expected_value_type_name = "subtitles type" _select_values = SUBTITLE_EXTENSIONS diff --git a/src/ytdl_sub/thread/log_entries_downloaded_listener.py b/src/ytdl_sub/thread/log_entries_downloaded_listener.py index a20038b5..ffa9c3b9 100644 --- a/src/ytdl_sub/thread/log_entries_downloaded_listener.py +++ b/src/ytdl_sub/thread/log_entries_downloaded_listener.py @@ -13,7 +13,7 @@ logger = Logger.get(name="downloader") class LogEntriesDownloadedListener(threading.Thread): - def __init__(self, working_directory: str, info_json_extractor: str, log_prefix: str): + def __init__(self, working_directory: str, log_prefix: str): """ To be ran in a thread while download via ytdl-sub. Listens for new .info.json files in the working directory, checks the extractor value, and if it matches the input arg, log the @@ -23,20 +23,18 @@ class LogEntriesDownloadedListener(threading.Thread): ---------- working_directory subscription download working directory - info_json_extractor - print the titles of the info.json file with this extractor log_prefix The message to print prefixed to the title, i.e. '{log_prefix} {title}' """ threading.Thread.__init__(self) self.working_directory = working_directory - self.info_json_extractor = info_json_extractor self.log_prefix = log_prefix self.complete = False self._files_read: Set[str] = set() - def _get_title_from_info_json(self, path: Path) -> Optional[str]: + @classmethod + def _get_title_from_info_json(cls, path: Path) -> Optional[str]: try: with open(path, "r", encoding="utf-8") as file: file_json = json.load(file) @@ -44,10 +42,7 @@ class LogEntriesDownloadedListener(threading.Thread): # swallow the error since this is only printing logs return None - if file_json.get("extractor") == self.info_json_extractor: - return file_json.get("title") - - return None + return file_json.get("title") @classmethod def _is_info_json(cls, path: Path) -> bool: diff --git a/src/ytdl_sub/utils/datetime.py b/src/ytdl_sub/utils/datetime.py index 157629e1..d7b1e07a 100644 --- a/src/ytdl_sub/utils/datetime.py +++ b/src/ytdl_sub/utils/datetime.py @@ -3,42 +3,9 @@ from typing import Optional from yt_dlp import DateRange from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_datetime import StringDatetimeValidator -def to_date_range_hack( - before: Optional[StringDatetimeValidator], after: Optional[StringDatetimeValidator] -) -> Optional[DateRange]: - """ - Workaround for channel before/after support. - - Returns - ------- - Date range if the 'before' or 'after' is defined. None otherwise. - """ - start: Optional[str] = None - end: Optional[str] = None - - if after: - start = after.apply_formatter(variable_dict={}) - - if before: - end = before.apply_formatter(variable_dict={}) - - if start or end: - logger = Logger.get(name="youtube-channel") - logger.warning( - "DEPRECATED: youtube.before/after will are deprecated and will be removed in v0.0.5. " - "Use the 'date_range' plugin instead: " - "https://ytdl-sub.readthedocs.io/en/latest/config.html#date-range" - ) - - return DateRange(start=start, end=end) - - return None - - def to_date_range( before: Optional[StringDatetimeValidator], after: Optional[StringDatetimeValidator], diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index 694ed6c7..e2235587 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -244,24 +244,3 @@ class LiteralDictValidator(DictValidator): def keys(self) -> List[str]: """Returns a sorted list of the dict's keys""" return super()._keys - - -class UniformDictValidator(DictValidator, Generic[ValidatorT], ABC): - """DictValidator where all values have the same validator""" - - uniform_validator_class: Type[ValidatorT] - - def __init__(self, name, value): - super().__init__(name, value) - - for key in self._keys: - _ = self._validate_key_if_present(key=key, validator=self.uniform_validator_class) - - @property - def validator_dict(self) -> Dict[str, ValidatorT]: - """ - Returns - ------- - Dict of name: validator - """ - return self._validator_dict diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index b39cacd7..97b20247 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -123,19 +123,6 @@ class DownloadArchive: file.write(f"{line}\n") return self - def contains(self, entry_id: str) -> bool: - """ - Parameters - ---------- - entry_id - Id of the entry - - Returns - ------- - True if the entry id is within this download archive. False otherwise. - """ - return any(entry_id in line for line in self._download_archive_lines) - def remove_entry(self, entry_id: str) -> "DownloadArchive": """ Parameters @@ -598,7 +585,7 @@ class DownloadArchiver: """ def __init__(self, enhanced_download_archive: EnhancedDownloadArchive): - self.__enhanced_download_archive = enhanced_download_archive + self._enhanced_download_archive = enhanced_download_archive @property def working_directory(self) -> str: @@ -607,7 +594,7 @@ class DownloadArchiver: ------- Path to the working directory """ - return self.__enhanced_download_archive.working_directory + return self._enhanced_download_archive.working_directory @property def is_dry_run(self) -> bool: @@ -616,7 +603,7 @@ class DownloadArchiver: ------- True if this session is a dry-run. False otherwise. """ - return self.__enhanced_download_archive.is_dry_run + return self._enhanced_download_archive.is_dry_run def save_file( self, @@ -640,7 +627,7 @@ class DownloadArchiver: entry Optional. Entry that the file belongs to """ - self.__enhanced_download_archive.save_file_to_output_directory( + self._enhanced_download_archive.save_file_to_output_directory( file_name=file_name, file_metadata=file_metadata, output_file_name=output_file_name, diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 6a4a5a95..af6239c1 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -85,6 +85,12 @@ def mock_entry_to_dict( "thumbnail_ext": thumbnail_ext, "info_json_ext": "info.json", "webpage_url": webpage_url, + "playlist": "entry ｛title｝", + "playlist_sanitized": "entry ｛title｝", + "playlist_index": 1, + "playlist_index_padded": "01", + "playlist_count": 1, + "playlist_max_upload_year": 2021, }