[BACKEND] Add generic.collection download strategy, use it for soundcloud, youtube channel/playlist (#230)

docs/config.rst

@@ -114,7 +114,7 @@ for available source variables to use.
 albums_and_singles
 __________________
 .. autoclass:: ytdl_sub.downloaders.soundcloud.albums_and_singles.SoundcloudAlbumsAndSinglesDownloadOptions()
-  :members:
+  :members: url, skip_premiere_tracks
   :member-order: bysource
   :inherited-members:
 

examples/kodi_tv_shows_config.yaml

@@ -85,31 +85,3 @@ presets:
     overrides:
       youtube_tv_shows_directory: "/path/to/youtube_tv_shows"
       episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {title_sanitized}"
-
-  ###############################################################################
-  # RECENT ARCHIVE
-  # This preset shows how you can download just the last 14 days-worth of videos
-  # on each download invocation.
-  yt_channel_as_tv__recent:
-    # `preset` can be set to any other preset in the config, and will inherit all its defined fields.
-    # This helps reduce copy-paste in the config.yaml
-    preset: "yt_channel_as_tv"
-
-    # We will add the `after` field onto `yt_channel_as_tv`'s YouTube channel download strategy.
-    # This is saying 'only download videos uploaded in the last 2 weeks'
-    youtube:
-      after: "today-2weeks"
-
-  ###############################################################################
-  # ROLLING ARCHIVE
-  # This example shows how to only keep the last 14-days worth of videos, and
-  # delete the rest.
-  yt_channel_as_tv__only_recent:
-    # Reuse `yt_channel_as_tv__recent` to only download the last 2 weeks' of videos
-    preset: "yt_channel_as_tv__recent"
-
-    # This is saying "only keep files if they were uploaded in the last 2 weeks".
-    # All other files that this subscription had previously downloaded will be
-    # deleted.
-    output_options:
-      keep_files_after: "today-2weeks"

examples/kodi_tv_shows_subscriptions.yaml

@@ -3,7 +3,7 @@
 # in each example.
 
 ###############################################################################
-# LEVEL 1 - FULL ARCHIVE
+# FULL ARCHIVE
 #
 # Subscription names are defined by you. We will call this one
 # john_smith_archive because it will download every single video in
@@ -25,27 +25,3 @@ john_smith_archive:
   # '_sanitized' version that is safe for file systems.
   overrides:
     tv_show_name: "John /\ Smith"
-
-###############################################################################
-# LEVEL 2 - RECENT ARCHIVE
-#
-# Use the "yt_channel_as_tv__recent" preset to only download the last two
-# weeks' worth of YouTube videos.
-john_smith_recent_archive:
-  preset: "yt_channel_as_tv__recent"
-  youtube:
-    channel_url: "https://youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
-  overrides:
-    tv_show_name: "John /\ Smith"
-
-###############################################################################
-# LEVEL 3 - ROLLING ARCHIVE
-#
-# Use the "yt_channel_as_tv__recent_only" preset to only download the last two
-# weeks' worth of YouTube videos, and delete any existing older videos.
-john_smith_rolling_archive:
-  preset: "yt_channel_as_tv__only_recent"
-  youtube:
-    channel_url: "https://youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
-  overrides:
-    tv_show_name: "John /\ Smith"

pyproject.toml

@@ -13,6 +13,7 @@ disable = [
     "R0903", # too-few-public-methods
     "R0801", # similar lines
     "R0913", # Too many arguments
+    "R0901", # too-many-ancestors
     "W0511", # TODO
 ]
 

src/ytdl_sub/config/preset.py

@@ -186,7 +186,6 @@ class Preset(StrictDictValidator):
 
     def __validate_and_get_plugins(self) -> PresetPlugins:
         preset_plugins = PresetPlugins()
-        source_variables = copy.deepcopy(self._source_variables)
 
         for key in self._keys:
             if key not in PluginMapping.plugins():
@@ -197,20 +196,30 @@ class Preset(StrictDictValidator):
 
             preset_plugins.add(plugin_type=plugin, plugin_options=plugin_options)
 
-        for plugin, plugin_options in sorted(
-            preset_plugins.zipped(),
-            key=lambda _plugin_and_options: _plugin_and_options[0].priority.modify_entry,
+        return preset_plugins
+
+    def __validate_added_variables(self):
+        source_variables = copy.deepcopy(self._source_variables)
+
+        # Validate added download option variables here since plugins could subsequently use them
+        self.downloader_options.validate_with_variables(
+            source_variables=source_variables,
+            override_variables=self.overrides.dict_with_format_strings,
+        )
+        source_variables.extend(self.downloader_options.added_source_variables())
+
+        for _, plugin_options in sorted(
+            self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry
         ):
             # Validate current plugin using source + added plugin variables
             plugin_options.validate_with_variables(
-                source_variables=source_variables, override_variables=self.overrides.keys
+                source_variables=source_variables,
+                override_variables=self.overrides.dict_with_format_strings,
             )
 
             # Extend existing source variables with ones created from this plugin
             source_variables.extend(plugin_options.added_source_variables())
 
-        return preset_plugins
-
     def __validate_override_string_formatter_validator(
         self,
         formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
@@ -224,7 +233,11 @@ class Preset(StrictDictValidator):
         # If the formatter supports source variables, set the formatter variables to include
         # both source and override variables
         if not isinstance(formatter_validator, OverridesStringFormatterValidator):
-            source_variables = {source_var: "dummy_string" for source_var in self._source_variables}
+            source_variables = {
+                source_var: "dummy_string"
+                for source_var in self._source_variables
+                + self.downloader_options.added_source_variables()
+            }
             variable_dict = dict(source_variables, **variable_dict)
 
             # For all plugins, add in any extra added source variables
@@ -330,6 +343,7 @@ class Preset(StrictDictValidator):
 
         self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
         self.plugins: PresetPlugins = self.__validate_and_get_plugins()
+        self.__validate_added_variables()
 
         # After all options are initialized, perform a recursive post-validate that requires
         # values from multiple validators

src/ytdl_sub/config/preset_options.py

@@ -1,4 +1,6 @@
+from abc import ABC
 from typing import Dict
+from typing import List
 from typing import Optional
 
 from yt_dlp.utils import sanitize_filename
@@ -13,6 +15,43 @@ from ytdl_sub.validators.validators import BoolValidator
 from ytdl_sub.validators.validators import LiteralDictValidator
 
 
+# pylint: disable=no-self-use
+# pylint: disable=unused-argument
+class AddsVariablesMixin(ABC):
+    """
+    Mixin for parts of the Preset that adds source variables
+    """
+
+    def added_source_variables(self) -> List[str]:
+        """
+        If the plugin adds source variables, list them here.
+
+        Returns
+        -------
+        List of added source variables this plugin creates
+        """
+        return []
+
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: Dict[str, str]
+    ) -> None:
+        """
+        Optional validation after init with the session's source and override variables.
+
+        Parameters
+        ----------
+        source_variables
+            Available source variables when running the plugin
+        override_variables
+            Available override variables when running the plugin
+        """
+        return None
+
+
+# pylint: enable=no-self-use
+# pylint: enable=unused-argument
+
+
 class YTDLOptions(LiteralDictValidator):
     """
     Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader.

src/ytdl_sub/downloaders/downloader.py

@@ -21,6 +21,7 @@ import yt_dlp as ytdl
 from yt_dlp.utils import ExistingVideoReached
 from yt_dlp.utils import RejectedVideoReached
 
+from ytdl_sub.config.preset_options import AddsVariablesMixin
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.base_entry import BaseEntry
@@ -36,7 +37,7 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
 download_logger = Logger.get(name="downloader")
 
 
-class DownloaderValidator(StrictDictValidator, ABC):
+class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
     """
     Placeholder class to define downloader options
     """
@@ -203,37 +204,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
             f"yt-dlp failed to download an entry with these arguments: {error_dict}"
         )
 
-    def _filter_entry_dicts(
-        self,
-        entry_dicts: List[Dict],
-        extractor: Optional[str] = None,
-        sort_by: Optional[str] = None,
-    ) -> List[Dict]:
-        """
-        Parameters
-        ----------
-        entry_dicts
-            entry dicts to filter
-        extractor
-            Optional. Extractor that the entry dicts must have. If None, defaults to the
-            entry type's extractor
-        sort_by
-            Optional. Sort the entry dicts on this key
-
-        Returns
-        -------
-        filtered entry dicts
-        """
-        if extractor is None:
-            extractor = self.downloader_entry_type.entry_extractor
-
-        output = [
-            entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor
-        ]
-        if sort_by:
-            output = sorted(output, key=lambda entry_dict: entry_dict[sort_by])
-        return output
-
     def _get_entry_dicts_from_info_json_files(self) -> List[Dict]:
         """
         Returns
@@ -265,7 +235,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
 
         info_json_listener = LogEntriesDownloadedListener(
             working_directory=self.working_directory,
-            info_json_extractor=self.downloader_entry_type.entry_extractor,
             log_prefix=log_prefix,
         )
 

src/ytdl_sub/downloaders/generic/collection.py

@@ -0,0 +1,286 @@
+import contextlib
+import os.path
+from typing import Dict
+from typing import Generator
+from typing import List
+from typing import Set
+
+from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.downloaders.downloader import Downloader
+from ytdl_sub.downloaders.downloader import DownloaderOptionsT
+from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.downloader import download_logger
+from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
+from ytdl_sub.entries.base_entry import BaseEntry
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.entries.entry_parent import EntryParent
+from ytdl_sub.utils.file_handler import FileHandler
+from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
+from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
+from ytdl_sub.validators.validators import ListValidator
+from ytdl_sub.validators.validators import StringValidator
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
+
+
+def _entry_key(entry: BaseEntry) -> str:
+    return entry.extractor + entry.uid
+
+
+def _get_parent_entry_variables(parent: EntryParent) -> Dict[str, str | int]:
+    """
+    Adds source variables to the child entry derived from the parent entry.
+    """
+    if not parent.child_entries:
+        return {}
+
+    return {
+        "playlist_max_upload_year": max(
+            child_entry.to_type(Entry).upload_year for child_entry in parent.child_entries
+        )
+    }
+
+
+class CollectionUrlValidator(StrictDictValidator):
+    _required_keys = {"url"}
+    _optional_keys = {"variables"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+
+        # TODO: url validate using yt-dlp IE
+        self._url = self._validate_key(key="url", validator=StringValidator)
+        variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator)
+        self._variables = variables.dict_with_format_strings if variables else {}
+
+    @property
+    def url(self) -> str:
+        """
+        Returns
+        -------
+        URL to download from
+        """
+        return self._url.value
+
+    @property
+    def variables(self) -> Dict[str, str]:
+        """
+        Variables to add to each entry
+        """
+        return self._variables
+
+
+class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]):
+    _inner_list_type = CollectionUrlValidator
+    _expected_value_type_name = "collection url list"
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+
+        added_variables: Dict[str, str] = self.list[0].variables
+
+        for idx, collection_url_validator in enumerate(self.list[1:]):
+            collection_variables = collection_url_validator.variables
+
+            # see if this collection contains new added vars (it should not)
+            for var in collection_variables.keys():
+                if var not in added_variables:
+                    raise self._validation_exception(
+                        f"Collection url {idx} contains the variable '{var}' that the first "
+                        f"collection url does not. The first collection url must define all added "
+                        f"variables."
+                    )
+
+            # see if this collection is missing any added vars (if so, inherit from the top)
+            for var in added_variables.keys():
+                if var not in collection_variables:
+                    collection_variables[var] = added_variables[var]
+
+
+class CollectionDownloadOptions(DownloaderValidator):
+    """
+    Downloads from multiple URLs. If an entry is returned from more than one URL, it will
+    resolve to the bottom-most URL settings.
+
+    Usage:
+
+    .. code-block:: yaml
+
+      presets:
+        my_example_preset:
+          generic:
+            # required
+            download_strategy: "collection"
+            urls:
+            - url: "soundcloud.com/tracks"
+              variables:
+                season: "1"
+                album: "{title}"
+            - url: "soundcloud.com/albums"
+              variables:
+                season: "1"
+                album: "{playlist_title}"
+    """
+
+    _required_keys = {"urls"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._urls = self._validate_key(key="urls", validator=CollectionUrlListValidator)
+
+    @property
+    def collection_urls(self) -> CollectionUrlListValidator:
+        """
+        Required. The Soundcloud user's url, i.e. ``soundcloud.com/the_username``
+        """
+        return self._urls
+
+    def added_source_variables(self) -> List[str]:
+        """
+        Returns
+        -------
+        List of variables added. The first collection url always contains all the variables.
+        """
+        return list(self._urls.list[0].variables.keys())
+
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: Dict[str, str]
+    ) -> None:
+        """
+        Ensures new variables added are not existing variables
+        """
+        for source_var_name in self.added_source_variables():
+            if source_var_name in source_variables:
+                raise self._validation_exception(
+                    f"'{source_var_name}' cannot be used as a variable name because it "
+                    f"is an existing source variable"
+                )
+
+        base_variables = dict(
+            override_variables, **{source_var: "dummy_string" for source_var in source_variables}
+        )
+
+        # Apply formatting to each new source variable, ensure it resolves
+        for collection_url in self.collection_urls.list:
+            for source_var_name, source_var_formatter_str in collection_url.variables.items():
+                _ = StringFormatterValidator(
+                    name=f"{self._name}.{source_var_name}", value=source_var_formatter_str
+                ).apply_formatter(base_variables)
+
+
+class CollectionDownloader(Downloader[CollectionDownloadOptions, Entry]):
+    downloader_options_type = CollectionDownloadOptions
+    downloader_entry_type = Entry
+
+    def __init__(
+        self,
+        download_options: DownloaderOptionsT,
+        enhanced_download_archive: EnhancedDownloadArchive,
+        ytdl_options_builder: YTDLOptionsBuilder,
+        overrides: Overrides,
+    ):
+        super().__init__(
+            download_options=download_options,
+            enhanced_download_archive=enhanced_download_archive,
+            ytdl_options_builder=ytdl_options_builder,
+            overrides=overrides,
+        )
+
+        self.parents: List[EntryParent] = []
+        self.downloaded_entries: Set[str] = set()
+
+    @contextlib.contextmanager
+    def _separate_download_archives(self):
+        """
+        Separate download archive writing between collection urls. This is so break_on_existing
+        does not break when downloading from subset urls.
+        """
+        archive_path = self._ytdl_options_builder.to_dict().get("download_archive", "")
+        backup_archive_path = f"{archive_path}.backup"
+
+        # If archive path exists, maintain download archive is enable
+        if archive_file_exists := archive_path and os.path.isfile(archive_path):
+            archive_file_exists = True
+
+            # If a backup exists, it's the one prior to any downloading, use that.
+            if os.path.isfile(backup_archive_path):
+                FileHandler.copy(src_file_path=backup_archive_path, dst_file_path=archive_path)
+            # If not, create the backup
+            else:
+                FileHandler.copy(src_file_path=archive_path, dst_file_path=backup_archive_path)
+
+        yield
+
+        # If an archive path did not exist at first, but now exists, delete it
+        if not archive_file_exists and os.path.isfile(archive_path):
+            FileHandler.delete(file_path=archive_path)
+        # If the archive file did exist, restore the backup
+        elif archive_file_exists:
+            FileHandler.copy(src_file_path=backup_archive_path, dst_file_path=archive_path)
+
+    def _download_entry(self, entry: Entry) -> Entry:
+        download_logger.info("Downloading entry %s", entry.title)
+        download_entry_dict = self.extract_info_with_retry(
+            is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded,
+            url=entry.webpage_url,
+            ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
+        )
+
+        # Workaround for the ytdlp issue
+        # pylint: disable=protected-access
+        entry._kwargs["requested_subtitles"] = download_entry_dict.get("requested_subtitles")
+        # pylint: enable=protected-access
+
+        return entry
+
+    def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]:
+        """Download in reverse order, that way we download older entries ones first"""
+        for entry_child in reversed(parent.entry_children()):
+            if _entry_key(entry_child) in self.downloaded_entries:
+                continue
+
+            yield self._download_entry(entry_child.to_type(Entry))
+            self.downloaded_entries.add(_entry_key(entry_child))
+
+        for parent_child in reversed(parent.parent_children()):
+            for entry_child in self._download_parent_entry(parent=parent_child):
+                yield entry_child
+
+    def download_url_metadata(self, collection_url: CollectionUrlValidator) -> List[EntryParent]:
+        """
+        Downloads only info.json files and forms EntryParent trees
+        """
+        with self._separate_download_archives():
+            entry_dicts = self.extract_info_via_info_json(
+                only_info_json=True,
+                url=collection_url.url,
+                log_prefix_on_info_json_dl="Downloading metadata for",
+            )
+
+        return EntryParent.from_entry_dicts(
+            entry_dicts=entry_dicts, working_directory=self.working_directory
+        )
+
+    def download_url(
+        self, collection_url: CollectionUrlValidator, parents: List[EntryParent]
+    ) -> Generator[Entry, None, None]:
+        """
+        Downloads the leaf entries from EntryParent trees
+        """
+        with self._separate_download_archives():
+            for parent in parents:
+                for entry_child in self._download_parent_entry(parent=parent):
+                    entry_child.add_variables(
+                        dict(_get_parent_entry_variables(parent), **collection_url.variables)
+                    )
+                    yield entry_child
+
+    def download(self) -> Generator[Entry, None, None]:
+        """
+        Soundcloud subscription to download albums and tracks as singles.
+        """
+        # download the bottom-most urls first since they are top-priority
+        for collection_url in reversed(self.download_options.collection_urls.list):
+            parents = self.download_url_metadata(collection_url=collection_url)
+            for entry in self.download_url(collection_url=collection_url, parents=parents):
+                yield entry

src/ytdl_sub/downloaders/soundcloud/abc.py

@@ -1,65 +0,0 @@
-from abc import ABC
-from typing import Generic
-from typing import TypeVar
-
-from ytdl_sub.downloaders.downloader import Downloader
-from ytdl_sub.downloaders.downloader import DownloaderValidator
-from ytdl_sub.entries.soundcloud import SoundcloudTrack
-from ytdl_sub.validators.validators import BoolValidator
-
-
-class SoundcloudDownloaderOptions(DownloaderValidator, ABC):
-    """
-    Abstract source validator for all soundcloud sources.
-    """
-
-    _optional_keys = {"skip_premiere_tracks"}
-
-    def __init__(self, name: str, value: dict):
-        super().__init__(name=name, value=value)
-        self._skip_premiere_tracks = self._validate_key(
-            "skip_premiere_tracks", BoolValidator, default=True
-        )
-
-    @property
-    def skip_premiere_tracks(self) -> bool:
-        """
-        Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
-        """
-        return self._skip_premiere_tracks.value
-
-
-SoundcloudDownloaderOptionsT = TypeVar(
-    "SoundcloudDownloaderOptionsT", bound=SoundcloudDownloaderOptions
-)
-
-
-class SoundcloudDownloader(
-    Downloader[SoundcloudDownloaderOptionsT, SoundcloudTrack],
-    Generic[SoundcloudDownloaderOptionsT],
-    ABC,
-):
-    """
-    Class that handles downloading soundcloud entries via ytdl and converting them into
-    SoundcloudTrack / SoundcloudAlbumTrack objects
-    """
-
-    downloader_entry_type = SoundcloudTrack
-
-    @classmethod
-    def artist_albums_url(cls, artist_url: str) -> str:
-        """
-        Returns
-        -------
-        Full artist album url
-        """
-        return f"{artist_url}/albums"
-
-    @classmethod
-    def artist_tracks_url(cls, artist_url: str) -> str:
-        """
-        Returns
-        -------
-        Full artist tracks url
-        """
-        return f"{artist_url}/tracks"

src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py

@@ -2,16 +2,16 @@ from typing import Dict
 from typing import Generator
 from typing import List
 
-from ytdl_sub.downloaders.downloader import download_logger
-from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloader
-from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloaderOptions
-from ytdl_sub.entries.entry_parent import EntryParent
-from ytdl_sub.entries.soundcloud import SoundcloudAlbumTrack
-from ytdl_sub.entries.soundcloud import SoundcloudTrack
+from ytdl_sub.downloaders.downloader import Downloader
+from ytdl_sub.downloaders.downloader import DownloaderValidator
+from ytdl_sub.downloaders.generic.collection import CollectionDownloader
+from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions
+from ytdl_sub.entries.entry import Entry
 from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator
+from ytdl_sub.validators.validators import BoolValidator
 
 
-class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):
+class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator):
     """
     Downloads a soundcloud user's entire discography. Groups together album tracks and considers
     any track not in an album as a single. Also includes any collaboration tracks.
@@ -32,12 +32,53 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):
     """
 
     _required_keys = {"url"}
+    _optional_keys = {"skip_premiere_tracks"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
         self._url = self._validate_key(
             key="url", validator=SoundcloudUsernameUrlValidator
         ).username_url
+        self._skip_premiere_tracks = self._validate_key(
+            "skip_premiere_tracks", BoolValidator, default=True
+        )
+
+        self.collection_validator = CollectionDownloadOptions(
+            name=self._name,
+            value={
+                "urls": [
+                    {
+                        "url": f"{self._url}/tracks",
+                        "variables": {
+                            "track_number": "1",
+                            "track_number_padded": "01",
+                            "track_count": "1",
+                            "album": "{title}",
+                            "album_sanitized": "{title_sanitized}",
+                            "album_year": "{upload_year}",
+                        },
+                    },
+                    {
+                        "url": f"{self._url}/albums",
+                        "variables": {
+                            "track_number": "{playlist_index}",
+                            "track_number_padded": "{playlist_index_padded}",
+                            "track_count": "{playlist_count}",
+                            "album": "{playlist}",
+                            "album_sanitized": "{playlist_sanitized}",
+                            "album_year": "{playlist_max_upload_year}",
+                        },
+                    },
+                ]
+            },
+        )
+
+    @property
+    def skip_premiere_tracks(self) -> bool:
+        """
+        Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True.
+        """
+        return self._skip_premiere_tracks.value
 
     @property
     def url(self) -> str:
@@ -46,11 +87,25 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(SoundcloudDownloaderOptions):
         """
         return self._url
 
+    def added_source_variables(self) -> List[str]:
+        """Validate using collection_validator"""
+        return self.collection_validator.added_source_variables()
+
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: Dict[str, str]
+    ) -> None:
+        """Validate using collection_validator"""
+        return self.collection_validator.validate_with_variables(
+            source_variables, override_variables
+        )
+
 
 class SoundcloudAlbumsAndSinglesDownloader(
-    SoundcloudDownloader[SoundcloudAlbumsAndSinglesDownloadOptions]
+    Downloader[SoundcloudAlbumsAndSinglesDownloadOptions, Entry]
 ):
     downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
+    downloader_entry_type = Entry
+
     supports_subtitles = False
     supports_chapters = False
 
@@ -72,104 +127,29 @@ class SoundcloudAlbumsAndSinglesDownloader(
             },
         )
 
-    def _get_singles(
-        self, entry_dicts: List[Dict], albums: List[EntryParent]
-    ) -> List[SoundcloudTrack]:
-        tracks: List[SoundcloudTrack] = []
+    def _should_skip(self, entry: Entry) -> bool:
+        if not self.download_options.skip_premiere_tracks:
+            return False
 
-        # Get all tracks that are not part of an album
-        for entry_dict in self._filter_entry_dicts(entry_dicts):
-            if not any(entry_dict in album for album in albums):
-                tracks.append(
-                    SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory)
-                )
+        for url in [entry.kwargs_get("url", ""), entry.webpage_url]:
+            if "/preview/" in url:
+                return True
 
-        return tracks
+        return False
 
-    def _get_albums(self) -> List[EntryParent]:
-        # Dry-run to get the info json files
-        artist_albums_url = self.artist_albums_url(artist_url=self.download_options.url)
-        entry_dicts = self.extract_info_via_info_json(
-            only_info_json=True,
-            log_prefix_on_info_json_dl="Downloading metadata for",
-            url=artist_albums_url,
-        )
-
-        albums: List[EntryParent] = []
-        for entry_dict in self._filter_entry_dicts(entry_dicts, extractor="soundcloud:set"):
-            albums.append(
-                EntryParent(
-                    entry_dict=entry_dict, working_directory=self.working_directory
-                ).read_children_from_entry_dicts(
-                    entry_dicts=entry_dicts, child_class=SoundcloudAlbumTrack
-                )
-            )
-
-        return albums
-
-    def _get_album_tracks(
-        self, albums: List[EntryParent]
-    ) -> Generator[SoundcloudAlbumTrack, None, None]:
-        for album in albums:
-            if album.child_count > 0:
-                download_logger.info("Downloading album %s", album.title)
-
-            for track in album.child_entries:
-                if self.download_options.skip_premiere_tracks and track.is_premiere():
-                    continue
-
-                download_logger.info(
-                    "Downloading album track %d/%d %s",
-                    track.track_number,
-                    track.track_count,
-                    track.title,
-                )
-                if not self.is_dry_run:
-                    _ = self.extract_info_with_retry(
-                        is_downloaded_fn=track.is_downloaded,
-                        url=album.webpage_url,
-                        ytdl_options_overrides={
-                            "playlist_items": str(track.kwargs("playlist_index")),
-                            "writeinfojson": False,
-                        },
-                    )
-
-                yield track
-
-    def _get_single_tracks(
-        self, albums: List[EntryParent]
-    ) -> Generator[SoundcloudTrack, None, None]:
-        artist_tracks_url = self.artist_tracks_url(artist_url=self.download_options.url)
-        tracks_entry_dicts = self.extract_info_via_info_json(
-            only_info_json=True,
-            log_prefix_on_info_json_dl="Downloading metadata for",
-            url=artist_tracks_url,
-        )
-
-        # Then, get all singles
-        tracks = self._get_singles(entry_dicts=tracks_entry_dicts, albums=albums)
-        for track in tracks:
-            # Filter any premiere tracks if specified
-            if self.download_options.skip_premiere_tracks and track.is_premiere():
-                continue
-
-            download_logger.info("Downloading single track %s", track.title)
-            if not self.is_dry_run:
-                _ = self.extract_info_with_retry(
-                    is_downloaded_fn=track.is_downloaded,
-                    url=track.webpage_url,
-                    ytdl_options_overrides={"writeinfojson": False},
-                )
-
-            yield track
-
-    def download(self) -> Generator[SoundcloudTrack, None, None]:
+    def download(self) -> Generator[Entry, None, None]:
         """
         Soundcloud subscription to download albums and tracks as singles.
         """
-        albums = self._get_albums()
-        for album_track in self._get_album_tracks(albums=albums):
-            yield album_track
+        downloader = CollectionDownloader(
+            download_options=self.download_options.collection_validator,
+            enhanced_download_archive=self._enhanced_download_archive,
+            ytdl_options_builder=self._ytdl_options_builder,
+            overrides=self.overrides,
+        )
 
-        for single_track in self._get_single_tracks(albums=albums):
-            yield single_track
+        for entry in downloader.download():
+            if self._should_skip(entry):
+                continue
+
+            yield entry

src/ytdl_sub/downloaders/youtube/channel.py

@@ -7,14 +7,14 @@ from typing import Optional
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.downloader import DownloaderOptionsT
 from ytdl_sub.downloaders.downloader import download_logger
+from ytdl_sub.downloaders.generic.collection import CollectionDownloader
+from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry_parent import EntryParent
 from ytdl_sub.entries.youtube import YoutubeVideo
-from ytdl_sub.utils.datetime import to_date_range_hack
 from ytdl_sub.utils.thumbnail import convert_url_thumbnail
-from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@@ -41,8 +41,6 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
 
     _required_keys = {"channel_url"}
     _optional_keys = {
-        "before",
-        "after",
         "channel_avatar_path",
         "channel_banner_path",
     }
@@ -58,8 +56,11 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
         self._channel_banner_path = self._validate_key_if_present(
             "channel_banner_path", OverridesStringFormatterValidator
         )
-        self._before = self._validate_key_if_present("before", StringDatetimeValidator)
-        self._after = self._validate_key_if_present("after", StringDatetimeValidator)
+
+        self.collection_validator = CollectionDownloadOptions(
+            name=self._name,
+            value={"urls": [{"url": self._channel_url}]},
+        )
 
     @property
     def channel_url(self) -> str:
@@ -84,22 +85,6 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
         """
         return self._channel_banner_path
 
-    @property
-    def before(self) -> Optional[StringDatetimeValidator]:
-        """
-        DEPRECATED: use the `date_range` plugin instead. Will be removed in version 0.5.0
-        Optional. Only download videos before this datetime.
-        """
-        return self._before
-
-    @property
-    def after(self) -> Optional[StringDatetimeValidator]:
-        """
-        DEPRECATED: use the `date_range` plugin instead. Will be removed in version 0.5.0
-        Optional. Only download videos after this datetime.
-        """
-        return self._after
-
 
 class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]):
     downloader_options_type = YoutubeChannelDownloaderOptions
@@ -162,31 +147,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
         """
         Downloads all videos from a channel
         """
-        ytdl_options_overrides = {}
-
-        # If a date range is specified when download a YT channel, add it into the ytdl options
-        source_date_range = to_date_range_hack(
-            before=self.download_options.before, after=self.download_options.after
+        downloader = CollectionDownloader(
+            download_options=self.download_options.collection_validator,
+            enhanced_download_archive=self._enhanced_download_archive,
+            ytdl_options_builder=self._ytdl_options_builder,
+            overrides=self.overrides,
         )
-        if source_date_range:
-            ytdl_options_overrides["daterange"] = source_date_range
+        collection_url = self.download_options.collection_validator.collection_urls.list[0]
 
-        # dry-run the entire channel download first, this will get the
-        # videos that will be downloaded. Afterwards, download each video one-by-one
-        entry_dicts = self.extract_info_via_info_json(
-            ytdl_options_overrides=ytdl_options_overrides,
-            only_info_json=True,
-            log_prefix_on_info_json_dl="Downloading metadata for",
-            url=self.download_options.channel_url,
-        )
-
-        self.channel = EntryParent.from_entry_dicts_with_children(
-            entry_dicts=entry_dicts,
-            working_directory=self.working_directory,
-            child_class=YoutubeVideo,
-            extractor="youtube:tab",
-        )
+        parents = downloader.download_url_metadata(collection_url=collection_url)
+        assert len(parents) == 1, "Channel should be the only entry parent"
+        self.channel = parents[0]
 
+        # TODO: Handle this better
         self.overrides.add_override_variables(
             variables_to_add={
                 "source_uploader": self.channel.kwargs_get("uploader", "__failed_to_scrape__"),
@@ -195,27 +168,11 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
             }
         )
 
-        # Iterate in descending order to process older videos first. In case an error occurs and a
-        # the channel must be redownloaded, it will fetch most recent metadata first, and break
-        # on the older video that's been processed and is in the download archive.
-        for idx, video in enumerate(reversed(self.channel.child_entries), start=1):
-            download_logger.info("Downloading %d/%d %s", idx, self.channel.child_count, video.title)
-
-            # Re-download the contents even if it's a dry-run as a single video. At this time,
-            # channels do not download subtitles or subtitle metadata
-            as_single_video_dict = self.extract_info_with_retry(
-                is_downloaded_fn=None if self.is_dry_run else video.is_downloaded,
-                ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
-                url=video.webpage_url,
-            )
-
-            # Workaround for the ytdlp issue
+        for entry in downloader.download_url(collection_url=collection_url, parents=parents):
             # pylint: disable=protected-access
-            video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles")
+            yield YoutubeVideo(entry_dict=entry._kwargs, working_directory=self.working_directory)
             # pylint: enable=protected-access
 
-            yield video
-
     def _download_thumbnail(
         self,
         thumbnail_url: str,

src/ytdl_sub/downloaders/youtube/playlist.py

@@ -2,10 +2,10 @@ from typing import Dict
 from typing import Generator
 from typing import List
 
-from ytdl_sub.downloaders.downloader import download_logger
+from ytdl_sub.downloaders.generic.collection import CollectionDownloader
+from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
-from ytdl_sub.entries.entry_parent import EntryParent
 from ytdl_sub.entries.youtube import YoutubePlaylistVideo
 from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
 
@@ -34,6 +34,11 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
             "playlist_url", YoutubePlaylistUrlValidator
         ).playlist_url
 
+        self.collection_validator = CollectionDownloadOptions(
+            name=self._name,
+            value={"urls": [{"url": self._playlist_url}]},
+        )
+
     @property
     def playlist_url(self) -> str:
         """
@@ -85,22 +90,20 @@ class YoutubePlaylistDownloader(
     def download(self) -> Generator[YoutubePlaylistVideo, None, None]:
         """
         Downloads all videos in a Youtube playlist.
-
-        Dry-run the entire playlist download first. This will get the videos that will be
-        downloaded. Afterwards, download each video one-by-one
         """
-        entry_dicts = self.extract_info_via_info_json(
-            only_info_json=True,
-            log_prefix_on_info_json_dl="Downloading metadata for",
-            url=self.download_options.playlist_url,
+        downloader = CollectionDownloader(
+            download_options=self.download_options.collection_validator,
+            enhanced_download_archive=self._enhanced_download_archive,
+            ytdl_options_builder=self._ytdl_options_builder,
+            overrides=self.overrides,
         )
+        collection_url = self.download_options.collection_validator.collection_urls.list[0]
 
-        playlist: EntryParent = EntryParent.from_entry_dicts_with_children(
-            entry_dicts=entry_dicts,
-            working_directory=self.working_directory,
-            child_class=YoutubePlaylistVideo,
-            extractor="youtube:tab",
-        )
+        parents = downloader.download_url_metadata(collection_url=collection_url)
+        assert len(parents) == 1, "Playlist should be the only entry parent"
+        playlist = parents[0]
+
+        # TODO: Handle this better
         self.overrides.add_override_variables(
             variables_to_add={
                 "source_title": playlist.title,
@@ -109,23 +112,9 @@ class YoutubePlaylistDownloader(
             }
         )
 
-        # Iterate in reverse order to process older videos first. In case an error occurs and a
-        # the playlist must be redownloaded, it will fetch most recent metadata first, and break
-        # on the older video that's been processed and is in the download archive.
-        for idx, video in enumerate(reversed(playlist.child_entries), start=1):
-            download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
-
-            # Re-download the contents even if it's a dry-run as a single video. At this time,
-            # playlists do not download subtitles or subtitle metadata
-            as_single_video_dict = self.extract_info_with_retry(
-                is_downloaded_fn=None if self.is_dry_run else video.is_downloaded,
-                ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
-                url=video.webpage_url,
-            )
-
-            # Workaround for the ytdlp issue
+        for entry in downloader.download_url(collection_url=collection_url, parents=parents):
             # pylint: disable=protected-access
-            video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles")
+            yield YoutubePlaylistVideo(
+                entry_dict=entry._kwargs, working_directory=self.working_directory
+            )
             # pylint: enable=protected-access
-
-            yield video

src/ytdl_sub/entries/base_entry.py

@@ -88,9 +88,6 @@ class BaseEntry(BaseEntryVariables, ABC):
     Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc).
     """
 
-    # The ytdl extractor type that the entry represents
-    entry_extractor: str
-
     def __init__(self, entry_dict: Dict, working_directory: str):
         """
         Initialize the entry using ytdl metadata
@@ -126,7 +123,7 @@ class BaseEntry(BaseEntryVariables, ABC):
         """
         Dict get on kwargs
         """
-        if not self.kwargs_contains(key):
+        if not self.kwargs_contains(key) or self.kwargs(key) is None:
             return default
         return self.kwargs(key)
 
@@ -192,19 +189,3 @@ class BaseEntry(BaseEntryVariables, ABC):
             source_var: getattr(self, source_var) for source_var in self.source_variables()
         }
         return dict(source_variable_dict, **self._added_variables())
-
-    # TODO: super typing
-    @classmethod
-    def from_entry_dicts(
-        cls, entry_dicts: List[Dict], working_directory: str, extractor: Optional[str]
-    ):
-        """
-        Load the entry from a list of dicts. There should only be one entry with the extractor
-        type
-        """
-        extractor = extractor or cls.extractor
-        output = [
-            entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor
-        ]
-        assert len(output) == 1, f"Expected a single entry with extractor of type '{extractor}'"
-        return cls(entry_dict=output[0], working_directory=working_directory)

src/ytdl_sub/entries/entry_parent.py

@@ -6,70 +6,49 @@ from typing import TypeVar
 
 from ytdl_sub.entries.base_entry import BaseEntry
 
-TChildEntry = TypeVar("TChildEntry", bound=BaseEntry)
+TBaseEntry = TypeVar("TBaseEntry", bound=BaseEntry)
 
 
 class EntryParent(BaseEntry):
     def __init__(self, entry_dict: Dict, working_directory: str):
         super().__init__(entry_dict=entry_dict, working_directory=working_directory)
-        self._child_entries: List[TChildEntry] = []
+        self.child_entries: List["EntryParent"] = []
 
-    # pylint: disable=no-self-use
-    def _get_children_entry_variables_to_add(
-        self, child_entries: List[TChildEntry]
-    ) -> Dict[str, str | int]:
+    def is_entry(self) -> bool:
         """
-        Adds source variables to the child entry derived from the parent entry.
-        """
-        if not child_entries:
-            return {}
-
-        return {"playlist_max_upload_year": max(entry.upload_year for entry in child_entries)}
-
-    # pylint: enable=no-self-use
-
-    def read_children_from_entry_dicts(
-        self, entry_dicts: List[Dict], child_class: Type[TChildEntry]
-    ) -> "EntryParent":
-        """
-        Parameters
-        ----------
-        entry_dicts
-            Entry dicts to look for children from
-        child_class
-            The class to convert the entry dict to
-
         Returns
         -------
-        List of children
+        True if the entry contains a media file. False otherwise.
         """
-        child_entries: List[TChildEntry] = []
+        return self.kwargs_contains("ext")
+
+    def parent_children(self) -> List["EntryParent"]:
+        """This parent's children that are also parents"""
+        return [child for child in self.child_entries if child.child_count() > 0]
+
+    def entry_children(self) -> List["EntryParent"]:
+        """This parent's children that are entries"""
+        return [child for child in self.child_entries if child.is_entry()]
+
+    def read_children_from_entry_dicts(self, entry_dicts: List[Dict]) -> "EntryParent":
+        """
+        Populates a tree of EntryParents that belong to this instance
+        """
+        child_entries: List["EntryParent"] = []
 
         for entry_dict in entry_dicts:
             if entry_dict.get("playlist_id") == self.uid:
                 child_entries.append(
-                    child_class(entry_dict=entry_dict, working_directory=self.working_directory())
+                    self.__class__(
+                        entry_dict=entry_dict,
+                        working_directory=self.working_directory(),
+                    )
                 )
+                child_entries[-1].read_children_from_entry_dicts(entry_dicts)
 
-        child_variables_to_add = self._get_children_entry_variables_to_add(child_entries)
-        for child_entry in child_entries:
-            child_entry.add_variables(variables_to_add=child_variables_to_add)
-
-        self._child_entries = sorted(
-            child_entries, key=lambda entry: entry.kwargs("playlist_index")
-        )
+        self.child_entries = sorted(child_entries, key=lambda entry: entry.kwargs("playlist_index"))
         return self
 
-    @property
-    def child_entries(self) -> List[TChildEntry]:
-        """
-        Returns
-        -------
-        List of child entities
-        """
-        return self._child_entries
-
-    @property
     def child_count(self) -> int:
         """
         Returns
@@ -96,37 +75,25 @@ class EntryParent(BaseEntry):
                 return thumbnail["url"]
         return None
 
-    def __contains__(self, item):
+    @classmethod
+    def from_entry_dicts(
+        cls, entry_dicts: List[Dict], working_directory: str
+    ) -> List["EntryParent"]:
+        """
+        Reads all entry dicts and builds a tree of EntryParents
+        """
+        return [
+            EntryParent(
+                entry_dict=entry_dict, working_directory=working_directory
+            ).read_children_from_entry_dicts(entry_dicts)
+            for entry_dict in entry_dicts
+            if "playlist_id" not in entry_dict
+        ]
+
+    def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry:
         """
         Returns
         -------
-        True if the the item (entry_dict) has the same id as one of the tracks. False otherwise.
+        Converted EntryParent to Entry-like class
         """
-        uid: Optional[str] = None
-        if isinstance(item, BaseEntry):
-            uid = item.uid
-        elif isinstance(item, dict):
-            uid = item.get("id")
-
-        if uid is not None:
-            return any(uid == child_entry.uid for child_entry in self._child_entries)
-        return False
-
-    @classmethod
-    def from_entry_dicts_with_children(
-        cls,
-        entry_dicts: List[Dict],
-        working_directory: str,
-        child_class: Type[TChildEntry],
-        extractor: Optional[str],
-    ) -> "EntryParent":
-        """
-        Load the parent entry and its children
-        """
-        entry_parent: EntryParent = cls.from_entry_dicts(
-            entry_dicts=entry_dicts, working_directory=working_directory, extractor=extractor
-        )
-        entry_parent.read_children_from_entry_dicts(
-            entry_dicts=entry_dicts, child_class=child_class
-        )
-        return entry_parent
+        return entry_type(entry_dict=self._kwargs, working_directory=self._working_directory)

src/ytdl_sub/entries/soundcloud.py

@@ -1,45 +1,11 @@
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.variables.soundcloud_variables import SoundcloudVariables
 
+# TODO: Delete since not used
+
 
 class SoundcloudTrack(SoundcloudVariables, Entry):
     """
     Entry object to represent a Soundcloud track yt-dlp that is a single, which implies
     it does not belong to an album.
     """
-
-    entry_extractor = "soundcloud"
-
-    def is_premiere(self) -> bool:
-        """
-        Returns whether the entry is a premier track. Check this by seeing if the track's url is
-        a preview.
-        """
-        return "/preview/" in self.kwargs("url")
-
-
-class SoundcloudAlbumTrack(SoundcloudTrack):
-    """
-    Entry object to represent a Soundcloud track yt-dlp that belongs to an album.
-    """
-
-    @property
-    def track_number(self) -> int:
-        """Returns the entry's track number"""
-        return self.kwargs("playlist_index")
-
-    @property
-    def track_count(self) -> int:
-        """Returns the entry's total tracks in album"""
-        return self.kwargs("playlist_count")
-
-    @property
-    def album(self) -> str:
-        """Returns the entry's album name, fetched from its internal album"""
-        return self.kwargs("playlist")
-
-    @property
-    def album_year(self) -> int:
-        """Returns the entry's album year, fetched from its internal album"""
-        # added from parent entry
-        return self._additional_variables["playlist_max_upload_year"]

src/ytdl_sub/entries/variables/entry_variables.py

@@ -1,3 +1,5 @@
+from yt_dlp.utils import sanitize_filename
+
 from ytdl_sub.entries.base_entry import BaseEntry
 from ytdl_sub.entries.base_entry import BaseEntryVariables
 
@@ -14,6 +16,72 @@ _days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
 
 
 class EntryVariables(BaseEntryVariables):
+    @property
+    def playlist(self: BaseEntry) -> str:
+        """
+        Returns
+        -------
+        str
+            Name of its parent playlist/channel if it exists, otherwise returns its title.
+        """
+        return self.kwargs_get("playlist", self.title)
+
+    @property
+    def playlist_sanitized(self) -> str:
+        """
+        Returns
+        -------
+        str
+            The playlist name, sanitized
+        """
+        return sanitize_filename(self.playlist)
+
+    @property
+    def playlist_index(self: BaseEntry) -> int:
+        """
+        Returns
+        -------
+        int
+            Playlist index if it exists, otherwise returns ``1``.
+
+            Note that for channels/playlists, an index of 1 implies it's the most recent
+            uploaded entry. It is recommended to not use this unless you know the channel/playlist
+            will never add new content, i.e. a music album.
+        """
+        return self.kwargs_get("playlist_index", 1)
+
+    @property
+    def playlist_index_padded(self) -> str:
+        """
+        Returns
+        -------
+        str
+            playlist_index padded two digits
+        """
+        return _pad(self.playlist_index, width=2)
+
+    @property
+    def playlist_count(self: BaseEntry) -> int:
+        """
+        Returns
+        -------
+        int
+            Playlist count if it exists, otherwise returns ``1``.
+        """
+        return self.kwargs_get("playlist_count", 1)
+
+    @property
+    def playlist_max_upload_year(self) -> int:
+        """
+        Returns
+        -------
+        int
+            Max upload_year for all entries in this entry's playlist if it exists, otherwise returns
+            ``upload_year``
+        """
+        # override in EntryParent
+        return self.upload_year
+
     @property
     def ext(self: BaseEntry) -> str:
         """

src/ytdl_sub/entries/youtube.py

@@ -10,8 +10,6 @@ class YoutubeVideo(YoutubeVideoVariables, Entry):
     Entry object to represent a Youtube video. Reserved for shared Youtube entry logic.
     """
 
-    entry_extractor = "youtube"
-
     @property
     def ext(self) -> str:
         """

src/ytdl_sub/plugins/music_tags.py

@@ -7,7 +7,6 @@ from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
-from ytdl_sub.validators.validators import StringValidator
 
 
 class MusicTagsOptions(PluginOptions):
@@ -34,17 +33,12 @@ class MusicTagsOptions(PluginOptions):
     """
 
     _required_keys = {"tags"}
-    _optional_keys = {"multi_value_separator"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
 
         self._tags = self._validate_key(key="tags", validator=DictFormatterValidator)
 
-        self.multi_value_separator = self._validate_key_if_present(
-            key="multi_value_separator", validator=StringValidator
-        )
-
     @property
     def tags(self) -> DictFormatterValidator:
         """

src/ytdl_sub/plugins/plugin.py

@@ -8,6 +8,7 @@ from typing import Type
 from typing import TypeVar
 from typing import final
 
+from ytdl_sub.config.preset_options import AddsVariablesMixin
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.file_handler import FileMetadata
@@ -39,40 +40,11 @@ class PluginPriority:
         return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
 
 
-class PluginOptions(StrictDictValidator):
+class PluginOptions(StrictDictValidator, AddsVariablesMixin, ABC):
     """
     Class that defines the parameters to a plugin
     """
 
-    # pylint: disable=no-self-use
-    def added_source_variables(self) -> List[str]:
-        """
-        If the plugin adds source variables, list them here.
-
-        Returns
-        -------
-        List of added source variables this plugin creates
-        """
-        return []
-
-    # pylint: disable=unused-argument
-    def validate_with_variables(
-        self, source_variables: List[str], override_variables: List[str]
-    ) -> None:
-        """
-        Optional validation after init with the session's source and override variables.
-
-        Parameters
-        ----------
-        source_variables
-            Available source variables when running the plugin
-        override_variables
-            Available override variables when running the plugin
-        """
-        return None
-
-    # pylint: enable=no-self-use,unused-argument
-
 
 PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
 

src/ytdl_sub/plugins/regex.py

@@ -165,7 +165,7 @@ class RegexOptions(PluginOptions):
         return self._skip_if_match_fails
 
     def validate_with_variables(
-        self, source_variables: List[str], override_variables: List[str]
+        self, source_variables: List[str], override_variables: Dict[str, str]
     ) -> None:
         """
         Ensures each source variable capture group is valid

src/ytdl_sub/plugins/subtitles.py

@@ -1,4 +1,3 @@
-from pathlib import Path
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -21,14 +20,6 @@ SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"}
 logger = Logger.get(name="subtitles")
 
 
-def _is_entry_subtitle_file(path: Path, entry: Entry) -> bool:
-    if path.is_file() and path.name.startswith(entry.uid):
-        for ext in SUBTITLE_EXTENSIONS:
-            if path.name.endswith(f".{ext}"):
-                return True
-    return False
-
-
 class SubtitlesTypeValidator(StringSelectValidator):
     _expected_value_type_name = "subtitles type"
     _select_values = SUBTITLE_EXTENSIONS

src/ytdl_sub/thread/log_entries_downloaded_listener.py

@@ -13,7 +13,7 @@ logger = Logger.get(name="downloader")
 
 
 class LogEntriesDownloadedListener(threading.Thread):
-    def __init__(self, working_directory: str, info_json_extractor: str, log_prefix: str):
+    def __init__(self, working_directory: str, log_prefix: str):
         """
         To be ran in a thread while download via ytdl-sub. Listens for new .info.json files in the
         working directory, checks the extractor value, and if it matches the input arg, log the
@@ -23,20 +23,18 @@ class LogEntriesDownloadedListener(threading.Thread):
         ----------
         working_directory
             subscription download working directory
-        info_json_extractor
-            print the titles of the info.json file with this extractor
         log_prefix
             The message to print prefixed to the title, i.e. '{log_prefix} {title}'
         """
         threading.Thread.__init__(self)
         self.working_directory = working_directory
-        self.info_json_extractor = info_json_extractor
         self.log_prefix = log_prefix
         self.complete = False
 
         self._files_read: Set[str] = set()
 
-    def _get_title_from_info_json(self, path: Path) -> Optional[str]:
+    @classmethod
+    def _get_title_from_info_json(cls, path: Path) -> Optional[str]:
         try:
             with open(path, "r", encoding="utf-8") as file:
                 file_json = json.load(file)
@@ -44,10 +42,7 @@ class LogEntriesDownloadedListener(threading.Thread):
             # swallow the error since this is only printing logs
             return None
 
-        if file_json.get("extractor") == self.info_json_extractor:
-            return file_json.get("title")
-
-        return None
+        return file_json.get("title")
 
     @classmethod
     def _is_info_json(cls, path: Path) -> bool:

src/ytdl_sub/utils/datetime.py

@@ -3,42 +3,9 @@ from typing import Optional
 from yt_dlp import DateRange
 
 from ytdl_sub.config.preset_options import Overrides
-from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 
 
-def to_date_range_hack(
-    before: Optional[StringDatetimeValidator], after: Optional[StringDatetimeValidator]
-) -> Optional[DateRange]:
-    """
-    Workaround for channel before/after support.
-
-    Returns
-    -------
-    Date range if the 'before' or 'after' is defined. None otherwise.
-    """
-    start: Optional[str] = None
-    end: Optional[str] = None
-
-    if after:
-        start = after.apply_formatter(variable_dict={})
-
-    if before:
-        end = before.apply_formatter(variable_dict={})
-
-    if start or end:
-        logger = Logger.get(name="youtube-channel")
-        logger.warning(
-            "DEPRECATED: youtube.before/after will are deprecated and will be removed in v0.0.5. "
-            "Use the 'date_range' plugin instead: "
-            "https://ytdl-sub.readthedocs.io/en/latest/config.html#date-range"
-        )
-
-        return DateRange(start=start, end=end)
-
-    return None
-
-
 def to_date_range(
     before: Optional[StringDatetimeValidator],
     after: Optional[StringDatetimeValidator],

src/ytdl_sub/validators/validators.py

@@ -244,24 +244,3 @@ class LiteralDictValidator(DictValidator):
     def keys(self) -> List[str]:
         """Returns a sorted list of the dict's keys"""
         return super()._keys
-
-
-class UniformDictValidator(DictValidator, Generic[ValidatorT], ABC):
-    """DictValidator where all values have the same validator"""
-
-    uniform_validator_class: Type[ValidatorT]
-
-    def __init__(self, name, value):
-        super().__init__(name, value)
-
-        for key in self._keys:
-            _ = self._validate_key_if_present(key=key, validator=self.uniform_validator_class)
-
-    @property
-    def validator_dict(self) -> Dict[str, ValidatorT]:
-        """
-        Returns
-        -------
-        Dict of name: validator
-        """
-        return self._validator_dict

src/ytdl_sub/ytdl_additions/enhanced_download_archive.py

@@ -123,19 +123,6 @@ class DownloadArchive:
                 file.write(f"{line}\n")
         return self
 
-    def contains(self, entry_id: str) -> bool:
-        """
-        Parameters
-        ----------
-        entry_id
-            Id of the entry
-
-        Returns
-        -------
-        True if the entry id is within this download archive. False otherwise.
-        """
-        return any(entry_id in line for line in self._download_archive_lines)
-
     def remove_entry(self, entry_id: str) -> "DownloadArchive":
         """
         Parameters
@@ -598,7 +585,7 @@ class DownloadArchiver:
     """
 
     def __init__(self, enhanced_download_archive: EnhancedDownloadArchive):
-        self.__enhanced_download_archive = enhanced_download_archive
+        self._enhanced_download_archive = enhanced_download_archive
 
     @property
     def working_directory(self) -> str:
@@ -607,7 +594,7 @@ class DownloadArchiver:
         -------
         Path to the working directory
         """
-        return self.__enhanced_download_archive.working_directory
+        return self._enhanced_download_archive.working_directory
 
     @property
     def is_dry_run(self) -> bool:
@@ -616,7 +603,7 @@ class DownloadArchiver:
         -------
         True if this session is a dry-run. False otherwise.
         """
-        return self.__enhanced_download_archive.is_dry_run
+        return self._enhanced_download_archive.is_dry_run
 
     def save_file(
         self,
@@ -640,7 +627,7 @@ class DownloadArchiver:
         entry
             Optional. Entry that the file belongs to
         """
-        self.__enhanced_download_archive.save_file_to_output_directory(
+        self._enhanced_download_archive.save_file_to_output_directory(
             file_name=file_name,
             file_metadata=file_metadata,
             output_file_name=output_file_name,

tests/unit/entries/conftest.py

@@ -85,6 +85,12 @@ def mock_entry_to_dict(
         "thumbnail_ext": thumbnail_ext,
         "info_json_ext": "info.json",
         "webpage_url": webpage_url,
+        "playlist": "entry ｛title｝",
+        "playlist_sanitized": "entry ｛title｝",
+        "playlist_index": 1,
+        "playlist_index_padded": "01",
+        "playlist_count": 1,
+        "playlist_max_upload_year": 2021,
     }