actually working

src/ytdl_sub/config/preset.py

@@ -83,6 +83,10 @@ class Preset(StrictDictValidator):
     # and ensure required keys are present.
     _optional_keys = PRESET_KEYS
 
+    @property
+    def _source_variables(self) -> List[str]:
+        return self.downloader.downloader_entry_type.source_variables()
+
     def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
         return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
             downloader_source=downloader_source
@@ -141,6 +145,7 @@ class Preset(StrictDictValidator):
 
             plugin = PluginMapping.get(plugin=key)
             plugin_options = self._validate_key(key=key, validator=plugin.plugin_options_type)
+            plugin_options.validate_with_source_variables(source_variables=self._source_variables)
 
             plugins.append((plugin, plugin_options))
 
@@ -156,12 +161,20 @@ class Preset(StrictDictValidator):
         # If the formatter supports source variables, set the formatter variables to include
         # both source and override variables
         if not isinstance(formatter_validator, OverridesStringFormatterValidator):
-            source_variables = {
+            source_variables = {source_var: "dummy_string" for source_var in self._source_variables}
-                source_var: "dummy_string"
-                for source_var in self.downloader.downloader_entry_type.source_variables()
-            }
             variable_dict = dict(source_variables, **variable_dict)
 
+        # For all plugins, add in any extra added source variables
+        for _, plugin_options in self.plugins:
+            added_plugin_variables = {
+                source_var: "dummy_string" for source_var in plugin_options.added_source_variables()
+            }
+            # sanity check plugin variables do not override source variables
+            expected_len = len(variable_dict) + len(added_plugin_variables)
+            variable_dict = dict(variable_dict, **added_plugin_variables)
+
+            assert len(variable_dict) == expected_len, "plugin variables overwrote source variables"
+
         _ = formatter_validator.apply_formatter(variable_dict=variable_dict)
 
     def __recursive_preset_validate(

src/ytdl_sub/config/preset_class_mappings.py

@@ -13,6 +13,7 @@ from ytdl_sub.plugins.music_tags import MusicTagsPlugin
 from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
 from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
 from ytdl_sub.plugins.plugin import Plugin
+from ytdl_sub.plugins.regex_capture import RegexCapturePlugin
 
 
 class DownloadStrategyMapping:
@@ -107,6 +108,7 @@ class PluginMapping:
         "music_tags": MusicTagsPlugin,
         "nfo_tags": NfoTagsPlugin,
         "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
+        "regex_capture": RegexCapturePlugin,
     }
 
     @classmethod

src/ytdl_sub/entries/base_entry.py

@@ -22,6 +22,8 @@ class BaseEntry(ABC):
         self._working_directory = working_directory
         self._kwargs = entry_dict
 
+        self._additional_variables: Dict[str, str] = {}
+
     def kwargs_contains(self, key: str) -> bool:
         """Returns whether internal kwargs contains the specified key"""
         return key in self._kwargs
@@ -39,3 +41,28 @@ class BaseEntry(ABC):
         The working directory
         """
         return self._working_directory
+
+    def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry":
+        """
+        Parameters
+        ----------
+        variables_to_add
+            Variables to add to this entry
+
+        Returns
+        -------
+        self
+
+        Raises
+        ------
+        ValueError
+            If a variable trying to be added already exists as a source variable
+        """
+        for variable_name in variables_to_add.keys():
+            if self.kwargs_contains(variable_name):
+                raise ValueError(
+                    f"Cannot add variable '{variable_name}': already exists in the kwargs"
+                )
+
+        self._additional_variables = dict(self._additional_variables, **variables_to_add)
+        return self

src/ytdl_sub/entries/variables/entry_variables.py

@@ -37,6 +37,14 @@ class SourceVariables:
         """
         return self.kwargs("extractor")
 
+    def _added_variables(self: BaseEntry) -> Dict[str, str]:
+        """
+        Returns
+        -------
+        Dict of variables added to this entry
+        """
+        return self._additional_variables
+
     @classmethod
     def source_variables(cls) -> List[str]:
         """
@@ -54,7 +62,10 @@ class SourceVariables:
         -------
         Dictionary containing all variables
         """
-        return {source_var: getattr(self, source_var) for source_var in self.source_variables()}
+        source_variable_dict = {
+            source_var: getattr(self, source_var) for source_var in self.source_variables()
+        }
+        return dict(source_variable_dict, **self._added_variables())
 
 
 class EntryVariables(SourceVariables):

src/ytdl_sub/plugins/plugin.py

@@ -1,5 +1,6 @@
 from abc import ABC
 from typing import Generic
+from typing import List
 from typing import Optional
 from typing import Type
 from typing import TypeVar
@@ -19,6 +20,32 @@ class PluginOptions(StrictDictValidator):
     Class that defines the parameters to a plugin
     """
 
+    # pylint: disable=no-self-use
+    def added_source_variables(self) -> List[str]:
+        """
+        If the plugin adds source variables, list them here.
+
+        Returns
+        -------
+        List of added source variables this plugin creates
+        """
+        return []
+
+    # pylint: disable=unused-argument
+    def validate_with_source_variables(self, source_variables: List[str]) -> None:
+        """
+        Performs validation after init using the source variables, in case the plugin
+        depends on specific source variables.
+
+        Parameters
+        ----------
+        source_variables
+            Source variables to be used when running the plugin
+        """
+        return None
+
+    # pylint: enable=no-self-use,unused-argument
+
 
 PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
 
@@ -43,6 +70,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
         # TODO pass yaml snake case name in the class somewhere, and use it for the logger
         self._logger = Logger.get(self.__class__.__name__)
 
+    # pylint: disable=no-self-use
     def modify_entry(self, entry: Entry) -> Optional[Entry]:
         """
         For each entry downloaded, modify the entry in some way before sending it to
@@ -57,6 +85,9 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC):
         -------
         The entry or None, indicating not to move it to the output directory
         """
+        return entry
+
+    # pylint: enable=no-self-use
 
     def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
         """

src/ytdl_sub/plugins/regex_capture.py

@@ -0,0 +1,177 @@
+from typing import Dict
+from typing import List
+from typing import Optional
+
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.plugins.plugin import Plugin
+from ytdl_sub.plugins.plugin import PluginOptions
+from ytdl_sub.utils.exceptions import ValidationException
+from ytdl_sub.validators.regex_validator import RegexListValidator
+from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
+from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
+
+
+def _source_var_name(source_variable: str, capture_group_idx: int) -> str:
+    return f"{source_variable}_capture_{capture_group_idx+1}"
+
+
+class SourceVariableRegexCapture(StrictDictValidator):
+
+    _required_keys = {"capture"}
+    _optional_keys = {"defaults"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._capture = self._validate_key(key="capture", validator=RegexListValidator)
+        self._defaults = self._validate_key_if_present(
+            key="defaults", validator=ListFormatterValidator
+        )
+
+        # If defaults are to be used, ensure there are the same number of defaults as there are
+        # capture groups
+        if self._defaults is not None and self._capture.num_capture_groups != len(
+            self._defaults.list
+        ):
+            raise self._validation_exception(
+                f"number of defaults must match number of capture groups, "
+                f"{len(self._defaults.list)} != {self._capture.num_capture_groups}"
+            )
+
+    @property
+    def capture_list(self) -> RegexListValidator:
+        """
+        Returns
+        -------
+        List of regex captures
+        """
+        return self._capture
+
+    @property
+    def has_defaults(self) -> bool:
+        """
+        Returns
+        -------
+        True if a validation exception should be raised if not captured. False otherwise.
+        """
+        return self._defaults is not None
+
+    @property
+    def defaults(self) -> Optional[List[StringFormatterValidator]]:
+        """
+        Returns
+        -------
+        List of string format validators to use for the defaults
+        """
+        return self._defaults.list if self.has_defaults else None
+
+
+class RegexCaptureOptions(PluginOptions):
+
+    _optional_keys = {"_"}
+    _allow_extra_keys = True
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+        self._source_variable_capture_dict: Dict[str, SourceVariableRegexCapture] = {}
+
+    def validate_with_source_variables(self, source_variables: List[str]) -> None:
+        """
+        Ensures each source variable capture group is valid
+
+        Parameters
+        ----------
+        source_variables
+            Variables to check against the provided capture groups
+        """
+        for key in self._keys:
+            if key not in source_variables:
+                raise self._validation_exception(
+                    f"cannot regex capture '{key}' because it is not a source variable"
+                )
+
+            self._source_variable_capture_dict[key] = self._validate_key(
+                key=key, validator=SourceVariableRegexCapture
+            )
+
+    @property
+    def source_variable_capture_dict(self) -> Dict[str, SourceVariableRegexCapture]:
+        """
+        Returns
+        -------
+        Dict of { source variable: capture options }
+        """
+        return self._source_variable_capture_dict
+
+    def added_source_variables(self) -> List[str]:
+        """
+        Returns
+        -------
+        List of new source variables created via regex capture
+        """
+        added_source_vars: List[str] = []
+        for source_var, regex_options in self.source_variable_capture_dict.items():
+            added_source_vars.extend(
+                _source_var_name(source_var, idx)
+                for idx in range(regex_options.capture_list.num_capture_groups)
+            )
+
+        return added_source_vars
+
+
+class RegexCapturePlugin(Plugin[RegexCaptureOptions]):
+    plugin_options_type = RegexCaptureOptions
+
+    def modify_entry(self, entry: Entry) -> Optional[Entry]:
+        """
+        Parameters
+        ----------
+        entry
+            Entry to add source variables to
+
+        Returns
+        -------
+        Entry with regex capture variables added to its source variables
+
+        Raises
+        ------
+        ValidationException
+            If no capture and no defaults
+        """
+        entry_variable_dict = entry.to_dict()
+
+        # Iterate each source var to capture and add to the entry
+        for source_var, regex_options in self.plugin_options.source_variable_capture_dict.items():
+            maybe_capture = regex_options.capture_list.capture_any(
+                input_str=entry_variable_dict[source_var]
+            )
+
+            # If no capture
+            if maybe_capture is None:
+                # and no defaults, then error
+                if not regex_options.has_defaults:
+                    raise ValidationException(
+                        f"Failed to capture {source_var} from an entry with the value:\n"
+                        f"{entry_variable_dict[source_var]}"
+                    )
+
+                # otherwise, use defaults (apply them using the original entry source dict)
+                entry.add_variables(
+                    variables_to_add={
+                        _source_var_name(source_var, i): default.apply_formatter(
+                            variable_dict=entry_variable_dict
+                        )
+                        for i, default in enumerate(regex_options.defaults)
+                    },
+                )
+            # There is a capture, add the source variables to the entry as
+            # {source_var}_capture_1, {source_var}_capture_2, ...
+            else:
+                entry.add_variables(
+                    variables_to_add={
+                        _source_var_name(source_var, i): capture
+                        for i, capture in enumerate(maybe_capture)
+                    },
+                )
+
+        return entry

src/ytdl_sub/subscriptions/subscription.py

@@ -271,6 +271,11 @@ class Subscription:
                 if isinstance(entry, tuple):
                     entry, entry_metadata = entry
 
+                # First, modify the entry with all plugins
+                for plugin in plugins:
+                    entry = plugin.modify_entry(entry)
+
+                # Then, post-process the entry with all plugins
                 for plugin in plugins:
                     optional_plugin_entry_metadata = plugin.post_process_entry(entry)
                     if optional_plugin_entry_metadata:

src/ytdl_sub/validators/regex_validator.py

@@ -70,6 +70,17 @@ class RegexListValidator(ListValidator[RegexValidator]):
                 "each regex in a list must have the same number of capture groups"
             )
 
+        self._num_capture_groups = self._list[0].num_capture_groups
+
+    @property
+    def num_capture_groups(self) -> int:
+        """
+        Returns
+        -------
+        Number of capture groups. All regexes in the list will have the same number.
+        """
+        return self._num_capture_groups
+
     def matches_any(self, input_str: str) -> bool:
         """
         Parameters

src/ytdl_sub/validators/string_formatter_validators.py

@@ -7,6 +7,7 @@ from typing import final
 
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
+from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.validators.validators import LiteralDictValidator
 from ytdl_sub.validators.validators import Validator
 
@@ -43,7 +44,7 @@ class StringFormatterValidator(Validator):
         "Format variable '{variable_name}' does not exist. Available variables: {available_fields}"
     )
 
-    __fields_validator = re.compile(r"{([a-z_]+?)}")
+    __fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}")
 
     __max_format_recursion = 3
 
@@ -75,8 +76,8 @@ class StringFormatterValidator(Validator):
 
         if len(format_variables) != open_bracket_count:
             raise self._validation_exception(
-                "{variable_names} should only contain "
+                "{variable_names} must start with a lowercase letter, should only contain lowercase"
-                "lowercase letters and underscores with a single open and close bracket.",
+                "letters, numbers, underscores, and have a single open and close bracket.",
                 exception_class=StringFormattingException,
             )
 
@@ -177,6 +178,10 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
 # pylint: enable=line-too-long
 
 
+class ListFormatterValidator(ListValidator[StringFormatterValidator]):
+    _inner_list_type = StringFormatterValidator
+
+
 class DictFormatterValidator(LiteralDictValidator):
     """
     A dict made up of

src/ytdl_sub/validators/validators.py

@@ -112,6 +112,15 @@ class ListValidator(Validator, ABC, Generic[T]):
             for i, val in enumerate(self._value)
         ]
 
+    @property
+    def list(self) -> List[T]:
+        """
+        Returns
+        -------
+        The list
+        """
+        return self._list
+
 
 class DictValidator(Validator):
     """

tests/e2e/conftest.py

@@ -2,8 +2,20 @@ import tempfile
 
 import pytest
 
+from ytdl_sub.config.config_file import ConfigFile
+
 
 @pytest.fixture()
 def output_directory():
     with tempfile.TemporaryDirectory() as temp_dir:
         yield temp_dir
+
+
+@pytest.fixture()
+def music_video_config():
+    return ConfigFile.from_file_path(config_path="examples/kodi_music_videos_config.yaml")
+
+
+@pytest.fixture()
+def channel_as_tv_show_config():
+    return ConfigFile.from_file_path(config_path="examples/kodi_tv_shows_config.yaml")

tests/e2e/plugins/test_regex_capture.py

@@ -0,0 +1,71 @@
+import pytest
+from e2e.expected_transaction_log import assert_transaction_log_matches
+
+from ytdl_sub.config.preset import Preset
+from ytdl_sub.subscriptions.subscription import Subscription
+
+
+@pytest.fixture
+def regex_capture_subscription_dict(output_directory):
+    return {
+        "preset": "yt_music_video_playlist",
+        "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory},
+        # download the worst format so it is fast
+        "ytdl_options": {
+            "format": "best[height<=480]",
+            "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
+        },
+        "regex_capture": {
+            "title": {
+                "capture": [
+                    "should not cap (.+) - (.+)",
+                    ".*\\[(.+) - (.+)]",
+                ],
+            },
+            "description": {"capture": [".*http:\\/\\/(.+).com.*"]},
+            "upload_date_standardized": {
+                "capture": ["([0-9]+)-([0-9]+)-27"],
+                "defaults": [
+                    "First",
+                    "Second",
+                ],
+            },
+            "artist": {"capture": ["Never (.*) capture"], "defaults": ["Always default"]},
+        },
+        "nfo_tags": {
+            "tags": {
+                "title_cap_1": "{title_capture_1}",
+                "title_cap_2": "{title_capture_2}",
+                "desc_cap": "{description_capture_1}",
+                "upload_date_both_caps": "{upload_date_standardized_capture_1} and {upload_date_standardized_capture_2}",
+                "artist_cap_always_default": "{artist_capture_1}",
+            }
+        },
+    }
+
+
+@pytest.fixture
+def playlist_subscription(music_video_config, regex_capture_subscription_dict):
+    playlist_preset = Preset.from_dict(
+        config=music_video_config,
+        preset_name="regex_capture_playlist_test",
+        preset_dict=regex_capture_subscription_dict,
+    )
+
+    return Subscription.from_preset(
+        preset=playlist_preset,
+        config=music_video_config,
+    )
+
+
+class TestRegexCapture:
+    def test_regex_capture_success(self, playlist_subscription, output_directory):
+        # Only dry run is needed to see if capture variables are created
+        transaction_log = playlist_subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/test_regex_mapping.txt",
+        )

tests/e2e/resources/transaction_log_summaries/plugins/test_regex_mapping.txt

@@ -0,0 +1,45 @@
+Files created in '{output_directory}'
+----------------------------------------
+.ytdl-sub-regex_capture_playlist_test-download-archive.json
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].mp4
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.1].nfo
+  NFO tags:
+    musicvideo:
+      album: Music Videos
+      artist: Project Zombie
+      artist_cap_always_default: Always default
+      desc_cap: www.jesseminecraft.webs
+      title: Jesse's Minecraft Server [Trailer - Feb.1]
+      title_cap_1: Trailer
+      title_cap_2: Feb.1
+      upload_date_both_caps: First and Second
+      year: 2011
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].mp4
+Project Zombie - Jesse's Minecraft Server [Trailer - Feb.27].nfo
+  NFO tags:
+    musicvideo:
+      album: Music Videos
+      artist: Project Zombie
+      artist_cap_always_default: Always default
+      desc_cap: jesseminecraft.webs
+      title: Jesse's Minecraft Server [Trailer - Feb.27]
+      title_cap_1: Trailer
+      title_cap_2: Feb.27
+      upload_date_both_caps: 2011 and 02
+      year: 2011
+Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
+Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].mp4
+Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
+  NFO tags:
+    musicvideo:
+      album: Music Videos
+      artist: Project Zombie
+      artist_cap_always_default: Always default
+      desc_cap: jesseminecraft.webs
+      title: Jesse's Minecraft Server [Trailer - Mar.21]
+      title_cap_1: Trailer
+      title_cap_2: Mar.21
+      upload_date_both_caps: First and Second
+      year: 2011

tests/e2e/youtube/test_playlist_as_kodi_music_videos.py

@@ -8,28 +8,12 @@ from e2e.expected_download import ExpectedDownloads
 from e2e.expected_transaction_log import assert_transaction_log_matches
 
 import ytdl_sub.downloaders.downloader
-from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
 @pytest.fixture
-def config_path():
+def subscription_dict(output_directory):
-    return "examples/kodi_music_videos_config.yaml"
-
-
-@pytest.fixture
-def subscription_name():
-    return "jmc"
-
-
-@pytest.fixture
-def config(config_path):
-    return ConfigFile.from_file_path(config_path=config_path)
-
-
-@pytest.fixture
-def subscription_dict(output_directory, subscription_name):
     return {
         "preset": "yt_music_video_playlist",
         "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
@@ -48,16 +32,16 @@ def subscription_dict(output_directory, subscription_name):
 
 
 @pytest.fixture
-def playlist_subscription(config, subscription_name, subscription_dict):
+def playlist_subscription(music_video_config, subscription_dict):
     playlist_preset = Preset.from_dict(
-        config=config,
+        config=music_video_config,
-        preset_name=subscription_name,
+        preset_name="music_video_playlist_test",
         preset_dict=subscription_dict,
     )
 
     return Subscription.from_preset(
         preset=playlist_preset,
-        config=config,
+        config=music_video_config,
     )
 
 
@@ -103,10 +87,10 @@ def single_video_subscription_dict(subscription_dict):
 
 
 @pytest.fixture
-def single_video_subscription(config, subscription_name, single_video_subscription_dict):
+def single_video_subscription(config, single_video_subscription_dict):
     single_video_preset = Preset.from_dict(
         config=config,
-        preset_name=subscription_name,
+        preset_name="music_video_single_video_test",
         preset_dict=single_video_subscription_dict,
     )
 

tests/e2e/youtube/test_split_video.py

@@ -1,24 +1,15 @@
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
-import mergedeep
 import pytest
-from conftest import assert_debug_log
 from e2e.expected_download import ExpectedDownloadFile
 from e2e.expected_download import ExpectedDownloads
 from e2e.expected_transaction_log import assert_transaction_log_matches
 
-import ytdl_sub.downloaders.downloader
-from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.preset import Preset
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
-@pytest.fixture
-def config_path():
-    return "examples/kodi_music_videos_config.yaml"
-
-
 @pytest.fixture
 def split_timestamps_file_path():
     timestamps = [
@@ -37,17 +28,7 @@ def split_timestamps_file_path():
 
 
 @pytest.fixture
-def subscription_name():
+def subscription_dict(output_directory, split_timestamps_file_path):
-    return "jmc"
-
-
-@pytest.fixture
-def config(config_path):
-    return ConfigFile.from_file_path(config_path=config_path)
-
-
-@pytest.fixture
-def subscription_dict(output_directory, subscription_name, split_timestamps_file_path):
     return {
         "preset": "yt_music_video",
         "youtube": {
@@ -73,16 +54,16 @@ def subscription_dict(output_directory, subscription_name, split_timestamps_file
 
 
 @pytest.fixture
-def single_video_subscription(config, subscription_name, subscription_dict):
+def single_video_subscription(music_video_config, subscription_dict):
     single_video_preset = Preset.from_dict(
-        config=config,
+        config=music_video_config,
-        preset_name=subscription_name,
+        preset_name="split_video_test",
         preset_dict=subscription_dict,
     )
 
     return Subscription.from_preset(
         preset=single_video_preset,
-        config=config,
+        config=music_video_config,
     )