From 4b6add06e1478bf194171e303bbf6102bd126af5 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 29 Sep 2022 00:09:56 -0700 Subject: [PATCH 1/3] [BUGFIX] Fix nested parents with multiple parents (#255) --- src/ytdl_sub/config/preset.py | 76 +++++++++++++++++++------------- tests/unit/config/test_preset.py | 10 ++++- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 89c5b74e..d046d62b 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -277,46 +277,60 @@ class Preset(StrictDictValidator): for validator_value in validator.dict.values(): self.__validate_override_string_formatter_validator(validator_value) + def _get_presets_to_merge( + self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigFile + ) -> List[Dict]: + presets_to_merge: List[Dict] = [] + + if isinstance(parent_presets, str): + parent_presets = [parent_presets] + + for parent_preset in reversed(parent_presets): + # Make sure we do not hit an infinite loop + if parent_preset in seen_presets: + raise self._validation_exception( + f"preset loop detected with the preset '{parent_preset}'" + ) + + # Make sure the parent preset actually exists + if parent_preset not in config.presets.keys: + raise self._validation_exception( + f"preset '{parent_preset}' does not exist in the provided config. " + f"Available presets: {', '.join(config.presets.keys)}" + ) + + parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset]) + presets_to_merge.append(parent_preset_dict) + + if "preset" in parent_preset_dict: + presets_to_merge.extend( + self._get_presets_to_merge( + parent_presets=parent_preset_dict["preset"], + seen_presets=seen_presets + [parent_preset], + config=config, + ) + ) + + return presets_to_merge + def __merge_parent_preset_dicts_if_present(self, config: ConfigFile): parent_preset_validator = self._validate_key_if_present( key="preset", validator=StringListValidator ) - if parent_preset_validator is None: return - presets_to_merge: List[Dict] = [] - for parent_preset in [preset.value for preset in parent_preset_validator.list]: - sub_parent_presets: Dict[str, Dict] = {} - - while parent_preset: - # Make sure the parent preset actually exists - if parent_preset not in config.presets.keys: - raise self._validation_exception( - f"preset '{parent_preset}' does not exist in the provided config. " - f"Available presets: {', '.join(config.presets.keys)}" - ) - - # Make sure we do not hit an infinite loop - if parent_preset in sub_parent_presets: - raise self._validation_exception( - f"preset loop detected with the preset '{parent_preset}'" - ) - - parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset]) - - sub_parent_presets[parent_preset] = parent_preset_dict - parent_preset = parent_preset_dict.get("preset") - - # Extend reversed, so top-most parents are first - if sub_parent_presets: - presets_to_merge.extend(reversed(sub_parent_presets.values())) - - # Append this preset (the subscription) last - presets_to_merge.append(copy.deepcopy(self._value)) + # Get list of all parent presets in depth-first search order, beginning with this preset + presets_to_merge: List[Dict] = [copy.deepcopy(self._value)] + self._get_presets_to_merge( + parent_presets=[preset.value for preset in parent_preset_validator.list], + seen_presets=[], + config=config, + ) # Merge all presets - self._value = mergedeep.merge({}, *presets_to_merge, strategy=mergedeep.Strategy.ADDITIVE) + self._value = mergedeep.merge( + {}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE + ) def __init__(self, config: ConfigFile, name: str, value: Any): super().__init__(name=name, value=value) diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index bb1d164e..3d469f1a 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -32,6 +32,7 @@ def config_file() -> ConfigFile: "tags": {"key-2": "preset_2", "key-3": "preset_2"}, } }, + "parent_preset_3": {"preset": ["parent_preset_1", "parent_preset_2"]}, "preset_self_loop": {"preset": "preset_self_loop"}, "preset_loop_0": {"preset": "preset_loop_1"}, "preset_loop_1": {"preset": "preset_loop_0"}, @@ -113,12 +114,17 @@ class TestPreset: assert tags_string_dict == {"key-1": "preset_0", "key-2": "this-preset"} - def test_preset_multiple_parents(self, config_file, output_options, youtube_video): + @pytest.mark.parametrize( + "preset_value", [["parent_preset_1", "parent_preset_2"], "parent_preset_3"] + ) + def test_preset_multiple_parents( + self, config_file, output_options, youtube_video, preset_value + ): preset = Preset( config=config_file, name="test", value={ - "preset": ["parent_preset_1", "parent_preset_2"], + "preset": preset_value, "youtube": youtube_video, "output_options": output_options, "nfo_tags": {"tags": {"key-3": "this-preset"}}, From 7d29b72010084a55030248b573cb62862d5944dc Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 29 Sep 2022 00:33:57 -0700 Subject: [PATCH 2/3] [BACKEND] `uid_sanitized` variable (#256) --- src/ytdl_sub/entries/base_entry.py | 10 ++++++++++ tests/unit/entries/conftest.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 950c7dfb..639d3528 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -40,6 +40,16 @@ class BaseEntryVariables: """ return self.kwargs(UID) + @property + def uid_sanitized(self: "BaseEntry") -> str: + """ + Returns + ------- + str + The sanitized uid of the entry, which is safe to use for Unix and Windows file names. + """ + return sanitize_filename(self.uid) + @property def extractor(self: "BaseEntry") -> str: """ diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index ab9b9e30..610ee599 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -61,6 +61,7 @@ def mock_entry_to_dict( ): return { "uid": uid, + "uid_sanitized": uid, "title": "entry ｛title｝", "title_sanitized": "entry ｛title｝", "ext": ext, From cd9a094e2c0dc90885e9e688969b8e646cb0ee34 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 29 Sep 2022 15:39:53 -0700 Subject: [PATCH 3/3] [BACKEND] `epoch` and `playlist_index_reversed` variables (#257) --- src/ytdl_sub/entries/base_entry.py | 35 ++++++++++++++++++- .../entries/variables/entry_variables.py | 30 ++++++++++++++++ src/ytdl_sub/entries/variables/kwargs.py | 2 ++ tests/unit/entries/conftest.py | 7 ++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 639d3528..9553bed7 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -1,4 +1,5 @@ from abc import ABC +from datetime import datetime from pathlib import Path from typing import Any from typing import Dict @@ -11,6 +12,8 @@ from typing import final from yt_dlp.utils import sanitize_filename from ytdl_sub.entries.variables.kwargs import DESCRIPTION +from ytdl_sub.entries.variables.kwargs import EPOCH +from ytdl_sub.entries.variables.kwargs import EXTRACTOR from ytdl_sub.entries.variables.kwargs import TITLE from ytdl_sub.entries.variables.kwargs import UID from ytdl_sub.entries.variables.kwargs import UPLOADER @@ -58,7 +61,37 @@ class BaseEntryVariables: str The ytdl extractor name """ - return self.kwargs("extractor") + return self.kwargs(EXTRACTOR) + + @property + def epoch(self: "BaseEntry") -> int: + """ + Returns + ------- + int + The unix epoch of when the metadata was scraped by yt-dlp. + """ + return self.kwargs(EPOCH) + + @property + def epoch_date(self: "BaseEntry") -> str: + """ + Returns + ------- + str + The epoch's date, in YYYYMMDD format. + """ + return datetime.utcfromtimestamp(self.epoch).strftime("%Y%m%d") + + @property + def epoch_hour(self: "BaseEntry") -> str: + """ + Returns + ------- + str + The epoch's hour, padded + """ + return datetime.utcfromtimestamp(self.epoch).strftime("%H") @property def title(self: "BaseEntry") -> str: diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py index 05aded33..a37a7eb2 100644 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ b/src/ytdl_sub/entries/variables/entry_variables.py @@ -169,6 +169,16 @@ class EntryVariables(BaseEntryVariables): """ return self.kwargs_get(PLAYLIST_INDEX, 1) + @property + def playlist_index_reversed(self: Self) -> int: + """ + Returns + ------- + int + Playlist index reversed via ``playlist_count - playlist_index + 1`` + """ + return self.playlist_count - self.playlist_index + 1 + @property def playlist_index_padded(self: Self) -> str: """ @@ -179,6 +189,16 @@ class EntryVariables(BaseEntryVariables): """ return _pad(self.playlist_index, width=2) + @property + def playlist_index_reversed_padded(self: Self) -> str: + """ + Returns + ------- + str + playlist_index_reversed padded two digits + """ + return _pad(self.playlist_index_reversed, width=2) + @property def playlist_index_padded6(self: Self) -> str: """ @@ -189,6 +209,16 @@ class EntryVariables(BaseEntryVariables): """ return _pad(self.playlist_index, width=6) + @property + def playlist_index_reversed_padded6(self: Self) -> str: + """ + Returns + ------- + str + playlist_index_reversed padded six digits. + """ + return _pad(self.playlist_index_reversed, width=6) + @property def playlist_count(self: Self) -> int: """ diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py index d222606f..80304ca2 100644 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -41,6 +41,8 @@ PLAYLIST_UPLOADER_ID = _("playlist_uploader_id") PLAYLIST_UPLOADER_URL = _("playlist_uploader_url") UID = _("id") +EXTRACTOR = _("extractor") +EPOCH = _("epoch") CHANNEL = _("channel") EXT = _("ext") TITLE = _("title") diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 610ee599..8be3a8b1 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -62,6 +62,9 @@ def mock_entry_to_dict( return { "uid": uid, "uid_sanitized": uid, + "epoch": 1596878400, + "epoch_date": "20200808", + "epoch_hour": "09", "title": "entry ｛title｝", "title_sanitized": "entry ｛title｝", "ext": ext, @@ -95,6 +98,9 @@ def mock_entry_to_dict( "playlist_index": 1, "playlist_index_padded": "01", "playlist_index_padded6": "000001", + "playlist_index_reversed": 1, + "playlist_index_reversed_padded": "01", + "playlist_index_reversed_padded6": "000001", "playlist_count": 1, "playlist_max_upload_year": 2021, "playlist_max_upload_year_truncated": 21, @@ -126,6 +132,7 @@ def mock_entry_kwargs( ): return { "id": uid, + "epoch": 1596878400, "extractor": extractor, "title": title, "ext": ext,