From 3ace86435d23a37e6ac3fa592e9109da28ecb721 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 17 Sep 2022 08:09:09 -0700 Subject: [PATCH] [BACKEND] Defined kwarg strings. Add playlist/source description variable (#237) --- .../soundcloud/albums_and_singles.py | 4 +- src/ytdl_sub/entries/base_entry.py | 18 +- src/ytdl_sub/entries/entry_parent.py | 54 +++-- .../entries/variables/entry_variables.py | 184 +++++++++++++++--- src/ytdl_sub/entries/variables/kwargs.py | 40 ++++ .../entries/variables/youtube_variables.py | 10 - tests/unit/entries/conftest.py | 15 +- tests/unit/entries/test_youtube_entries.py | 7 - 8 files changed, 261 insertions(+), 71 deletions(-) create mode 100644 src/ytdl_sub/entries/variables/kwargs.py diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index 79470afb..2a143d81 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -65,8 +65,8 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator): "track_number": "{playlist_index}", "track_number_padded": "{playlist_index_padded}", "track_count": "{playlist_count}", - "album": "{playlist}", - "album_sanitized": "{playlist_sanitized}", + "album": "{playlist_title}", + "album_sanitized": "{playlist_title_sanitized}", "album_year": "{playlist_max_upload_year}", }, }, diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 4c5e8248..c79dc60f 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -10,6 +10,10 @@ from typing import final from yt_dlp.utils import sanitize_filename +from ytdl_sub.entries.variables.kwargs import DESCRIPTION +from ytdl_sub.entries.variables.kwargs import TITLE +from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL + # pylint: disable=no-member @@ -50,7 +54,7 @@ class BaseEntryVariables: str The title of the entry. If a title does not exist, returns its unique ID. """ - return self.kwargs_get("title", self.uid) + return self.kwargs_get(TITLE, self.uid) @property def title_sanitized(self) -> str: @@ -70,7 +74,7 @@ class BaseEntryVariables: str The url to the webpage. """ - return self.kwargs("webpage_url") + return self.kwargs(WEBPAGE_URL) @property def info_json_ext(self) -> str: @@ -82,6 +86,16 @@ class BaseEntryVariables: """ return "info.json" + @property + def description(self: "BaseEntry") -> str: + """ + Returns + ------- + str + The description if it exists. Otherwise, returns an emtpy string. + """ + return self.kwargs_get(DESCRIPTION, "") + # pylint: enable=no-member diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 06e7c674..24b227b0 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -9,6 +9,22 @@ import mergedeep from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import TBaseEntry from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT +from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION +from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY +from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX +from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR +from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED +from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE +from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID +from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL +from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT +from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION +from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY +from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX +from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE +from ytdl_sub.entries.variables.kwargs import SOURCE_UID +from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL class ParentType: @@ -18,7 +34,7 @@ class ParentType: def _sort_entries(entries: List[TBaseEntry]) -> List[TBaseEntry]: """Try sorting by playlist_id first, then fall back to uid""" - return sorted(entries, key=lambda ent: (ent.kwargs_get("playlist_index", math.inf), ent.uid)) + return sorted(entries, key=lambda ent: (ent.kwargs_get(PLAYLIST_INDEX, math.inf), ent.uid)) class EntryParent(BaseEntry): @@ -39,23 +55,29 @@ class EntryParent(BaseEntry): ) def _playlist_variables(self, idx: int, children: List[TBaseEntry], parent_type: str) -> Dict: - _count = self.kwargs_get("playlist_count", len(children)) - _index = children[idx].kwargs_get("playlist_index", idx + 1) + _count = self.kwargs_get(PLAYLIST_COUNT, len(children)) + _index = children[idx].kwargs_get(PLAYLIST_INDEX, idx + 1) if parent_type == ParentType.SOURCE: - return {"source_index": _index, "source_count": _count} + return {SOURCE_INDEX: _index, SOURCE_COUNT: _count} return { - "source_index": self.kwargs_get("source_index", 1), - "source_count": self.kwargs_get("source_count", 1), - "playlist_index": _index, - "playlist_count": _count, + SOURCE_INDEX: self.kwargs_get(SOURCE_INDEX, 1), + SOURCE_COUNT: self.kwargs_get(SOURCE_INDEX, 1), + PLAYLIST_INDEX: _index, + PLAYLIST_COUNT: _count, } def _parent_variables(self, parent_type: str) -> Dict: - return dict( - {f"{parent_type}_entry": self._kwargs}, - **{f"{parent_type}_{key}": value for key, value in self.base_variable_dict().items()}, - ) + def _(source_key: str, playlist_key: str) -> str: + return playlist_key if parent_type == ParentType.PLAYLIST else source_key + + return { + _(SOURCE_ENTRY, PLAYLIST_ENTRY): self._kwargs, + _(SOURCE_TITLE, PLAYLIST_TITLE): self.title, + _(SOURCE_WEBPAGE_URL, PLAYLIST_WEBPAGE_URL): self.webpage_url, + _(SOURCE_UID, PLAYLIST_UID): self.uid, + _(SOURCE_DESCRIPTION, PLAYLIST_DESCRIPTION): self.description, + } def _get_entry_children_variable_list(self, variable_name: str) -> List[str | int]: return [getattr(entry_child, variable_name) for entry_child in self.entry_children()] @@ -65,8 +87,8 @@ class EntryParent(BaseEntry): return {} return { - "playlist_max_upload_year": max(self._get_entry_children_variable_list("upload_year")), - "playlist_max_upload_year_truncated": max( + PLAYLIST_MAX_UPLOAD_YEAR: max(self._get_entry_children_variable_list("upload_year")), + PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED: max( self._get_entry_children_variable_list("upload_year_truncated") ), } @@ -114,7 +136,7 @@ class EntryParent(BaseEntry): """ Populates a tree of EntryParents that belong to this instance """ - child_entries = [ + self.child_entries = [ EntryParent( entry_dict=entry_dict, working_directory=self.working_directory(), @@ -122,8 +144,6 @@ class EntryParent(BaseEntry): for entry_dict in entry_dicts if entry_dict in self ] - - self.child_entries = sorted(child_entries, key=lambda entry: entry.kwargs("playlist_index")) return self def get_thumbnail_url(self, thumbnail_id: str) -> Optional[str]: diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py index 658208f6..e0f94b1e 100644 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ b/src/ytdl_sub/entries/variables/entry_variables.py @@ -1,7 +1,22 @@ +from typing import Union + from yt_dlp.utils import sanitize_filename from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import BaseEntryVariables +from ytdl_sub.entries.variables.kwargs import EXT +from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT +from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION +from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX +from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR +from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED +from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE +from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL +from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT +from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION +from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX +from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE +from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL # This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion # pylint: disable=no-member @@ -14,30 +29,106 @@ def _pad(num: int, width: int = 2): _days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] +Self = Union[BaseEntry, "EntryVariables"] + class EntryVariables(BaseEntryVariables): @property - def playlist(self: BaseEntry) -> str: + def source_title(self: Self) -> str: + """ + Returns + ------- + str + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + """ + return self.kwargs_get(SOURCE_TITLE, self.playlist_title) + + @property + def source_title_sanitized(self: Self) -> str: + """ + Returns + ------- + str + The source title, sanitized + """ + return sanitize_filename(self.source_title) + + @property + def source_index(self: Self) -> int: + """ + Returns + ------- + int + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + """ + return self.kwargs_get(SOURCE_INDEX, self.playlist_index) + + @property + def source_index_padded(self: Self) -> str: + """ + Returns + ------- + int + The source index, padded. + """ + return _pad(self.source_index, 2) + + @property + def source_count(self: Self) -> int: + """ + Returns + ------- + int + The source count if it exists, otherwise returns the playlist count. + """ + return self.kwargs_get(SOURCE_COUNT, self.playlist_count) + + @property + def source_webpage_url(self: Self) -> str: + """ + Returns + ------- + str + The source webpage url if it exists, otherwise returns the playlist webpage url. + """ + return self.kwargs_get(SOURCE_WEBPAGE_URL, self.playlist_webpage_url) + + @property + def source_description(self: Self) -> str: + """ + Returns + ------- + str + The source description if it exists, otherwise returns the playlist description. + """ + return self.kwargs_get(SOURCE_DESCRIPTION, self.playlist_description) + + @property + def playlist_title(self: Self) -> str: """ Returns ------- str Name of its parent playlist/channel if it exists, otherwise returns its title. """ - return self.kwargs_get("playlist", self.title) + return self.kwargs_get(PLAYLIST_TITLE, self.title) @property - def playlist_sanitized(self) -> str: + def playlist_title_sanitized(self: Self) -> str: """ Returns ------- str The playlist name, sanitized """ - return sanitize_filename(self.playlist) + return sanitize_filename(self.playlist_title) @property - def playlist_index(self: BaseEntry) -> int: + def playlist_index(self: Self) -> int: """ Returns ------- @@ -48,10 +139,10 @@ class EntryVariables(BaseEntryVariables): uploaded entry. It is recommended to not use this unless you know the channel/playlist will never add new content, i.e. a music album. """ - return self.kwargs_get("playlist_index", 1) + return self.kwargs_get(PLAYLIST_INDEX, 1) @property - def playlist_index_padded(self) -> str: + def playlist_index_padded(self: Self) -> str: """ Returns ------- @@ -61,17 +152,37 @@ class EntryVariables(BaseEntryVariables): return _pad(self.playlist_index, width=2) @property - def playlist_count(self: BaseEntry) -> int: + def playlist_count(self: Self) -> int: """ Returns ------- int Playlist count if it exists, otherwise returns ``1``. """ - return self.kwargs_get("playlist_count", 1) + return self.kwargs_get(PLAYLIST_COUNT, 1) @property - def playlist_max_upload_year(self) -> int: + def playlist_description(self: Self) -> str: + """ + Returns + ------- + str + The playlist description if it exists, otherwise returns the entry's description. + """ + return self.kwargs_get(PLAYLIST_DESCRIPTION, self.description) + + @property + def playlist_webpage_url(self: Self) -> str: + """ + Returns + ------- + str + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + """ + return self.kwargs_get(PLAYLIST_WEBPAGE_URL, self.webpage_url) + + @property + def playlist_max_upload_year(self: Self) -> int: """ Returns ------- @@ -80,20 +191,31 @@ class EntryVariables(BaseEntryVariables): ``upload_year`` """ # override in EntryParent - return self.upload_year + return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR, self.upload_year) @property - def ext(self: BaseEntry) -> str: + def playlist_max_upload_year_truncated(self: Self) -> int: + """ + Returns + ------- + int + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + """ + return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED, self.upload_year_truncated) + + @property + def ext(self: Self) -> str: """ Returns ------- str The downloaded entry's file extension """ - return self.kwargs("ext") + return self.kwargs(EXT) @property - def thumbnail_ext(self: BaseEntry) -> str: + def thumbnail_ext(self: Self) -> str: """ Returns ------- @@ -104,7 +226,7 @@ class EntryVariables(BaseEntryVariables): return "jpg" @property - def upload_date(self: BaseEntry) -> str: + def upload_date(self: Self) -> str: """ Returns ------- @@ -114,7 +236,7 @@ class EntryVariables(BaseEntryVariables): return self.kwargs("upload_date") @property - def upload_year(self) -> int: + def upload_year(self: Self) -> int: """ Returns ------- @@ -124,7 +246,7 @@ class EntryVariables(BaseEntryVariables): return int(self.upload_date[:4]) @property - def upload_year_truncated(self) -> int: + def upload_year_truncated(self: Self) -> int: """ Returns ------- @@ -134,7 +256,7 @@ class EntryVariables(BaseEntryVariables): return int(str(self.upload_year)[-2:]) @property - def upload_year_truncated_reversed(self) -> int: + def upload_year_truncated_reversed(self: Self) -> int: """ Returns ------- @@ -145,7 +267,7 @@ class EntryVariables(BaseEntryVariables): return 100 - self.upload_year_truncated @property - def upload_month_reversed(self) -> int: + def upload_month_reversed(self: Self) -> int: """ Returns ------- @@ -155,7 +277,7 @@ class EntryVariables(BaseEntryVariables): return 13 - self.upload_month @property - def upload_month_reversed_padded(self) -> str: + def upload_month_reversed_padded(self: Self) -> str: """ Returns ------- @@ -165,7 +287,7 @@ class EntryVariables(BaseEntryVariables): return _pad(self.upload_month_reversed) @property - def upload_month_padded(self) -> str: + def upload_month_padded(self: Self) -> str: """ Returns ------- @@ -175,7 +297,7 @@ class EntryVariables(BaseEntryVariables): return self.upload_date[4:6] @property - def upload_day_padded(self) -> str: + def upload_day_padded(self: Self) -> str: """ Returns ------- @@ -185,7 +307,7 @@ class EntryVariables(BaseEntryVariables): return self.upload_date[6:8] @property - def upload_month(self) -> int: + def upload_month(self: Self) -> int: """ Returns ------- @@ -195,7 +317,7 @@ class EntryVariables(BaseEntryVariables): return int(self.upload_month_padded.lstrip("0")) @property - def upload_day(self) -> int: + def upload_day(self: Self) -> int: """ Returns ------- @@ -205,7 +327,7 @@ class EntryVariables(BaseEntryVariables): return int(self.upload_day_padded.lstrip("0")) @property - def upload_day_reversed(self) -> int: + def upload_day_reversed(self: Self) -> int: """ Returns ------- @@ -220,7 +342,7 @@ class EntryVariables(BaseEntryVariables): return total_days_in_month + 1 - self.upload_day @property - def upload_day_reversed_padded(self) -> str: + def upload_day_reversed_padded(self: Self) -> str: """ Returns ------- @@ -230,7 +352,7 @@ class EntryVariables(BaseEntryVariables): return _pad(self.upload_day_reversed) @property - def upload_day_of_year(self) -> int: + def upload_day_of_year(self: Self) -> int: """ Returns ------- @@ -244,7 +366,7 @@ class EntryVariables(BaseEntryVariables): return output @property - def upload_day_of_year_padded(self) -> str: + def upload_day_of_year_padded(self: Self) -> str: """ Returns ------- @@ -254,7 +376,7 @@ class EntryVariables(BaseEntryVariables): return _pad(self.upload_day_of_year, width=3) @property - def upload_day_of_year_reversed(self) -> int: + def upload_day_of_year_reversed(self: Self) -> int: """ Returns ------- @@ -269,7 +391,7 @@ class EntryVariables(BaseEntryVariables): return total_days_in_year + 1 - self.upload_day_of_year @property - def upload_day_of_year_reversed_padded(self) -> str: + def upload_day_of_year_reversed_padded(self: Self) -> str: """ Returns ------- @@ -279,7 +401,7 @@ class EntryVariables(BaseEntryVariables): return _pad(self.upload_day_of_year_reversed, width=3) @property - def upload_date_standardized(self) -> str: + def upload_date_standardized(self: Self) -> str: """ Returns ------- diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py new file mode 100644 index 00000000..b4fa836d --- /dev/null +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -0,0 +1,40 @@ +from typing import List + + +class KwargKeys: + keys: List[str] = [] + backend_keys: List[str] = [] + + +def _(key: str, backend: bool = False) -> str: + if backend: + assert key not in KwargKeys.backend_keys + KwargKeys.backend_keys.append(key) + else: + assert key not in KwargKeys.keys + KwargKeys.keys.append(key) + return key + + +SOURCE_ENTRY = _("source_entry", backend=True) +SOURCE_INDEX = _("source_index") +SOURCE_COUNT = _("source_count") +SOURCE_TITLE = _("source_title") +SOURCE_UID = _("source_uid") +SOURCE_DESCRIPTION = _("source_description") +SOURCE_WEBPAGE_URL = _("source_webpage_url") + +PLAYLIST_ENTRY = _("playlist_entry", backend=True) +PLAYLIST_WEBPAGE_URL = _("playlist_webpage_url") +PLAYLIST_INDEX = _("playlist_index") +PLAYLIST_COUNT = _("playlist_count") +PLAYLIST_MAX_UPLOAD_YEAR = _("playlist_max_upload_year") +PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED = _("playlist_max_upload_year_truncated") +PLAYLIST_TITLE = _("playlist_title") +PLAYLIST_DESCRIPTION = _("playlist_description") +PLAYLIST_UID = _("playlist_uid") + +EXT = _("ext") +TITLE = _("title") +DESCRIPTION = _("description") +WEBPAGE_URL = _("webpage_url") diff --git a/src/ytdl_sub/entries/variables/youtube_variables.py b/src/ytdl_sub/entries/variables/youtube_variables.py index 616b2683..7446e1dc 100644 --- a/src/ytdl_sub/entries/variables/youtube_variables.py +++ b/src/ytdl_sub/entries/variables/youtube_variables.py @@ -94,13 +94,3 @@ class YoutubeVideoVariables(EntryVariables): 1. """ return 1 - - @property - def description(self: BaseEntry) -> str: - """ - Returns - ------- - str - The video description. - """ - return self.kwargs("description") diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index af6239c1..9339b8f3 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -64,6 +64,7 @@ def mock_entry_to_dict( "title": "entry ｛title｝", "title_sanitized": "entry ｛title｝", "ext": ext, + "description": "", "extractor": extractor, "upload_date": upload_date, "upload_date_standardized": "2021-01-12", @@ -85,12 +86,22 @@ def mock_entry_to_dict( "thumbnail_ext": thumbnail_ext, "info_json_ext": "info.json", "webpage_url": webpage_url, - "playlist": "entry ｛title｝", - "playlist_sanitized": "entry ｛title｝", "playlist_index": 1, "playlist_index_padded": "01", "playlist_count": 1, "playlist_max_upload_year": 2021, + "playlist_max_upload_year_truncated": 21, + "playlist_title": "entry ｛title｝", + "playlist_title_sanitized": "entry ｛title｝", + "playlist_description": "", + "playlist_webpage_url": "https://yourname.here", + "source_count": 1, + "source_description": "", + "source_index": 1, + "source_index_padded": "01", + "source_title": "entry ｛title｝", + "source_title_sanitized": "entry ｛title｝", + "source_webpage_url": "https://yourname.here", } diff --git a/tests/unit/entries/test_youtube_entries.py b/tests/unit/entries/test_youtube_entries.py index 6dfcb8a9..67f7ea79 100644 --- a/tests/unit/entries/test_youtube_entries.py +++ b/tests/unit/entries/test_youtube_entries.py @@ -4,11 +4,6 @@ from ytdl_sub.entries.soundcloud import SoundcloudTrack from ytdl_sub.entries.youtube import YoutubeVideo -@pytest.fixture -def description(): - return "a description" - - @pytest.fixture def playlist_index(): return 1 @@ -49,7 +44,6 @@ def mock_youtube_video_to_dict( "track_title_sanitized": track_title, "artist": artist, "artist_sanitized": artist, - "description": description, } ) @@ -66,7 +60,6 @@ def mock_youtube_video_kwargs( "channel": channel, "track": track_title, "artist": artist, - "description": description, } )