From de1e68e5aa34d29b656ab9cf0a893217347dc10a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 27 Mar 2024 23:43:35 -0700 Subject: [PATCH] test in tv show collection --- src/ytdl_sub/downloaders/url/downloader.py | 29 ++++++++++++++----- src/ytdl_sub/downloaders/url/validators.py | 19 ++++++++++-- .../entries/script/custom_functions.py | 9 ++++-- .../entries/script/function_scripts.py | 1 + .../tv_show/tv_show_collection.yaml | 4 +++ 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index a0ceb535..7f773e36 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -25,6 +25,7 @@ from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.thumbnail import ThumbnailTypes from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail @@ -245,9 +246,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): .to_dict() ) - @property - def metadata_ytdl_options(self) -> Dict: + def metadata_ytdl_options(self, scrape_reverse: bool) -> Dict: """ + Parameters + ---------- + scrape_reverse + Whether to scrape in reverse order + Returns ------- YTDL options dict for fetching metadata @@ -255,6 +260,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): return ( self._metadata_ytdl_options_builder.clone() .add(self.ytdl_option_defaults(), before=True) + .add({"playlistreverse": scrape_reverse} if scrape_reverse else None, before=True) .to_dict() ) @@ -390,16 +396,19 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): yield entry_child def _download_url_metadata( - self, url: str, include_sibling_metadata: bool + self, url: str, include_sibling_metadata: bool, scrape_reverse: bool ) -> Tuple[List[EntryParent], List[Entry]]: """ Downloads only info.json files and forms EntryParent trees """ + with self._separate_download_archives(): entry_dicts = YTDLP.extract_info_via_info_json( working_directory=self.working_directory, - ytdl_options_overrides=self.metadata_ytdl_options, - log_prefix_on_info_json_dl="Downloading metadata for", + ytdl_options_overrides=self.metadata_ytdl_options(scrape_reverse=scrape_reverse), + log_prefix_on_info_json_dl=( + f"Downloading metadata {'in reverse ' if scrape_reverse else ''}for" + ), url=url, ) @@ -448,7 +457,11 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): continue parents, orphan_entries = self._download_url_metadata( - url=url, include_sibling_metadata=collection_url.include_sibling_metadata + url=url, + include_sibling_metadata=collection_url.include_sibling_metadata, + scrape_reverse=ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(collection_url.scrape_reverse) + ), ) # TODO: Encapsulate this logic into its own class @@ -462,7 +475,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): for entry in self._iterate_entries( parents=parents, orphans=orphan_entries, - download_reversed=collection_url.download_reverse, + download_reversed=ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(collection_url.download_reverse) + ), ): entry.initialize_script(self.overrides).add( {v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)} diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 53330306..38fc00ab 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -8,6 +8,7 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.script.parser import parse from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator @@ -49,6 +50,7 @@ class UrlValidator(StrictDictValidator): "source_thumbnails", "playlist_thumbnails", "download_reverse", + "scrape_reverse", "include_sibling_metadata", } @@ -77,7 +79,10 @@ class UrlValidator(StrictDictValidator): key="playlist_thumbnails", validator=UrlThumbnailListValidator, default=[] ) self._download_reverse = self._validate_key( - key="download_reverse", validator=BoolValidator, default=True + key="download_reverse", validator=OverridesBooleanFormatterValidator, default="True" + ) + self._scrape_reverse = self._validate_key( + key="scrape_reverse", validator=OverridesBooleanFormatterValidator, default="False" ) self._include_sibling_metadata = self._validate_key( key="include_sibling_metadata", validator=BoolValidator, default=False @@ -148,12 +153,20 @@ class UrlValidator(StrictDictValidator): return self._source_thumbnails @property - def download_reverse(self) -> bool: + def download_reverse(self) -> OverridesBooleanFormatterValidator: """ Optional. Whether to download entries in the reverse order of the metadata downloaded. Defaults to True. """ - return self._download_reverse.value + return self._download_reverse + + @property + def scrape_reverse(self) -> OverridesBooleanFormatterValidator: + """ + Optional. Whether to scrape entry metadata in the reverse order. + Defaults to False. + """ + return self._scrape_reverse @property def include_sibling_metadata(self) -> bool: diff --git a/src/ytdl_sub/entries/script/custom_functions.py b/src/ytdl_sub/entries/script/custom_functions.py index b34e51af..0024ae60 100644 --- a/src/ytdl_sub/entries/script/custom_functions.py +++ b/src/ytdl_sub/entries/script/custom_functions.py @@ -30,6 +30,11 @@ class CustomFunctions: Queries a playlist-based URL using yt-dlp to see if it is ordered from newest (lower playlist number) to oldest. """ + # Top-level allow-list of notorious playlists that can be + # ordered in either direction + if not any(["youtube.com/playlist" in url.value]): + return Boolean("True") + info_only_kwargs = { "skip_download": True, "writethumbnail": False, @@ -47,7 +52,7 @@ class CustomFunctions: if ( not isinstance(url_info, dict) or not isinstance(url_info.get("entries"), list) - or not len(url_info["entries"]) + or len(url_info["entries"]) == 0 ): return Boolean(False) @@ -66,7 +71,7 @@ class CustomFunctions: return Boolean(True) - except Exception: + except Exception: # pylint: disable=broad-except return Boolean(False) @staticmethod diff --git a/src/ytdl_sub/entries/script/function_scripts.py b/src/ytdl_sub/entries/script/function_scripts.py index 68019a00..58f950bb 100644 --- a/src/ytdl_sub/entries/script/function_scripts.py +++ b/src/ytdl_sub/entries/script/function_scripts.py @@ -5,6 +5,7 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions v: VariableDefinitions = VARIABLES +# TODO: Make this a proper class with docstrings CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { ############################################################################################# # SIBLING GETTER diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index 0999161c..816ec42e 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -38,6 +38,10 @@ presets: uid: "avatar_uncropped" - name: "{tv_show_fanart_file_name}" uid: "banner_uncropped" + scrape_reverse: >- + { %not( %is_playlist_ordered_by_newest( collection_season_1_url )) } + download_reverse: >- + { %is_playlist_ordered_by_newest( collection_season_1_url ) } output_directory_nfo_tags: tags: