From 28c29683cbb5e64508d26f562ed33e18d52a4d34 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 5 Oct 2024 01:03:52 -0700 Subject: [PATCH] [FEATURE] Filter Keywords prebuilt preset (#1086) Adds ``Filter Keywords``, a preset that can include or exclude media with any of the listed keywords. Both keywords and title/description are lower-cased before filtering. Supports the following override variables: * ``title_include_keywords`` * ``title_exclude_keywords`` * ``description_include_keywords`` * ``description_exclude_keywords`` For best usage, use the `~` tilda subscription mode to set a subscription's list override variables. Tilda mode allows override variables to be set directly underneath it. ``` Plex TV Show by Date | Filter Keywords: = Documentaries: "~NOVA PBS": url: "https://www.youtube.com/@novapbs" title_exclude_keywords: - "preview" - "trailer" "~To Catch a Smuggler": url: "https://www.youtube.com/@NatGeo" title_include_keywords: - "To Catch a Smuggler" ``` --- .../prebuilt_presets/common.rst | 5 + docs/source/faq/index.rst | 7 +- docs/source/prebuilt_presets/helpers.rst | 34 ++++ .../prebuilt_presets/helpers/filtering.yaml | 52 ++++++ .../script/functions/error_functions.py | 17 +- src/ytdl_sub/script/types/function.py | 2 +- src/ytdl_sub/script/utils/type_checking.py | 4 + .../prebuilt_presets/test_filter_keywords.py | 171 ++++++++++++++++++ .../description_filter_keywords_exclude.txt | 1 + .../description_filter_keywords_include.txt | 66 +++++++ .../filter_keywords_empty.txt | 66 +++++++ .../title_filter_keywords_exclude.txt | 51 ++++++ .../title_filter_keywords_include.txt | 20 ++ 13 files changed, 487 insertions(+), 9 deletions(-) create mode 100644 src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml create mode 100644 tests/integration/prebuilt_presets/test_filter_keywords.py create mode 100644 tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_exclude.txt create mode 100644 tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_include.txt create mode 100644 tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt create mode 100644 tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_exclude.txt create mode 100644 tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_include.txt diff --git a/docs/source/config_reference/prebuilt_presets/common.rst b/docs/source/config_reference/prebuilt_presets/common.rst index 0a6c8e8a..ab1fd545 100644 --- a/docs/source/config_reference/prebuilt_presets/common.rst +++ b/docs/source/config_reference/prebuilt_presets/common.rst @@ -4,6 +4,11 @@ Common .. highlight:: yaml +Filtering +------------- + +.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml + Media Quality ------------- diff --git a/docs/source/faq/index.rst b/docs/source/faq/index.rst index 1822194f..f8ae6653 100644 --- a/docs/source/faq/index.rst +++ b/docs/source/faq/index.rst @@ -55,7 +55,12 @@ See `yt-dl's recommended way ` +See the prebuilt preset :doc:`chunk_initial_download `. + +...filter to include or exclude based on certain keywords? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See the prebuilt preset :doc:`Filter Keywords `. There is a bug where... ----------------------- diff --git a/docs/source/prebuilt_presets/helpers.rst b/docs/source/prebuilt_presets/helpers.rst index 745cdb97..fa6d25f9 100644 --- a/docs/source/prebuilt_presets/helpers.rst +++ b/docs/source/prebuilt_presets/helpers.rst @@ -35,6 +35,40 @@ Add the following preset to download the best available audio and video quality, ``max_1080p`` +Filter Keywords +--------------- + +``Filter Keywords`` can include or exclude media with any of the listed keywords. Both keywords and title/description are lower-cased before filtering. + +Supports the following override variables: + +* ``title_include_keywords`` +* ``title_exclude_keywords`` +* ``description_include_keywords`` +* ``description_exclude_keywords`` + +.. tip:: + + Use the `~` tilda subscription mode to set a subscription's list override variables. + Tilda mode allows override variables to be set directly underneath it. + + .. code-block:: yaml + + Plex TV Show by Date | Filter Keywords: + + = Documentaries: + "~NOVA PBS": + url: "https://www.youtube.com/@novapbs" + title_exclude_keywords: + - "preview" + - "trailer" + + "~To Catch a Smuggler": + url: "https://www.youtube.com/@NatGeo" + title_include_keywords: + - "To Catch a Smuggler" + + Chunk Initial Download ---------------------- diff --git a/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml b/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml new file mode 100644 index 00000000..b964cc8c --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml @@ -0,0 +1,52 @@ +presets: + + ############################################################################# + # Include Keywords + # Include or exclude media with any of the listed keywords in their titles + # Keywords will check a lower-cased title or description + Filter Keywords: + overrides: + # default filter lists to be empty + title_include_keywords: "{ [] }" + title_exclude_keywords: "{ [] }" + description_include_keywords: "{ [] }" + description_exclude_keywords: "{ [] }" + + "%ensure_string": >- + { + %assert_then( + %is_string($0), + %lower($0), + "filter keywords must be strings" + ) + } + "%ensure_lower_array": >- + { + %assert_then( + %is_array($0), + %array_apply( + $0, + %ensure_string + ), + %concat($1," must be an array") + ) + } + + # $0 - var to evaluate + # $1 - keyword list + # $2 - variable name for error messages + # $3 - default return if keyword list is empty + "%contains_keywords": >- + { + %if( + %bool( $1 ), + %contains_any( %lower($0), %ensure_lower_array($1, $2) ), + $3 + ) + } + + filter_exclude: + - "{ %not( %contains_keywords(title, title_include_keywords, 'title_include_keywords', true) ) }" + - "{ %not( %contains_keywords(description, description_include_keywords, 'description_include_keywords', true) ) }" + - "{ %contains_keywords(title, title_exclude_keywords, 'title_exclude_keywords', false) }" + - "{ %contains_keywords(description, description_exclude_keywords, 'description_exclude_keywords',false) }" \ No newline at end of file diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py index 3a1be9f5..34b92a74 100644 --- a/src/ytdl_sub/script/functions/error_functions.py +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -20,9 +20,10 @@ class ErrorFunctions: Explicitly throw an error with the provided assert message if ``value`` evaluates to False. If it evaluates to True, it will return ``value``. """ - if not bool(value.value): + evaluated_val = value.value() + if not bool(evaluated_val.value): raise UserThrownRuntimeError(assert_message) - return value + return evaluated_val @staticmethod def assert_then( @@ -35,7 +36,7 @@ class ErrorFunctions: """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) - return ret + return ret.value() @staticmethod def assert_eq( @@ -46,9 +47,10 @@ class ErrorFunctions: Explicitly throw an error with the provided assert message if ``value`` does not equal ``equals``. If they do equal, then return ``value``. """ - if not value.value == equals.value: + evaluated_val = value.value() + if not evaluated_val.value == equals.value: raise UserThrownRuntimeError(assert_message) - return value + return evaluated_val @staticmethod def assert_ne( @@ -59,6 +61,7 @@ class ErrorFunctions: Explicitly throw an error with the provided assert message if ``value`` equals ``equals``. If they do equal, then return ``value``. """ - if value.value == equals.value: + evaluated_value = value.value() + if evaluated_value.value == equals.value: raise UserThrownRuntimeError(assert_message) - return value + return evaluated_value diff --git a/src/ytdl_sub/script/types/function.py b/src/ytdl_sub/script/types/function.py index a61fc7d4..7ecd1999 100644 --- a/src/ytdl_sub/script/types/function.py +++ b/src/ytdl_sub/script/types/function.py @@ -259,7 +259,7 @@ class BuiltInFunction(Function, BuiltInFunctionType): resolved_variables: Dict[Variable, Resolvable], custom_functions: Dict[str, "VariableDependency"], ) -> Resolvable: - # TODO: Make conditionals not execute all branches!!! + # Ensure conditionals do not execute all branches conditional_return_args = self.function_spec.conditional_arg_indices( num_input_args=len(self.args) ) diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py index 2489760b..e8a8bc79 100644 --- a/src/ytdl_sub/script/utils/type_checking.py +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -208,6 +208,10 @@ class FunctionSpec: return list(range(1, num_input_args, 2)) + [num_input_args - 1] if self.function_name == "if_passthrough": return [0, 1] # true-passthrough, false-passthrough + if self.function_name in ("assert", "assert_eq", "assert_ne"): + return [0] + if self.function_name == "assert_then": + return [1] return [] @property diff --git a/tests/integration/prebuilt_presets/test_filter_keywords.py b/tests/integration/prebuilt_presets/test_filter_keywords.py new file mode 100644 index 00000000..41761b03 --- /dev/null +++ b/tests/integration/prebuilt_presets/test_filter_keywords.py @@ -0,0 +1,171 @@ +import re + +import pytest +from expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import ValidationException + + +@pytest.fixture +def filter_subscription_dict(output_directory): + return { + "preset": [ + "Plex TV Show by Date", + "Filter Keywords", + ], + "overrides": {"url": "https://your.name.here", "tv_show_directory": output_directory}, + } + + +class TestFilterKeywords: + + def test_no_overrides( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + ): + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/filter_keywords_empty.txt", + ) + + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) + def test_title( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + filter_mode: str, + ): + filter_subscription_dict["overrides"][f"title_{filter_mode}_keywords"] = [ + "not included", + "MOCK ENTRY 20-3", + ] + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/title_filter_keywords_{filter_mode}.txt", + ) + + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) + def test_description( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + filter_mode: str, + ): + filter_subscription_dict["overrides"][f"description_{filter_mode}_keywords"] = [ + "no filter here", + "description", + ] + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/description_filter_keywords_{filter_mode}.txt", + ) + + @pytest.mark.parametrize( + "keyword_variable", + [ + "title_include_keywords", + "title_exclude_keywords", + "description_include_keywords", + "description_exclude_keywords", + ], + ) + def test_error_not_list_type( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + keyword_variable, + ): + filter_subscription_dict["overrides"][keyword_variable] = "not array" + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with ( + mock_download_collection_entries(is_youtube_channel=False, num_urls=1, is_dry_run=True), + pytest.raises(UserThrownRuntimeError, match=f"{keyword_variable} must be an array"), + ): + _ = subscription.download(dry_run=True) + + @pytest.mark.parametrize( + "keyword_variable", + [ + "title_include_keywords", + "title_exclude_keywords", + "description_include_keywords", + "description_exclude_keywords", + ], + ) + def test_error_not_string_keyword( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + keyword_variable, + ): + filter_subscription_dict["overrides"][keyword_variable] = "{['str', ['nested array not']]}" + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with ( + mock_download_collection_entries(is_youtube_channel=False, num_urls=1, is_dry_run=True), + pytest.raises(UserThrownRuntimeError, match="filter keywords must be strings"), + ): + _ = subscription.download(dry_run=True) diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_exclude.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_exclude.txt new file mode 100644 index 00000000..7192b6c7 --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_exclude.txt @@ -0,0 +1 @@ +No new, modified, or removed files in '{output_directory}' \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_include.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_include.txt new file mode 100644 index 00000000..8c3ebf9a --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/description_filter_keywords_include.txt @@ -0,0 +1,66 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/Season 2020 + s2020.e080701 - Mock Entry ２０-３-thumb.jpg + s2020.e080701 - Mock Entry ２０-３.info.json + s2020.e080701 - Mock Entry ２０-３.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-07 + episode_id: 80701 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-3.com + + The Description + title: 2020-08-07 - Mock Entry 20-3 + year: 2020 + s2020.e080801 - Mock Entry ２０-２-thumb.jpg + s2020.e080801 - Mock Entry ２０-２.info.json + s2020.e080801 - Mock Entry ２０-２.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-2.com + + The Description + title: 2020-08-08 - Mock Entry 20-2 + year: 2020 + s2020.e080802 - Mock Entry ２０-１-thumb.jpg + s2020.e080802 - Mock Entry ２０-１.info.json + s2020.e080802 - Mock Entry ２０-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80802 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-1.com + + The Description + title: 2020-08-08 - Mock Entry 20-1 + year: 2020 +{output_directory}/Season 2021 + s2021.e080801 - Mock Entry ２１-１-thumb.jpg + s2021.e080801 - Mock Entry ２１-１.info.json + s2021.e080801 - Mock Entry ２１-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2021-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://21-1.com + + The Description + title: 2021-08-08 - Mock Entry 21-1 + year: 2021 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt new file mode 100644 index 00000000..8c3ebf9a --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt @@ -0,0 +1,66 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/Season 2020 + s2020.e080701 - Mock Entry ２０-３-thumb.jpg + s2020.e080701 - Mock Entry ２０-３.info.json + s2020.e080701 - Mock Entry ２０-３.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-07 + episode_id: 80701 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-3.com + + The Description + title: 2020-08-07 - Mock Entry 20-3 + year: 2020 + s2020.e080801 - Mock Entry ２０-２-thumb.jpg + s2020.e080801 - Mock Entry ２０-２.info.json + s2020.e080801 - Mock Entry ２０-２.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-2.com + + The Description + title: 2020-08-08 - Mock Entry 20-2 + year: 2020 + s2020.e080802 - Mock Entry ２０-１-thumb.jpg + s2020.e080802 - Mock Entry ２０-１.info.json + s2020.e080802 - Mock Entry ２０-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80802 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-1.com + + The Description + title: 2020-08-08 - Mock Entry 20-1 + year: 2020 +{output_directory}/Season 2021 + s2021.e080801 - Mock Entry ２１-１-thumb.jpg + s2021.e080801 - Mock Entry ２１-１.info.json + s2021.e080801 - Mock Entry ２１-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2021-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://21-1.com + + The Description + title: 2021-08-08 - Mock Entry 21-1 + year: 2021 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_exclude.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_exclude.txt new file mode 100644 index 00000000..36d8f389 --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_exclude.txt @@ -0,0 +1,51 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/Season 2020 + s2020.e080801 - Mock Entry ２０-２-thumb.jpg + s2020.e080801 - Mock Entry ２０-２.info.json + s2020.e080801 - Mock Entry ２０-２.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-2.com + + The Description + title: 2020-08-08 - Mock Entry 20-2 + year: 2020 + s2020.e080802 - Mock Entry ２０-１-thumb.jpg + s2020.e080802 - Mock Entry ２０-１.info.json + s2020.e080802 - Mock Entry ２０-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80802 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-1.com + + The Description + title: 2020-08-08 - Mock Entry 20-1 + year: 2020 +{output_directory}/Season 2021 + s2021.e080801 - Mock Entry ２１-１-thumb.jpg + s2021.e080801 - Mock Entry ２１-１.info.json + s2021.e080801 - Mock Entry ２１-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2021-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://21-1.com + + The Description + title: 2021-08-08 - Mock Entry 21-1 + year: 2021 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_include.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_include.txt new file mode 100644 index 00000000..cf4f5275 --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/title_filter_keywords_include.txt @@ -0,0 +1,20 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/Season 2020 + s2020.e080701 - Mock Entry ２０-３-thumb.jpg + s2020.e080701 - Mock Entry ２０-３.info.json + s2020.e080701 - Mock Entry ２０-３.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-07 + episode_id: 80701 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-3.com + + The Description + title: 2020-08-07 - Mock Entry 20-3 + year: 2020 \ No newline at end of file