diff --git a/docs/source/config_reference/prebuilt_presets/common.rst b/docs/source/config_reference/prebuilt_presets/common.rst index 0a6c8e8a..ab1fd545 100644 --- a/docs/source/config_reference/prebuilt_presets/common.rst +++ b/docs/source/config_reference/prebuilt_presets/common.rst @@ -4,6 +4,11 @@ Common .. highlight:: yaml +Filtering +------------- + +.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml + Media Quality ------------- diff --git a/docs/source/prebuilt_presets/helpers.rst b/docs/source/prebuilt_presets/helpers.rst index 745cdb97..6f67693f 100644 --- a/docs/source/prebuilt_presets/helpers.rst +++ b/docs/source/prebuilt_presets/helpers.rst @@ -35,6 +35,34 @@ Add the following preset to download the best available audio and video quality, ``max_1080p`` +Filter Keywords +--------------- + +Include or exclude media with any of the listed keywords in their titles. Both keywords and title/description are lower-cased before filtering. + +``Filter Keywords`` + +.. tip:: + + Use the `~` tilda subscription mode to set a subscription's list override variables. + Tilda mode allows override variables to be set directly underneath it. + + .. code-block:: yaml + Plex TV Show by Date: + + = Documentaries: + "~NOVA PBS": + url: "https://www.youtube.com/@novapbs" + title_exclude_keywords: + - "preview" + - "trailer" + + "~To Catch a Smuggler": + url: "https://www.youtube.com/@NatGeo" + title_include_keywords: + - "To Catch a Smuggler" + + Chunk Initial Download ---------------------- diff --git a/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml b/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml new file mode 100644 index 00000000..b964cc8c --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml @@ -0,0 +1,52 @@ +presets: + + ############################################################################# + # Include Keywords + # Include or exclude media with any of the listed keywords in their titles + # Keywords will check a lower-cased title or description + Filter Keywords: + overrides: + # default filter lists to be empty + title_include_keywords: "{ [] }" + title_exclude_keywords: "{ [] }" + description_include_keywords: "{ [] }" + description_exclude_keywords: "{ [] }" + + "%ensure_string": >- + { + %assert_then( + %is_string($0), + %lower($0), + "filter keywords must be strings" + ) + } + "%ensure_lower_array": >- + { + %assert_then( + %is_array($0), + %array_apply( + $0, + %ensure_string + ), + %concat($1," must be an array") + ) + } + + # $0 - var to evaluate + # $1 - keyword list + # $2 - variable name for error messages + # $3 - default return if keyword list is empty + "%contains_keywords": >- + { + %if( + %bool( $1 ), + %contains_any( %lower($0), %ensure_lower_array($1, $2) ), + $3 + ) + } + + filter_exclude: + - "{ %not( %contains_keywords(title, title_include_keywords, 'title_include_keywords', true) ) }" + - "{ %not( %contains_keywords(description, description_include_keywords, 'description_include_keywords', true) ) }" + - "{ %contains_keywords(title, title_exclude_keywords, 'title_exclude_keywords', false) }" + - "{ %contains_keywords(description, description_exclude_keywords, 'description_exclude_keywords',false) }" \ No newline at end of file diff --git a/tests/integration/prebuilt_presets/test_filter_keywords.py b/tests/integration/prebuilt_presets/test_filter_keywords.py new file mode 100644 index 00000000..5c68cbf5 --- /dev/null +++ b/tests/integration/prebuilt_presets/test_filter_keywords.py @@ -0,0 +1,130 @@ +import re + +import pytest + +from expected_transaction_log import assert_transaction_log_matches +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import ValidationException + + +@pytest.fixture +def filter_subscription_dict(output_directory): + return { + "preset": [ + "Plex TV Show by Date", + "Filter Keywords", + ], + "overrides": { + "url": "https://your.name.here", + "tv_show_directory": output_directory + } + } + +class TestFilterKeywords: + + def test_no_overrides( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + ): + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/filter_keywords_empty.txt", + ) + + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) + def test_title( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + filter_mode: str, + ): + filter_subscription_dict["overrides"][f"title_{filter_mode}_keywords"] = [ + "not included", + "MOCK ENTRY 20-3" + ] + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/filter_keywords_{filter_mode}.txt", + ) + + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) + def test_description( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + filter_mode: str, + ): + filter_subscription_dict["overrides"][f"description_{filter_mode}_keywords"] = [ + "not included", + "MOCK ENTRY 20-3" + ] + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/filter_keywords_{filter_mode}.txt", + ) + + def test_error_not_list_type( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + ): + filter_subscription_dict["overrides"][f"description_include_keywords"] = "not list" + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) \ No newline at end of file diff --git a/tests/resources.py b/tests/resources.py index 8a426c9e..bb9808fd 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -2,7 +2,7 @@ import os import shutil from pathlib import Path -REGENERATE_FIXTURES: bool = False +REGENERATE_FIXTURES: bool = True RESOURCE_PATH: Path = Path("tests") / "resources" _FILE_FIXTURE_PATH: Path = RESOURCE_PATH / "file_fixtures" diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt new file mode 100644 index 00000000..8c3ebf9a --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_empty.txt @@ -0,0 +1,66 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/Season 2020 + s2020.e080701 - Mock Entry ２０-３-thumb.jpg + s2020.e080701 - Mock Entry ２０-３.info.json + s2020.e080701 - Mock Entry ２０-３.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-07 + episode_id: 80701 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-3.com + + The Description + title: 2020-08-07 - Mock Entry 20-3 + year: 2020 + s2020.e080801 - Mock Entry ２０-２-thumb.jpg + s2020.e080801 - Mock Entry ２０-２.info.json + s2020.e080801 - Mock Entry ２０-２.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-2.com + + The Description + title: 2020-08-08 - Mock Entry 20-2 + year: 2020 + s2020.e080802 - Mock Entry ２０-１-thumb.jpg + s2020.e080802 - Mock Entry ２０-１.info.json + s2020.e080802 - Mock Entry ２０-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80802 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-1.com + + The Description + title: 2020-08-08 - Mock Entry 20-1 + year: 2020 +{output_directory}/Season 2021 + s2021.e080801 - Mock Entry ２１-１-thumb.jpg + s2021.e080801 - Mock Entry ２１-１.info.json + s2021.e080801 - Mock Entry ２１-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2021-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://21-1.com + + The Description + title: 2021-08-08 - Mock Entry 21-1 + year: 2021 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_exclude.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_exclude.txt new file mode 100644 index 00000000..8c3ebf9a --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_exclude.txt @@ -0,0 +1,66 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/Season 2020 + s2020.e080701 - Mock Entry ２０-３-thumb.jpg + s2020.e080701 - Mock Entry ２０-３.info.json + s2020.e080701 - Mock Entry ２０-３.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-07 + episode_id: 80701 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-3.com + + The Description + title: 2020-08-07 - Mock Entry 20-3 + year: 2020 + s2020.e080801 - Mock Entry ２０-２-thumb.jpg + s2020.e080801 - Mock Entry ２０-２.info.json + s2020.e080801 - Mock Entry ２０-２.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-2.com + + The Description + title: 2020-08-08 - Mock Entry 20-2 + year: 2020 + s2020.e080802 - Mock Entry ２０-１-thumb.jpg + s2020.e080802 - Mock Entry ２０-１.info.json + s2020.e080802 - Mock Entry ２０-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2020-08-08 + episode_id: 80802 + genre: ytdl-sub + show: subscription_test + synopsis: + https://20-1.com + + The Description + title: 2020-08-08 - Mock Entry 20-1 + year: 2020 +{output_directory}/Season 2021 + s2021.e080801 - Mock Entry ２１-１-thumb.jpg + s2021.e080801 - Mock Entry ２１-１.info.json + s2021.e080801 - Mock Entry ２１-１.mp4 + Video Tags: + contentRating: TV-14 + date: 2021-08-08 + episode_id: 80801 + genre: ytdl-sub + show: subscription_test + synopsis: + https://21-1.com + + The Description + title: 2021-08-08 - Mock Entry 21-1 + year: 2021 \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_include.txt b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_include.txt new file mode 100644 index 00000000..7192b6c7 --- /dev/null +++ b/tests/resources/transaction_log_summaries/integration/prebuilt_presets/filter_keywords_include.txt @@ -0,0 +1 @@ +No new, modified, or removed files in '{output_directory}' \ No newline at end of file