diff --git a/tests/resources.py b/tests/resources.py index 8a426c9e..bb9808fd 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -2,7 +2,7 @@ import os import shutil from pathlib import Path -REGENERATE_FIXTURES: bool = False +REGENERATE_FIXTURES: bool = True RESOURCE_PATH: Path = Path("tests") / "resources" _FILE_FIXTURE_PATH: Path = RESOURCE_PATH / "file_fixtures" diff --git a/tests/resources/transaction_log_summaries/plugins/filter/test_regex.txt b/tests/resources/transaction_log_summaries/plugins/filter/test_regex.txt new file mode 100644 index 00000000..a4c9bb7c --- /dev/null +++ b/tests/resources/transaction_log_summaries/plugins/filter/test_regex.txt @@ -0,0 +1,105 @@ +Files created: +---------------------------------------- +{output_directory} + .ytdl-sub-subscription_test-download-archive.json +{output_directory}/subscription_test + Mock Entry 20-1.info.json + Mock Entry 20-1.jpg + Mock Entry 20-1.mp4 + Video Tags: + album: Music Videos + artist: subscription_test + genre: ytdl-sub + premiered: 2020-08-08 + title: Mock Entry 20-1 + year: 2020 + Mock Entry 20-1.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: subscription_test + desc_cap: Description + genre: ytdl-sub + override_with_capture_variable: contains 20 + override_with_capture_variable_sanitized: contains 20 + premiered: 2020-08-08 + title: Mock Entry 20-1 + title_cap_1: 20 + title_cap_1_sanitized: 20 + title_cap_2: 1 + upload_date_both_caps: First and Second containing in regex default + Mock Entry 20-2.info.json + Mock Entry 20-2.jpg + Mock Entry 20-2.mp4 + Video Tags: + album: Music Videos + artist: subscription_test + genre: ytdl-sub + premiered: 2020-08-08 + title: Mock Entry 20-2 + year: 2020 + Mock Entry 20-2.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: subscription_test + desc_cap: Description + genre: ytdl-sub + override_with_capture_variable: contains 20 + override_with_capture_variable_sanitized: contains 20 + premiered: 2020-08-08 + title: Mock Entry 20-2 + title_cap_1: 20 + title_cap_1_sanitized: 20 + title_cap_2: 2 + upload_date_both_caps: First and Second containing in regex default + Mock Entry 20-3.info.json + Mock Entry 20-3.jpg + Mock Entry 20-3.mp4 + Video Tags: + album: Music Videos + artist: subscription_test + genre: ytdl-sub + premiered: 2020-08-07 + title: Mock Entry 20-3 + year: 2020 + Mock Entry 20-3.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: subscription_test + desc_cap: Description + genre: ytdl-sub + override_with_capture_variable: contains 20 + override_with_capture_variable_sanitized: contains 20 + premiered: 2020-08-07 + title: Mock Entry 20-3 + title_cap_1: 20 + title_cap_1_sanitized: 20 + title_cap_2: 3 + upload_date_both_caps: First and Second containing in regex default + Mock Entry 21-1.info.json + Mock Entry 21-1.jpg + Mock Entry 21-1.mp4 + Video Tags: + album: Music Videos + artist: subscription_test + genre: ytdl-sub + premiered: 2021-08-08 + title: Mock Entry 21-1 + year: 2021 + Mock Entry 21-1.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: subscription_test + desc_cap: Description + genre: ytdl-sub + override_with_capture_variable: contains 21 + override_with_capture_variable_sanitized: contains 21 + premiered: 2021-08-08 + title: Mock Entry 21-1 + title_cap_1: 21 + title_cap_1_sanitized: 21 + title_cap_2: 1 + upload_date_both_caps: First and Second containing in regex default \ No newline at end of file diff --git a/tests/unit/plugins/test_filter.py b/tests/unit/plugins/test_filter.py new file mode 100644 index 00000000..6ca968b9 --- /dev/null +++ b/tests/unit/plugins/test_filter.py @@ -0,0 +1,304 @@ +import re +from typing import Any +from typing import Dict + +import mergedeep +import pytest +from expected_transaction_log import assert_transaction_log_matches +from unit.conftest import mock_download_collection_entries + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def regex_subscription_dict_base(output_directory): + return { + "preset": "Jellyfin Music Videos", + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + "format": "best[height<=480]", # download the worst format so it is fast + "filter_include": ["{%not(%is_null(description_website))}"], + "overrides": { + "in_regex_default": "in regex default", + "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "upload_capture": """{ + %regex_capture_many_with_defaults( + upload_date_standardized, + ["([0-9]+)-([0-9]+)-27"], + ["First", %concat("Second containing ", in_regex_default)] + ) + }""", + "upload_captured_year": "{%array_at(upload_capture, 1)}", + "upload_captured_month": "{%array_at(upload_capture, 2)}", + "description_capture": """{ + %regex_capture_many_with_defaults( + description, + ["The (.*)"], + [null] + ) + }""", + "description_website": "{%array_at(description_capture, 1)}", + }, + } + + +@pytest.fixture +def regex_subscription_dict(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "filter_exclude": ["{%is_null(title_type)}"], + "overrides": { + "title_capture_list": """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", "Mock Entry (.*)-(.*)" ], + [ null, null ] + ) + }""", + "title_type": "{%array_at(title_capture_list, 1)}", + "title_date": "{%array_at(title_capture_list, 2)}", + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "filter_exclude": [ + """{ + %regex_search_any( + title, + [ "should not cap", ".*Feb.*" ] + ) + }""" + ] + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + "exclude": [ + "should not cap", + ".*27.*", # should filter out Feb 27th video + ], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude_override_variable( + regex_subscription_dict_base, output_directory +): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "override_title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + "override_description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["override_description_website"], + }, + }, + }, + "overrides": {"override_title": "{title}", "override_description": "{description}"}, + }, + ) + + +@pytest.fixture +def playlist_subscription(config: ConfigFile, subscription_name: str, regex_subscription_dict: Dict[str, Any]): + return Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_no_match_fails( + config: ConfigFile, subscription_name: str, regex_subscription_dict: Dict[str, Any] +): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_required( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ] + ) + }""" + + return Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_exclude( + config: ConfigFile, subscription_name: str, regex_subscription_dict_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=regex_subscription_dict_exclude, + ) + + +@pytest.fixture +def playlist_subscription_overrides( + config: ConfigFile, + subscription_name: str, + regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any], +) -> Subscription: + return Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=regex_subscription_dict_match_and_exclude_override_variable, + ) + + +@pytest.fixture +def playlist_subscription_match_and_exclude( + config: ConfigFile, subscription_name: str, regex_subscription_dict_match_and_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=regex_subscription_dict_match_and_exclude, + ) + + +class TestFilter: + def test_regex_success(self, mock_download_collection_entries, playlist_subscription, output_directory): + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_extracted_audio=False, is_dry_run=True + ): + transaction_log = playlist_subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/filter/test_regex.txt", + ) + + def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_exclude.txt", + ) + + def test_regex_match_and_excludes_success( + self, playlist_subscription_match_and_exclude, output_directory + ): + # Should only contain the Feb 1st video + transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", + ) + + def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_overrides.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_overrides.txt", + ) + + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): + with pytest.raises( + RuntimeException, + match=re.escape( + "no regex strings were captured for input string Jesse's Minecraft Server [Trailer - Mar.21]" + ), + ): + _ = playlist_subscription_no_match_fails.download(dry_run=True) + + def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ], + [ "one default, expects >= 2" ] + ) + }""" + + subscription = Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + with pytest.raises( + RuntimeException, + match=re.escape( + "When using %regex_capture_array, number of regex capture groups must be less than or equal to the number of defaults" + ), + ): + _ = subscription.download(dry_run=True)