moved tests

src/ytdl_sub/config/preset_class_mappings.py

@@ -11,6 +11,7 @@ from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader
 from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader
 from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin
+from ytdl_sub.plugins.date_range import DateRangePlugin
 from ytdl_sub.plugins.music_tags import MusicTagsPlugin
 from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
 from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
@@ -111,6 +112,7 @@ class PluginMapping:
 
     _MAPPING: Dict[str, Type[Plugin]] = {
         "audio_extract": AudioExtractPlugin,
+        "date_range": DateRangePlugin,
         "music_tags": MusicTagsPlugin,
         "video_tags": VideoTagsPlugin,
         "nfo_tags": NfoTagsPlugin,

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -10,6 +10,7 @@ from ytdl_sub.downloaders.downloader import Downloader
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin
+from ytdl_sub.plugins.date_range import DateRangePlugin
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.subtitles import SubtitlesPlugin
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@@ -111,6 +112,13 @@ class SubscriptionYTDLOptions:
 
         return chapters_plugin.ytdl_options()
 
+    @property
+    def _date_range_options(self) -> Dict:
+        if not (date_range_plugin := self._get_plugin(DateRangePlugin)):
+            return {}
+
+        return date_range_plugin.ytdl_options()
+
     @property
     def _user_ytdl_options(self) -> Dict:
         return self._preset.ytdl_options.dict
@@ -125,18 +133,20 @@ class SubscriptionYTDLOptions:
         ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options)
         if self._dry_run:
             ytdl_options_builder.add(
+                self._date_range_options,
                 self._subtitle_options,
                 self._chapter_options,
-                self._user_ytdl_options,
+                self._user_ytdl_options,  # user ytdl options...
-                self._dry_run_options,
+                self._dry_run_options,  # then dry-run
             )
         else:
             ytdl_options_builder.add(
                 self._output_options,
+                self._date_range_options,
                 self._subtitle_options,
                 self._chapter_options,
                 self._audio_extract_options,
-                self._user_ytdl_options,
+                self._user_ytdl_options,  # user ytdl options last
             )
 
         return ytdl_options_builder

tests/e2e/plugins/test_date_range.py

@@ -0,0 +1,201 @@
+import copy
+
+import mergedeep
+import pytest
+from conftest import assert_debug_log
+from e2e.expected_download import assert_expected_downloads
+from e2e.expected_transaction_log import assert_transaction_log_matches
+
+import ytdl_sub.downloaders.downloader
+from ytdl_sub.subscriptions.subscription import Subscription
+
+
+@pytest.fixture
+def recent_preset_dict(output_directory):
+    return {
+        "preset": "yt_channel_as_tv",
+        "youtube": {"channel_url": "https://youtube.com/channel/UCcRSMoQqXc_JrBZRHDFGbqA"},
+        "date_range": {"after": "20150101"},
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory},
+        # download the worst format so it is fast
+        "ytdl_options": {
+            "format": "worst[ext=mp4]",
+            "max_views": 100000,  # do not download the popular PJ concert
+        },
+        "subtitles": {
+            "subtitles_name": "{episode_name}.{lang}.{subtitles_ext}",
+            "allow_auto_generated_subtitles": True,
+        },
+        "overrides": {"tv_show_name": "Project / Zombie"},
+    }
+
+
+@pytest.fixture
+def rolling_recent_channel_preset_dict(recent_preset_dict):
+    preset_dict = copy.deepcopy(recent_preset_dict)
+    return mergedeep.merge(
+        preset_dict,
+        {
+            "output_options": {"keep_files_after": "20181101"},
+        },
+    )
+
+
+class TestDateRange:
+    @pytest.mark.parametrize("dry_run", [True, False])
+    def test_recent_channel_download(
+        self,
+        recent_preset_dict,
+        channel_as_tv_show_config,
+        output_directory,
+        dry_run,
+    ):
+        recent_channel_subscription = Subscription.from_dict(
+            config=channel_as_tv_show_config,
+            preset_name="recent",
+            preset_dict=recent_preset_dict,
+        )
+
+        transaction_log = recent_channel_subscription.download(dry_run=dry_run)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/date_range/test_channel_recent.txt",
+        )
+        assert_expected_downloads(
+            output_directory=output_directory,
+            dry_run=dry_run,
+            expected_download_summary_file_name="plugins/date_range/test_channel_recent.json",
+        )
+        if not dry_run:
+            # try downloading again, ensure nothing more was downloaded
+            with assert_debug_log(
+                logger=ytdl_sub.downloaders.downloader.download_logger,
+                expected_message="ExistingVideoReached, stopping additional downloads",
+            ):
+                transaction_log = recent_channel_subscription.download()
+                assert_transaction_log_matches(
+                    output_directory=output_directory,
+                    transaction_log=transaction_log,
+                    transaction_log_summary_file_name=("plugins/date_range/no_downloads.txt"),
+                )
+                assert_expected_downloads(
+                    output_directory=output_directory,
+                    dry_run=dry_run,
+                    expected_download_summary_file_name="plugins/date_range/test_channel_recent.json",
+                )
+
+    @pytest.mark.parametrize("dry_run", [True, False])
+    def test_recent_channel_download__no_vids_in_range(
+        self,
+        channel_as_tv_show_config,
+        recent_preset_dict,
+        output_directory,
+        dry_run,
+    ):
+        recent_preset_dict["date_range"]["after"] = "21000101"
+
+        recent_channel_no_vids_in_range_subscription = Subscription.from_dict(
+            config=channel_as_tv_show_config,
+            preset_name="recent",
+            preset_dict=recent_preset_dict,
+        )
+
+        # Run twice, ensure nothing changes between runsyoutube
+        for _ in range(2):
+            transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
+            assert_transaction_log_matches(
+                output_directory=output_directory,
+                transaction_log=transaction_log,
+                transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
+            )
+            assert_expected_downloads(
+                output_directory=output_directory,
+                dry_run=dry_run,
+                expected_download_summary_file_name="plugins/date_range/no_downloads.json",
+            )
+
+    @pytest.mark.parametrize("dry_run", [True, False])
+    def test_rolling_recent_channel_download(
+        self,
+        channel_as_tv_show_config,
+        recent_preset_dict,
+        rolling_recent_channel_preset_dict,
+        output_directory,
+        dry_run,
+    ):
+        recent_channel_subscription = Subscription.from_dict(
+            config=channel_as_tv_show_config,
+            preset_name="recent",
+            preset_dict=recent_preset_dict,
+        )
+        rolling_recent_channel_subscription = Subscription.from_dict(
+            config=channel_as_tv_show_config,
+            preset_name="recent",
+            preset_dict=rolling_recent_channel_preset_dict,
+        )
+
+        # First, download recent vids. Always download since we want to test dry-run
+        # on the rolling recent portion.
+        with assert_debug_log(
+            logger=ytdl_sub.downloaders.downloader.download_logger,
+            expected_message="RejectedVideoReached, stopping additional downloads",
+        ):
+            transaction_log = recent_channel_subscription.download(dry_run=False)
+
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/date_range/test_channel_recent.txt",
+        )
+        assert_expected_downloads(
+            output_directory=output_directory,
+            dry_run=False,
+            expected_download_summary_file_name="plugins/date_range/test_channel_recent.json",
+        )
+
+        # Then, download the rolling recent vids subscription. This should remove one of the
+        # two videos
+        with assert_debug_log(
+            logger=ytdl_sub.downloaders.downloader.download_logger,
+            expected_message="ExistingVideoReached, stopping additional downloads",
+        ):
+            transaction_log = rolling_recent_channel_subscription.download(dry_run=dry_run)
+
+        expected_downloads_summary = (
+            "plugins/date_range/test_channel_recent.json"
+            if dry_run
+            else "plugins/date_range/test_channel_rolling_recent.json"
+        )
+
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="plugins/date_range/test_channel_rolling_recent.txt",
+        )
+        assert_expected_downloads(
+            output_directory=output_directory,
+            dry_run=False,
+            expected_download_summary_file_name=expected_downloads_summary,
+        )
+
+        # Invoke the rolling download again, ensure downloading stopped early from it already
+        # existing
+        if not dry_run:
+            with assert_debug_log(
+                logger=ytdl_sub.downloaders.downloader.download_logger,
+                expected_message="ExistingVideoReached, stopping additional downloads",
+            ):
+                transaction_log = rolling_recent_channel_subscription.download()
+
+            assert_transaction_log_matches(
+                output_directory=output_directory,
+                transaction_log=transaction_log,
+                transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
+            )
+            assert_expected_downloads(
+                output_directory=output_directory,
+                dry_run=False,
+                expected_download_summary_file_name=expected_downloads_summary,
+            )

tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json

@@ -1,5 +1,5 @@
 {
-  ".ytdl-sub-pz-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
+  ".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
   "fanart.jpg": "129c6639b47299bc48062f0365e670ee",
   "poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
   "tvshow.nfo": "85cef9db54e9afb7af8e4912b5262d3f"

tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json

@@ -1,5 +1,5 @@
 {
-  ".ytdl-sub-pz-download-archive.json": "756b60d7a6c47d8163e3283404493a8d",
+  ".ytdl-sub-recent-download-archive.json": "756b60d7a6c47d8163e3283404493a8d",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82",

tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json

@@ -1,5 +1,5 @@
 {
-  ".ytdl-sub-pz-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08",
+  ".ytdl-sub-recent-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",

tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt

@@ -1,6 +1,6 @@
 Files created in '{output_directory}'
 ----------------------------------------
-.ytdl-sub-pz-download-archive.json
+.ytdl-sub-recent-download-archive.json
 fanart.jpg
 poster.jpg
 tvshow.nfo

tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_recent.txt

@@ -1,6 +1,6 @@
 Files created in '{output_directory}'
 ----------------------------------------
-.ytdl-sub-pz-download-archive.json
+.ytdl-sub-recent-download-archive.json
 Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg
 Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4
 Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo

tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt

@@ -1,6 +1,6 @@
 Files created in '{output_directory}'
 ----------------------------------------
-.ytdl-sub-pz-download-archive.json
+.ytdl-sub-recent-download-archive.json
 fanart.jpg
 poster.jpg
 tvshow.nfo

tests/e2e/youtube/test_channel.py

@@ -1,21 +1,10 @@
-import copy
-from typing import Dict
-
-import mergedeep
 import pytest
-from conftest import assert_debug_log
 from e2e.expected_download import assert_expected_downloads
 from e2e.expected_transaction_log import assert_transaction_log_matches
 
-import ytdl_sub.downloaders.downloader
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
-@pytest.fixture
-def subscription_name():
-    return "pz"
-
-
 @pytest.fixture
 def channel_preset_dict(output_directory):
     return {
@@ -37,46 +26,6 @@ def channel_preset_dict(output_directory):
     }
 
 
-@pytest.fixture
-def channel_subscription_generator(channel_as_tv_show_config, subscription_name):
-    def _channel_subscription_generator(preset_dict: Dict):
-        return Subscription.from_dict(
-            config=channel_as_tv_show_config, preset_name=subscription_name, preset_dict=preset_dict
-        )
-
-    return _channel_subscription_generator
-
-
-####################################################################################################
-# RECENT CHANNEL FIXTURES
-@pytest.fixture
-def recent_channel_preset_dict(channel_preset_dict):
-    # TODO: remove this hack by using a different channel
-    channel_preset_dict = copy.deepcopy(channel_preset_dict)
-    del channel_preset_dict["ytdl_options"]["break_on_reject"]
-    return mergedeep.merge(
-        channel_preset_dict,
-        {
-            "preset": "yt_channel_as_tv__recent",
-            "youtube": {"after": "20150101"},
-        },
-    )
-
-
-####################################################################################################
-# ROLLING RECENT CHANNEL FIXTURES
-@pytest.fixture
-def rolling_recent_channel_preset_dict(recent_channel_preset_dict):
-    recent_channel_preset_dict = copy.deepcopy(recent_channel_preset_dict)
-    return mergedeep.merge(
-        recent_channel_preset_dict,
-        {
-            "preset": "yt_channel_as_tv__only_recent",
-            "output_options": {"keep_files_after": "20181101"},
-        },
-    )
-
-
 class TestChannelAsKodiTvShow:
     """
     Downloads my old minecraft youtube channel. Ensure the above files exist and have the
@@ -86,12 +35,14 @@ class TestChannelAsKodiTvShow:
     @pytest.mark.parametrize("dry_run", [True, False])
     def test_full_channel_download(
         self,
-        channel_subscription_generator,
+        channel_as_tv_show_config,
         channel_preset_dict,
         output_directory,
         dry_run,
     ):
-        full_channel_subscription = channel_subscription_generator(preset_dict=channel_preset_dict)
+        full_channel_subscription = Subscription.from_dict(
+            config=channel_as_tv_show_config, preset_name="pz", preset_dict=channel_preset_dict
+        )
         transaction_log = full_channel_subscription.download(dry_run=dry_run)
         assert_transaction_log_matches(
             output_directory=output_directory,
@@ -103,153 +54,3 @@ class TestChannelAsKodiTvShow:
             dry_run=dry_run,
             expected_download_summary_file_name="youtube/test_channel_full.json",
         )
-
-    @pytest.mark.parametrize("dry_run", [True, False])
-    def test_recent_channel_download(
-        self,
-        channel_subscription_generator,
-        recent_channel_preset_dict,
-        output_directory,
-        dry_run,
-    ):
-        recent_channel_subscription = channel_subscription_generator(
-            preset_dict=recent_channel_preset_dict
-        )
-
-        transaction_log = recent_channel_subscription.download(dry_run=dry_run)
-        assert_transaction_log_matches(
-            output_directory=output_directory,
-            transaction_log=transaction_log,
-            transaction_log_summary_file_name="youtube/test_channel_recent.txt",
-        )
-        assert_expected_downloads(
-            output_directory=output_directory,
-            dry_run=dry_run,
-            expected_download_summary_file_name="youtube/test_channel_recent.json",
-        )
-        if not dry_run:
-            # try downloading again, ensure nothing more was downloaded
-            with assert_debug_log(
-                logger=ytdl_sub.downloaders.downloader.download_logger,
-                expected_message="ExistingVideoReached, stopping additional downloads",
-            ):
-                transaction_log = recent_channel_subscription.download()
-                assert_transaction_log_matches(
-                    output_directory=output_directory,
-                    transaction_log=transaction_log,
-                    transaction_log_summary_file_name=(
-                        "youtube/test_channel_no_additional_downloads.txt"
-                    ),
-                )
-                assert_expected_downloads(
-                    output_directory=output_directory,
-                    dry_run=dry_run,
-                    expected_download_summary_file_name="youtube/test_channel_recent.json",
-                )
-
-    @pytest.mark.parametrize("dry_run", [True, False])
-    def test_recent_channel_download__no_vids_in_range(
-        self,
-        channel_subscription_generator,
-        recent_channel_preset_dict,
-        output_directory,
-        dry_run,
-    ):
-        recent_channel_preset_dict["youtube"]["after"] = "21000101"
-
-        recent_channel_no_vids_in_range_subscription = channel_subscription_generator(
-            preset_dict=recent_channel_preset_dict
-        )
-        # Run twice, ensure nothing changes between runs
-        for _ in range(2):
-            transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
-            assert_transaction_log_matches(
-                output_directory=output_directory,
-                transaction_log=transaction_log,
-                transaction_log_summary_file_name="youtube/test_channel_no_additional_downloads.txt",
-            )
-            assert_expected_downloads(
-                output_directory=output_directory,
-                dry_run=dry_run,
-                expected_download_summary_file_name="youtube/test_channel_no_additional_downloads.json",
-            )
-
-    @pytest.mark.parametrize("dry_run", [True, False])
-    def test_rolling_recent_channel_download(
-        self,
-        channel_subscription_generator,
-        recent_channel_preset_dict,
-        rolling_recent_channel_preset_dict,
-        output_directory,
-        dry_run,
-    ):
-        recent_channel_subscription = channel_subscription_generator(
-            preset_dict=recent_channel_preset_dict
-        )
-        rolling_recent_channel_subscription = channel_subscription_generator(
-            preset_dict=rolling_recent_channel_preset_dict
-        )
-
-        # First, download recent vids. Always download since we want to test dry-run
-        # on the rolling recent portion.
-        with assert_debug_log(
-            logger=ytdl_sub.downloaders.downloader.download_logger,
-            expected_message="RejectedVideoReached, stopping additional downloads",
-        ):
-            transaction_log = recent_channel_subscription.download(dry_run=False)
-
-        assert_transaction_log_matches(
-            output_directory=output_directory,
-            transaction_log=transaction_log,
-            transaction_log_summary_file_name="youtube/test_channel_recent.txt",
-        )
-        assert_expected_downloads(
-            output_directory=output_directory,
-            dry_run=False,
-            expected_download_summary_file_name="youtube/test_channel_recent.json",
-        )
-
-        # Then, download the rolling recent vids subscription. This should remove one of the
-        # two videos
-        with assert_debug_log(
-            logger=ytdl_sub.downloaders.downloader.download_logger,
-            expected_message="ExistingVideoReached, stopping additional downloads",
-        ):
-            transaction_log = rolling_recent_channel_subscription.download(dry_run=dry_run)
-
-        expected_downloads_summary = (
-            "youtube/test_channel_recent.json"
-            if dry_run
-            else "youtube/test_channel_rolling_recent.json"
-        )
-
-        assert_transaction_log_matches(
-            output_directory=output_directory,
-            transaction_log=transaction_log,
-            transaction_log_summary_file_name="youtube/test_channel_rolling_recent.txt",
-        )
-        assert_expected_downloads(
-            output_directory=output_directory,
-            dry_run=False,
-            expected_download_summary_file_name=expected_downloads_summary,
-        )
-
-        # Invoke the rolling download again, ensure downloading stopped early from it already
-        # existing
-        if not dry_run:
-            with assert_debug_log(
-                logger=ytdl_sub.downloaders.downloader.download_logger,
-                expected_message="ExistingVideoReached, stopping additional downloads",
-            ):
-                transaction_log = rolling_recent_channel_subscription.download()
-
-            assert_transaction_log_matches(
-                output_directory=output_directory,
-                transaction_log=transaction_log,
-                transaction_log_summary_file_name="youtube/test_channel_no_additional_downloads.txt",
-            )
-            assert_expected_downloads(
-                output_directory=output_directory,
-                dry_run=False,
-                expected_download_summary_file_name=expected_downloads_summary,
-            )