Merge branch 'master' into j/string-validator

docker/Dockerfile

@@ -15,6 +15,7 @@ RUN mkdir -p /config && \
       g++ \
       nano \
       make \
+      libffi-dev \
       "python3>=3.10" \
       py3-pip \
       fontconfig \
@@ -46,6 +47,7 @@ RUN mkdir -p /config && \
     apk del \
       g++ \
       make \
+      libffi-dev \
       py3-pip \
       py3-setuptools
 

setup.cfg

@@ -27,7 +27,7 @@ package_dir =
 packages=find:
 
 install_requires =
-    yt-dlp==2023.10.13
+    yt-dlp==2023.11.16
     argparse==1.4.0
     colorama==0.4.6
     mergedeep==1.3.4

tests/conftest.py

@@ -14,6 +14,9 @@ from unittest.mock import patch
 
 import pytest
 from expected_download import _get_files_in_directory
+from resources import copy_file_fixture
+from resources import file_fixture_path
+from yt_dlp.utils import sanitize_filename
 
 from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.subscriptions.subscription_download import SubscriptionDownload
@@ -67,13 +70,13 @@ def working_directory() -> str:
 
 
 @pytest.fixture()
-def output_directory() -> Path:
+def output_directory() -> str:
     with tempfile.TemporaryDirectory() as temp_dir:
         yield temp_dir
 
 
 @pytest.fixture()
-def reformat_directory() -> Path:
+def reformat_directory() -> str:
     with tempfile.TemporaryDirectory() as temp_dir:
         yield temp_dir
 
@@ -163,6 +166,38 @@ def preset_dict_to_subscription_yaml_generator() -> Callable:
     return _preset_dict_to_subscription_yaml_generator
 
 
+###################################################################################################
+# Staging a mock already-existing download
+
+
+@pytest.fixture
+def pz_channel_mock_downloaded_with_archive_factory(output_directory: Path) -> Callable:
+    def _pz_channel_mock_downloaded_with_archive_factory(tv_show_name: str, archive_file_name: str):
+        subscription_path = Path(output_directory) / sanitize_filename(tv_show_name)
+        copy_file_fixture(
+            fixture_name="pz_download_archive.json",
+            output_file_path=subscription_path / archive_file_name,
+        )
+
+        with open(
+            file_fixture_path("pz_download_archive.json"), "r", encoding="utf-8"
+        ) as archive_file:
+            archive_dict = json.load(archive_file)
+
+        assert isinstance(archive_dict, dict)
+        for uid, metadata in archive_dict.items():
+            assert isinstance(metadata, dict)
+            for filename in metadata["file_names"]:
+                assert isinstance(filename, str)
+                output_file_path = subscription_path / filename
+                if filename.endswith(".mp4"):
+                    copy_file_fixture("sample_vid.mp4", output_file_path)
+                else:
+                    copy_file_fixture("empty.txt", output_file_path)
+
+    return _pz_channel_mock_downloaded_with_archive_factory
+
+
 ###################################################################################################
 # Example config fixtures
 

tests/e2e/youtube/test_channel.py

@@ -1,7 +1,11 @@
+from typing import Callable
+from typing import Dict
+
 import pytest
 from expected_download import assert_expected_downloads
 from expected_transaction_log import assert_transaction_log_matches
 
+from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
@@ -65,3 +69,22 @@ class TestChannel:
             dry_run=dry_run,
             expected_download_summary_file_name="youtube/test_channel_full.json",
         )
+
+    def test_full_channel_existing_archive_downloads_nothing(
+        self,
+        pz_channel_mock_downloaded_with_archive_factory: Callable,
+        tv_show_config: ConfigFile,
+        channel_preset_dict: Dict,
+    ):
+        subscription_name = "pz"
+        full_channel_subscription = Subscription.from_dict(
+            config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
+        )
+        tv_show_name = channel_preset_dict["overrides"]["tv_show_name"]
+        archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json"
+
+        pz_channel_mock_downloaded_with_archive_factory(
+            tv_show_name=tv_show_name, archive_file_name=archive_file_name
+        )
+        transaction_log = full_channel_subscription.download(dry_run=True)
+        assert transaction_log.is_empty

tests/expected_transaction_log.py

@@ -11,7 +11,7 @@ _TRANSACTION_LOG_SUMMARY_PATH = RESOURCE_PATH / "transaction_log_summaries"
 
 
 def assert_transaction_log_matches(
-    output_directory: Path,
+    output_directory: Path | str,
     transaction_log: FileHandlerTransactionLog,
     transaction_log_summary_file_name: str,
 ):

tests/resources.py

@@ -1,3 +1,4 @@
+import os
 import shutil
 from pathlib import Path
 
@@ -7,5 +8,10 @@ RESOURCE_PATH: Path = Path("tests") / "resources"
 _FILE_FIXTURE_PATH: Path = RESOURCE_PATH / "file_fixtures"
 
 
+def file_fixture_path(fixture_name: str) -> Path:
+    return _FILE_FIXTURE_PATH / fixture_name
+
+
 def copy_file_fixture(fixture_name: str, output_file_path: Path) -> None:
-    shutil.copy(_FILE_FIXTURE_PATH / fixture_name, output_file_path)
+    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
+    shutil.copy(file_fixture_path(fixture_name), output_file_path)

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
-  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "de33d8f05d743c1f19091ff8b0115d3d",
+  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "991b0eb62683c2194c5bdfa9a61ab18e",
   "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
 }

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-single_song_best_test-download-archive.json": "f179ccfe0a9b3a76ae62122a3ccb58fd",
-  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.m4a": "0144f43ed99468d8ef38c6374b8784a0",
+  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.m4a": "114b35c29df84146cd98207f807b837b",
   "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
 }

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
-  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "7d0473bd154637a151210c5116a63174",
+  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "11376667a11bb71565b520f8ce5fa303",
   "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
 }

tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json

@@ -1,6 +1,6 @@
 {
   ".ytdl-sub-subtitles_embedded_test-download-archive.json": "a74ecea9f7844be23f470bbe702788f3",
   "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.jpg": "50ee47c80f679029f5d3503bb91b045a",
-  "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "6366af35ac989142a48af4eb89ed2be6",
+  "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "d5df2fa121748a54d6954c58e3d5884f",
   "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "e6ac56ce52c747e2e271f12208f9a538"
 }

tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json

@@ -3,6 +3,6 @@
   "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt": "b343c3bb9257b7ee7ba38f570a115b37",
   "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt": "fe8c6ee92cae6e059fd80fd61691adbe",
   "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.jpg": "50ee47c80f679029f5d3503bb91b045a",
-  "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "6366af35ac989142a48af4eb89ed2be6",
+  "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "d5df2fa121748a54d6954c58e3d5884f",
   "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "e6ac56ce52c747e2e271f12208f9a538"
 }

tests/resources/expected_downloads_summaries/youtube/test_channel_full.json

@@ -34,7 +34,7 @@
   "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479",
   "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
   "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "INFO_JSON",
-  "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4": "0101d64db7720efc133c9bf9beb157ab",
+  "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4": "df6a6901b168cd191c3b6fce2dde8f7a",
   "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo": "e2b078891b27cfee4c791598d046be24",
   "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg": "e29d49433175de8a761af35c5307791f",
   "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json": "INFO_JSON",

tests/resources/file_fixtures/empty.txt

@@ -0,0 +1 @@
+empty

tests/resources/file_fixtures/pz_download_archive.json

@@ -0,0 +1,123 @@
+{
+  "0SVukUyys10": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg",
+      "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json",
+      "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4",
+      "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo"
+    ],
+    "upload_date": "2011-02-01"
+  },
+  "6sxlggREhMc": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c-thumb.jpg",
+      "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c.info.json",
+      "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c.mp4",
+      "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c.nfo"
+    ],
+    "upload_date": "2011-06-30"
+  },
+  "DBjFvs6HafU": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg",
+      "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json",
+      "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4",
+      "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo"
+    ],
+    "upload_date": "2011-03-21"
+  },
+  "HKTNxEqsN3Q": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1-thumb.jpg",
+      "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1.info.json",
+      "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1.mp4",
+      "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1.nfo"
+    ],
+    "upload_date": "2010-08-13"
+  },
+  "IV9Z4wcA-z8": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2-thumb.jpg",
+      "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2.info.json",
+      "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2.mp4",
+      "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2.nfo"
+    ],
+    "upload_date": "2010-12-02"
+  },
+  "LN2e6idGluI": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id-thumb.jpg",
+      "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.en.srt",
+      "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.info.json",
+      "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.mp4",
+      "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.nfo"
+    ],
+    "upload_date": "2018-11-02"
+  },
+  "c_PZdc0Zi7M": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c-thumb.jpg",
+      "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c.info.json",
+      "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c.mp4",
+      "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c.nfo"
+    ],
+    "upload_date": "2013-07-19"
+  },
+  "hP-PL_V2wR8": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer-thumb.jpg",
+      "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer.info.json",
+      "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer.mp4",
+      "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer.nfo"
+    ],
+    "upload_date": "2018-10-29"
+  },
+  "hxV9b-1hjfw": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net)-thumb.jpg",
+      "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net).info.json",
+      "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net).mp4",
+      "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net).nfo"
+    ],
+    "upload_date": "2011-05-29"
+  },
+  "qPybBrXspds": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg",
+      "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json",
+      "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4",
+      "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo"
+    ],
+    "upload_date": "2011-02-27"
+  },
+  "qbMJh2df1M4": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC]-thumb.jpg",
+      "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC].info.json",
+      "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC].mp4",
+      "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC].nfo"
+    ],
+    "upload_date": "2011-11-21"
+  },
+  "y5-3ovwQQ_U": {
+    "extractor": "youtube",
+    "file_names": [
+      "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c-thumb.jpg",
+      "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c.info.json",
+      "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c.mp4",
+      "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c.nfo"
+    ],
+    "upload_date": "2012-01-23"
+  }
+}