From 4d9a30779c3e364306571246fcd37f6a036783ea Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 14 Dec 2023 17:04:03 -0800 Subject: [PATCH] [BUGFIX] Use proper extractor key for download archive (#840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For less popular sites, ytdl-sub's custom download archive was not working. This was because we weren't using the right extraction key (though it still worked for YouTube, Bandcamp, Soundcloud, and others). This bugfix should now exactly match yt-dlp's and never be a problem again. Huge thanks to @Svagtlys for root-causing this 🤝 --- src/ytdl_sub/entries/base_entry.py | 5 ++--- .../soundcloud/test_soundcloud_discography.py | 18 ++++++++++++++++++ .../bandcamp/test_artist_url.json | 2 +- tests/unit/conftest.py | 1 + tests/unit/entries/conftest.py | 1 + 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 697bb1f6..e5e25151 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -13,7 +13,6 @@ from yt_dlp.utils import sanitize_filename from ytdl_sub.entries.variables.kwargs import DESCRIPTION from ytdl_sub.entries.variables.kwargs import EPOCH -from ytdl_sub.entries.variables.kwargs import EXTRACTOR from ytdl_sub.entries.variables.kwargs import IE_KEY from ytdl_sub.entries.variables.kwargs import TITLE from ytdl_sub.entries.variables.kwargs import UID @@ -100,12 +99,12 @@ class BaseEntryVariables: Returns ------- str - The ytdl extractor name + The ytdl extractor name used in the download archive """ # pylint: disable=line-too-long # Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514 # pylint: enable=line-too-long - return self.kwargs_get(EXTRACTOR) or self.kwargs(IE_KEY) + return str(self.kwargs_get("extractor_key") or self.kwargs(IE_KEY)).lower() @property def epoch(self: "BaseEntry") -> int: diff --git a/tests/e2e/soundcloud/test_soundcloud_discography.py b/tests/e2e/soundcloud/test_soundcloud_discography.py index de1f4a4f..8e7f5345 100644 --- a/tests/e2e/soundcloud/test_soundcloud_discography.py +++ b/tests/e2e/soundcloud/test_soundcloud_discography.py @@ -1,7 +1,9 @@ import pytest +from conftest import assert_logs from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches +from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.subscriptions.subscription import Subscription @@ -48,3 +50,19 @@ class TestSoundcloudDiscography: dry_run=dry_run, expected_download_summary_file_name="soundcloud/test_soundcloud_discography.json", ) + + # Ensure another invocation will hit ExistingVideoReached + if not dry_run: + with assert_logs( + logger=YTDLP.logger, + expected_message="ExistingVideoReached, stopping additional downloads", + log_level="debug", + ): + transaction_log = discography_subscription.download() + + assert transaction_log.is_empty + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name="soundcloud/test_soundcloud_discography.json", + ) diff --git a/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json b/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json index d1dd90b1..9c2ea6ef 100644 --- a/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json +++ b/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-Sithu Aye-download-archive.json": "0c58919b660f699f6aea2ec523c812c5", + ".ytdl-sub-Sithu Aye-download-archive.json": "ca37a404a860a2a6b6b0f38b659c9f17", "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/01 - Double Helix Reimagined.mp3": "56f7ee579031f4795230e68b63b15f6b", "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/02 - Skye Reimagined.mp3": "dfb24e0ef03e203d471bb81f854f5cd3", "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/03 - Baryofusion.mp3": "95bd9ab2238e5372f445c59daafd0138", diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index f25845a7..672361dd 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -68,6 +68,7 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable: PLAYLIST_INDEX: playlist_index, PLAYLIST_COUNT: playlist_count, EXTRACTOR: "mock-entry-dict", + "extractor_key": "mock-entry-dict", TITLE: f"Mock Entry {uid}", EXT: "mp4", UPLOAD_DATE: upload_date, diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 16199c7d..f4895fac 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -163,6 +163,7 @@ def mock_entry_kwargs( "id": uid, "epoch": 1596878400, "extractor": extractor, + "extractor_key": extractor, "title": title, "ext": ext, "upload_date": upload_date,