danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Perform match-filter on download, not metadata pull (#612)

Browse source 
* [FEATURE] Apply match-filter on download instead of metadata

* correct ytdl_option ordering

* lint

* test that downloads
This commit is contained in:
Jesse Bannon 2023-05-18 09:52:24 -07:00 committed by GitHub
parent 38e64a5bd5
commit 0497f2b749
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 114 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -11,6 +11,8 @@ from typing import Optional
from typing import Set
from typing import Tuple
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.base_downloader import BaseDownloader
from ytdl_sub.downloaders.base_downloader import BaseDownloaderOptionsT
@ -31,6 +33,7 @@ from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
@ -523,6 +526,11 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
Returns
-------
The entry that was downloaded successfully
Raises
------
RejectedVideoReached
If a video was rejected and was not from match_filter
"""
download_logger.info(
"Downloading entry %d/%d: %s",
@ -530,7 +538,17 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
self._url_state.entries_total,
entry.title,
)
download_entry = self._extract_entry_info_with_retry(entry=entry)
# Match-filters are applied at the download stage (not metadata stage).
# If the download is rejected, and match_filter is present in the ytdl options,
# then filter downstream in the match_filter plugin
try:
download_entry = self._extract_entry_info_with_retry(entry=entry)
except RejectedVideoReached:
if "match_filter" in self.download_ytdl_options:
entry.add_kwargs({YTDL_SUB_MATCH_FILTER_REJECT: True})
return entry
raise
upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
upload_date_standardized=entry.upload_date_standardized

1
src/ytdl_sub/entries/variables/kwargs.py
View file

@ -47,6 +47,7 @@ REQUESTED_SUBTITLES = _("requested_subtitles", backend=True)
CHAPTERS = _("chapters", backend=True)
YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)
YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True)
YTDL_SUB_MATCH_FILTER_REJECT = _("ytdl_sub_match_filter_reject", backend=True)
SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True)
SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True)
COMMENTS = _("comments", backend=True)

19
src/ytdl_sub/plugins/match_filters.py
View file

@ -5,8 +5,11 @@ from typing import Optional
from yt_dlp import match_filter_func
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_MATCH_FILTER_REJECT
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.plugins.plugin import PluginPriority
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.validators import StringListValidator
@ -63,6 +66,7 @@ class MatchFiltersOptions(PluginOptions):
class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
plugin_options_type = MatchFiltersOptions
priority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_FIRST)
def ytdl_options(self) -> Optional[Dict]:
"""
@ -75,4 +79,17 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
logger.debug("Adding match-filter %s", filter_str)
match_filters.append(filter_str)
return {"match_filter": match_filter_func(match_filters)}
return {
"match_filter": match_filter_func(match_filters),
}
def modify_entry(self, entry: Entry) -> Optional[Entry]:
"""
If an entry is marked as not being downloaded due to a match_filter reject,
do not propagate the entry further (especially since there is no entry file!)
"""
if entry.kwargs_get(YTDL_SUB_MATCH_FILTER_REJECT, False):
logger.info("Entry rejected by match-filter, skipping ..")
return None
return entry

4
src/ytdl_sub/plugins/plugin.py
View file

@ -24,7 +24,9 @@ class PluginPriority:
# If modify_entry priority is >= to this value, run after split
MODIFY_ENTRY_AFTER_SPLIT = 10
def __init__(self, modify_entry: int = 0, post_process: int = 0):
MODIFY_ENTRY_FIRST = 0
def __init__(self, modify_entry: int = 5, post_process: int = 5):
self.modify_entry = modify_entry
self.post_process = post_process

8
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -78,6 +78,10 @@ class SubscriptionYTDLOptions:
"writeinfojson": True,
}
@property
def _download_only_options(self) -> Dict:
return {"break_on_reject": True}
@property
def _output_options(self) -> Dict:
ytdl_options = {}
@ -110,7 +114,6 @@ class SubscriptionYTDLOptions:
self._global_options,
self._output_options,
self._plugin_ytdl_options(DateRangePlugin),
self._plugin_ytdl_options(MatchFiltersPlugin),
self._user_ytdl_options, # user ytdl options...
self._info_json_only_options, # then info_json_only options
)
@ -125,12 +128,13 @@ class SubscriptionYTDLOptions:
ytdl_options_builder = YTDLOptionsBuilder().add(
self._global_options,
self._output_options,
self._plugin_ytdl_options(DateRangePlugin),
self._plugin_ytdl_options(MatchFiltersPlugin),
self._plugin_ytdl_options(FileConvertPlugin),
self._plugin_ytdl_options(SubtitlesPlugin),
self._plugin_ytdl_options(ChaptersPlugin),
self._plugin_ytdl_options(AudioExtractPlugin),
self._user_ytdl_options, # user ytdl options...
self._download_only_options, # then download_only options
)
# Add dry run options last if enabled
if self._dry_run:

46
tests/e2e/plugins/test_match_filters.py
View file

@ -1,4 +1,6 @@
import pytest
from expected_download import assert_expected_downloads
from expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription
@ -17,9 +19,23 @@ def preset_dict(output_directory):
}
@pytest.fixture
def playlist_preset_dict(output_directory):
return {
"preset": "music_video",
"download": {"url": "https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
"output_options": {"output_directory": output_directory},
# download the worst format so it is fast
"ytdl_options": {
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
},
"match_filters": {"filters": ["view_count > 20000"]},
}
class TestFileConvert:
@pytest.mark.parametrize("dry_run", [True, False])
def test_match_filters(
def test_match_filters_empty(
self,
music_video_config,
preset_dict,
@ -34,3 +50,31 @@ class TestFileConvert:
transaction_log = subscription.download(dry_run=dry_run)
assert transaction_log.is_empty
@pytest.mark.parametrize("dry_run", [True, False])
def test_match_filters_partial(
self,
music_video_config,
playlist_preset_dict,
output_directory,
dry_run,
):
subscription = Subscription.from_dict(
config=music_video_config,
preset_name="match_filter_test",
preset_dict=playlist_preset_dict,
)
transaction_log = subscription.download(dry_run=dry_run)
summary_path = "plugins/match_filters"
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name=f"{summary_path}/test_match_filters_partial.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name=f"{summary_path}/test_match_filters_partial.json",
)

7
tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json Normal file
View file

@ -0,0 +1,7 @@
{
".ytdl-sub-match_filter_test-download-archive.json": "6017105735dd8fdfe18932c9c048dccf",
"Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "ddfda0c60aafd9779da692d6c165385b",
"Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "17c303f07e9b3af77c90975f843c95bd",
"Project Zombie/Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].webm": "ad90ebef1ea6cbdb924670cdd7d1b9b9"
}

15
tests/resources/transaction_log_summaries/plugins/match_filters/test_match_filters_partial.txt Normal file
View file

@ -0,0 +1,15 @@
Files created:
----------------------------------------
{output_directory}
.ytdl-sub-match_filter_test-download-archive.json
{output_directory}/Project Zombie
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].info.json
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].nfo
NFO tags:
musicvideo:
album: Music Videos
artist: Project Zombie
title: Jesse's Minecraft Server [Trailer - Mar.21]
year: 2011
Project Zombie - Jesse's Minecraft Server [Trailer - Mar.21].webm