danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BUGFIX] Fix match-filter edge case and date_range.before (#750)

Browse source 
Fixes yet another bug that is match-filter related. The `date_range.before` value would get ignored if you specified any other match-filters. That is fixed by making them `&` behind-the-scenes.
This commit is contained in:
Jesse Bannon 2023-10-03 11:46:42 -07:00 committed by GitHub
parent c255f40348
commit c119e6f644
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 94 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/ytdl_sub/config/plugin.py
View file

@ -72,15 +72,6 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
Class to define the new plugin functionality
"""
@classmethod
def default_ytdl_options(cls) -> Dict:
"""
Returns
-------
ytdl options to enable if the plugin is not specified in the download
"""
return {}
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
"""
Returns

95
src/ytdl_sub/plugins/match_filters.py
View file

@ -1,11 +1,8 @@
import copy
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from yt_dlp import match_filter_func
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.utils.logger import Logger
@ -13,7 +10,50 @@ from ytdl_sub.validators.validators import StringListValidator
logger = Logger.get("match_filters")
_DEFAULT_DOWNLOAD_MATCH_FILTERS: List[str] = ["!is_live & !is_upcoming & !post_live"]
def default_filters() -> Tuple[List[str], List[str]]:
"""
Returns
-------
Default filters and breaking filters to always use
"""
return ["!is_live & !is_upcoming & !post_live"], []
def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
"""
Parameters
----------
filters
User-defined match-filters
to_combine
Filters that need to be combined via AND to the original filters.
These are derived from plugins
Returns
-------
merged filters
Raises
------
ValueError
Only supports combining 1 filter at this time. Should never be hit by users
"""
if len(to_combine) == 0:
return filters
if not filters:
return copy.deepcopy(to_combine)
if len(to_combine) > 1:
raise ValueError("Match-filters to combine only supports 1 at this time")
output_filters: List[str] = []
filter_to_combine: str = to_combine[0]
for match_filter in filters:
output_filters.append(f"{match_filter} & {filter_to_combine}")
return output_filters
class MatchFiltersOptions(OptionsDictValidator):
@ -46,24 +86,18 @@ class MatchFiltersOptions(OptionsDictValidator):
# - "availability=?public"
"""
_optional_keys = {"filters", "download_filters"}
_optional_keys = {"filters"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""Ensure filters looks right"""
if isinstance(value, dict):
value["filters"] = value.get("filters", [""])
value["download_filters"] = value.get("download_filters", [""])
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._filters = self._validate_key_if_present(key="filters", validator=StringListValidator)
self._download_filters = self._validate_key(
key="download_filters",
validator=StringListValidator,
default=_DEFAULT_DOWNLOAD_MATCH_FILTERS,
)
@property
def filters(self) -> List[str]:
@ -74,35 +108,10 @@ class MatchFiltersOptions(OptionsDictValidator):
"""
return [validator.value for validator in self._filters.list] if self._filters else []
@property
def download_filters(self) -> List[str]:
"""
Filters to apply during the download stage. This can be useful when building presets
that contain match-filters that you do not want to conflict with metadata-based
match-filters since they act as logical OR's.
By default, if no download_filters are present, then the filter
``"!is_live & !is_upcoming & !post_live"`` is added.
"""
return [validator.value for validator in self._download_filters.list]
class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
plugin_options_type = MatchFiltersOptions
@classmethod
def default_ytdl_options(cls) -> Dict:
"""
Returns
-------
match-filter to filter out live + upcoming videos when downloading
"""
return {
"match_filter": match_filter_func(
filters=[], breaking_filters=_DEFAULT_DOWNLOAD_MATCH_FILTERS
),
}
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
"""
Returns
@ -110,15 +119,3 @@ class MatchFiltersPlugin(Plugin[MatchFiltersOptions]):
match_filters to apply at the metadata stage
"""
return self.plugin_options.filters, []
def ytdl_options(self) -> Optional[Dict]:
"""
Returns
-------
match_filters to apply at the download stage
"""
return {
"match_filter": match_filter_func(
filters=[], breaking_filters=self.plugin_options.download_filters
),
}

56
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -15,12 +15,17 @@ from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.file_convert import FileConvertPlugin
from ytdl_sub.plugins.format import FormatPlugin
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
from ytdl_sub.plugins.match_filters import combine_filters
from ytdl_sub.plugins.match_filters import default_filters
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.logger import Logger
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
PluginT = TypeVar("PluginT", bound=Plugin)
logger = Logger.get("ytdl-options")
class SubscriptionYTDLOptions:
def __init__(
@ -89,10 +94,10 @@ class SubscriptionYTDLOptions:
return ytdl_options
def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict:
if not (plugin_obj := self._get_plugin(plugin)):
return plugin.default_ytdl_options()
if plugin_obj := self._get_plugin(plugin):
return plugin_obj.ytdl_options()
return plugin_obj.ytdl_options()
return {}
@property
def _user_ytdl_options(self) -> Dict:
@ -100,14 +105,50 @@ class SubscriptionYTDLOptions:
@property
def _plugin_match_filters(self) -> Dict:
match_filters: List[str] = []
breaking_match_filters: List[str] = []
"""
All match-filters from every plugin to fetch metadata.
In order for other plugins to not collide with user-defined match-filters, do
match_filters = user_match_filters or {}
for plugin in plugins:
for filter in match_filters:
AND plugin's match filters onto the existing filters
Otherwise, the filters separately act as an OR
"""
match_filters, breaking_match_filters = default_filters()
match_filters_plugin = self._get_plugin(MatchFiltersPlugin)
if match_filters_plugin:
(
user_match_filters,
user_breaking_match_filters,
) = match_filters_plugin.ytdl_options_match_filters()
match_filters = combine_filters(filters=user_match_filters, to_combine=match_filters)
breaking_match_filters = combine_filters(
filters=user_breaking_match_filters, to_combine=breaking_match_filters
)
for plugin in self._plugins:
# Do not re-add original match-filters plugin
if isinstance(plugin, MatchFiltersPlugin):
continue
pl_match_filters, pl_breaking_match_filters = plugin.ytdl_options_match_filters()
match_filters.extend(pl_match_filters)
breaking_match_filters.extend(pl_breaking_match_filters)
match_filters = combine_filters(filters=match_filters, to_combine=pl_match_filters)
breaking_match_filters = combine_filters(
filters=breaking_match_filters, to_combine=pl_breaking_match_filters
)
logger.debug(
"Setting match-filters: %s",
"\n - ".join([""] + match_filters) if match_filters else "[]",
)
logger.debug(
"Setting breaking-match-filters: %s",
"\n - ".join([""] + breaking_match_filters) if breaking_match_filters else "[]",
)
return {
"match_filter": match_filter_func(
filters=match_filters, breaking_filters=breaking_match_filters
@ -145,7 +186,6 @@ class SubscriptionYTDLOptions:
self._plugin_ytdl_options(ChaptersPlugin),
self._plugin_ytdl_options(AudioExtractPlugin),
self._plugin_ytdl_options(FormatPlugin),
self._plugin_ytdl_options(MatchFiltersPlugin),
self._user_ytdl_options, # user ytdl options...
self._download_only_options, # then download_only options
)