diff --git a/requirements.txt b/requirements.txt index fc84dd08..4d0fc968 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ pylint==2.13.3 pytest==7.1.1 pyYAML==5.4.1 sanitize-filename==1.2.0 -yt-dlp==2021.9.2 \ No newline at end of file +yt-dlp==2022.4.8 \ No newline at end of file diff --git a/subscriptions.yaml b/subscriptions.yaml index f1745adc..3de8d90d 100644 --- a/subscriptions.yaml +++ b/subscriptions.yaml @@ -61,10 +61,11 @@ recent_channel_test: preset: "yt_channel" ytdl_options: - daterange: now-2weeks + break_on_reject: True youtube: download_strategy: "channel" channel_id: "UCRcCVDu_4oARsAKFkKYydAQ" + after: today-8months output_options: output_directory: "/tmp/dunkey" overrides: diff --git a/ytdl_subscribe/downloaders/youtube_downloader.py b/ytdl_subscribe/downloaders/youtube_downloader.py index ab1555f5..eae37fa7 100644 --- a/ytdl_subscribe/downloaders/youtube_downloader.py +++ b/ytdl_subscribe/downloaders/youtube_downloader.py @@ -3,6 +3,8 @@ import os from pathlib import Path from typing import List +from yt_dlp.utils import RejectedVideoReached + from ytdl_subscribe.downloaders.downloader import Downloader from ytdl_subscribe.entries.youtube import YoutubeVideo @@ -38,7 +40,11 @@ class YoutubeDownloader(Downloader): "download_archive": str(Path(self.output_directory) / "ytdl-download-archive.txt"), "writeinfojson": True, } - _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url) + + try: + _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url) + except RejectedVideoReached: + pass def download_video(self, video_id: str) -> YoutubeVideo: """Download a single Youtube video""" diff --git a/ytdl_subscribe/entries/entry.py b/ytdl_subscribe/entries/entry.py index 18406a32..67102e68 100644 --- a/ytdl_subscribe/entries/entry.py +++ b/ytdl_subscribe/entries/entry.py @@ -120,9 +120,14 @@ class Entry: "uid": self.uid, "title": self.title, "sanitized_title": self.sanitized_title, + "description": self.description, "ext": self.ext, "upload_date": self.upload_date, "upload_year": self.upload_year, + "upload_month": self.upload_month, + "upload_month_padded": self.upload_month_padded, + "upload_day": self.upload_day, + "upload_day_padded": self.upload_day_padded, "thumbnail": self.thumbnail, "thumbnail_ext": self.thumbnail_ext, } diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index a9790b55..d40ea49b 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -2,9 +2,12 @@ import os from abc import ABC from pathlib import Path from shutil import copyfile -from typing import Generic, overload, Optional +from typing import Dict +from typing import Generic +from typing import Optional from typing import Type from typing import TypeVar +from typing import overload import dicttoxml import music_tag @@ -55,12 +58,17 @@ class Subscription(Generic[S], ABC): """Returns the source options defined for this subscription""" return self.__preset_options.subscription_source - def get_downloader(self, downloader_type: Type[D]) -> D: + def get_downloader( + self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None + ) -> D: """Returns the downloader that will be used to download media for this subscription""" - return downloader_type( - output_directory=self.working_directory, - ytdl_options=self.__preset_options.ytdl_options.dict, - ) + + # if source_ytdl_options are present, override the ytdl_options with them + ytdl_options = self.__preset_options.ytdl_options.dict + if source_ytdl_options: + ytdl_options = dict(ytdl_options, **source_ytdl_options) + + return downloader_type(output_directory=self.working_directory, ytdl_options=ytdl_options) @property def output_options(self) -> OutputOptionsValidator: @@ -82,7 +90,9 @@ class Subscription(Generic[S], ABC): """Returns the directory that the downloader saves files to""" return str(Path(self.__config_options.working_directory.value) / Path(self.name)) - def _apply_formatter(self, formatter: StringFormatterValidator, entry: Optional[Entry] = None) -> str: + def _apply_formatter( + self, formatter: StringFormatterValidator, entry: Optional[Entry] = None + ) -> str: """ Returns the format_string after .format has been called on it using entry (if provided) and override values @@ -182,4 +192,4 @@ class Subscription(Generic[S], ABC): self._post_process_nfo(nfo_options=self.metadata_options.nfo, entry=entry) if self.metadata_options.output_directory_nfo: - self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo) \ No newline at end of file + self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo) diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py index 747ed919..d45cfeac 100644 --- a/ytdl_subscribe/subscriptions/youtube.py +++ b/ytdl_subscribe/subscriptions/youtube.py @@ -1,7 +1,12 @@ +from yt_dlp import DateRange + from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader from ytdl_subscribe.subscriptions.subscription import Subscription from ytdl_subscribe.validators.config.source_options.youtube_validators import ( - YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator, + YoutubeChannelSourceValidator, +) +from ytdl_subscribe.validators.config.source_options.youtube_validators import ( + YoutubePlaylistSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( YoutubeVideoSourceValidator, @@ -20,9 +25,15 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]): class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]): def extract_info(self): - entries = self.get_downloader(YoutubeDownloader).download_channel( - channel_id=self.source_options.channel_id.value - ) + source_ytdl_options = {} + if self.source_options.before or self.source_options.after: + source_ytdl_options["daterange"] = DateRange( + start=self.source_options.after.datetime_str if self.source_options.after else None, + end=self.source_options.before.datetime_str if self.source_options.before else None, + ) + + downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options) + entries = downloader.download_channel(channel_id=self.source_options.channel_id.value) for entry in entries: self.post_process_entry(entry) diff --git a/ytdl_subscribe/validators/base/string_datetime.py b/ytdl_subscribe/validators/base/string_datetime.py new file mode 100644 index 00000000..3b034d45 --- /dev/null +++ b/ytdl_subscribe/validators/base/string_datetime.py @@ -0,0 +1,26 @@ +from yt_dlp.utils import datetime_from_str + +from ytdl_subscribe.validators.base.validators import Validator +from ytdl_subscribe.validators.exceptions import ValidationException + + +class StringDatetimeValidator(Validator): + """ + Validates a ytdl datetime string value + """ + + _expected_value_type = str + _expected_value_type_name = "datetime string" + + def __init__(self, name, value): + super().__init__(name, value) + + try: + _ = datetime_from_str(self._value) + except Exception as e: + raise ValidationException(e) + + @property + def datetime_str(self) -> str: + """Returns the datetime as a string""" + return self._value diff --git a/ytdl_subscribe/validators/base/string_formatter_validators.py b/ytdl_subscribe/validators/base/string_formatter_validators.py index a70e521b..39e6b240 100644 --- a/ytdl_subscribe/validators/base/string_formatter_validators.py +++ b/ytdl_subscribe/validators/base/string_formatter_validators.py @@ -78,10 +78,12 @@ class StringFormatterValidator(Validator): """ return self._value - @final - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: + def __apply_formatter( + self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str] + ) -> "StringFormatterValidator": + """TODO: test recursive case where variable not found""" # Ensure the variable names exist within the entry and overrides - for variable_name in self.format_variables: + for variable_name in formatter.format_variables: if variable_name not in variable_dict: available_fields = ", ".join(sorted(variable_dict.keys())) raise self._validation_exception( @@ -90,16 +92,20 @@ class StringFormatterValidator(Validator): exception_class=StringFormattingException, ) + return StringFormatterValidator( + name=self._name, + value=formatter.format_string.format(**OrderedDict(variable_dict)), + ) + + @final + def apply_formatter(self, variable_dict: Dict[str, str]) -> str: # Keep formatting the format string until no format_variables are present formatter = self recursion_depth = 0 max_depth = StringFormatterValidator.__max_format_recursion while formatter.format_variables and recursion_depth < max_depth: - formatter = StringFormatterValidator( - name=self._name, - value=formatter.format_string.format(**OrderedDict(variable_dict)), - ) + formatter = self.__apply_formatter(formatter=formatter, variable_dict=variable_dict) recursion_depth += 1 if formatter.format_variables: diff --git a/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py b/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py index 27b097b1..bcf1c107 100644 --- a/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py +++ b/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py @@ -13,4 +13,6 @@ class MetadataOptionsValidator(StrictDictValidator): self.nfo = self._validate_key_if_present(key="nfo", validator=NFOValidator) # TODO: Ensure this does not depend on entry variables, only overrides - self.output_directory_nfo = self._validate_key_if_present(key="output_directory_nfo", validator=NFOValidator) + self.output_directory_nfo = self._validate_key_if_present( + key="output_directory_nfo", validator=NFOValidator + ) diff --git a/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py b/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py index 27282b8c..5d8e6b63 100644 --- a/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py +++ b/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py @@ -11,7 +11,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor ) from ytdl_subscribe.validators.config.source_options.source_validators import SourceValidator from ytdl_subscribe.validators.config.source_options.youtube_validators import ( - YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator, + YoutubeChannelSourceValidator, +) +from ytdl_subscribe.validators.config.source_options.youtube_validators import ( + YoutubePlaylistSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( YoutubeVideoSourceValidator, diff --git a/ytdl_subscribe/validators/config/source_options/youtube_validators.py b/ytdl_subscribe/validators/config/source_options/youtube_validators.py index f4a8f60b..2fe4585f 100644 --- a/ytdl_subscribe/validators/config/source_options/youtube_validators.py +++ b/ytdl_subscribe/validators/config/source_options/youtube_validators.py @@ -1,3 +1,4 @@ +from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator from ytdl_subscribe.validators.base.validators import StringValidator from ytdl_subscribe.validators.config.source_options.source_validators import YoutubeSourceValidator @@ -20,7 +21,10 @@ class YoutubeVideoSourceValidator(YoutubeSourceValidator): class YoutubeChannelSourceValidator(YoutubeSourceValidator): _required_keys = {"channel_id"} + _optional_keys = {"before", "after"} def __init__(self, name, value): super().__init__(name, value) - self.channel_id = self._validate_key("channel_id", StringValidator) \ No newline at end of file + self.channel_id = self._validate_key("channel_id", StringValidator) + self.before = self._validate_key_if_present("before", StringDatetimeValidator) + self.after = self._validate_key_if_present("after", StringDatetimeValidator) diff --git a/ytdl_subscribe/validators/config/subscription_validator.py b/ytdl_subscribe/validators/config/subscription_validator.py index bc83d680..50cf77ce 100644 --- a/ytdl_subscribe/validators/config/subscription_validator.py +++ b/ytdl_subscribe/validators/config/subscription_validator.py @@ -9,8 +9,8 @@ from mergedeep import mergedeep from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription from ytdl_subscribe.subscriptions.subscription import Subscription -from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription, \ - YoutubeChannelSubscription +from ytdl_subscribe.subscriptions.youtube import YoutubeChannelSubscription +from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.base.validators import StringValidator @@ -23,7 +23,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor SoundcloudAlbumsAndSinglesSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( - YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator, + YoutubeChannelSourceValidator, +) +from ytdl_subscribe.validators.config.source_options.youtube_validators import ( + YoutubePlaylistSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( YoutubeVideoSourceValidator,