channel support with range

requirements.txt

@@ -12,4 +12,4 @@ pylint==2.13.3
 pytest==7.1.1
 pyYAML==5.4.1
 sanitize-filename==1.2.0
-yt-dlp==2021.9.2
+yt-dlp==2022.4.8

subscriptions.yaml

@@ -61,10 +61,11 @@
 recent_channel_test:
   preset: "yt_channel"
   ytdl_options:
-    daterange: now-2weeks
+    break_on_reject: True
   youtube:
     download_strategy: "channel"
     channel_id: "UCRcCVDu_4oARsAKFkKYydAQ"
+    after: today-8months
   output_options:
     output_directory: "/tmp/dunkey"
   overrides:

ytdl_subscribe/downloaders/youtube_downloader.py

@@ -3,6 +3,8 @@ import os
 from pathlib import Path
 from typing import List
 
+from yt_dlp.utils import RejectedVideoReached
+
 from ytdl_subscribe.downloaders.downloader import Downloader
 from ytdl_subscribe.entries.youtube import YoutubeVideo
 
@@ -38,7 +40,11 @@ class YoutubeDownloader(Downloader):
             "download_archive": str(Path(self.output_directory) / "ytdl-download-archive.txt"),
             "writeinfojson": True,
         }
-        _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url)
+
+        try:
+            _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url)
+        except RejectedVideoReached:
+            pass
 
     def download_video(self, video_id: str) -> YoutubeVideo:
         """Download a single Youtube video"""

ytdl_subscribe/entries/entry.py

@@ -120,9 +120,14 @@ class Entry:
             "uid": self.uid,
             "title": self.title,
             "sanitized_title": self.sanitized_title,
+            "description": self.description,
             "ext": self.ext,
             "upload_date": self.upload_date,
             "upload_year": self.upload_year,
+            "upload_month": self.upload_month,
+            "upload_month_padded": self.upload_month_padded,
+            "upload_day": self.upload_day,
+            "upload_day_padded": self.upload_day_padded,
             "thumbnail": self.thumbnail,
             "thumbnail_ext": self.thumbnail_ext,
         }

ytdl_subscribe/subscriptions/subscription.py

@@ -2,9 +2,12 @@ import os
 from abc import ABC
 from pathlib import Path
 from shutil import copyfile
-from typing import Generic, overload, Optional
+from typing import Dict
+from typing import Generic
+from typing import Optional
 from typing import Type
 from typing import TypeVar
+from typing import overload
 
 import dicttoxml
 import music_tag
@@ -55,12 +58,17 @@ class Subscription(Generic[S], ABC):
         """Returns the source options defined for this subscription"""
         return self.__preset_options.subscription_source
 
-    def get_downloader(self, downloader_type: Type[D]) -> D:
+    def get_downloader(
+        self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None
+    ) -> D:
         """Returns the downloader that will be used to download media for this subscription"""
-        return downloader_type(
+
-            output_directory=self.working_directory,
+        # if source_ytdl_options are present, override the ytdl_options with them
-            ytdl_options=self.__preset_options.ytdl_options.dict,
+        ytdl_options = self.__preset_options.ytdl_options.dict
-        )
+        if source_ytdl_options:
+            ytdl_options = dict(ytdl_options, **source_ytdl_options)
+
+        return downloader_type(output_directory=self.working_directory, ytdl_options=ytdl_options)
 
     @property
     def output_options(self) -> OutputOptionsValidator:
@@ -82,7 +90,9 @@ class Subscription(Generic[S], ABC):
         """Returns the directory that the downloader saves files to"""
         return str(Path(self.__config_options.working_directory.value) / Path(self.name))
 
-    def _apply_formatter(self, formatter: StringFormatterValidator, entry: Optional[Entry] = None) -> str:
+    def _apply_formatter(
+        self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
+    ) -> str:
         """
         Returns the format_string after .format has been called on it using entry (if provided) and
         override values
@@ -182,4 +192,4 @@ class Subscription(Generic[S], ABC):
             self._post_process_nfo(nfo_options=self.metadata_options.nfo, entry=entry)
 
         if self.metadata_options.output_directory_nfo:
-            self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo)
+            self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo)

ytdl_subscribe/subscriptions/youtube.py

@@ -1,7 +1,12 @@
+from yt_dlp import DateRange
+
 from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader
 from ytdl_subscribe.subscriptions.subscription import Subscription
 from ytdl_subscribe.validators.config.source_options.youtube_validators import (
-    YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator,
+    YoutubeChannelSourceValidator,
+)
+from ytdl_subscribe.validators.config.source_options.youtube_validators import (
+    YoutubePlaylistSourceValidator,
 )
 from ytdl_subscribe.validators.config.source_options.youtube_validators import (
     YoutubeVideoSourceValidator,
@@ -20,9 +25,15 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]):
 
 class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]):
     def extract_info(self):
-        entries = self.get_downloader(YoutubeDownloader).download_channel(
+        source_ytdl_options = {}
-            channel_id=self.source_options.channel_id.value
+        if self.source_options.before or self.source_options.after:
-        )
+            source_ytdl_options["daterange"] = DateRange(
+                start=self.source_options.after.datetime_str if self.source_options.after else None,
+                end=self.source_options.before.datetime_str if self.source_options.before else None,
+            )
+
+        downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options)
+        entries = downloader.download_channel(channel_id=self.source_options.channel_id.value)
 
         for entry in entries:
             self.post_process_entry(entry)

ytdl_subscribe/validators/base/string_datetime.py

@@ -0,0 +1,26 @@
+from yt_dlp.utils import datetime_from_str
+
+from ytdl_subscribe.validators.base.validators import Validator
+from ytdl_subscribe.validators.exceptions import ValidationException
+
+
+class StringDatetimeValidator(Validator):
+    """
+    Validates a ytdl datetime string value
+    """
+
+    _expected_value_type = str
+    _expected_value_type_name = "datetime string"
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+
+        try:
+            _ = datetime_from_str(self._value)
+        except Exception as e:
+            raise ValidationException(e)
+
+    @property
+    def datetime_str(self) -> str:
+        """Returns the datetime as a string"""
+        return self._value

ytdl_subscribe/validators/base/string_formatter_validators.py

@@ -78,10 +78,12 @@ class StringFormatterValidator(Validator):
         """
         return self._value
 
-    @final
+    def __apply_formatter(
-    def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
+        self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str]
+    ) -> "StringFormatterValidator":
+        """TODO: test recursive case where variable not found"""
         # Ensure the variable names exist within the entry and overrides
-        for variable_name in self.format_variables:
+        for variable_name in formatter.format_variables:
             if variable_name not in variable_dict:
                 available_fields = ", ".join(sorted(variable_dict.keys()))
                 raise self._validation_exception(
@@ -90,16 +92,20 @@ class StringFormatterValidator(Validator):
                     exception_class=StringFormattingException,
                 )
 
+        return StringFormatterValidator(
+            name=self._name,
+            value=formatter.format_string.format(**OrderedDict(variable_dict)),
+        )
+
+    @final
+    def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
         # Keep formatting the format string until no format_variables are present
         formatter = self
         recursion_depth = 0
         max_depth = StringFormatterValidator.__max_format_recursion
 
         while formatter.format_variables and recursion_depth < max_depth:
-            formatter = StringFormatterValidator(
+            formatter = self.__apply_formatter(formatter=formatter, variable_dict=variable_dict)
-                name=self._name,
-                value=formatter.format_string.format(**OrderedDict(variable_dict)),
-            )
             recursion_depth += 1
 
         if formatter.format_variables:

ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py

@@ -13,4 +13,6 @@ class MetadataOptionsValidator(StrictDictValidator):
         self.nfo = self._validate_key_if_present(key="nfo", validator=NFOValidator)
 
         # TODO: Ensure this does not depend on entry variables, only overrides
-        self.output_directory_nfo = self._validate_key_if_present(key="output_directory_nfo", validator=NFOValidator)
+        self.output_directory_nfo = self._validate_key_if_present(
+            key="output_directory_nfo", validator=NFOValidator
+        )

ytdl_subscribe/validators/config/source_options/download_strategy_validators.py

@@ -11,7 +11,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor
 )
 from ytdl_subscribe.validators.config.source_options.source_validators import SourceValidator
 from ytdl_subscribe.validators.config.source_options.youtube_validators import (
-    YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator,
+    YoutubeChannelSourceValidator,
+)
+from ytdl_subscribe.validators.config.source_options.youtube_validators import (
+    YoutubePlaylistSourceValidator,
 )
 from ytdl_subscribe.validators.config.source_options.youtube_validators import (
     YoutubeVideoSourceValidator,

ytdl_subscribe/validators/config/source_options/youtube_validators.py

@@ -1,3 +1,4 @@
+from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator
 from ytdl_subscribe.validators.base.validators import StringValidator
 from ytdl_subscribe.validators.config.source_options.source_validators import YoutubeSourceValidator
 
@@ -20,7 +21,10 @@ class YoutubeVideoSourceValidator(YoutubeSourceValidator):
 
 class YoutubeChannelSourceValidator(YoutubeSourceValidator):
     _required_keys = {"channel_id"}
+    _optional_keys = {"before", "after"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
-        self.channel_id = self._validate_key("channel_id", StringValidator)
+        self.channel_id = self._validate_key("channel_id", StringValidator)
+        self.before = self._validate_key_if_present("before", StringDatetimeValidator)
+        self.after = self._validate_key_if_present("after", StringDatetimeValidator)

ytdl_subscribe/validators/config/subscription_validator.py

@@ -9,8 +9,8 @@ from mergedeep import mergedeep
 
 from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription
 from ytdl_subscribe.subscriptions.subscription import Subscription
-from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription, \
+from ytdl_subscribe.subscriptions.youtube import YoutubeChannelSubscription
-    YoutubeChannelSubscription
+from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription
 from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription
 from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator
 from ytdl_subscribe.validators.base.validators import StringValidator
@@ -23,7 +23,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor
     SoundcloudAlbumsAndSinglesSourceValidator,
 )
 from ytdl_subscribe.validators.config.source_options.youtube_validators import (
-    YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator,
+    YoutubeChannelSourceValidator,
+)
+from ytdl_subscribe.validators.config.source_options.youtube_validators import (
+    YoutubePlaylistSourceValidator,
 )
 from ytdl_subscribe.validators.config.source_options.youtube_validators import (
     YoutubeVideoSourceValidator,