danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

channel support with range

Browse source
This commit is contained in:
jbannon 2022-04-09 15:46:28 +00:00
parent a1ccb0669e
commit e3239153ec
12 changed files with 105 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
requirements.txt
View file

@ -12,4 +12,4 @@ pylint==2.13.3
pytest==7.1.1
pyYAML==5.4.1
sanitize-filename==1.2.0
yt-dlp==2021.9.2
yt-dlp==2022.4.8

3
subscriptions.yaml
View file

@ -61,10 +61,11 @@
recent_channel_test:
preset: "yt_channel"
ytdl_options:
daterange: now-2weeks
break_on_reject: True
youtube:
download_strategy: "channel"
channel_id: "UCRcCVDu_4oARsAKFkKYydAQ"
after: today-8months
output_options:
output_directory: "/tmp/dunkey"
overrides:

8
ytdl_subscribe/downloaders/youtube_downloader.py
View file

@ -3,6 +3,8 @@ import os
from pathlib import Path
from typing import List
from yt_dlp.utils import RejectedVideoReached
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.entries.youtube import YoutubeVideo
@ -38,7 +40,11 @@ class YoutubeDownloader(Downloader):
"download_archive": str(Path(self.output_directory) / "ytdl-download-archive.txt"),
"writeinfojson": True,
}
_ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url)
try:
_ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url)
except RejectedVideoReached:
pass
def download_video(self, video_id: str) -> YoutubeVideo:
"""Download a single Youtube video"""

5
ytdl_subscribe/entries/entry.py
View file

@ -120,9 +120,14 @@ class Entry:
"uid": self.uid,
"title": self.title,
"sanitized_title": self.sanitized_title,
"description": self.description,
"ext": self.ext,
"upload_date": self.upload_date,
"upload_year": self.upload_year,
"upload_month": self.upload_month,
"upload_month_padded": self.upload_month_padded,
"upload_day": self.upload_day,
"upload_day_padded": self.upload_day_padded,
"thumbnail": self.thumbnail,
"thumbnail_ext": self.thumbnail_ext,
}

26
ytdl_subscribe/subscriptions/subscription.py
View file

@ -2,9 +2,12 @@ import os
from abc import ABC
from pathlib import Path
from shutil import copyfile
from typing import Generic, overload, Optional
from typing import Dict
from typing import Generic
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import overload
import dicttoxml
import music_tag
@ -55,12 +58,17 @@ class Subscription(Generic[S], ABC):
"""Returns the source options defined for this subscription"""
return self.__preset_options.subscription_source
def get_downloader(self, downloader_type: Type[D]) -> D:
def get_downloader(
self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None
) -> D:
"""Returns the downloader that will be used to download media for this subscription"""
return downloader_type(
output_directory=self.working_directory,
ytdl_options=self.__preset_options.ytdl_options.dict,
)
# if source_ytdl_options are present, override the ytdl_options with them
ytdl_options = self.__preset_options.ytdl_options.dict
if source_ytdl_options:
ytdl_options = dict(ytdl_options, **source_ytdl_options)
return downloader_type(output_directory=self.working_directory, ytdl_options=ytdl_options)
@property
def output_options(self) -> OutputOptionsValidator:
@ -82,7 +90,9 @@ class Subscription(Generic[S], ABC):
"""Returns the directory that the downloader saves files to"""
return str(Path(self.__config_options.working_directory.value) / Path(self.name))
def _apply_formatter(self, formatter: StringFormatterValidator, entry: Optional[Entry] = None) -> str:
def _apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> str:
"""
Returns the format_string after .format has been called on it using entry (if provided) and
override values
@ -182,4 +192,4 @@ class Subscription(Generic[S], ABC):
self._post_process_nfo(nfo_options=self.metadata_options.nfo, entry=entry)
if self.metadata_options.output_directory_nfo:
self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo)
self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo)

19
ytdl_subscribe/subscriptions/youtube.py
View file

@ -1,7 +1,12 @@
from yt_dlp import DateRange
from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader
from ytdl_subscribe.subscriptions.subscription import Subscription
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator,
YoutubeChannelSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubePlaylistSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubeVideoSourceValidator,
@ -20,9 +25,15 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]):
class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]):
def extract_info(self):
entries = self.get_downloader(YoutubeDownloader).download_channel(
channel_id=self.source_options.channel_id.value
)
source_ytdl_options = {}
if self.source_options.before or self.source_options.after:
source_ytdl_options["daterange"] = DateRange(
start=self.source_options.after.datetime_str if self.source_options.after else None,
end=self.source_options.before.datetime_str if self.source_options.before else None,
)
downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options)
entries = downloader.download_channel(channel_id=self.source_options.channel_id.value)
for entry in entries:
self.post_process_entry(entry)

26
ytdl_subscribe/validators/base/string_datetime.py Normal file
View file

@ -0,0 +1,26 @@
from yt_dlp.utils import datetime_from_str
from ytdl_subscribe.validators.base.validators import Validator
from ytdl_subscribe.validators.exceptions import ValidationException
class StringDatetimeValidator(Validator):
"""
Validates a ytdl datetime string value
"""
_expected_value_type = str
_expected_value_type_name = "datetime string"
def __init__(self, name, value):
super().__init__(name, value)
try:
_ = datetime_from_str(self._value)
except Exception as e:
raise ValidationException(e)
@property
def datetime_str(self) -> str:
"""Returns the datetime as a string"""
return self._value

20
ytdl_subscribe/validators/base/string_formatter_validators.py
View file

@ -78,10 +78,12 @@ class StringFormatterValidator(Validator):
"""
return self._value
@final
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
def __apply_formatter(
self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str]
) -> "StringFormatterValidator":
"""TODO: test recursive case where variable not found"""
# Ensure the variable names exist within the entry and overrides
for variable_name in self.format_variables:
for variable_name in formatter.format_variables:
if variable_name not in variable_dict:
available_fields = ", ".join(sorted(variable_dict.keys()))
raise self._validation_exception(
@ -90,16 +92,20 @@ class StringFormatterValidator(Validator):
exception_class=StringFormattingException,
)
return StringFormatterValidator(
name=self._name,
value=formatter.format_string.format(**OrderedDict(variable_dict)),
)
@final
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
# Keep formatting the format string until no format_variables are present
formatter = self
recursion_depth = 0
max_depth = StringFormatterValidator.__max_format_recursion
while formatter.format_variables and recursion_depth < max_depth:
formatter = StringFormatterValidator(
name=self._name,
value=formatter.format_string.format(**OrderedDict(variable_dict)),
)
formatter = self.__apply_formatter(formatter=formatter, variable_dict=variable_dict)
recursion_depth += 1
if formatter.format_variables:

4
ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py
View file

@ -13,4 +13,6 @@ class MetadataOptionsValidator(StrictDictValidator):
self.nfo = self._validate_key_if_present(key="nfo", validator=NFOValidator)
# TODO: Ensure this does not depend on entry variables, only overrides
self.output_directory_nfo = self._validate_key_if_present(key="output_directory_nfo", validator=NFOValidator)
self.output_directory_nfo = self._validate_key_if_present(
key="output_directory_nfo", validator=NFOValidator
)

5
ytdl_subscribe/validators/config/source_options/download_strategy_validators.py
View file

@ -11,7 +11,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor
)
from ytdl_subscribe.validators.config.source_options.source_validators import SourceValidator
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator,
YoutubeChannelSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubePlaylistSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubeVideoSourceValidator,

6
ytdl_subscribe/validators/config/source_options/youtube_validators.py
View file

@ -1,3 +1,4 @@
from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator
from ytdl_subscribe.validators.base.validators import StringValidator
from ytdl_subscribe.validators.config.source_options.source_validators import YoutubeSourceValidator
@ -20,7 +21,10 @@ class YoutubeVideoSourceValidator(YoutubeSourceValidator):
class YoutubeChannelSourceValidator(YoutubeSourceValidator):
_required_keys = {"channel_id"}
_optional_keys = {"before", "after"}
def __init__(self, name, value):
super().__init__(name, value)
self.channel_id = self._validate_key("channel_id", StringValidator)
self.channel_id = self._validate_key("channel_id", StringValidator)
self.before = self._validate_key_if_present("before", StringDatetimeValidator)
self.after = self._validate_key_if_present("after", StringDatetimeValidator)

9
ytdl_subscribe/validators/config/subscription_validator.py
View file

@ -9,8 +9,8 @@ from mergedeep import mergedeep
from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription
from ytdl_subscribe.subscriptions.subscription import Subscription
from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription, \
YoutubeChannelSubscription
from ytdl_subscribe.subscriptions.youtube import YoutubeChannelSubscription
from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription
from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription
from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.base.validators import StringValidator
@ -23,7 +23,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor
SoundcloudAlbumsAndSinglesSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator,
YoutubeChannelSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubePlaylistSourceValidator,
)
from ytdl_subscribe.validators.config.source_options.youtube_validators import (
YoutubeVideoSourceValidator,