Add inheritance for presets (#24)
This commit is contained in:
parent
29ef377cc7
commit
d17f677db3
10 changed files with 325 additions and 227 deletions
|
|
@ -48,6 +48,8 @@ _______
|
||||||
|
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
.. _YouTube Playlist:
|
||||||
|
|
||||||
playlist
|
playlist
|
||||||
________
|
________
|
||||||
.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloaderOptions()
|
.. autoclass:: ytdl_sub.downloaders.youtube_downloader.YoutubePlaylistDownloaderOptions()
|
||||||
|
|
@ -103,6 +105,24 @@ overrides
|
||||||
"""""""""
|
"""""""""
|
||||||
.. autoclass:: ytdl_sub.config.preset_options.Overrides()
|
.. autoclass:: ytdl_sub.config.preset_options.Overrides()
|
||||||
|
|
||||||
|
.. _parent preset:
|
||||||
|
|
||||||
|
preset
|
||||||
|
""""""
|
||||||
|
Presets support inheritance by defining a parent preset:
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
presets:
|
||||||
|
parent_preset:
|
||||||
|
...
|
||||||
|
child_preset:
|
||||||
|
preset: "parent_preset"
|
||||||
|
|
||||||
|
In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
|
||||||
|
It is advantageous to use parent presets where possible to reduce duplicate yaml
|
||||||
|
definitions.
|
||||||
|
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -133,6 +153,47 @@ nfo_output_directory
|
||||||
|
|
||||||
-------------------------------------------------------------------------------
|
-------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
subscription.yaml
|
||||||
|
-----------------
|
||||||
|
The ``subscription.yaml`` file is where we use our `presets`_ in the `config.yaml`_
|
||||||
|
to define a `subscription`: something we want to recurrently download such as a specific
|
||||||
|
channel or playlist.
|
||||||
|
|
||||||
|
The only difference between a ``subscription`` and ``preset`` is that the subscription
|
||||||
|
must have all required fields and ``{variables}`` defined so it can perform a download.
|
||||||
|
|
||||||
|
Below is an example that downloads YouTube videos:
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
:caption: config.yaml
|
||||||
|
|
||||||
|
presets:
|
||||||
|
playlist_preset_ex:
|
||||||
|
youtube:
|
||||||
|
download_strategy: "playlist"
|
||||||
|
output_options:
|
||||||
|
output_directory: "{output_directory}/{playlist_name}"
|
||||||
|
file_name: "{playlist_name}.{title}.{ext}"
|
||||||
|
overrides:
|
||||||
|
output_directory: "/path/to/ytdl-sub-videos"
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
:caption: subscription.yaml
|
||||||
|
|
||||||
|
my_subscription_name:
|
||||||
|
preset: "playlist_preset_ex"
|
||||||
|
youtube:
|
||||||
|
playlist_id: "UCsvn_Po0SmunchJYtttWpOxMg"
|
||||||
|
overrides:
|
||||||
|
playlist_name: "diy-playlist"
|
||||||
|
|
||||||
|
Our preset ``playlist_preset_ex`` uses the `YouTube Playlist`_ download strategy, and defines two
|
||||||
|
custom variables: ``{output_directory}`` and ``{playlist_name}``. The subscription sets
|
||||||
|
the `parent preset`_ to ``playlist_preset_ex``, and must define the ``playlist_id`` field and
|
||||||
|
the ``{playlist_name}`` variable since the preset did not.
|
||||||
|
|
||||||
|
-------------------------------------------------------------------------------
|
||||||
|
|
||||||
.. _source-variables:
|
.. _source-variables:
|
||||||
|
|
||||||
Source Variables
|
Source Variables
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ configuration:
|
||||||
working_directory: '.ytdl-sub-downloads'
|
working_directory: '.ytdl-sub-downloads'
|
||||||
|
|
||||||
presets:
|
presets:
|
||||||
yt_music_video:
|
yt_music_video_playlist:
|
||||||
# Youtube playlists are our source/download strategy. However, this
|
# Youtube playlists are our source/download strategy. However, this
|
||||||
# can be overwritten to download music videos from a 'channel' or a
|
# can be overwritten to download music videos from a 'channel' or a
|
||||||
# single 'video'
|
# single 'video'
|
||||||
|
|
@ -66,3 +66,11 @@ presets:
|
||||||
overrides:
|
overrides:
|
||||||
music_video_directory: "path/to/Music Videos"
|
music_video_directory: "path/to/Music Videos"
|
||||||
music_video_name: "{sanitized_artist} - {sanitized_title}"
|
music_video_name: "{sanitized_artist} - {sanitized_title}"
|
||||||
|
|
||||||
|
# It is not always ideal to download all of an artist's music videos.
|
||||||
|
# Maybe you only like one song of theirs. We can reuse our preset above
|
||||||
|
# to download a single video instead.
|
||||||
|
yt_music_video:
|
||||||
|
preset: "yt_music_video_playlist"
|
||||||
|
youtube:
|
||||||
|
download_strategy: "video"
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@
|
||||||
john_smith:
|
john_smith:
|
||||||
# We must define a preset to use from our config. We named the one in the
|
# We must define a preset to use from our config. We named the one in the
|
||||||
# config example "yt_music_video", so set that here.
|
# config example "yt_music_video", so set that here.
|
||||||
preset: "yt_music_video"
|
preset: "yt_music_video_playlist"
|
||||||
|
|
||||||
# Since our preset download strategy is set to 'playlist', set the playlist id.
|
# Since our preset download strategy is set to 'playlist', set the playlist id.
|
||||||
youtube:
|
youtube:
|
||||||
|
|
@ -37,9 +37,8 @@ john_smith:
|
||||||
# to download a single video instead.
|
# to download a single video instead.
|
||||||
#
|
#
|
||||||
# The only difference between this example and the one above is
|
# The only difference between this example and the one above is
|
||||||
# - youtube.download_strategy
|
# - preset
|
||||||
# We defined this as 'playlist' in the config. We must override it
|
# Use the music video preset, not the playlist preset
|
||||||
# with 'video' to perform a single video download
|
|
||||||
# - youtube.video_id
|
# - youtube.video_id
|
||||||
# The id of the video to download
|
# The id of the video to download
|
||||||
#
|
#
|
||||||
|
|
@ -48,18 +47,13 @@ john_smith:
|
||||||
# command:
|
# command:
|
||||||
#
|
#
|
||||||
# ytdl-sub dl \
|
# ytdl-sub dl \
|
||||||
# --preset "yt_channel_as_tv" \
|
# --preset "yt_music_video" \
|
||||||
# --youtube.download_strategy "video" \
|
|
||||||
# --youtube.video_id "QhY6r6oAErg" \
|
# --youtube.video_id "QhY6r6oAErg" \
|
||||||
# --overrides.artist "John Smith and the Instrument Players"
|
# --overrides.artist "John Smith and the Instrument Players"
|
||||||
#
|
#
|
||||||
# The --preset and --youtube.download_strategy args will always be the
|
|
||||||
# same. We will try to simplify the dl command to make it shorter.
|
|
||||||
|
|
||||||
john_smith_one_hit_wonder:
|
john_smith_one_hit_wonder:
|
||||||
preset: "yt_channel_as_tv"
|
preset: "yt_music_video"
|
||||||
youtube:
|
youtube:
|
||||||
download_strategy: "video"
|
|
||||||
video_id: "QhY6r6oAErg"
|
video_id: "QhY6r6oAErg"
|
||||||
overrides:
|
overrides:
|
||||||
artist: "John Smith and the Instrument Players"
|
artist: "John Smith and the Instrument Players"
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,14 @@ configuration:
|
||||||
working_directory: '.ytdl-sub-downloads'
|
working_directory: '.ytdl-sub-downloads'
|
||||||
|
|
||||||
presets:
|
presets:
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# LEVEL 1 - FULL ARCHIVE
|
||||||
|
#
|
||||||
|
# We will call this preset `yt_channel_as_tv`, and it will download every single video in
|
||||||
|
# a YouTube channel.
|
||||||
yt_channel_as_tv:
|
yt_channel_as_tv:
|
||||||
# Youtube channels are our source/download strategy
|
# YouTube channels are our source/download strategy
|
||||||
# Use the channel avatar and banner images for Kodi
|
# Use the channel avatar and banner images for Kodi
|
||||||
youtube:
|
youtube:
|
||||||
download_strategy: "channel"
|
download_strategy: "channel"
|
||||||
|
|
@ -82,3 +88,53 @@ presets:
|
||||||
overrides:
|
overrides:
|
||||||
youtube_tv_shows_directory: "/path/to/youtube_tv_shows"
|
youtube_tv_shows_directory: "/path/to/youtube_tv_shows"
|
||||||
episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {sanitized_title}"
|
episode_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {sanitized_title}"
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# LEVEL 2 - RECENT ARCHIVE
|
||||||
|
#
|
||||||
|
# It is not always ideal to go full data-hoarder on a channel's videos.
|
||||||
|
# This example shows how you can only download the last 14 days-worth
|
||||||
|
# of videos on each download invocation.
|
||||||
|
yt_channel_as_tv__recent:
|
||||||
|
# `preset` can be set to any other preset in the config, and will inherit all its defined fields.
|
||||||
|
# This helps reduce copy-paste in the config.yaml
|
||||||
|
preset: "yt_channel_as_tv"
|
||||||
|
|
||||||
|
# We will add the `after` field onto `yt_channel_as_tv`'s YouTube channel download strategy.
|
||||||
|
# This is saying 'only download videos uploaded in the last 2 weeks'
|
||||||
|
youtube:
|
||||||
|
after: "today-2weeks"
|
||||||
|
|
||||||
|
# This is getting into YTDL voodoo. By default, YTDL will try to
|
||||||
|
# download all channel videos beginning with the most recent one.
|
||||||
|
# By setting break_on_reject, we will break this full-download on
|
||||||
|
# the first video that gets rejected. Since we have youtube.after
|
||||||
|
# defined, all videos after today-14days will be rejected. Therefore,
|
||||||
|
# the first video that is out of range that it tries to download, it
|
||||||
|
# will stop there, and save a significant amount of time.
|
||||||
|
#
|
||||||
|
# Similar to break_on_reject, but instead, breaks if a video has
|
||||||
|
# already been downloaded. If you were to perform a download on this
|
||||||
|
# subscription one-after-the-other, the second invocation would stop
|
||||||
|
# after looking at the first (most recent) video since it would exist
|
||||||
|
# in the download archive.
|
||||||
|
ytdl_options:
|
||||||
|
break_on_reject: True
|
||||||
|
break_on_existing: True
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# LEVEL 3 - ROLLING ARCHIVE
|
||||||
|
|
||||||
|
# If you put the recent archive example in a cron job, then in a year or so
|
||||||
|
# you will basically be datahoarding that channel unless you manually delete
|
||||||
|
# old videos. This example shows how to only keep the last 14-days worth of videos,
|
||||||
|
# and delete the rest.
|
||||||
|
yt_channel_as_tv__only_recent:
|
||||||
|
# Reuse `yt_channel_as_tv__recent` to only download the last 2 weeks' of videos
|
||||||
|
preset: "yt_channel_as_tv__recent"
|
||||||
|
|
||||||
|
# This is saying "only keep files if they were uploaded in the last 2 weeks".
|
||||||
|
# All other files that this subscription had previously downloaded will be
|
||||||
|
# deleted.
|
||||||
|
output_options:
|
||||||
|
keep_files_after: "today-2weeks"
|
||||||
|
|
@ -4,13 +4,13 @@
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# LEVEL 1 - FULL ARCHIVE
|
# LEVEL 1 - FULL ARCHIVE
|
||||||
|
#
|
||||||
# Subscription names are defined by you. We will call this one
|
# Subscription names are defined by you. We will call this one
|
||||||
# john_smith_archive because it will download every single video in
|
# john_smith_archive because it will download every single video in
|
||||||
# john_smith's channel.
|
# john_smith's channel.
|
||||||
john_smith_archive:
|
john_smith_archive:
|
||||||
# We must define a preset to use from our config. We named the one in the
|
# We must define a preset to use from our config. The one that downloads the
|
||||||
# config example "yt_channel_as_tv", so set that here.
|
# entire YouTube channel is called "yt_channel_as_tv", so set that here.
|
||||||
preset: "yt_channel_as_tv"
|
preset: "yt_channel_as_tv"
|
||||||
|
|
||||||
# Our download strategy was Youtube channels. Define the channel id here
|
# Our download strategy was Youtube channels. Define the channel id here
|
||||||
|
|
@ -28,61 +28,24 @@ john_smith_archive:
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# LEVEL 2 - RECENT ARCHIVE
|
# LEVEL 2 - RECENT ARCHIVE
|
||||||
|
|
||||||
# It is not always ideal to go full datahoarder on a channel's videos.
|
|
||||||
# This example shows how you can only download the last 14 days-worth
|
|
||||||
# of videos on each download invocation.
|
|
||||||
#
|
#
|
||||||
# The only difference between this example and the one above is
|
# Use the "yt_channel_as_tv__recent" preset to only download the last two
|
||||||
# - youtube.after
|
# weeks' worth of YouTube videos.
|
||||||
# This is saying 'only download videos in the last 14 days'
|
|
||||||
# - ytdl_options.break_on_reject
|
|
||||||
# This is getting into YTDL voodoo. By default, YTDL will try to
|
|
||||||
# download all channel videos beginning with the most recent one.
|
|
||||||
# By setting break_on_reject, we will break this full-download on
|
|
||||||
# the first video that gets rejected. Since we have youtube.after
|
|
||||||
# defined, all videos after today-14days will be rejected. Therefore,
|
|
||||||
# the first video that is out of range that it tries to download, it
|
|
||||||
# will stop there, and save a significant amount of time.
|
|
||||||
# - ytdl_options.break_on_existing
|
|
||||||
# Similar to break_on_reject, but instead, breaks if a video has
|
|
||||||
# already been downloaded. If you were to perform a download on this
|
|
||||||
# subscription one-after-the-other, the second invocation would stop
|
|
||||||
# after looking at the first (most recent) video since it would exist
|
|
||||||
# in the download archive.
|
|
||||||
john_smith_recent_archive:
|
john_smith_recent_archive:
|
||||||
preset: "yt_channel_as_tv"
|
preset: "yt_channel_as_tv__recent"
|
||||||
youtube:
|
youtube:
|
||||||
channel_id: "UCsvn_Po0SmunchJYtttWpOxMg"
|
channel_id: "UCsvn_Po0SmunchJYtttWpOxMg"
|
||||||
after: today-14days
|
|
||||||
overrides:
|
overrides:
|
||||||
tv_show_name: "John /\ Smith"
|
tv_show_name: "John /\ Smith"
|
||||||
ytdl_options:
|
|
||||||
break_on_reject: True
|
|
||||||
break_on_existing: True
|
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# LEVEL 3 - ROLLING ARCHIVE
|
# LEVEL 3 - ROLLING ARCHIVE
|
||||||
|
|
||||||
# If you put the recent archive example in a cron job, then in a year or so
|
|
||||||
# you will basically be datahoarding that channel unless you manually delete
|
|
||||||
# old videos. We automate because we are lazy. This example shows how to
|
|
||||||
# only keep the last 14-days worth of videos, and delete the rest.
|
|
||||||
#
|
#
|
||||||
# The only difference between this example and the one above is
|
# Use the "yt_channel_as_tv__recent_only" preset to only download the last two
|
||||||
# - output_options.keep_files_after
|
# weeks' worth of YouTube videos, and delete any existing older videos.
|
||||||
# This is saying "only keep files if they were uploaded in the last 14 days".
|
|
||||||
# All other files that this subscription had previously downloaded will be
|
|
||||||
# deleted.
|
|
||||||
john_smith_rolling_archive:
|
john_smith_rolling_archive:
|
||||||
preset: "yt_channel_as_tv"
|
preset: "yt_channel_as_tv__only_recent"
|
||||||
youtube:
|
youtube:
|
||||||
channel_id: "UCsvn_Po0SmunchJYtttWpOxMg"
|
channel_id: "UCsvn_Po0SmunchJYtttWpOxMg"
|
||||||
after: today-2weeks
|
|
||||||
overrides:
|
overrides:
|
||||||
tv_show_name: "John /\ Smith"
|
tv_show_name: "John /\ Smith"
|
||||||
ytdl_options:
|
|
||||||
break_on_reject: True
|
|
||||||
break_on_existing: True
|
|
||||||
output_options:
|
|
||||||
keep_files_after: today-2weeks
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import copy
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
@ -5,6 +6,10 @@ from typing import Optional
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
from typing import Type
|
from typing import Type
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from mergedeep import mergedeep
|
||||||
|
|
||||||
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping
|
from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping
|
||||||
from ytdl_sub.config.preset_class_mappings import PluginMapping
|
from ytdl_sub.config.preset_class_mappings import PluginMapping
|
||||||
from ytdl_sub.config.preset_options import OutputOptions
|
from ytdl_sub.config.preset_options import OutputOptions
|
||||||
|
|
@ -21,8 +26,9 @@ from ytdl_sub.validators.validators import DictValidator
|
||||||
from ytdl_sub.validators.validators import StringValidator
|
from ytdl_sub.validators.validators import StringValidator
|
||||||
from ytdl_sub.validators.validators import Validator
|
from ytdl_sub.validators.validators import Validator
|
||||||
|
|
||||||
PRESET_REQUIRED_KEYS = {"output_options"}
|
PRESET_KEYS = {
|
||||||
PRESET_OPTIONAL_KEYS = {
|
"preset",
|
||||||
|
"output_options",
|
||||||
"ytdl_options",
|
"ytdl_options",
|
||||||
"overrides",
|
"overrides",
|
||||||
*DownloadStrategyMapping.sources(),
|
*DownloadStrategyMapping.sources(),
|
||||||
|
|
@ -69,8 +75,10 @@ class DownloadStrategyValidator(StrictDictValidator):
|
||||||
|
|
||||||
|
|
||||||
class Preset(StrictDictValidator):
|
class Preset(StrictDictValidator):
|
||||||
_required_keys = PRESET_REQUIRED_KEYS
|
# Have all present keys optional since parent presets could not have all the
|
||||||
_optional_keys = PRESET_OPTIONAL_KEYS
|
# required keys. They will get validated in the init after the mergedeep of dicts
|
||||||
|
# and ensure required keys are present.
|
||||||
|
_optional_keys = PRESET_KEYS
|
||||||
|
|
||||||
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
|
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
|
||||||
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
|
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
|
||||||
|
|
@ -176,9 +184,43 @@ class Preset(StrictDictValidator):
|
||||||
if isinstance(validator, OverridesStringFormatterValidator):
|
if isinstance(validator, OverridesStringFormatterValidator):
|
||||||
self.__validate_override_string_formatter_validator(validator)
|
self.__validate_override_string_formatter_validator(validator)
|
||||||
|
|
||||||
def __init__(self, name: str, value: Any):
|
def __merge_parent_preset_dicts_if_present(self, config: ConfigFile):
|
||||||
|
parent_presets = set()
|
||||||
|
parent_preset_validator = self._validate_key_if_present(
|
||||||
|
key="preset", validator=StringValidator
|
||||||
|
)
|
||||||
|
parent_preset = parent_preset_validator.value if parent_preset_validator else None
|
||||||
|
|
||||||
|
while parent_preset:
|
||||||
|
# Make sure the parent preset actually exists
|
||||||
|
if parent_preset not in config.presets.keys:
|
||||||
|
raise self._validation_exception(
|
||||||
|
f"preset '{parent_preset}' does not exist in the provided config. "
|
||||||
|
f"Available presets: {', '.join(config.presets.keys)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Make sure we do not hit an infinite loop
|
||||||
|
if parent_preset in parent_presets:
|
||||||
|
raise self._validation_exception(
|
||||||
|
f"preset loop detected with the preset '{parent_preset}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset])
|
||||||
|
|
||||||
|
parent_presets.add(parent_preset)
|
||||||
|
parent_preset = parent_preset_dict.get("preset")
|
||||||
|
|
||||||
|
# Override the parent preset with the contents of this preset
|
||||||
|
self._value = mergedeep.merge(
|
||||||
|
parent_preset_dict, self._value, strategy=mergedeep.Strategy.REPLACE
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(self, config: ConfigFile, name: str, value: Any):
|
||||||
super().__init__(name=name, value=value)
|
super().__init__(name=name, value=value)
|
||||||
|
|
||||||
|
# Perform the merge of parent presets before validating any keys
|
||||||
|
self.__merge_parent_preset_dicts_if_present(config=config)
|
||||||
|
|
||||||
self.downloader, self.downloader_options = self.__validate_and_get_downloader_and_options()
|
self.downloader, self.downloader_options = self.__validate_and_get_downloader_and_options()
|
||||||
|
|
||||||
self.output_options = self._validate_key(
|
self.output_options = self._validate_key(
|
||||||
|
|
@ -196,3 +238,60 @@ class Preset(StrictDictValidator):
|
||||||
# After all options are initialized, perform a recursive post-validate that requires
|
# After all options are initialized, perform a recursive post-validate that requires
|
||||||
# values from multiple validators
|
# values from multiple validators
|
||||||
self.__recursive_preset_validate()
|
self.__recursive_preset_validate()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name(self) -> str:
|
||||||
|
"""
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
Name of the preset
|
||||||
|
"""
|
||||||
|
return self._name
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Preset":
|
||||||
|
"""
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
config:
|
||||||
|
Validated instance of the config
|
||||||
|
preset_name:
|
||||||
|
Name of the preset
|
||||||
|
preset_dict:
|
||||||
|
The preset config in dict format
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The Subscription validator
|
||||||
|
"""
|
||||||
|
return cls(config=config, name=preset_name, value=preset_dict)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_file_path(cls, config: ConfigFile, subscription_path: str) -> List["Preset"]:
|
||||||
|
"""
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
config:
|
||||||
|
Validated instance of the config
|
||||||
|
subscription_path:
|
||||||
|
File path to the subscription yaml file
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
List of presets, for each one in the subscription yaml
|
||||||
|
"""
|
||||||
|
# TODO: Create separate yaml file loader class
|
||||||
|
with open(subscription_path, "r", encoding="utf-8") as file:
|
||||||
|
subscription_dict = yaml.safe_load(file)
|
||||||
|
|
||||||
|
subscriptions: List["Preset"] = []
|
||||||
|
for subscription_key, subscription_object in subscription_dict.items():
|
||||||
|
subscriptions.append(
|
||||||
|
Preset.from_dict(
|
||||||
|
config=config,
|
||||||
|
preset_name=subscription_key,
|
||||||
|
preset_dict=subscription_object,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return subscriptions
|
||||||
|
|
|
||||||
|
|
@ -1,131 +0,0 @@
|
||||||
import copy
|
|
||||||
from typing import Any
|
|
||||||
from typing import Dict
|
|
||||||
from typing import List
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
from mergedeep import mergedeep
|
|
||||||
|
|
||||||
from ytdl_sub.config.config_file import ConfigFile
|
|
||||||
from ytdl_sub.config.preset import PRESET_OPTIONAL_KEYS
|
|
||||||
from ytdl_sub.config.preset import PRESET_REQUIRED_KEYS
|
|
||||||
from ytdl_sub.config.preset import Preset
|
|
||||||
from ytdl_sub.config.preset_options import Overrides
|
|
||||||
from ytdl_sub.subscriptions.subscription import Subscription
|
|
||||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
|
||||||
from ytdl_sub.validators.validators import StringValidator
|
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionValidator(StrictDictValidator):
|
|
||||||
"""
|
|
||||||
A Subscription is a preset but overrides it with specific values
|
|
||||||
"""
|
|
||||||
|
|
||||||
_required_keys = {"preset"}
|
|
||||||
_optional_keys = PRESET_REQUIRED_KEYS.union(PRESET_OPTIONAL_KEYS)
|
|
||||||
|
|
||||||
def __init__(self, config: ConfigFile, name: str, value: Any):
|
|
||||||
super().__init__(name, value)
|
|
||||||
self.config = config
|
|
||||||
|
|
||||||
# Ensure the overrides defined here are valid
|
|
||||||
_ = self._validate_key(
|
|
||||||
key="overrides",
|
|
||||||
validator=Overrides,
|
|
||||||
default={},
|
|
||||||
)
|
|
||||||
|
|
||||||
preset_name = self._validate_key(
|
|
||||||
key="preset",
|
|
||||||
validator=StringValidator,
|
|
||||||
).value
|
|
||||||
|
|
||||||
if preset_name not in self.config.presets.keys:
|
|
||||||
raise self._validation_exception(
|
|
||||||
f"preset '{preset_name}' does not exist in the provided config. "
|
|
||||||
f"Available presets: {', '.join(self.config.presets.keys)}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# A little hacky, we will override the preset with the contents of this subscription,
|
|
||||||
# then validate it
|
|
||||||
preset_dict = copy.deepcopy(self.config.presets.dict[preset_name])
|
|
||||||
preset_dict = mergedeep.merge(preset_dict, self._dict, strategy=mergedeep.Strategy.REPLACE)
|
|
||||||
del preset_dict["preset"]
|
|
||||||
|
|
||||||
self.preset = Preset(
|
|
||||||
name=f"{self._name}.{preset_name}",
|
|
||||||
value=preset_dict,
|
|
||||||
)
|
|
||||||
|
|
||||||
def to_subscription(self) -> Subscription:
|
|
||||||
"""
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
The subscription after the config and preset have been validated
|
|
||||||
"""
|
|
||||||
return Subscription(
|
|
||||||
name=self._name,
|
|
||||||
config_options=self.config.config_options,
|
|
||||||
preset_options=self.preset,
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def name(self) -> str:
|
|
||||||
"""
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
Name of the subscription
|
|
||||||
"""
|
|
||||||
return self._name
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_dict(
|
|
||||||
cls, config: ConfigFile, subscription_name: str, subscription_dict: Dict
|
|
||||||
) -> "SubscriptionValidator":
|
|
||||||
"""
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
config:
|
|
||||||
Validated instance of the config
|
|
||||||
subscription_name:
|
|
||||||
Name of the subscription
|
|
||||||
subscription_dict:
|
|
||||||
The subscription config in dict format
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
The Subscription validator
|
|
||||||
"""
|
|
||||||
return SubscriptionValidator(config=config, name=subscription_name, value=subscription_dict)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_file_path(
|
|
||||||
cls, config: ConfigFile, subscription_path: str
|
|
||||||
) -> List["SubscriptionValidator"]:
|
|
||||||
"""
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
config:
|
|
||||||
Validated instance of the config
|
|
||||||
subscription_path:
|
|
||||||
File path to the subscription yaml file
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
List of subscription validators, for each one in the subscription yaml
|
|
||||||
"""
|
|
||||||
# TODO: Create separate yaml file loader class
|
|
||||||
with open(subscription_path, "r", encoding="utf-8") as file:
|
|
||||||
subscription_dict = yaml.safe_load(file)
|
|
||||||
|
|
||||||
subscriptions: List["SubscriptionValidator"] = []
|
|
||||||
for subscription_key, subscription_object in subscription_dict.items():
|
|
||||||
subscriptions.append(
|
|
||||||
SubscriptionValidator.from_dict(
|
|
||||||
config=config,
|
|
||||||
subscription_name=subscription_key,
|
|
||||||
subscription_dict=subscription_object,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return subscriptions
|
|
||||||
|
|
@ -5,7 +5,8 @@ from typing import List
|
||||||
from ytdl_sub.cli.download_args_parser import DownloadArgsParser
|
from ytdl_sub.cli.download_args_parser import DownloadArgsParser
|
||||||
from ytdl_sub.cli.main_args_parser import parser
|
from ytdl_sub.cli.main_args_parser import parser
|
||||||
from ytdl_sub.config.config_file import ConfigFile
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.config.subscription import SubscriptionValidator
|
from ytdl_sub.config.preset import Preset
|
||||||
|
from ytdl_sub.subscriptions.subscription import Subscription
|
||||||
from ytdl_sub.utils.exceptions import ValidationException
|
from ytdl_sub.utils.exceptions import ValidationException
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
|
|
||||||
|
|
@ -19,17 +20,17 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N
|
||||||
:param config: Configuration file
|
:param config: Configuration file
|
||||||
:param args: Arguments from argparse
|
:param args: Arguments from argparse
|
||||||
"""
|
"""
|
||||||
subscription_paths: List[str] = args.subscription_paths
|
preset_paths: List[str] = args.subscription_paths
|
||||||
subscriptions: List[SubscriptionValidator] = []
|
presets: List[Preset] = []
|
||||||
|
|
||||||
for subscription_path in subscription_paths:
|
for preset_path in preset_paths:
|
||||||
subscriptions += SubscriptionValidator.from_file_path(
|
presets += Preset.from_file_path(config=config, subscription_path=preset_path)
|
||||||
config=config, subscription_path=subscription_path
|
|
||||||
)
|
for preset in presets:
|
||||||
|
subscription = Subscription.from_preset(preset=preset, config=config)
|
||||||
|
|
||||||
for subscription in subscriptions:
|
|
||||||
logger.info("Beginning subscription download for %s", subscription.name)
|
logger.info("Beginning subscription download for %s", subscription.name)
|
||||||
subscription.to_subscription().download()
|
subscription.download()
|
||||||
|
|
||||||
|
|
||||||
def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None:
|
def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None:
|
||||||
|
|
@ -43,11 +44,18 @@ def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -
|
||||||
subscription_args_dict = dl_args_parser.to_subscription_dict()
|
subscription_args_dict = dl_args_parser.to_subscription_dict()
|
||||||
|
|
||||||
subscription_name = f"cli-dl-{dl_args_parser.get_args_hash()}"
|
subscription_name = f"cli-dl-{dl_args_parser.get_args_hash()}"
|
||||||
SubscriptionValidator.from_dict(
|
subscription_preset = Preset.from_dict(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_name=subscription_name,
|
preset_name=subscription_name,
|
||||||
subscription_dict=subscription_args_dict,
|
preset_dict=subscription_args_dict,
|
||||||
).to_subscription().download()
|
)
|
||||||
|
|
||||||
|
subscription = Subscription.from_preset(
|
||||||
|
preset=subscription_preset,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
subscription.download()
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ from typing import List
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
from typing import Type
|
from typing import Type
|
||||||
|
|
||||||
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.config.config_file import ConfigOptions
|
from ytdl_sub.config.config_file import ConfigOptions
|
||||||
from ytdl_sub.config.preset import Preset
|
from ytdl_sub.config.preset import Preset
|
||||||
from ytdl_sub.config.preset_options import OutputOptions
|
from ytdl_sub.config.preset_options import OutputOptions
|
||||||
|
|
@ -253,3 +254,25 @@ class Subscription:
|
||||||
downloader.post_download(
|
downloader.post_download(
|
||||||
overrides=self.overrides, output_directory=self.output_directory
|
overrides=self.overrides, output_directory=self.output_directory
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_preset(cls, preset: Preset, config: ConfigFile) -> "Subscription":
|
||||||
|
"""
|
||||||
|
Creates a subscription from a preset
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
preset
|
||||||
|
Preset to make the subscription out of
|
||||||
|
config
|
||||||
|
The config file that should contain this preset
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
Initialized subscription
|
||||||
|
"""
|
||||||
|
return cls(
|
||||||
|
name=preset.name,
|
||||||
|
preset_options=preset,
|
||||||
|
config_options=config.config_options,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,8 @@ import pytest
|
||||||
from e2e.expected_download import ExpectedDownload
|
from e2e.expected_download import ExpectedDownload
|
||||||
|
|
||||||
from ytdl_sub.config.config_file import ConfigFile
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.config.subscription import SubscriptionValidator
|
from ytdl_sub.config.preset import Preset
|
||||||
|
from ytdl_sub.subscriptions.subscription import Subscription
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
|
|
@ -52,11 +53,16 @@ def subscription_dict(output_directory, subscription_name):
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def full_channel_subscription(config, subscription_name, subscription_dict):
|
def full_channel_subscription(config, subscription_name, subscription_dict):
|
||||||
return SubscriptionValidator.from_dict(
|
full_channel_preset = Preset.from_dict(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_name=subscription_name,
|
preset_name=subscription_name,
|
||||||
subscription_dict=subscription_dict,
|
preset_dict=subscription_dict,
|
||||||
).to_subscription()
|
)
|
||||||
|
|
||||||
|
return Subscription.from_preset(
|
||||||
|
preset=full_channel_preset,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|
@ -133,22 +139,24 @@ def recent_channel_subscription_dict(subscription_dict):
|
||||||
return mergedeep.merge(
|
return mergedeep.merge(
|
||||||
subscription_dict,
|
subscription_dict,
|
||||||
{
|
{
|
||||||
|
"preset": "yt_channel_as_tv__recent",
|
||||||
"youtube": {"after": "20150101"},
|
"youtube": {"after": "20150101"},
|
||||||
"ytdl_options": {
|
|
||||||
"break_on_reject": True,
|
|
||||||
"break_on_existing": True,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict):
|
def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict):
|
||||||
return SubscriptionValidator.from_dict(
|
recent_channel_preset = Preset.from_dict(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_name=subscription_name,
|
preset_name=subscription_name,
|
||||||
subscription_dict=recent_channel_subscription_dict,
|
preset_dict=recent_channel_subscription_dict,
|
||||||
).to_subscription()
|
)
|
||||||
|
|
||||||
|
return Subscription.from_preset(
|
||||||
|
preset=recent_channel_preset,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|
@ -183,7 +191,11 @@ def expected_recent_channel_download():
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
|
def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
|
||||||
return mergedeep.merge(
|
return mergedeep.merge(
|
||||||
recent_channel_subscription_dict, {"output_options": {"keep_files_after": "20181101"}}
|
recent_channel_subscription_dict,
|
||||||
|
{
|
||||||
|
"preset": "yt_channel_as_tv__only_recent",
|
||||||
|
"output_options": {"keep_files_after": "20181101"},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -191,11 +203,16 @@ def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
|
||||||
def rolling_recent_channel_subscription(
|
def rolling_recent_channel_subscription(
|
||||||
config, subscription_name, rolling_recent_channel_subscription_dict
|
config, subscription_name, rolling_recent_channel_subscription_dict
|
||||||
):
|
):
|
||||||
return SubscriptionValidator.from_dict(
|
rolling_recent_channel_preset = Preset.from_dict(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_name=subscription_name,
|
preset_name=subscription_name,
|
||||||
subscription_dict=rolling_recent_channel_subscription_dict,
|
preset_dict=rolling_recent_channel_subscription_dict,
|
||||||
).to_subscription()
|
)
|
||||||
|
|
||||||
|
return Subscription.from_preset(
|
||||||
|
preset=rolling_recent_channel_preset,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue