danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

separate preset options

Browse source
This commit is contained in:
jbannon 2022-04-22 20:52:15 +00:00
parent f2ec156138
commit 24fac00064
8 changed files with 158 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
config.yaml
View file

@ -56,7 +56,6 @@ presets:
convert_thumbnail: "jpeg"
thumbnail_name: "{output_file_name}.jpg"
maintain_download_archive: True
maintain_stale_file_deletion: True
convert_thumbnail:
to: "jpg"
nfo_tags:

108
ytdl_subscribe/config/preset.py
View file

@ -1,4 +1,3 @@
from abc import ABC
from typing import Any
from typing import Dict
from typing import List
@ -6,25 +5,21 @@ from typing import Optional
from typing import Tuple
from typing import Type
from yt_dlp.utils import sanitize_filename
from ytdl_subscribe.config.preset_class_mappings import DownloadStrategyMapping
from ytdl_subscribe.config.preset_class_mappings import PluginMapping
from ytdl_subscribe.config.preset_options import DownloadStrategyValidator
from ytdl_subscribe.config.preset_options import OutputOptions
from ytdl_subscribe.config.preset_options import Overrides
from ytdl_subscribe.config.preset_options import YTDLOptions
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.utils.exceptions import StringFormattingVariableNotFoundException
from ytdl_subscribe.utils.exceptions import ValidationException
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator
from ytdl_subscribe.validators.validators import BoolValidator
from ytdl_subscribe.validators.validators import DictValidator
from ytdl_subscribe.validators.validators import LiteralDictValidator
from ytdl_subscribe.validators.validators import StringValidator
from ytdl_subscribe.validators.validators import Validator
PRESET_REQUIRED_KEYS = {"output_options"}
@ -36,100 +31,7 @@ PRESET_OPTIONAL_KEYS = {
}
class YTDLOptions(LiteralDictValidator):
"""Ensures `ytdl_options` is a dict"""
class Overrides(DictFormatterValidator):
"""Ensures `overrides` is a dict"""
def __init__(self, name, value):
super().__init__(name, value)
for key in self._keys:
sanitized_key_name = f"sanitized_{key}"
# First, sanitize the format string
self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string)
# Then, convert it into a StringFormatterValidator
self._value[sanitized_key_name] = StringFormatterValidator(
name="__should_never_fail__",
value=self._value[sanitized_key_name],
)
def apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> str:
"""
Returns the format_string after .format has been called on it using entry (if provided) and
override values
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
return formatter.apply_formatter(variable_dict)
class OutputOptions(StrictDictValidator):
"""Where to output the final files and thumbnails"""
_required_keys = {"output_directory", "file_name"}
_optional_keys = {
"thumbnail_name",
"maintain_download_archive",
"maintain_stale_file_deletion",
}
def __init__(self, name, value):
super().__init__(name, value)
# Output directory should resolve without any entry variables.
# This is to check the directory for any download-archives before any downloads begin
self.output_directory: OverridesStringFormatterValidator = self._validate_key(
key="output_directory", validator=OverridesStringFormatterValidator
)
# file name and thumbnails however can use entry variables
self.file_name: StringFormatterValidator = self._validate_key(
key="file_name", validator=StringFormatterValidator
)
self.thumbnail_name = self._validate_key_if_present(
key="thumbnail_name", validator=StringFormatterValidator
)
self.maintain_download_archive = self._validate_key_if_present(
key="maintain_download_archive", validator=BoolValidator, default=False
)
self.maintain_stale_file_deletion = self._validate_key_if_present(
key="maintain_stale_file_deletion", validator=BoolValidator, default=False
)
if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value:
raise self._validation_exception(
"maintain_stale_file_deletion requires maintain_download_archive set to True"
)
class DownloadStrategyValidator(StrictDictValidator, ABC):
"""
Ensures a download strategy exists for a source. Does not validate any more than that.
The respective Downloader's option validator will do that.
"""
# All media sources must define a download strategy
_required_keys = {"download_strategy"}
# Extra fields will be strict-validated using other StictDictValidators
_allow_extra_keys = True
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
self.name = self._validate_key(
key="download_strategy",
validator=StringValidator,
).value
class PresetValidator(StrictDictValidator):
class Preset(StrictDictValidator):
_required_keys = PRESET_REQUIRED_KEYS
_optional_keys = PRESET_OPTIONAL_KEYS

108
ytdl_subscribe/config/preset_options.py Normal file
View file

@ -0,0 +1,108 @@
from abc import ABC
from typing import Any
from typing import Optional
from yt_dlp.utils import sanitize_filename
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.validators.date_range_validator import DateRangeValidator
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator
from ytdl_subscribe.validators.validators import BoolValidator
from ytdl_subscribe.validators.validators import LiteralDictValidator
from ytdl_subscribe.validators.validators import StringValidator
class YTDLOptions(LiteralDictValidator):
"""Ensures `ytdl_options` is a dict"""
class Overrides(DictFormatterValidator):
"""Ensures `overrides` is a dict"""
def __init__(self, name, value):
super().__init__(name, value)
for key in self._keys:
sanitized_key_name = f"sanitized_{key}"
# First, sanitize the format string
self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string)
# Then, convert it into a StringFormatterValidator
self._value[sanitized_key_name] = StringFormatterValidator(
name="__should_never_fail__",
value=self._value[sanitized_key_name],
)
def apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> str:
"""
Returns the format_string after .format has been called on it using entry (if provided) and
override values
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
return formatter.apply_formatter(variable_dict)
class OutputOptions(StrictDictValidator):
"""Where to output the final files and thumbnails"""
_required_keys = {"output_directory", "file_name"}
_optional_keys = {
"thumbnail_name",
"maintain_download_archive",
"maintain_stale_file_deletion",
}
def __init__(self, name, value):
super().__init__(name, value)
# Output directory should resolve without any entry variables.
# This is to check the directory for any download-archives before any downloads begin
self.output_directory: OverridesStringFormatterValidator = self._validate_key(
key="output_directory", validator=OverridesStringFormatterValidator
)
# file name and thumbnails however can use entry variables
self.file_name: StringFormatterValidator = self._validate_key(
key="file_name", validator=StringFormatterValidator
)
self.thumbnail_name = self._validate_key_if_present(
key="thumbnail_name", validator=StringFormatterValidator
)
self.maintain_download_archive = self._validate_key_if_present(
key="maintain_download_archive", validator=BoolValidator, default=False
)
self.maintain_stale_file_deletion = self._validate_key_if_present(
key="maintain_stale_file_deletion", validator=DateRangeValidator, default=False
)
if self.maintain_stale_file_deletion and not self.maintain_download_archive:
raise self._validation_exception(
"maintain_stale_file_deletion requires maintain_download_archive set to True"
)
class DownloadStrategyValidator(StrictDictValidator, ABC):
"""
Ensures a download strategy exists for a source. Does not validate any more than that.
The respective Downloader's option validator will do that.
"""
# All media sources must define a download strategy
_required_keys = {"download_strategy"}
# Extra fields will be strict-validated using other StictDictValidators
_allow_extra_keys = True
def __init__(self, name: str, value: Any):
super().__init__(name=name, value=value)
self.name = self._validate_key(
key="download_strategy",
validator=StringValidator,
).value

6
ytdl_subscribe/config/subscription.py
View file

@ -9,8 +9,8 @@ from mergedeep import mergedeep
from ytdl_subscribe.config.config_file import ConfigFile
from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS
from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset import PresetValidator
from ytdl_subscribe.config.preset import Preset
from ytdl_subscribe.config.preset_options import Overrides
from ytdl_subscribe.subscriptions.subscription import Subscription
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.validators import StringValidator
@ -52,7 +52,7 @@ class SubscriptionValidator(StrictDictValidator):
preset_dict = mergedeep.merge(preset_dict, self._dict, strategy=mergedeep.Strategy.REPLACE)
del preset_dict["preset"]
self.preset = PresetValidator(
self.preset = Preset(
name=f"{self._name}.{preset_name}",
value=preset_dict,
)

60
ytdl_subscribe/downloaders/youtube_downloader.py
View file

@ -13,7 +13,7 @@ from yt_dlp.utils import RejectedVideoReached
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.youtube import YoutubeVideo
from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource
from ytdl_subscribe.validators.date_range_validator import DateRangeValidator
from ytdl_subscribe.validators.validators import StringValidator
###############################################################################
@ -54,14 +54,19 @@ class YoutubeDownloader(
"""Returns full channel url"""
return f"https://youtube.com/channel/{channel_id}"
def _download_with_metadata(
self, url: str, ytdl_options_overrides: Optional[Dict] = None
) -> None:
def _download_using_metadata(
self,
url: str,
ignore_prefix: str,
ytdl_options_overrides: Optional[Dict] = None,
) -> List[YoutubeVideo]:
"""
Do not get entries from the extract info, let it write to the info.json file and load
that instead. This is because if the video is already downloaded in a playlist, it will
not fetch the metadata (maybe there is a way??)
"""
entries: List[YoutubeVideo] = []
ytdl_overrides = {
"writeinfojson": True,
}
@ -73,6 +78,14 @@ class YoutubeDownloader(
except RejectedVideoReached:
pass
# Load the entries from info.json, ignore the playlist entry
for file_name in os.listdir(self.working_directory):
if file_name.endswith(".info.json") and not file_name.startswith(ignore_prefix):
with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file:
entries.append(YoutubeVideo(**json.load(file)))
return entries
def download_video(self, video_id: str) -> YoutubeVideo:
"""Download a single Youtube video"""
entry = self.extract_info(url=self.video_url(video_id))
@ -82,20 +95,9 @@ class YoutubeDownloader(
"""
Downloads all videos in a Youtube playlist
"""
playlist_url = self.playlist_url(playlist_id=playlist_id)
self._download_with_metadata(url=playlist_url)
# Load the entries from info.json, ignore the playlist entry
entries: List[YoutubeVideo] = []
# Load the entries from info.json, ignore the playlist entry
for file_name in os.listdir(self.working_directory):
if file_name.endswith(".info.json") and not file_name.startswith(playlist_id):
with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file:
entries.append(YoutubeVideo(**json.load(file)))
return entries
return self._download_using_metadata(
url=self.playlist_url(playlist_id=playlist_id), ignore_prefix=playlist_id
)
def download_channel(
self, channel_id: str, ytdl_options_overrides: Optional[Dict] = None
@ -103,22 +105,12 @@ class YoutubeDownloader(
"""
Downloads all videos from a channel
"""
self._download_with_metadata(
url=self.channel_url(channel_id), ytdl_options_overrides=ytdl_options_overrides
return self._download_using_metadata(
url=self.channel_url(channel_id),
ignore_prefix=channel_id,
ytdl_options_overrides=ytdl_options_overrides,
)
# Load the entries from info.json
entries: List[YoutubeVideo] = []
# Load the entries from info.json
# TODO dupe code between this and playlist
for file_name in os.listdir(self.working_directory):
if file_name.endswith(".info.json") and not file_name.startswith(channel_id):
with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file:
entries.append(YoutubeVideo(**json.load(file)))
return entries
###############################################################################
# Youtube single video downloader + options
@ -163,13 +155,13 @@ class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptio
# Youtube channel downloader + options
class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DownloadDateRangeSource):
class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidator):
_required_keys = {"channel_id"}
_optional_keys = {"before", "after"}
def __init__(self, name, value):
YoutubeDownloaderOptions.__init__(self, name, value)
DownloadDateRangeSource.__init__(self, name, value)
DateRangeValidator.__init__(self, name, value)
self.channel_id = self._validate_key("channel_id", StringValidator)

2
ytdl_subscribe/plugins/plugin.py
View file

@ -4,7 +4,7 @@ from typing import Type
from typing import TypeVar
from typing import final
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset_options import Overrides
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive

32
ytdl_subscribe/subscriptions/subscription.py
View file

@ -3,23 +3,20 @@ import os
import shutil
from pathlib import Path
from shutil import copyfile
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_subscribe.config.config_file import ConfigOptions
from ytdl_subscribe.config.preset import OutputOptions
from ytdl_subscribe.config.preset import Overrides
from ytdl_subscribe.config.preset import PresetValidator
from ytdl_subscribe.config.preset import YTDLOptions
from ytdl_subscribe.config.preset import Preset
from ytdl_subscribe.config.preset_options import OutputOptions
from ytdl_subscribe.config.preset_options import Overrides
from ytdl_subscribe.config.preset_options import YTDLOptions
from ytdl_subscribe.downloaders.downloader import Downloader
from ytdl_subscribe.downloaders.downloader import DownloaderValidator
from ytdl_subscribe.entries.entry import Entry
from ytdl_subscribe.plugins.plugin import Plugin
from ytdl_subscribe.plugins.plugin import PluginOptions
from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource
from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -41,7 +38,7 @@ class Subscription:
self,
name: str,
config_options: ConfigOptions,
preset_options: PresetValidator,
preset_options: Preset,
):
"""
Parameters
@ -49,7 +46,7 @@ class Subscription:
name: str
Name of the subscription
config_options: ConfigOptions
preset_options: PresetValidator
preset_options: Preset
"""
self.name = name
self.__config_options = config_options
@ -143,15 +140,14 @@ class Subscription:
yield
# TODO: add date range in output options
# date_range = None
# if isinstance(self.source_options, DownloadDateRangeSource):
# date_range = self.source_options.get_date_range()
#
# if date_range and self.output_options.maintain_stale_file_deletion:
# self._enhanced_download_archive.remove_stale_files(date_range=date_range)
#
# self._enhanced_download_archive.save_download_archive()
# If output options maintains stale file deletion, perform the delete here prior to saving
# the download archive
if self.output_options.maintain_stale_file_deletion:
self._enhanced_download_archive.remove_stale_files(
date_range=self.output_options.maintain_stale_file_deletion.get_date_range()
)
self._enhanced_download_archive.save_download_archive()
def _initialize_plugins(self) -> List[Plugin]:
plugins: List[Plugin] = []

2
ytdl_subscribe/validators/date_range_validator.py
View file

@ -7,7 +7,7 @@ from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.string_datetime import StringDatetimeValidator
class DownloadDateRangeSource(StrictDictValidator, ABC):
class DateRangeValidator(StrictDictValidator, ABC):
optional_keys = {"before", "after"}
def __init__(self, name, value):