danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ytdl options builder

Browse source
This commit is contained in:
jbannon 2022-08-10 23:39:27 +00:00
parent cb439a4d4d
commit 3b3c23963e
7 changed files with 124 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/ytdl_sub/config/preset.py
View file

@ -79,6 +79,8 @@ class DownloadStrategyValidator(StrictDictValidator):
raise self._validation_exception(error_message=value_exc)
# Preset is the meat of ytdl-sub, the linter is wrong here
# pylint: disable=too-many-instance-attributes
class Preset(StrictDictValidator):
# Have all present keys optional since parent presets could not have all the
# required keys. They will get validated in the init after the mergedeep of dicts

59
src/ytdl_sub/downloaders/downloader.py
View file

@ -1,6 +1,5 @@
import abc
import contextlib
import copy
import json
import os
import time
@ -22,6 +21,7 @@ from yt_dlp.utils import ExistingVideoReached
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener
@ -70,25 +70,11 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
"""
return {"ignoreerrors": True}
@classmethod
def _configure_ytdl_options(
cls,
ytdl_options: Optional[Dict],
) -> Dict:
"""Configure the ytdl options for the downloader"""
if ytdl_options is None:
ytdl_options = {}
# Overwrite defaults with input
ytdl_options = dict(cls.ytdl_option_defaults(), **ytdl_options)
return ytdl_options
def __init__(
self,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options: Optional[Dict] = None,
ytdl_options_builder: YTDLOptionsBuilder,
):
"""
Parameters
@ -97,13 +83,14 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
Options validator for this downloader
enhanced_download_archive
Download archive
ytdl_options
YTDL options validator
ytdl_options_builder
YTDL options builder
"""
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.ytdl_options = self._configure_ytdl_options(
ytdl_options=ytdl_options,
self._ytdl_options_builder = ytdl_options_builder.clone().add(
self.ytdl_option_defaults(), before=True
)
@contextmanager
@ -111,9 +98,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
"""
Context manager to interact with yt_dlp.
"""
ytdl_options = self.ytdl_options
if ytdl_options_overrides is not None:
ytdl_options = dict(ytdl_options, **ytdl_options_overrides)
ytdl_options = self._ytdl_options_builder.clone().add(ytdl_options_overrides).to_dict()
download_logger.debug("ytdl_options: %s", str(ytdl_options))
with Logger.handle_external_logs(name="yt-dlp"):
@ -127,7 +112,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
-------
True if dry-run is enabled. False otherwise.
"""
return self.ytdl_options.get("skip_download", False)
return self._ytdl_options_builder.to_dict().get("skip_download", False)
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
"""
@ -173,7 +158,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
"""
num_tries = 0
entry_files_exist = False
ytdl_options_overrides = copy.deepcopy(ytdl_options_overrides)
while not entry_files_exist and num_tries < self._extract_entry_num_retries:
entry_dict = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
@ -185,7 +169,11 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
# Remove the download archive so it can retry without thinking its already downloaded,
# even though it is not
ytdl_options_overrides["download_archive"] = None
ytdl_options_overrides = (
YTDLOptionsBuilder()
.add(ytdl_options_overrides, {"download_archive": None})
.to_dict()
)
if num_tries < self._extract_entry_retry_wait_sec:
download_logger.debug(
@ -299,20 +287,25 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
**kwargs
arguments passed directory to YoutubeDL extract_info
"""
ytdl_options_builder = self._ytdl_options_builder.clone()
if ytdl_options_overrides is None:
ytdl_options_overrides = {}
extract_info_ytdl_options = {"writeinfojson": True}
ytdl_options_builder.add({"writeinfojson": True}, ytdl_options_overrides)
if only_info_json:
extract_info_ytdl_options["skip_download"] = True
extract_info_ytdl_options["writethumbnail"] = False
extract_info_ytdl_options["writesubtitles"] = False
ytdl_options_overrides = dict(ytdl_options_overrides, **extract_info_ytdl_options)
ytdl_options_builder.add(
{
"skip_download": True,
"writethumbnail": False,
"writesubtitles": False,
}
)
try:
with self._listen_and_log_downloaded_info_json(log_prefix=log_prefix_on_info_json_dl):
_ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
_ = self.extract_info(
ytdl_options_overrides=ytdl_options_builder.to_dict(), **kwargs
)
except RejectedVideoReached:
download_logger.debug("RejectedVideoReached, stopping additional downloads")
except ExistingVideoReached:

5
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -9,6 +9,7 @@ from ytdl_sub.downloaders.downloader import DownloaderOptionsT
from ytdl_sub.downloaders.downloader import download_logger
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.youtube import YoutubeChannel
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.thumbnail import convert_url_thumbnail
@ -115,12 +116,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
self,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options: Optional[Dict] = None,
ytdl_options_builder: YTDLOptionsBuilder,
):
super().__init__(
download_options=download_options,
enhanced_download_archive=enhanced_download_archive,
ytdl_options=ytdl_options,
ytdl_options_builder=ytdl_options_builder,
)
self.channel: Optional[YoutubeChannel] = None

59
src/ytdl_sub/downloaders/ytdl_options_builder.py Normal file
View file

@ -0,0 +1,59 @@
import copy
from typing import Dict
from typing import Optional
import mergedeep
class YTDLOptionsBuilder:
"""
A class to track any modifications made to ytdl options
"""
def __init__(self):
self._ytdl_options: Dict = {}
def add(
self,
*ytdl_option_dicts: Optional[Dict],
before: bool = False,
strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE
) -> "YTDLOptionsBuilder":
"""
Parameters
----------
*ytdl_option_dicts
One or many ytdl_option dicts. Can also contain None's for convenience
before
Optional. Whether to add these dicts before or after the original
strategy
Optional. mergedeep strategy. Defaults to TYPESAFE_ADDITIVE
Returns
-------
instance with the added ytdl_option dict(s)
"""
non_empty = [ytdl_options for ytdl_options in ytdl_option_dicts if ytdl_options is not None]
if before:
non_empty.append(self.to_dict())
self._ytdl_options = {}
mergedeep.merge(self._ytdl_options, *non_empty, strategy=strategy)
return self
def clone(self) -> "YTDLOptionsBuilder":
"""
Returns
-------
Deep-copied instance
"""
return copy.deepcopy(self)
def to_dict(self) -> Dict:
"""
Returns
-------
Deep-copied dict of the current builder state
"""
return copy.deepcopy(self._ytdl_options)

6
src/ytdl_sub/subscriptions/subscription.py
View file

@ -287,19 +287,19 @@ class Subscription:
directory.
"""
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
ytdl_options = SubscriptionYTDLOptions(
ytdl_options_builder = SubscriptionYTDLOptions(
preset=self.__preset_options,
enhanced_download_archive=self._enhanced_download_archive,
working_directory=self.working_directory,
dry_run=dry_run,
).to_dict()
).builder()
plugins = self._initialize_plugins()
with self._subscription_download_context_managers():
downloader = self.downloader_class(
download_options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
ytdl_options=ytdl_options,
ytdl_options_builder=ytdl_options_builder,
)
for entry in downloader.download():

54
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -1,9 +1,8 @@
from pathlib import Path
from typing import Dict
import mergedeep
from ytdl_sub.config.preset import Preset
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -22,11 +21,26 @@ class SubscriptionYTDLOptions:
@property
def _global_options(self) -> Dict:
return {
"""
Returns
-------
ytdl-options to apply to every run no matter what
"""
ytdl_options = {
# Download all files in the format of {id}.{ext}
"outtmpl": str(Path(self._working_directory) / "%(id)s.%(ext)s")
}
if (
self._preset.downloader.supports_download_archive
and self._preset.output_options.maintain_download_archive
):
ytdl_options["download_archive"] = str(
Path(self._working_directory) / self._enhanced_download_archive.archive_file_name
)
return ytdl_options
@property
def _dry_run_options(self) -> Dict:
return {
@ -42,13 +56,6 @@ class SubscriptionYTDLOptions:
if output_options.thumbnail_name:
ytdl_options["writethumbnail"] = True
if (
self._preset.downloader.supports_download_archive
and output_options.maintain_download_archive
):
ytdl_options["download_archive"] = str(
Path(self._working_directory) / self._enhanced_download_archive.archive_file_name
)
return ytdl_options
@ -83,22 +90,19 @@ class SubscriptionYTDLOptions:
def _user_ytdl_options(self) -> Dict:
return self._preset.ytdl_options.dict
def to_dict(self) -> Dict:
ytdl_options = self._global_options
def builder(self) -> YTDLOptionsBuilder:
"""
Returns
-------
YTDLOptionsBuilder
Builder with values set based on the subscription
"""
ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options)
if self._dry_run:
mergedeep.merge(
ytdl_options,
self._dry_run_options,
self._user_ytdl_options,
strategy=mergedeep.Strategy.TYPESAFE_ADDITIVE,
)
ytdl_options_builder.add(self._dry_run_options, self._user_ytdl_options)
else:
mergedeep.merge(
ytdl_options,
self._output_options,
self._subtitle_options,
self._user_ytdl_options,
strategy=mergedeep.Strategy.TYPESAFE_ADDITIVE,
ytdl_options_builder.add(
self._output_options, self._subtitle_options, self._user_ytdl_options
)
return ytdl_options
return ytdl_options_builder

2
tests/unit/entries/conftest.py
View file

@ -54,10 +54,12 @@ def ext():
def thumbnail_ext():
return "jpg"
@pytest.fixture
def subtitles_ext():
return "srt"
@pytest.fixture
def download_thumbnail_name(uid, thumbnail_ext):
return f"{uid}.{thumbnail_ext}"