danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

plugins done

Browse source
This commit is contained in:
Jesse Bannon 2023-12-28 14:55:16 -08:00
parent 724faaac9e
commit 0669ac3e32
30 changed files with 860 additions and 750 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
docs/source/config_reference/config_yaml.rst
View file

@ -53,51 +53,7 @@ presets
~~~~~~~
``presets`` define a `formula` for how to format downloaded media and metadata.
download_strategy
"""""""""""""""""
Download strategies dictate what is getting downloaded from a source. Each
download strategy has its own set of parameters.
.. _url:
url
'''
.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions()
:members: url, playlist_thumbnails, source_thumbnails, download_reverse
:member-order: bysource
multi_url
'''''''''
.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions()
:members: urls, variables
-------------------------------------------------------------------------------
output_options
""""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
:members:
:member-order: bysource
:exclude-members: get_upload_date_range_to_keep, partial_validate
-------------------------------------------------------------------------------
.. _ytdl_options:
ytdl_options
""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions()
-------------------------------------------------------------------------------
.. _overrides:
overrides
"""""""""
.. autoclass:: ytdl_sub.config.overrides.Overrides()
.. _parent preset:
This section is work-in-progress!
preset
""""""

741
docs/source/config_reference/plugins.rst
View file

File diff suppressed because it is too large Load diff

17
docs/source/config_reference/scripting/config_types.rst
View file

@ -1,17 +0,0 @@
==================
Config Field Types
==================
The ``config.yaml`` uses various types for its configurable fields. Below is a definition for each type.
.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator()
.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator()
.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator()

5
docs/source/config_reference/scripting/index.rst
View file

@ -2,8 +2,11 @@
Scripting
=========
Work in progress! Explanation of how to define scripts/variables will be added here.
.. toctree::
:maxdepth: 1
entry_variables
override_variables
scripting_functions
config_types

6
src/ytdl_sub/config/overrides.py
View file

@ -23,9 +23,9 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
class Overrides(DictFormatterValidator, Scriptable):
"""
Optional. This section allows you to define variables that can be used in any string formatter.
For example, if you want your file and thumbnail files to match without copy-pasting a large
format string, you can define something like:
Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter.
:Usage:
.. code-block:: yaml

94
src/ytdl_sub/config/preset_options.py
View file

@ -15,12 +15,12 @@ from ytdl_sub.validators.validators import LiteralDictValidator
class YTDLOptions(LiteralDictValidator):
"""
Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader.
Allows you to add any ytdl argument to ytdl-sub's downloader.
The argument names can differ slightly from the command-line argument names. See
`this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_
for more details.
ytdl_options should be formatted like:
:Usage:
.. code-block:: yaml
@ -58,7 +58,7 @@ class OutputOptions(StrictDictValidator):
"""
Defines where to output files and thumbnails after all post-processing has completed.
Usage:
:Usage:
.. code-block:: yaml
@ -156,97 +156,119 @@ class OutputOptions(StrictDictValidator):
@property
def output_directory(self) -> OverridesStringFormatterValidator:
"""
Required. The output directory to store all media files downloaded.
:expected type: OverridesFormatter
:description:
The output directory to store all media files downloaded.
"""
return self._output_directory
@property
def file_name(self) -> StringFormatterValidator:
"""
Required. The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
:expected type: EntryFormatter
:description:
The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
"""
return self._file_name
@property
def thumbnail_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
"""
return self._thumbnail_name
@property
def info_json_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
"""
return self._info_json_name
@property
def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
:expected type: Optional[OverridesFormatter]
:description:
The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
"""
return self._download_archive_name
@property
def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this name
first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
:expected type: Optional[OverridesFormatter]
:description:
Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this
name first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
"""
return self._migrated_download_archive_name
@property
def maintain_download_archive(self) -> bool:
"""
Optional. Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
:expected type: Optional[Boolean]
:description:
Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
create a ytdl download-archive file when invoking a download on a subscription. This will
prevent ytdl from redownloading media already downloaded.
The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
create a ytdl download-archive file when invoking a download on a subscription. This will
prevent ytdl from redownloading media already downloaded.
Defaults to False.
Defaults to False.
"""
return self._maintain_download_archive.value
@property
def keep_files_before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``.
"""
return self._keep_files_before
@property
def keep_files_after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``.
"""
return self._keep_files_after
@property
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
"""
return self._keep_max_files

37
src/ytdl_sub/downloaders/url/multi_url.py
View file

@ -1,37 +0,0 @@
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
# TODO: Remove later - keep for docstring
class MultiUrlDownloadOptions(MultiUrlValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""

25
src/ytdl_sub/downloaders/url/url.py
View file

@ -1,25 +0,0 @@
from ytdl_sub.downloaders.url.validators import UrlValidator
# TODO: Remove later - keep for docstring
class UrlDownloadOptions(UrlValidator):
"""
Downloads from a single URL supported by yt-dlp.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
# optional
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
download_reverse: True
"""

49
src/ytdl_sub/downloaders/url/validators.py
View file

@ -215,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]):
class MultiUrlValidator(OptionsValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Sets the URL(s) to download from. Can be used in many forms, including
:Single URL:
.. code-block:: yaml
download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
:Multi URL:
.. code-block:: yaml
download:
- "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
- "https://www.youtube.com/watch?v=3BFTio5296w"
:Thumbnails + Variables:
All variables must be defined for the top-most url. All subsequent URL variables can be either
overwritten or default to the top-most value.
If an entry is returned from more than one URL, it will use the variables in the bottom-most
URL.
.. code-block:: yaml
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""
@classmethod

2
src/ytdl_sub/plugins/chapters.py
View file

@ -64,7 +64,7 @@ class ChaptersOptions(OptionsDictValidator):
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex.
:usage:
:Usage:
.. code-block:: yaml

2
src/ytdl_sub/plugins/date_range.py
View file

@ -22,7 +22,7 @@ class DateRangeOptions(OptionsDictValidator):
Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest
granularity possible.
:usage:
:Usage:
.. code-block:: yaml

6
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -20,13 +20,11 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
"""
Whether to embed thumbnails to the audio/video file or not.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
embed_thumbnail: True
embed_thumbnail: True
"""

61
src/ytdl_sub/plugins/file_convert.py
View file

@ -32,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator):
"""
Converts video files from one extension to another.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mp4"
file_convert:
convert_to: "mp4"
Supports custom ffmpeg conversions:
Also supports custom ffmpeg conversions:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
ffmpeg_post_process_args: >
-bitexact
-vcodec copy
-acodec copy
-scodec mov_text
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
ffmpeg_post_process_args: >
-bitexact
-vcodec copy
-acodec copy
-scodec mov_text
"""
_required_keys = {"convert_to"}
@ -89,35 +87,38 @@ class FileConvertOptions(OptionsDictValidator):
@property
def convert_to(self) -> str:
"""
Convert to a desired file type. Supports:
:expected type: String
:description:
Convert to a desired file type. Supports
* Video: avi, flv, mkv, mov, mp4, webm
* Audio: aac, flac, mp3, m4a, opus, vorbis, wav
- Video: avi, flv, mkv, mov, mp4, webm
- Audio: aac, flac, mp3, m4a, opus, vorbis, wav
"""
return self._convert_to
@property
def convert_with(self) -> Optional[str]:
"""
Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
:expected type: Optional[String]
:description:
Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
"""
return self._convert_with
@property
def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the
form of:
:expected type: Optional[OverridesFormatter]
:description:
ffmpeg args to post-process an entry file with. The args will be inserted in the
form of
.. code-block:: bash
``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``.
ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.
"""
return self._ffmpeg_post_process_args

14
src/ytdl_sub/plugins/filter_exclude.py
View file

@ -19,17 +19,15 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
filter_exclude:
- >-
{ %contains( %lower(title), '#short' ) }
- >-
{ %contains( %lower(description), '#short' ) }
filter_exclude:
- >-
{ %contains( %lower(title), '#short' ) }
- >-
{ %contains( %lower(description), '#short' ) }
"""

30
src/ytdl_sub/plugins/filter_include.py
View file

@ -19,25 +19,23 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
filter_include:
- >-
{description}
- >-
{
%regex_search_any(
title,
[
"Full Episode",
"FULL",
]
)
}
filter_include:
- >-
{description}
- >-
{
%regex_search_any(
title,
[
"Full Episode",
"FULL",
]
)
}
"""

4
src/ytdl_sub/plugins/format.py
View file

@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator):
.. code-block:: yaml
presets:
my_example_preset:
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
"""
def __init__(self, name, value):

32
src/ytdl_sub/plugins/match_filters.py
View file

@ -58,32 +58,20 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
class MatchFiltersOptions(OptionsDictValidator):
"""
Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp.
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
For logical AND's between match filters, use the ``&`` operator in a single match filter.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
match_filters:
filters: "original_url!*=/shorts/"
Supports one or multiple filters:
.. code-block:: yaml
presets:
my_example_preset:
match_filters:
filters:
- "age_limit<?18"
- "like_count>?100"
# Other common match-filters
# - "original_url!*=/shorts/ & !is_live"
# - "age_limit<?18"
# - "availability=?public"
match_filters:
filters:
- "age_limit<?18 & like_count>?100"
# Other common match-filters
# - "original_url!*=/shorts/ & !is_live"
# - "availability=?public"
"""
_optional_keys = {"filters"}

21
src/ytdl_sub/plugins/music_tags.py
View file

@ -75,23 +75,22 @@ class MusicTagsOptions(OptionsDictValidator):
a full list of tags for various file types in MediaFile's
`source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
music_tags:
tags:
artist: "{artist}"
album: "{album}"
# Supports id3v2.4 multi-tags
genres:
- "{genre}"
- "ytdl-sub"
albumartists:
- "{artist}"
- "ytdl-sub"
artist: "{artist}"
album: "{album}"
# Supports id3v2.4 multi-tags
genres:
- "{genre}"
- "ytdl-sub"
albumartists:
- "{artist}"
- "ytdl-sub"
"""
_optional_keys = {"tags", "embed_thumbnail"}

94
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -60,7 +60,9 @@ class SharedNfoTagsOptions(OptionsDictValidator):
@property
def nfo_name(self) -> StringFormatterFileNameValidator:
"""
The NFO file name.
:expected type: EntryFormatter
:description:
The NFO file name.
"""
return self._nfo_name
@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator):
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Kodi does not support > 3-byte unicode characters, which include
emojis and some foreign language characters. Setting this to True will replace those
characters with '□'.
"""
return self._kodi_safe
@ -190,22 +194,18 @@ class NfoTagsOptions(SharedNfoTagsOptions):
Adds an NFO file for every download file. An NFO file is simply an XML file
with a ``.nfo`` extension. You can add any values into the NFO.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
nfo_tags:
# required
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
# optional
kodi_safe: False
nfo_tags:
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
kodi_safe: False
"""
_formatter_validator = StringFormatterValidator
@ -215,50 +215,54 @@ class NfoTagsOptions(SharedNfoTagsOptions):
@property
def nfo_root(self) -> StringFormatterValidator:
"""
The root tag of the NFO's XML. In the usage above, it would look like
:expected type: EntryFormatter
:description:
The root tag of the NFO's XML. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
</episodedetails>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
</episodedetails>
"""
return self._nfo_root
@property
def tags(self) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
:expected type: NfoTags
:description:
Tags within the nfo_root tag. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
<title>Awesome Youtube Video</title>
<season>2022</season>
<episode>502</episode>
</episodedetails>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
<title>Awesome Youtube Video</title>
<season>2022</season>
<episode>502</episode>
</episodedetails>
Also supports xml attributes and duplicate keys:
Also supports xml attributes and duplicate keys:
.. code-block:: yaml
.. code-block:: yaml
tags:
season:
attributes:
name: "Best Year"
tag: "{upload_year}"
genre:
- "Comedy"
- "Drama"
tags:
season:
attributes:
name: "Best Year"
tag: "{upload_year}"
genre:
- "Comedy"
- "Drama"
Which translates to
Which translates to
.. code-block:: xml
.. code-block:: xml
<season name="Best Year">2022</season>
<genre>Comedy</genre>
<genre>Drama</genre>
<season name="Best Year">2022</season>
<genre>Comedy</genre>
<genre>Drama</genre>
"""
return self._tags

56
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -37,48 +37,52 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
@property
def nfo_root(self) -> StringFormatterValidator:
"""
The root tag of the NFO's XML. In the usage above, it would look like
:expected type: EntryFormatter
:description:
The root tag of the NFO's XML. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
</tvshow>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
</tvshow>
"""
return self._nfo_root
@property
def tags(self) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
:expected type: NfoTags
:description:
Tags within the nfo_root tag. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
<title>Sweet youtube TV show</title>
</tvshow>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
<title>Sweet youtube TV show</title>
</tvshow>
Also supports xml attributes and duplicate keys:
Also supports xml attributes and duplicate keys:
.. code-block:: yaml
.. code-block:: yaml
tags:
named_season:
- tag: "{source_title}"
attributes:
number: "{collection_index}"
genre:
- "Comedy"
- "Drama"
tags:
named_season:
- tag: "{source_title}"
attributes:
number: "{collection_index}"
genre:
- "Comedy"
- "Drama"
Which translates to
Which translates to
.. code-block:: xml
.. code-block:: xml
<title year="2022">Sweet youtube TV show</season>
<genre>Comedy</genre>
<genre>Drama</genre>
<title year="2022">Sweet youtube TV show</season>
<genre>Comedy</genre>
<genre>Drama</genre>
"""
return self._tags

115
src/ytdl_sub/plugins/regex.py
View file

@ -122,6 +122,41 @@ class FromSourceVariablesRegex(DictValidator):
class RegexOptions(OptionsDictValidator):
r"""
.. attention::
This plugin will eventually be deprecated and replaced by scripting functions.
You can replicate the example below using the following.
.. code-block:: yaml
# Only includes videos with 'Official Video'
filter_include:
- >-
{ %contains( %lower(title), "official video" ) }
# Excludes videos with '#short' in its description
filter_exclude:
- >-
{ %contains( %lower(description), '#short' ) }
# Creates a capture array with defaults, and assigns
# each capture group to its own variable
overrides:
description_date_capture: >-
{
%regex_capture_many_with_defaults(
description,
[ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
[ upload_year, upload_month, upload_day ]
)
}
captured_upload_year: >-
{ %array_at(description_date_capture, 1) }
captured_upload_month: >-
{ %array_at(description_date_capture, 2) }
captured_upload_day: >-
{ %array_at(description_date_capture, 3) }
Performs regex matching on an entry's source or override variables. Regex can be used to filter
entries from proceeding with download or capture groups to create new source variables.
@ -137,51 +172,49 @@ class RegexOptions(OptionsDictValidator):
and using ``title_and_description`` can regex match/exclude from either ``title`` or
``description``.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
regex:
# By default, if any match fails and has no defaults, the entry will
# be skipped. If False, ytdl-sub will error and stop all downloads
# from proceeding.
skip_if_match_fails: True
regex:
# By default, if any match fails and has no defaults, the entry will
# be skipped. If False, ytdl-sub will error and stop all downloads
# from proceeding.
skip_if_match_fails: True
from:
# For each entry's `title` value...
title:
# Perform this regex match on it to act as a filter.
# This will only download videos with "[Official Video]" in it. Note that we
# double backslash to make YAML happy
match:
- '\\[Official Video\\]'
from:
# For each entry's `title` value...
title:
# Perform this regex match on it to act as a filter.
# This will only download videos with "[Official Video]" in it. Note that we
# double backslash to make YAML happy
match:
- '\\[Official Video\\]'
# For each entry's `description` value...
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new
# source variables
match:
- '([0-9]{4})-([0-9]{2})-([0-9]{2})'
# Exclude any entry where the description contains #short
exclude:
- '#short'
# For each entry's `description` value...
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new
# source variables
match:
- '([0-9]{4})-([0-9]{2})-([0-9]{2})'
# Exclude any entry where the description contains #short
exclude:
- '#short'
# Each capture group creates these new source variables, respectively,
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# Each capture group creates these new source variables, respectively,
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# And if the string does not match, use these as respective default
# values for the new source variables.
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
# And if the string does not match, use these as respective default
# values for the new source variables.
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
"""
_required_keys = {"from"}
@ -209,8 +242,10 @@ class RegexOptions(OptionsDictValidator):
@property
def skip_if_match_fails(self) -> Optional[bool]:
"""
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
:expected type: Optional[Boolean]
:description:
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
"""
return self._skip_if_match_fails

29
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -48,23 +48,22 @@ class WhenNoChaptersValidator(StringSelectValidator):
class SplitByChaptersOptions(OptionsDictValidator):
"""
Splits a file by chapters into multiple files. Each file becomes its own entry with the
new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``,
``chapter_index_padded``, ``chapter_count``.
new variables
If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
- ``chapter_title``
- ``chapter_index``
- ``chapter_index_padded``
- ``chapter_count``
Note that when using this plugin and performing dry-run, it assumes embedded chapters are being
used with no modifications.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
split_by_chapters:
when_no_chapters: "pass"
split_by_chapters:
when_no_chapters: "pass"
"""
_required_keys = {"when_no_chapters"}
@ -100,8 +99,16 @@ class SplitByChaptersOptions(OptionsDictValidator):
@property
def when_no_chapters(self) -> str:
"""
Behavior to perform when no chapters are present. Supports "pass" (continue processing),
"drop" (exclude it from output), and "error" (stop processing for everything).
:expected type: String
:description:
Behavior to perform when no chapters are present. Supports
- "pass" (continue processing),
- "drop" (exclude it from output)
- "error" (stop processing for everything).
If a file has no chapters and is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
"""
return self._when_no_chapters

48
src/ytdl_sub/plugins/subtitles.py
View file

@ -37,18 +37,18 @@ class SubtitleOptions(OptionsDictValidator):
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
It will set the respective language to the correct subtitle file.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
subtitles:
subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}"
subtitles_type: "srt"
embed_subtitles: False
languages: "en" # supports list of multiple languages
allow_auto_generated_subtitles: False
subtitles:
subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}"
subtitles_type: "srt"
embed_subtitles: False
languages:
- "en" # supports multiple languages
- "de"
allow_auto_generated_subtitles: False
"""
_optional_keys = {
@ -82,40 +82,50 @@ class SubtitleOptions(OptionsDictValidator):
@property
def subtitles_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's subtitles if they are present. This can include
directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and
will be placed in the output directory. ``lang`` is dynamic since you can download multiple
subtitles. It will set the respective language to the correct subtitle file.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's subtitles if they are present. This can include
directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``,
and will be placed in the output directory. ``lang`` is dynamic since you can download
multiple subtitles. It will set the respective language to the correct subtitle file.
"""
return self._subtitles_name
@property
def subtitles_type(self) -> Optional[str]:
"""
Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt"
:expected type: Optional[String]
:description:
Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc".
"""
return self._subtitles_type
@property
def embed_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to embed the subtitles into the video file. Defaults to False.
NOTE: webm files can only embed "vtt" subtitle types.
:expected type: Optional[Boolean]
:description:
Defaults to False. Whether to embed the subtitles into the video file. Note that
webm files can only embed "vtt" subtitle types.
"""
return self._embed_subtitles
@property
def languages(self) -> Optional[List[str]]:
"""
Optional. Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to "en".
:expected type: Optional[List[String]]
:description:
Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to only "en".
"""
return [lang.value for lang in self._languages]
@property
def allow_auto_generated_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to allow auto generated subtitles. Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Whether to allow auto generated subtitles.
"""
return self._allow_auto_generated_subtitles

24
src/ytdl_sub/plugins/throttle_protection.py
View file

@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
range-based values, a random number will be chosen within the range to avoid sleeps looking
scripted.
Usage:
:Usage:
.. code-block:: yaml
@ -110,31 +110,39 @@ class ThrottleProtectionOptions(OptionsDictValidator):
@property
def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]:
"""
Number in seconds to sleep between each download. Does not include time it takes for
ytdl-sub to perform post-processing.
:expected type: Optional[Range]
:description:
Number in seconds to sleep between each download. Does not include time it takes for
ytdl-sub to perform post-processing.
"""
return self._sleep_per_download_s
@property
def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]:
"""
Number in seconds to sleep between each subscription.
:expected type: Optional[Range]
:description:
Number in seconds to sleep between each subscription.
"""
return self._sleep_per_subscription_s
@property
def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]:
"""
Number of downloads to perform per subscription.
:expected type: Optional[Range]
:description:
Number of downloads to perform per subscription.
"""
return self._max_downloads_per_subscription
@property
def subscription_download_probability(self) -> Optional[ProbabilityValidator]:
"""
Probability to perform any downloads, recomputed for each subscription. This is only
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
guaranteed over time to eventually download the subscription.
:expected type: Optional[Float]
:description:
Probability to perform any downloads, recomputed for each subscription. This is only
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
guaranteed over time to eventually download the subscription.
"""
return self._subscription_download_probability

12
src/ytdl_sub/plugins/video_tags.py
View file

@ -17,16 +17,14 @@ class VideoTagsOptions(OptionsDictValidator):
"""
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
video_tags:
title: "{title}"
date: "{upload_date}"
description: "{description}"
video_tags:
title: "{title}"
date: "{upload_date}"
description: "{description}"
"""
_optional_keys = {"tags"}

2
tools/docgen/docgen.py
View file

@ -1,7 +1,7 @@
from abc import abstractmethod
from pathlib import Path
REGENERATE_DOCS: bool = True
REGENERATE_DOCS: bool = False
class DocGen:

3
tools/docgen/entry_variables.py
View file

@ -4,8 +4,9 @@ from typing import Dict
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import camel_case_to_human, line_section
from tools.docgen.utils import camel_case_to_human
from tools.docgen.utils import get_function_docs
from tools.docgen.utils import line_section
from tools.docgen.utils import properties
from tools.docgen.utils import section
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions

26
tools/docgen/plugins.py
View file

@ -1,6 +1,8 @@
import inspect
from pathlib import Path
from typing import Dict, Any, Optional
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from tools.docgen.docgen import DocGen
@ -12,6 +14,20 @@ from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
def should_filter_all_properties(plugin_name: str) -> bool:
return plugin_name in (
"format",
"match_filters",
"music_tags",
"filter_include",
"filter_exclude",
"embed_thumbnail",
"video_tags",
"download",
)
def should_filter_property(property_name: str) -> bool:
@ -26,9 +42,7 @@ def should_filter_property(property_name: str) -> bool:
)
def get_function_docs(
function_name: str, obj: Any, level: int
) -> str:
def get_function_docs(function_name: str, obj: Any, level: int) -> str:
docs = f"\n``{function_name}``\n\n"
docs += inspect.cleandoc(getattr(obj, function_name).__doc__)
docs += "\n\n"
@ -42,6 +56,9 @@ def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int
docs += inspect.cleandoc(options.__doc__)
docs += "\n"
if should_filter_all_properties(name):
return docs
property_names = [prop for prop in properties(options) if not should_filter_property(prop)]
for property_name in sorted(property_names):
docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1)
@ -59,6 +76,7 @@ class PluginsDocGen(DocGen):
"output_options": OutputOptions,
"ytdl_options": YTDLOptions,
"overrides": Overrides,
"download": MultiUrlValidator,
}
for plugin_name, plugin_type in PluginMapping._MAPPING.items():
if plugin_name.startswith("_"):

3
tools/docgen/scripting_functions.py
View file

@ -6,7 +6,8 @@ from typing import Optional
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import camel_case_to_human, line_section
from tools.docgen.utils import camel_case_to_human
from tools.docgen.utils import line_section
from tools.docgen.utils import section
from tools.docgen.utils import static_methods
from ytdl_sub.entries.script.custom_functions import CustomFunctions