danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[DOCS] Tooling to auto-generate custom sphinx docs (#849)

Browse source 
Tooling + updating docs
This commit is contained in:
Jesse Bannon 2023-12-28 22:39:59 -08:00 committed by GitHub
parent 1fcaba9274
commit 731d4e444c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
66 changed files with 2660 additions and 1967 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -149,3 +149,5 @@ docker/testing/volumes
ffmpeg.exe
ffprobe.exe
tools/docgen/out

0
docs/source/_static/custom.css Normal file
View file

1
docs/source/conf.py
View file

@ -53,6 +53,7 @@ html_theme_options = {
}
html_static_path = ["_static"]
html_css_files = ["custom.css"]
# Make sure the all autosectionlabel targets are unique

46
docs/source/config_reference/config_yaml.rst
View file

@ -53,51 +53,7 @@ presets
~~~~~~~
``presets`` define a `formula` for how to format downloaded media and metadata.
download_strategy
"""""""""""""""""
Download strategies dictate what is getting downloaded from a source. Each
download strategy has its own set of parameters.
.. _url:
url
'''
.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions()
:members: url, playlist_thumbnails, source_thumbnails, download_reverse
:member-order: bysource
multi_url
'''''''''
.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions()
:members: urls, variables
-------------------------------------------------------------------------------
output_options
""""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
:members:
:member-order: bysource
:exclude-members: get_upload_date_range_to_keep, partial_validate
-------------------------------------------------------------------------------
.. _ytdl_options:
ytdl_options
""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions()
-------------------------------------------------------------------------------
.. _overrides:
overrides
"""""""""
.. autoclass:: ytdl_sub.config.overrides.Overrides()
.. _parent preset:
This section is work-in-progress!
preset
""""""

688
docs/source/config_reference/plugins.rst
View file

File diff suppressed because it is too large Load diff

17
docs/source/config_reference/scripting/config_types.rst
View file

@ -1,17 +0,0 @@
==================
Config Field Types
==================
The ``config.yaml`` uses various types for its configurable fields. Below is a definition for each type.
.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator()
.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator()
.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator()

13
docs/source/config_reference/scripting/entry_variables.rst
View file

@ -1,3 +1,4 @@
Entry Variables
===============
@ -111,6 +112,8 @@ webpage_url
~~~~~~~~~~~
The url to the webpage.
----------------------------------------------------------------------------------------------------
Metadata Variables
------------------
@ -130,6 +133,8 @@ source_metadata
~~~~~~~~~~~~~~~
Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry)
----------------------------------------------------------------------------------------------------
Playlist Variables
------------------
@ -210,6 +215,8 @@ playlist_webpage_url
~~~~~~~~~~~~~~~~~~~~
The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
----------------------------------------------------------------------------------------------------
Release Date Variables
----------------------
@ -285,6 +292,8 @@ release_year_truncated_reversed
The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
----------------------------------------------------------------------------------------------------
Source Variables
----------------
@ -332,6 +341,8 @@ source_webpage_url
~~~~~~~~~~~~~~~~~~
The source webpage url if it exists, otherwise returns the playlist webpage url.
----------------------------------------------------------------------------------------------------
Upload Date Variables
---------------------
@ -406,6 +417,8 @@ upload_year_truncated_reversed
The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
----------------------------------------------------------------------------------------------------
Ytdl-Sub Variables
------------------

5
docs/source/config_reference/scripting/index.rst
View file

@ -2,8 +2,11 @@
Scripting
=========
Work in progress! Explanation of how to define scripts/variables will be added here.
.. toctree::
:maxdepth: 1
entry_variables
override_variables
scripting_functions
config_types

2
docs/source/config_reference/scripting/override_variables.rst
View file

@ -1,3 +1,4 @@
Override Variables
==================
@ -41,4 +42,3 @@ For subscriptions in the form of
``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...``
and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to
``subscription_value``.

293
docs/source/config_reference/scripting/scripting_functions.rst
View file

@ -7,414 +7,571 @@ Array Functions
array
~~~~~
``array(maybe_array: AnyArgument) -> Array``
:spec: ``array(maybe_array: AnyArgument) -> Array``
:description:
Tries to cast an unknown variable type to an Array.
array_apply
~~~~~~~~~~~
``array_apply(array: Array, lambda_function: Lambda) -> Array``
:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array``
:description:
Apply a lambda function on every element in the Array.
:usage:
.. code-block:: python
{
%array_apply( [1, 2, 3] , %string )
}
# ["1", "2", "3"]
array_apply_fixed
~~~~~~~~~~~~~~~~~
:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array``
:description:
Apply a lambda function on every element in the Array, with ``fixed_argument``
passed as a second argument to every invocation.
array_at
~~~~~~~~
``array_at(array: Array, idx: Integer) -> AnyArgument``
:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument``
:description:
Return the element in the Array at index ``idx``.
array_contains
~~~~~~~~~~~~~~
``array_contains(array: Array, value: AnyArgument) -> Boolean``
:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean``
:description:
Return True if the value exists in the Array. False otherwise.
array_enumerate
~~~~~~~~~~~~~~~
``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array``
:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array``
:description:
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
array_extend
~~~~~~~~~~~~
``array_extend(arrays: Array, ...) -> Array``
:spec: ``array_extend(arrays: Array, ...) -> Array``
:description:
Combine multiple Arrays into a single Array.
array_first
~~~~~~~~~~~
:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument``
:description:
Returns the first element whose boolean conversion is True. Returns fallback
if all elements evaluate to False.
array_flatten
~~~~~~~~~~~~~
``array_flatten(array: Array) -> Array``
:spec: ``array_flatten(array: Array) -> Array``
:description:
Flatten any nested Arrays into a single-dimensional Array.
array_index
~~~~~~~~~~~
``array_index(array: Array, value: AnyArgument) -> Integer``
:spec: ``array_index(array: Array, value: AnyArgument) -> Integer``
:description:
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
array_overlay
~~~~~~~~~~~~~
:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array``
:description:
Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
array_product
~~~~~~~~~~~~~
``array_product(arrays: Array, ...) -> Array``
:spec: ``array_product(arrays: Array, ...) -> Array``
:description:
Returns the Cartesian product of elements from different arrays
array_reduce
~~~~~~~~~~~~
``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument``
:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument``
:description:
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
array_reverse
~~~~~~~~~~~~~
``array_reverse(array: Array) -> Array``
:spec: ``array_reverse(array: Array) -> Array``
:description:
Reverse an Array.
array_size
~~~~~~~~~~
``array_size(array: Array) -> Integer``
:spec: ``array_size(array: Array) -> Integer``
:description:
Returns the size of an Array.
array_slice
~~~~~~~~~~~
``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array``
:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array``
:description:
Returns the slice of the Array.
----------------------------------------------------------------------------------------------------
Boolean Functions
-----------------
and
~~~
``and(values: AnyArgument, ...) -> Boolean``
:spec: ``and(values: AnyArgument, ...) -> Boolean``
:description:
``and`` operator. Returns True if all values evaluate to True. False otherwise.
bool
~~~~
``bool(value: AnyArgument) -> Boolean``
:spec: ``bool(value: AnyArgument) -> Boolean``
:description:
Cast any type to a Boolean.
eq
~~
``eq(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``==`` operator. Returns True if left == right. False otherwise.
gt
~~
``gt(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``>`` operator. Returns True if left > right. False otherwise.
gte
~~~
``gte(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``>=`` operator. Returns True if left >= right. False otherwise.
is_null
~~~~~~~
:spec: ``is_null(value: AnyArgument) -> Boolean``
:description:
Returns True if a value is null (i.e. an empty string). False otherwise.
lt
~~
``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``<`` operator. Returns True if left < right. False otherwise.
lte
~~~
``lte(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``<=`` operator. Returns True if left <= right. False otherwise.
ne
~~
``ne(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``!=`` operator. Returns True if left != right. False otherwise.
not
~~~
``not(value: Boolean) -> Boolean``
:spec: ``not(value: Boolean) -> Boolean``
:description:
``not`` operator. Returns the opposite of value.
or
~~
``or(values: AnyArgument, ...) -> Boolean``
:spec: ``or(values: AnyArgument, ...) -> Boolean``
:description:
``or`` operator. Returns True if any value evaluates to True. False otherwise.
xor
~~~
``xor(values: AnyArgument, ...) -> Boolean``
:spec: ``xor(values: AnyArgument, ...) -> Boolean``
:description:
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
----------------------------------------------------------------------------------------------------
Conditional Functions
---------------------
if
~~
``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:description:
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
if_passthrough
~~~~~~~~~~~~~~
``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:description:
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
----------------------------------------------------------------------------------------------------
Date Functions
--------------
datetime_strftime
~~~~~~~~~~~~~~~~~
``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String``
:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String``
:description:
Converts a posix timestamp to a date using strftime formatting.
----------------------------------------------------------------------------------------------------
Error Functions
---------------
assert
~~~~~~
``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument``
:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument``
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``value``.
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``value``.
assert_eq
~~~~~~~~~
:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` does not equal
``equals``. If they do equal, then return ``value``.
assert_ne
~~~~~~~~~
:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` equals
``equals``. If they do equal, then return ``value``.
assert_then
~~~~~~~~~~~
:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``ret``.
throw
~~~~~
``throw(error_message: String) -> AnyArgument``
:spec: ``throw(error_message: String) -> AnyArgument``
:description:
Explicitly throw an error with the provided error message.
----------------------------------------------------------------------------------------------------
Json Functions
--------------
from_json
~~~~~~~~~
``from_json(argument: String) -> AnyArgument``
:spec: ``from_json(argument: String) -> AnyArgument``
:description:
Converts a JSON string into an actual type.
----------------------------------------------------------------------------------------------------
Map Functions
-------------
map
~~~
``map(maybe_mapping: AnyArgument) -> Map``
:spec: ``map(maybe_mapping: AnyArgument) -> Map``
:description:
Tries to cast an unknown variable type to a Map.
map_apply
~~~~~~~~~
``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array``
:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array``
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
map_contains
~~~~~~~~~~~~
``map_contains(mapping: Map, key: AnyArgument) -> Boolean``
:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean``
:description:
Returns True if the key is in the Map. False otherwise.
map_enumerate
~~~~~~~~~~~~~
``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array``
:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array``
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
map_get
~~~~~~~
``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
:description:
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
map_get_non_empty
~~~~~~~~~~~~~~~~~
``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument``
:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument``
:description:
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
map_size
~~~~~~~~
``map_size(mapping: Map) -> Integer``
:spec: ``map_size(mapping: Map) -> Integer``
:description:
Returns the size of a Map.
----------------------------------------------------------------------------------------------------
Numeric Functions
-----------------
add
~~~
``add(values: Numeric, ...) -> Numeric``
:spec: ``add(values: Numeric, ...) -> Numeric``
:description:
``+`` operator. Returns the sum of all values.
div
~~~
``div(left: Numeric, right: Numeric) -> Numeric``
:spec: ``div(left: Numeric, right: Numeric) -> Numeric``
:description:
``/`` operator. Returns ``left / right``.
float
~~~~~
``float(value: AnyArgument) -> Float``
:spec: ``float(value: AnyArgument) -> Float``
:description:
Cast to Float.
int
~~~
``int(value: AnyArgument) -> Integer``
:spec: ``int(value: AnyArgument) -> Integer``
:description:
Cast to Integer.
max
~~~
``max(values: Numeric, ...) -> Numeric``
:spec: ``max(values: Numeric, ...) -> Numeric``
:description:
Returns max of all values.
min
~~~
``min(values: Numeric, ...) -> Numeric``
:spec: ``min(values: Numeric, ...) -> Numeric``
:description:
Returns min of all values.
mod
~~~
``mod(left: Numeric, right: Numeric) -> Numeric``
:spec: ``mod(left: Numeric, right: Numeric) -> Numeric``
:description:
``%`` operator. Returns ``left % right``.
mul
~~~
``mul(values: Numeric, ...) -> Numeric``
:spec: ``mul(values: Numeric, ...) -> Numeric``
:description:
``*`` operator. Returns the product of all values.
pow
~~~
``pow(base: Numeric, exponent: Numeric) -> Numeric``
:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric``
:description:
``**`` operator. Returns the exponential of the base and exponent value.
sub
~~~
``sub(values: Numeric, ...) -> Numeric``
:spec: ``sub(values: Numeric, ...) -> Numeric``
:description:
``-`` operator. Subtracts all values from left to right.
----------------------------------------------------------------------------------------------------
Regex Functions
---------------
regex_capture_groups
~~~~~~~~~~~~~~~~~~~~
:spec: ``regex_capture_groups(regex: String) -> Integer``
:description:
Returns number of capture groups in regex
regex_fullmatch
~~~~~~~~~~~~~~~
``regex_fullmatch(regex: String, string: String) -> Array``
:spec: ``regex_fullmatch(regex: String, string: String) -> Array``
:description:
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_match
~~~~~~~~~~~
``regex_match(regex: String, string: String) -> Array``
:spec: ``regex_match(regex: String, string: String) -> Array``
:description:
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_search
~~~~~~~~~~~~
``regex_search(regex: String, string: String) -> Array``
:spec: ``regex_search(regex: String, string: String) -> Array``
:description:
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
----------------------------------------------------------------------------------------------------
String Functions
----------------
capitalize
~~~~~~~~~~
``capitalize(string: String) -> String``
:spec: ``capitalize(string: String) -> String``
:description:
Capitalize the first character in the string.
concat
~~~~~~
``concat(values: String, ...) -> String``
:spec: ``concat(values: String, ...) -> String``
:description:
Concatenate multiple Strings into a single String.
contains
~~~~~~~~
:spec: ``contains(string: String, contains: String) -> Boolean``
:description:
Returns True if ``contains`` is in ``string``. False otherwise.
lower
~~~~~
``lower(string: String) -> String``
:spec: ``lower(string: String) -> String``
:description:
Lower-case the entire String.
pad
~~~
``pad(string: String, length: Integer, char: String) -> String``
:spec: ``pad(string: String, length: Integer, char: String) -> String``
:description:
Pads the string to the given length
pad_zero
~~~~~~~~
``pad_zero(numeric: Numeric, length: Integer) -> String``
:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String``
:description:
Pads a numeric with zeros to the given length
replace
~~~~~~~
``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String``
:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String``
:description:
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
slice
~~~~~
``slice(string: String, start: Integer, end: Optional[Integer]) -> String``
:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String``
:description:
Returns the slice of the Array.
string
~~~~~~
``string(value: AnyArgument) -> String``
:spec: ``string(value: AnyArgument) -> String``
:description:
Cast to String.
titlecase
~~~~~~~~~
``titlecase(string: String) -> String``
:spec: ``titlecase(string: String) -> String``
:description:
Capitalize each word in the string.
upper
~~~~~
``upper(string: String) -> String``
:spec: ``upper(string: String) -> String``
:description:
Upper-case the entire String.
----------------------------------------------------------------------------------------------------
Ytdl-Sub Functions
------------------
legacy_bracket_safety
~~~~~~~~~~~~~~~~~~~~~
``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument``
:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument``
ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not
interfere with its legacy variable scripting system. This function replicates that
@ -422,14 +579,14 @@ behavior.
sanitize
~~~~~~~~
``sanitize(value: AnyArgument) -> String``
:spec: ``sanitize(value: AnyArgument) -> String``
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS.
sanitize_plex_episode
~~~~~~~~~~~~~~~~~~~~~
``sanitize_plex_episode(string: String) -> String``
:spec: ``sanitize_plex_episode(string: String) -> String``
Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width
numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the
@ -437,7 +594,7 @@ season and/or episode.
to_date_metadata
~~~~~~~~~~~~~~~~
``to_date_metadata(yyyymmdd: String) -> Map``
:spec: ``to_date_metadata(yyyymmdd: String) -> Map``
Takes a date in the form of YYYYMMDD and returns a Map containing:
@ -461,14 +618,14 @@ Takes a date in the form of YYYYMMDD and returns a Map containing:
to_native_filepath
~~~~~~~~~~~~~~~~~~
``to_native_filepath(filepath: String) -> String``
:spec: ``to_native_filepath(filepath: String) -> String``
Convert any unix-based path separators ('/') with the OS's native
separator.
truncate_filepath_if_too_long
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``truncate_filepath_if_too_long(filepath: String) -> String``
:spec: ``truncate_filepath_if_too_long(filepath: String) -> String``
If a file-path is too long for the OS, this function will truncate it while preserving
the extension.

2
docs/source/prebuilt_presets/tv_shows.rst
View file

@ -18,7 +18,7 @@ Jellyfin
Kodi
~~~~
* Everything that the Jellyfin version does
* Turns on :ref:`config_reference/plugins:kodi_safe`, replacing characters that would break kodi with safer characters
* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□``
Plex
~~~~

14
src/ytdl_sub/config/overrides.py
View file

@ -8,7 +8,7 @@ import mergedeep
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME
from ytdl_sub.entries.variables.override_variables import OverrideVariables
from ytdl_sub.entries.variables.override_variables import OverrideHelpers
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@ -23,9 +23,9 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
class Overrides(DictFormatterValidator, Scriptable):
"""
Optional. This section allows you to define variables that can be used in any string formatter.
For example, if you want your file and thumbnail files to match without copy-pasting a large
format string, you can define something like:
Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter.
:Usage:
.. code-block:: yaml
@ -89,7 +89,7 @@ class Overrides(DictFormatterValidator, Scriptable):
"""
Ensures the variable name does not collide with any entry variables or built-in functions.
"""
if not OverrideVariables.is_valid_name(name):
if not OverrideHelpers.is_valid_name(name):
override_type = "function" if name.startswith("%") else "variable"
raise self._validation_exception(
f"Override {override_type} with name {name} is invalid. Names must be"
@ -97,14 +97,14 @@ class Overrides(DictFormatterValidator, Scriptable):
exception_class=InvalidVariableNameException,
)
if OverrideVariables.is_entry_variable_name(name):
if OverrideHelpers.is_entry_variable_name(name):
raise self._validation_exception(
f"Override variable with name {name} cannot be used since it is a"
" built-in ytdl-sub entry variable name.",
exception_class=InvalidVariableNameException,
)
if OverrideVariables.is_function_name(name):
if OverrideHelpers.is_function_name(name):
raise self._validation_exception(
f"Override function definition with name {name} cannot be used since it is"
" a built-in ytdl-sub function name.",

52
src/ytdl_sub/config/preset_options.py
View file

@ -15,12 +15,12 @@ from ytdl_sub.validators.validators import LiteralDictValidator
class YTDLOptions(LiteralDictValidator):
"""
Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader.
Allows you to add any ytdl argument to ytdl-sub's downloader.
The argument names can differ slightly from the command-line argument names. See
`this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_
for more details.
ytdl_options should be formatted like:
:Usage:
.. code-block:: yaml
@ -58,7 +58,7 @@ class OutputOptions(StrictDictValidator):
"""
Defines where to output files and thumbnails after all post-processing has completed.
Usage:
:Usage:
.. code-block:: yaml
@ -156,14 +156,18 @@ class OutputOptions(StrictDictValidator):
@property
def output_directory(self) -> OverridesStringFormatterValidator:
"""
Required. The output directory to store all media files downloaded.
:expected type: OverridesFormatter
:description:
The output directory to store all media files downloaded.
"""
return self._output_directory
@property
def file_name(self) -> StringFormatterValidator:
"""
Required. The file name for the media file. This can include directories such as
:expected type: EntryFormatter
:description:
The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
"""
return self._file_name
@ -171,7 +175,9 @@ class OutputOptions(StrictDictValidator):
@property
def thumbnail_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's thumbnail image. This can include directories such
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
"""
@ -180,7 +186,9 @@ class OutputOptions(StrictDictValidator):
@property
def info_json_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's info json file. This can include directories such
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
"""
@ -189,7 +197,9 @@ class OutputOptions(StrictDictValidator):
@property
def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. The file name to store a subscriptions download archive placed relative to
:expected type: Optional[OverridesFormatter]
:description:
The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
"""
return self._download_archive_name
@ -197,9 +207,11 @@ class OutputOptions(StrictDictValidator):
@property
def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this name
first, and fallback to ``download_archive_name``. It will always save to this file
:expected type: Optional[OverridesFormatter]
:description:
Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this
name first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
"""
return self._migrated_download_archive_name
@ -207,7 +219,9 @@ class OutputOptions(StrictDictValidator):
@property
def maintain_download_archive(self) -> bool:
"""
Optional. Maintains a download archive file in the output directory for a subscription.
:expected type: Optional[Boolean]
:description:
Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
@ -222,7 +236,10 @@ class OutputOptions(StrictDictValidator):
@property
def keep_files_before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. Can be used in conjunction with
@ -233,7 +250,10 @@ class OutputOptions(StrictDictValidator):
@property
def keep_files_after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. Can be used in conjunction with
@ -244,7 +264,9 @@ class OutputOptions(StrictDictValidator):
@property
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.

37
src/ytdl_sub/downloaders/url/multi_url.py
View file

@ -1,37 +0,0 @@
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
# TODO: Remove later - keep for docstring
class MultiUrlDownloadOptions(MultiUrlValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""

25
src/ytdl_sub/downloaders/url/url.py
View file

@ -1,25 +0,0 @@
from ytdl_sub.downloaders.url.validators import UrlValidator
# TODO: Remove later - keep for docstring
class UrlDownloadOptions(UrlValidator):
"""
Downloads from a single URL supported by yt-dlp.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
# optional
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
download_reverse: True
"""

49
src/ytdl_sub/downloaders/url/validators.py
View file

@ -215,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]):
class MultiUrlValidator(OptionsValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Sets the URL(s) to download from. Can be used in many forms, including
:Single URL:
.. code-block:: yaml
download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
:Multi URL:
.. code-block:: yaml
download:
- "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
- "https://www.youtube.com/watch?v=3BFTio5296w"
:Thumbnails + Variables:
All variables must be defined for the top-most url. All subsequent URL variables can be either
overwritten or default to the top-most value.
If an entry is returned from more than one URL, it will use the variables in the bottom-most
URL.
.. code-block:: yaml
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""
@classmethod

948
src/ytdl_sub/entries/script/variable_definitions.py
View file

File diff suppressed because it is too large Load diff

18
src/ytdl_sub/entries/variables/override_variables.py
View file

@ -10,15 +10,15 @@ SUBSCRIPTION_ARRAY = "subscription_array"
class OverrideVariables:
@classmethod
def subscription_name(cls) -> str:
@staticmethod
def subscription_name() -> str:
"""
Name of the subscription
"""
return SUBSCRIPTION_NAME
@classmethod
def subscription_value(cls) -> str:
@staticmethod
def subscription_value() -> str:
"""
For subscriptions in the form of
@ -30,8 +30,8 @@ class OverrideVariables:
"""
return SUBSCRIPTION_VALUE
@classmethod
def subscription_indent_i(cls, index: int) -> str:
@staticmethod
def subscription_indent_i(index: int) -> str:
"""
For subscriptions in the form of
@ -46,8 +46,8 @@ class OverrideVariables:
"""
return f"subscription_indent_{index + 1}"
@classmethod
def subscription_value_i(cls, index: int) -> str:
@staticmethod
def subscription_value_i(index: int) -> str:
"""
For subscriptions in the form of
@ -63,6 +63,8 @@ class OverrideVariables:
"""
return f"subscription_value_{index + 1}"
class OverrideHelpers:
@classmethod
def is_entry_variable_name(cls, name: str) -> bool:
"""

8
src/ytdl_sub/plugins/audio_extract.py
View file

@ -25,12 +25,10 @@ class AudioExtractOptions(OptionsDictValidator):
"""
Extracts audio from a video file.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
audio_extract:
codec: "mp3"
quality: 128
@ -56,6 +54,8 @@ class AudioExtractOptions(OptionsDictValidator):
@property
def codec(self) -> str:
"""
:expected type: String
:description:
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
opus, vorbis, wav, and best to grab the best possible format at runtime.
"""
@ -64,6 +64,8 @@ class AudioExtractOptions(OptionsDictValidator):
@property
def quality(self) -> Optional[float]:
"""
:expected type: Float
:description:
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
"""

34
src/ytdl_sub/plugins/chapters.py
View file

@ -64,12 +64,10 @@ class ChaptersOptions(OptionsDictValidator):
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
chapters:
# Embedded Chapter Fields
embed_chapters: True
@ -89,7 +87,6 @@ class ChaptersOptions(OptionsDictValidator):
- "intro"
remove_sponsorblock_categories: "all"
force_key_frames: False
"""
_optional_keys = {
@ -135,22 +132,28 @@ class ChaptersOptions(OptionsDictValidator):
@property
def embed_chapters(self) -> Optional[bool]:
"""
Optional. Embed chapters into the file. Defaults to True.
:expected type: Optional[Boolean]
:description:
Defaults to True. Embed chapters into the file.
"""
return self._embed_chapters
@property
def allow_chapters_from_comments(self) -> bool:
"""
Optional. If chapters do not exist in the video/description itself, attempt to scrape
comments to find the chapters. Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. If chapters do not exist in the video/description itself, attempt to
scrape comments to find the chapters.
"""
return self._allow_chapters_from_comments
@property
def remove_chapters_regex(self) -> Optional[List[re.Pattern]]:
"""
Optional. List of regex patterns to match chapter titles against and remove them from the
:expected type: Optional[List[RegexString]
:description:
List of regex patterns to match chapter titles against and remove them from the
entry.
"""
if self._remove_chapters_regex:
@ -160,7 +163,9 @@ class ChaptersOptions(OptionsDictValidator):
@property
def sponsorblock_categories(self) -> Optional[List[str]]:
"""
Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor",
:expected type: Optional[List[String]]
:description:
List of SponsorBlock categories to embed as chapters. Supports "sponsor",
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
"poi_highlight", or "all" to include all categories.
"""
@ -174,7 +179,9 @@ class ChaptersOptions(OptionsDictValidator):
@property
def remove_sponsorblock_categories(self) -> Optional[List[str]]:
"""
Optional. List of SponsorBlock categories to remove from the output file. Can only remove
:expected type: Optional[List[String]]
:description:
List of SponsorBlock categories to remove from the output file. Can only remove
categories that are specified in ``sponsorblock_categories`` or "all", which removes
everything specified in ``sponsorblock_categories``.
"""
@ -190,9 +197,10 @@ class ChaptersOptions(OptionsDictValidator):
@property
def force_key_frames(self) -> bool:
"""
Optional. Force keyframes at cuts when removing sections. This is slow due to needing a
re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to
False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Force keyframes at cuts when removing sections. This is slow due to
needing a re-encode, but the resulting video may have fewer artifacts around the cuts.
"""
return self._force_key_frames

22
src/ytdl_sub/plugins/date_range.py
View file

@ -11,13 +11,21 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator
class DateRangeOptions(OptionsDictValidator):
"""
Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs:
Usage:
.. code-block:: Markdown
A string in the format YYYYMMDD or
(now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this.
Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest
granularity possible.
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
date_range:
before: "now"
after: "today-2weeks"
@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator):
@property
def before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Only download videos before this datetime.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
"""
return self._before
@property
def after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Only download videos after this datetime.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
"""
return self._after

4
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -20,12 +20,10 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
"""
Whether to embed thumbnails to the audio/video file or not.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
embed_thumbnail: True
"""

33
src/ytdl_sub/plugins/file_convert.py
View file

@ -32,21 +32,19 @@ class FileConvertOptions(OptionsDictValidator):
"""
Converts video files from one extension to another.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mp4"
Supports custom ffmpeg conversions:
Also supports custom ffmpeg conversions:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
@ -89,32 +87,35 @@ class FileConvertOptions(OptionsDictValidator):
@property
def convert_to(self) -> str:
"""
Convert to a desired file type. Supports:
:expected type: String
:description:
Convert to a desired file type. Supports
* Video: avi, flv, mkv, mov, mp4, webm
* Audio: aac, flac, mp3, m4a, opus, vorbis, wav
- Video: avi, flv, mkv, mov, mp4, webm
- Audio: aac, flac, mp3, m4a, opus, vorbis, wav
"""
return self._convert_to
@property
def convert_with(self) -> Optional[str]:
"""
Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
:expected type: Optional[String]
:description:
Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
"""
return self._convert_with
@property
def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the
form of:
:expected type: Optional[OverridesFormatter]
:description:
ffmpeg args to post-process an entry file with. The args will be inserted in the
form of
.. code-block:: bash
ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext
``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``.
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.

10
src/ytdl_sub/plugins/filter_exclude.py
View file

@ -19,15 +19,15 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
filter_exclude:
- { %contains( %lower(title), '#short' ) }
- { %contains( %lower(description), '#short' ) }
- >-
{ %contains( %lower(title), '#short' ) }
- >-
{ %contains( %lower(description), '#short' ) }
"""

7
src/ytdl_sub/plugins/filter_include.py
View file

@ -19,14 +19,13 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
filter_include:
- {description}
- >-
{description}
- >-
{
%regex_search_any(

2
src/ytdl_sub/plugins/format.py
View file

@ -15,8 +15,6 @@ class FormatOptions(OptionsValidator):
.. code-block:: yaml
presets:
my_example_preset:
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
"""

22
src/ytdl_sub/plugins/match_filters.py
View file

@ -58,31 +58,19 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
class MatchFiltersOptions(OptionsDictValidator):
"""
Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp.
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
For logical AND's between match filters, use the ``&`` operator in a single match filter.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
match_filters:
filters: "original_url!*=/shorts/"
Supports one or multiple filters:
.. code-block:: yaml
presets:
my_example_preset:
match_filters:
filters:
- "age_limit<?18"
- "like_count>?100"
- "age_limit<?18 & like_count>?100"
# Other common match-filters
# - "original_url!*=/shorts/ & !is_live"
# - "age_limit<?18"
# - "availability=?public"
"""

3
src/ytdl_sub/plugins/music_tags.py
View file

@ -75,14 +75,13 @@ class MusicTagsOptions(OptionsDictValidator):
a full list of tags for various file types in MediaFile's
`source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
music_tags:
tags:
artist: "{artist}"
album: "{album}"
# Supports id3v2.4 multi-tags

20
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -60,6 +60,8 @@ class SharedNfoTagsOptions(OptionsDictValidator):
@property
def nfo_name(self) -> StringFormatterFileNameValidator:
"""
:expected type: EntryFormatter
:description:
The NFO file name.
"""
return self._nfo_name
@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator):
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with '□'.
Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Kodi does not support > 3-byte unicode characters, which include
emojis and some foreign language characters. Setting this to True will replace those
characters with '□'.
"""
return self._kodi_safe
@ -190,21 +194,17 @@ class NfoTagsOptions(SharedNfoTagsOptions):
Adds an NFO file for every download file. An NFO file is simply an XML file
with a ``.nfo`` extension. You can add any values into the NFO.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
nfo_tags:
# required
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
# optional
kodi_safe: False
"""
@ -215,6 +215,8 @@ class NfoTagsOptions(SharedNfoTagsOptions):
@property
def nfo_root(self) -> StringFormatterValidator:
"""
:expected type: EntryFormatter
:description:
The root tag of the NFO's XML. In the usage above, it would look like
.. code-block:: xml
@ -228,6 +230,8 @@ class NfoTagsOptions(SharedNfoTagsOptions):
@property
def tags(self) -> NfoTagsValidator:
"""
:expected type: NfoTags
:description:
Tags within the nfo_root tag. In the usage above, it would look like
.. code-block:: xml

4
src/ytdl_sub/plugins/output_directory_nfo_tags.py
View file

@ -37,6 +37,8 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
@property
def nfo_root(self) -> StringFormatterValidator:
"""
:expected type: EntryFormatter
:description:
The root tag of the NFO's XML. In the usage above, it would look like
.. code-block:: xml
@ -50,6 +52,8 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
@property
def tags(self) -> NfoTagsValidator:
"""
:expected type: NfoTags
:description:
Tags within the nfo_root tag. In the usage above, it would look like
.. code-block:: xml

41
src/ytdl_sub/plugins/regex.py
View file

@ -122,6 +122,41 @@ class FromSourceVariablesRegex(DictValidator):
class RegexOptions(OptionsDictValidator):
r"""
.. attention::
This plugin will eventually be deprecated and replaced by scripting functions.
You can replicate the example below using the following.
.. code-block:: yaml
# Only includes videos with 'Official Video'
filter_include:
- >-
{ %contains( %lower(title), "official video" ) }
# Excludes videos with '#short' in its description
filter_exclude:
- >-
{ %contains( %lower(description), '#short' ) }
# Creates a capture array with defaults, and assigns
# each capture group to its own variable
overrides:
description_date_capture: >-
{
%regex_capture_many_with_defaults(
description,
[ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
[ upload_year, upload_month, upload_day ]
)
}
captured_upload_year: >-
{ %array_at(description_date_capture, 1) }
captured_upload_month: >-
{ %array_at(description_date_capture, 2) }
captured_upload_day: >-
{ %array_at(description_date_capture, 3) }
Performs regex matching on an entry's source or override variables. Regex can be used to filter
entries from proceeding with download or capture groups to create new source variables.
@ -137,12 +172,10 @@ class RegexOptions(OptionsDictValidator):
and using ``title_and_description`` can regex match/exclude from either ``title`` or
``description``.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
regex:
# By default, if any match fails and has no defaults, the entry will
# be skipped. If False, ytdl-sub will error and stop all downloads
@ -209,6 +242,8 @@ class RegexOptions(OptionsDictValidator):
@property
def skip_if_match_fails(self) -> Optional[bool]:
"""
:expected type: Optional[Boolean]
:description:
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
"""

25
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -48,21 +48,20 @@ class WhenNoChaptersValidator(StringSelectValidator):
class SplitByChaptersOptions(OptionsDictValidator):
"""
Splits a file by chapters into multiple files. Each file becomes its own entry with the
new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``,
``chapter_index_padded``, ``chapter_count``.
new variables
If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
- ``chapter_title``
- ``chapter_index``
- ``chapter_index_padded``
- ``chapter_count``
Note that when using this plugin and performing dry-run, it assumes embedded chapters are being
used with no modifications.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
split_by_chapters:
when_no_chapters: "pass"
"""
@ -100,8 +99,16 @@ class SplitByChaptersOptions(OptionsDictValidator):
@property
def when_no_chapters(self) -> str:
"""
Behavior to perform when no chapters are present. Supports "pass" (continue processing),
"drop" (exclude it from output), and "error" (stop processing for everything).
:expected type: String
:description:
Behavior to perform when no chapters are present. Supports
- "pass" (continue processing),
- "drop" (exclude it from output)
- "error" (stop processing for everything).
If a file has no chapters and is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
"""
return self._when_no_chapters

38
src/ytdl_sub/plugins/subtitles.py
View file

@ -37,17 +37,17 @@ class SubtitleOptions(OptionsDictValidator):
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
It will set the respective language to the correct subtitle file.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
subtitles:
subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}"
subtitles_type: "srt"
embed_subtitles: False
languages: "en" # supports list of multiple languages
languages:
- "en" # supports multiple languages
- "de"
allow_auto_generated_subtitles: False
"""
@ -82,40 +82,50 @@ class SubtitleOptions(OptionsDictValidator):
@property
def subtitles_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's subtitles if they are present. This can include
directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and
will be placed in the output directory. ``lang`` is dynamic since you can download multiple
subtitles. It will set the respective language to the correct subtitle file.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's subtitles if they are present. This can include
directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``,
and will be placed in the output directory. ``lang`` is dynamic since you can download
multiple subtitles. It will set the respective language to the correct subtitle file.
"""
return self._subtitles_name
@property
def subtitles_type(self) -> Optional[str]:
"""
Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt"
:expected type: Optional[String]
:description:
Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc".
"""
return self._subtitles_type
@property
def embed_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to embed the subtitles into the video file. Defaults to False.
NOTE: webm files can only embed "vtt" subtitle types.
:expected type: Optional[Boolean]
:description:
Defaults to False. Whether to embed the subtitles into the video file. Note that
webm files can only embed "vtt" subtitle types.
"""
return self._embed_subtitles
@property
def languages(self) -> Optional[List[str]]:
"""
Optional. Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to "en".
:expected type: Optional[List[String]]
:description:
Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to only "en".
"""
return [lang.value for lang in self._languages]
@property
def allow_auto_generated_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to allow auto generated subtitles. Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Whether to allow auto generated subtitles.
"""
return self._allow_auto_generated_subtitles

10
src/ytdl_sub/plugins/throttle_protection.py
View file

@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
range-based values, a random number will be chosen within the range to avoid sleeps looking
scripted.
Usage:
:Usage:
.. code-block:: yaml
@ -110,6 +110,8 @@ class ThrottleProtectionOptions(OptionsDictValidator):
@property
def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]:
"""
:expected type: Optional[Range]
:description:
Number in seconds to sleep between each download. Does not include time it takes for
ytdl-sub to perform post-processing.
"""
@ -118,6 +120,8 @@ class ThrottleProtectionOptions(OptionsDictValidator):
@property
def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]:
"""
:expected type: Optional[Range]
:description:
Number in seconds to sleep between each subscription.
"""
return self._sleep_per_subscription_s
@ -125,6 +129,8 @@ class ThrottleProtectionOptions(OptionsDictValidator):
@property
def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]:
"""
:expected type: Optional[Range]
:description:
Number of downloads to perform per subscription.
"""
return self._max_downloads_per_subscription
@ -132,6 +138,8 @@ class ThrottleProtectionOptions(OptionsDictValidator):
@property
def subscription_download_probability(self) -> Optional[ProbabilityValidator]:
"""
:expected type: Optional[Float]
:description:
Probability to perform any downloads, recomputed for each subscription. This is only
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
guaranteed over time to eventually download the subscription.

4
src/ytdl_sub/plugins/video_tags.py
View file

@ -17,12 +17,10 @@ class VideoTagsOptions(OptionsDictValidator):
"""
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
video_tags:
title: "{title}"
date: "{upload_date}"

30
src/ytdl_sub/script/functions/array_functions.py
View file

@ -19,12 +19,8 @@ class ArrayFunctions:
@staticmethod
def array(maybe_array: AnyArgument) -> Array:
"""
:description:
Tries to cast an unknown variable type to an Array.
Raises
------
FunctionRuntimeException
If the input type is not actually an Array.
"""
if not isinstance(maybe_array, Array):
raise FunctionRuntimeException(
@ -35,6 +31,7 @@ class ArrayFunctions:
@staticmethod
def array_size(array: Array) -> Integer:
"""
:description:
Returns the size of an Array.
"""
return Integer(len(array.value))
@ -42,6 +39,7 @@ class ArrayFunctions:
@staticmethod
def array_extend(*arrays: Array) -> Array:
"""
:description:
Combine multiple Arrays into a single Array.
"""
output: List[Resolvable] = []
@ -55,6 +53,7 @@ class ArrayFunctions:
array: Array, overlap: Array, only_missing: Optional[Boolean] = None
) -> Array:
"""
:description:
Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
"""
output: List[Resolvable] = []
@ -76,6 +75,7 @@ class ArrayFunctions:
@staticmethod
def array_at(array: Array, idx: Integer) -> AnyArgument:
"""
:description:
Return the element in the Array at index ``idx``.
"""
return array.value[idx.value]
@ -83,6 +83,7 @@ class ArrayFunctions:
@staticmethod
def array_first(array: Array, fallback: AnyArgument) -> AnyArgument:
"""
:description:
Returns the first element whose boolean conversion is True. Returns fallback
if all elements evaluate to False.
"""
@ -95,6 +96,7 @@ class ArrayFunctions:
@staticmethod
def array_contains(array: Array, value: AnyArgument) -> Boolean:
"""
:description:
Return True if the value exists in the Array. False otherwise.
"""
return Boolean(value in array.value)
@ -102,6 +104,7 @@ class ArrayFunctions:
@staticmethod
def array_index(array: Array, value: AnyArgument) -> Integer:
"""
:description:
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
"""
@ -118,6 +121,7 @@ class ArrayFunctions:
@staticmethod
def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array:
"""
:description:
Returns the slice of the Array.
"""
if end is not None:
@ -127,6 +131,7 @@ class ArrayFunctions:
@staticmethod
def array_flatten(array: Array) -> Array:
"""
:description:
Flatten any nested Arrays into a single-dimensional Array.
"""
output: List[Resolvable] = []
@ -141,6 +146,7 @@ class ArrayFunctions:
@staticmethod
def array_reverse(array: Array) -> Array:
"""
:description:
Reverse an Array.
"""
return Array(list(reversed(array.value)))
@ -148,6 +154,7 @@ class ArrayFunctions:
@staticmethod
def array_product(*arrays: Array) -> Array:
"""
:description:
Returns the Cartesian product of elements from different arrays
"""
out: List[Resolvable] = []
@ -161,7 +168,17 @@ class ArrayFunctions:
@staticmethod
def array_apply(array: Array, lambda_function: Lambda) -> Array:
"""
:description:
Apply a lambda function on every element in the Array.
:usage:
.. code-block:: python
{
%array_apply( [1, 2, 3] , %string )
}
# ["1", "2", "3"]
"""
return Array([Array([val]) for val in array.value])
@ -173,6 +190,7 @@ class ArrayFunctions:
reverse_args: Optional[Boolean] = None,
) -> Array:
"""
:description:
Apply a lambda function on every element in the Array, with ``fixed_argument``
passed as a second argument to every invocation.
"""
@ -184,6 +202,7 @@ class ArrayFunctions:
@staticmethod
def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array:
"""
:description:
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
"""
@ -192,6 +211,7 @@ class ArrayFunctions:
@staticmethod
def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument:
"""
:description:
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
"""

12
src/ytdl_sub/script/functions/boolean_functions.py
View file

@ -13,6 +13,7 @@ class BooleanFunctions:
@staticmethod
def bool(value: AnyArgument) -> Boolean:
"""
:description:
Cast any type to a Boolean.
"""
return Boolean(bool(value.value))
@ -20,6 +21,7 @@ class BooleanFunctions:
@staticmethod
def eq(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
:description:
``==`` operator. Returns True if left == right. False otherwise.
"""
return Boolean(left.value == right.value)
@ -27,6 +29,7 @@ class BooleanFunctions:
@staticmethod
def ne(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
:description:
``!=`` operator. Returns True if left != right. False otherwise.
"""
return Boolean(left.value != right.value)
@ -34,6 +37,7 @@ class BooleanFunctions:
@staticmethod
def lt(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
:description:
``<`` operator. Returns True if left < right. False otherwise.
"""
return Boolean(left.value < right.value)
@ -41,6 +45,7 @@ class BooleanFunctions:
@staticmethod
def lte(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
:description:
``<=`` operator. Returns True if left <= right. False otherwise.
"""
return Boolean(left.value <= right.value)
@ -48,6 +53,7 @@ class BooleanFunctions:
@staticmethod
def gt(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
:description:
``>`` operator. Returns True if left > right. False otherwise.
"""
return Boolean(left.value > right.value)
@ -55,6 +61,7 @@ class BooleanFunctions:
@staticmethod
def gte(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
:description:
``>=`` operator. Returns True if left >= right. False otherwise.
"""
return Boolean(left.value >= right.value)
@ -62,6 +69,7 @@ class BooleanFunctions:
@staticmethod
def and_(*values: AnyArgument) -> Boolean:
"""
:description:
``and`` operator. Returns True if all values evaluate to True. False otherwise.
"""
return Boolean(all(bool(val.value) for val in values))
@ -69,6 +77,7 @@ class BooleanFunctions:
@staticmethod
def or_(*values: AnyArgument) -> Boolean:
"""
:description:
``or`` operator. Returns True if any value evaluates to True. False otherwise.
"""
return Boolean(any(bool(val.value) for val in values))
@ -76,6 +85,7 @@ class BooleanFunctions:
@staticmethod
def xor(*values: AnyArgument) -> Boolean:
"""
:description:
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
"""
bit_array = [bool(val.value) for val in values]
@ -85,6 +95,7 @@ class BooleanFunctions:
@staticmethod
def not_(value: Boolean) -> Boolean:
"""
:description:
``not`` operator. Returns the opposite of value.
"""
return Boolean(not value.value)
@ -92,6 +103,7 @@ class BooleanFunctions:
@staticmethod
def is_null(value: AnyArgument) -> Boolean:
"""
:description:
Returns True if a value is null (i.e. an empty string). False otherwise.
"""
return Boolean(isinstance(value, String) and value.value == "")

2
src/ytdl_sub/script/functions/conditional_functions.py
View file

@ -11,6 +11,7 @@ class ConditionalFunctions:
condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB
) -> Union[ReturnableArgumentA, ReturnableArgumentB]:
"""
:description:
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
"""
@ -23,6 +24,7 @@ class ConditionalFunctions:
maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB
) -> Union[ReturnableArgumentA, ReturnableArgumentB]:
"""
:description:
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
"""

1
src/ytdl_sub/script/functions/date_functions.py
View file

@ -8,6 +8,7 @@ class DateFunctions:
@staticmethod
def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String:
"""
:description:
Converts a posix timestamp to a date using strftime formatting.
"""
return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value))

13
src/ytdl_sub/script/functions/error_functions.py
View file

@ -8,6 +8,7 @@ class ErrorFunctions:
@staticmethod
def throw(error_message: String) -> AnyArgument:
"""
:description:
Explicitly throw an error with the provided error message.
"""
raise UserThrownRuntimeError(error_message)
@ -15,8 +16,9 @@ class ErrorFunctions:
@staticmethod
def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``value``.
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``value``.
"""
if not bool(value.value):
raise UserThrownRuntimeError(assert_message)
@ -27,8 +29,9 @@ class ErrorFunctions:
value: AnyArgument, ret: ReturnableArgument, assert_message: String
) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``ret``.
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``ret``.
"""
if not bool(value.value):
raise UserThrownRuntimeError(assert_message)
@ -39,6 +42,7 @@ class ErrorFunctions:
value: ReturnableArgument, equals: AnyArgument, assert_message: String
) -> ReturnableArgument:
"""
:description:
Explicitly throw an error with the provided assert message if ``value`` does not equal
``equals``. If they do equal, then return ``value``.
"""
@ -51,6 +55,7 @@ class ErrorFunctions:
value: ReturnableArgument, equals: AnyArgument, assert_message: String
) -> ReturnableArgument:
"""
:description:
Explicitly throw an error with the provided assert message if ``value`` equals
``equals``. If they do equal, then return ``value``.
"""

1
src/ytdl_sub/script/functions/json_functions.py
View file

@ -35,6 +35,7 @@ class JsonFunctions:
@staticmethod
def from_json(argument: String) -> AnyArgument:
"""
:description:
Converts a JSON string into an actual type.
"""
return _from_json(json.loads(argument.value))

12
src/ytdl_sub/script/functions/map_functions.py
View file

@ -18,12 +18,8 @@ class MapFunctions:
@staticmethod
def map(maybe_mapping: AnyArgument) -> Map:
"""
:description:
Tries to cast an unknown variable type to a Map.
Raises
------
FunctionRuntimeException
If the input type is not actually a Map.
"""
if not isinstance(maybe_mapping, Map):
raise FunctionRuntimeException(
@ -34,6 +30,7 @@ class MapFunctions:
@staticmethod
def map_size(mapping: Map) -> Integer:
"""
:description:
Returns the size of a Map.
"""
return Integer(len(mapping.value))
@ -41,6 +38,7 @@ class MapFunctions:
@staticmethod
def map_contains(mapping: Map, key: AnyArgument) -> Boolean:
"""
:description:
Returns True if the key is in the Map. False otherwise.
"""
if not isinstance(key, Hashable):
@ -55,6 +53,7 @@ class MapFunctions:
mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None
) -> AnyArgument:
"""
:description:
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
"""
@ -70,6 +69,7 @@ class MapFunctions:
@staticmethod
def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument:
"""
:description:
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
"""
@ -83,6 +83,7 @@ class MapFunctions:
@staticmethod
def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array:
"""
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
"""
@ -91,6 +92,7 @@ class MapFunctions:
@staticmethod
def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array:
"""
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
"""

10
src/ytdl_sub/script/functions/numeric_functions.py
View file

@ -16,6 +16,7 @@ class NumericFunctions:
@staticmethod
def float(value: AnyArgument) -> Float:
"""
:description:
Cast to Float.
"""
return Float(value=float(value.value))
@ -23,6 +24,7 @@ class NumericFunctions:
@staticmethod
def int(value: AnyArgument) -> Integer:
"""
:description:
Cast to Integer.
"""
return Integer(value=int(value.value))
@ -30,6 +32,7 @@ class NumericFunctions:
@staticmethod
def add(*values: Numeric) -> Numeric:
"""
:description:
``+`` operator. Returns the sum of all values.
"""
return _to_numeric(sum(val.value for val in values))
@ -37,6 +40,7 @@ class NumericFunctions:
@staticmethod
def sub(*values: Numeric) -> Numeric:
"""
:description:
``-`` operator. Subtracts all values from left to right.
"""
output = values[0].value
@ -48,6 +52,7 @@ class NumericFunctions:
@staticmethod
def mul(*values: Numeric) -> Numeric:
"""
:description:
``*`` operator. Returns the product of all values.
"""
return _to_numeric(math.prod([val.value for val in values]))
@ -55,6 +60,7 @@ class NumericFunctions:
@staticmethod
def pow(base: Numeric, exponent: Numeric) -> Numeric:
"""
:description:
``**`` operator. Returns the exponential of the base and exponent value.
"""
return _to_numeric(math.pow(base.value, exponent.value))
@ -62,6 +68,7 @@ class NumericFunctions:
@staticmethod
def div(left: Numeric, right: Numeric) -> Numeric:
"""
:description:
``/`` operator. Returns ``left / right``.
"""
return _to_numeric(left.value / right.value)
@ -69,6 +76,7 @@ class NumericFunctions:
@staticmethod
def mod(left: Numeric, right: Numeric) -> Numeric:
"""
:description:
``%`` operator. Returns ``left % right``.
"""
return _to_numeric(value=left.value % right.value)
@ -76,6 +84,7 @@ class NumericFunctions:
@staticmethod
def max(*values: Numeric) -> Numeric:
"""
:description:
Returns max of all values.
"""
return _to_numeric(max(val.value for val in values))
@ -83,6 +92,7 @@ class NumericFunctions:
@staticmethod
def min(*values: Numeric) -> Numeric:
"""
:description:
Returns min of all values.
"""
return _to_numeric(min(val.value for val in values))

4
src/ytdl_sub/script/functions/regex_functions.py
View file

@ -18,6 +18,7 @@ class RegexFunctions:
@staticmethod
def regex_match(regex: String, string: String) -> Array:
"""
:description:
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
@ -27,6 +28,7 @@ class RegexFunctions:
@staticmethod
def regex_search(regex: String, string: String) -> Array:
"""
:description:
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
@ -36,6 +38,7 @@ class RegexFunctions:
@staticmethod
def regex_fullmatch(regex: String, string: String) -> Array:
"""
:description:
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
@ -45,6 +48,7 @@ class RegexFunctions:
@staticmethod
def regex_capture_groups(regex: String) -> Integer:
"""
:description:
Returns number of capture groups in regex
"""
return Integer(re.compile(regex.value).groups)

11
src/ytdl_sub/script/functions/string_functions.py
View file

@ -11,6 +11,7 @@ class StringFunctions:
@staticmethod
def string(value: AnyArgument) -> String:
"""
:description:
Cast to String.
"""
return String(value=str(value.value))
@ -18,6 +19,7 @@ class StringFunctions:
@staticmethod
def contains(string: String, contains: String) -> Boolean:
"""
:description:
Returns True if ``contains`` is in ``string``. False otherwise.
"""
return Boolean(contains.value in string.value)
@ -25,6 +27,7 @@ class StringFunctions:
@staticmethod
def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String:
"""
:description:
Returns the slice of the Array.
"""
if end is not None:
@ -34,6 +37,7 @@ class StringFunctions:
@staticmethod
def lower(string: String) -> String:
"""
:description:
Lower-case the entire String.
"""
return String(string.value.lower())
@ -41,6 +45,7 @@ class StringFunctions:
@staticmethod
def upper(string: String) -> String:
"""
:description:
Upper-case the entire String.
"""
return String(string.value.upper())
@ -48,6 +53,7 @@ class StringFunctions:
@staticmethod
def capitalize(string: String) -> String:
"""
:description:
Capitalize the first character in the string.
"""
return String(string.value.capitalize())
@ -55,6 +61,7 @@ class StringFunctions:
@staticmethod
def titlecase(string: String) -> String:
"""
:description:
Capitalize each word in the string.
"""
return String(string.value.title())
@ -64,6 +71,7 @@ class StringFunctions:
string: String, old: String, new: String, count: Optional[Integer] = None
) -> String:
"""
:description:
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
"""
@ -75,6 +83,7 @@ class StringFunctions:
@staticmethod
def concat(*values: String) -> String:
"""
:description:
Concatenate multiple Strings into a single String.
"""
return String("".join(val.value for val in values))
@ -82,6 +91,7 @@ class StringFunctions:
@staticmethod
def pad(string: String, length: Integer, char: String) -> String:
"""
:description:
Pads the string to the given length
"""
output = string.value
@ -93,6 +103,7 @@ class StringFunctions:
@staticmethod
def pad_zero(numeric: Numeric, length: Integer) -> String:
"""
:description:
Pads a numeric with zeros to the given length
"""
return StringFunctions.pad(

28
src/ytdl_sub/script/utils/exception_formatters.py
View file

@ -1,16 +1,11 @@
import sys
from typing import List
from typing import Type
from typing import TypeVar
from typing import Union
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import NamedType
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.type_checking import FunctionSpec
from ytdl_sub.script.utils.type_checking import get_optional_type
from ytdl_sub.script.utils.type_checking import is_optional
from ytdl_sub.script.utils.type_checking import is_union
TUserException = TypeVar("TUserException", bound=UserException)
@ -103,28 +98,10 @@ class FunctionArgumentsExceptionFormatter:
input_spec: FunctionSpec,
function_instance: BuiltInFunctionType,
):
self._args = input_spec.args
self._varargs = input_spec.varargs
self._input_spec = input_spec
self._name = function_instance.name
self._input_args = function_instance.args
@classmethod
def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str:
if is_optional(python_type):
return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]"
if is_union(python_type):
return ", ".join(
sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__)
)
return python_type.type_name()
def _expected_args_str(self) -> str:
if self._args is not None:
return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})"
if self._varargs is not None:
return f"({self._to_human_readable_name(self._varargs)}, ...)"
return "()"
def _received_args_str(self) -> str:
received_type_names: List[str] = []
for arg in self._input_args:
@ -149,5 +126,6 @@ class FunctionArgumentsExceptionFormatter:
"""
return IncompatibleFunctionArguments(
f"Incompatible arguments passed to function {self._name}.\n"
f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}"
f"Expected {self._input_spec.human_readable_input_args()}\n"
f"Received {self._received_args_str()}"
)

39
src/ytdl_sub/script/utils/type_checking.py
View file

@ -119,6 +119,7 @@ def is_type_compatible(
@dataclass(frozen=True)
class FunctionSpec:
return_type: Type[Resolvable]
arg_names: List[str]
args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None
varargs: Optional[Type[Resolvable]] = None
@ -223,6 +224,42 @@ class FunctionSpec:
return l_type
return None
@classmethod
def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str:
if is_optional(python_type):
return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]"
if is_union(python_type):
args = ", ".join(
sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__)
)
return f"Union[{args}]"
return python_type.type_name()
def human_readable_input_args(self) -> str:
"""
Returns
-------
input arg string in human-readable format
"""
if self.args is not None:
args = ", ".join(
f"{name}: {self._to_human_readable_name(type_)}"
for name, type_ in zip(self.arg_names, self.args)
)
elif self.varargs is not None:
args = f"{self.arg_names[0]}: {self._to_human_readable_name(self.varargs)}, ..."
else:
args = ""
return f"({args})"
def human_readable_output_type(self) -> str:
"""
Returns
-------
output type string in human-readable format
"""
return self._to_human_readable_name(self.return_type)
@classmethod
def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec":
"""
@ -234,10 +271,12 @@ class FunctionSpec:
if arg_spec.varargs:
return FunctionSpec(
return_type=arg_spec.annotations["return"],
arg_names=[arg_spec.varargs],
varargs=arg_spec.annotations[arg_spec.varargs],
)
return FunctionSpec(
return_type=arg_spec.annotations["return"],
arg_names=arg_spec.args,
args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args],
)

7
src/ytdl_sub/utils/file_handler.py
View file

@ -33,6 +33,13 @@ def get_file_extension(file_name: Path | str) -> str:
return file_name.rsplit(".", maxsplit=1)[-1]
def get_md5_hash(contents: str) -> str:
"""
Helper function to compute md5 hash
"""
return hashlib.md5(contents.encode()).hexdigest()
def get_file_md5_hash(full_file_path: Path | str) -> str:
"""
Parameters

2
tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt
View file

@ -1,5 +1,5 @@
{
".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475",
"Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c",
"Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "d53121df33ac8c4a4699ec8919196552",
"Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
}

2
tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json
View file

@ -2,6 +2,6 @@
".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75",
"Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8",
"Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b",
"Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "b2d6388b4ddf8e3fbca042cb123672c3",
"Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975",
"Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530"
}

4
tests/resources/expected_downloads_summaries/youtube/test_channel_full.json
View file

@ -2,7 +2,7 @@
"Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584",
"Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
"Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "INFO_JSON",
"Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "6fb0ce965b75035079f82c84ad341e85",
"Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "246fa05b6443337785575987904848df",
"Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3",
"Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1",
"Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "INFO_JSON",
@ -10,7 +10,7 @@
"Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6",
"Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON",
"Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "43f271ef8d3a19877f0dc9bc5040b42a",
"Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "ae1d5e2e3979cea3c96e6a4cfcae8073",
"Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee",
"Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON",

2
tests/resources/expected_downloads_summaries/youtube/test_playlist.json
View file

@ -2,7 +2,7 @@
"JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON",

2
tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json
View file

@ -2,7 +2,7 @@
"JMC/.ytdl-sub-JMC-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON",

2
tests/resources/expected_downloads_summaries/youtube/test_video.json
View file

@ -1,5 +1,5 @@
{
"JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e",
"JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b",
"JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6",
"JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
}

2
tests/resources/expected_downloads_summaries/youtube/test_video_cli.json
View file

@ -1,5 +1,5 @@
{
"JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e",
"JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b",
"JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6",
"JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
}

2
tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json
View file

@ -1,4 +1,4 @@
{
"JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "718c187e6196c85eea73d16ebd489c91",
"JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d9d2d12feee44ee97729b39ba981c542",
"JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
}

0
tests/unit/docgen/__init__.py Normal file
View file

30
tests/unit/docgen/test_docgen.py Normal file
View file

@ -0,0 +1,30 @@
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.entry_variables import EntryVariablesDocGen
from tools.docgen.override_variables import OverrideVariablesDocGen
from tools.docgen.plugins import PluginsDocGen
from tools.docgen.scripting_functions import ScriptingFunctionsDocGen
from ytdl_sub.utils.file_handler import get_md5_hash
def _test_doc_gen(doc_gen: Type[DocGen]) -> None:
expected_md5_hash = get_md5_hash(doc_gen.generate_and_maybe_write_to_file())
with open(doc_gen.LOCATION, "r", encoding="utf-8") as file_doc:
md5_hash = get_md5_hash(file_doc.read())
assert md5_hash == expected_md5_hash
class TestDocGen:
def test_entry_variables_generated(self):
_test_doc_gen(EntryVariablesDocGen)
def test_override_variables_generated(self):
_test_doc_gen(OverrideVariablesDocGen)
def test_scripting_functions_generated(self):
_test_doc_gen(ScriptingFunctionsDocGen)
def test_plugins_generated(self):
_test_doc_gen(PluginsDocGen)

5
tests/unit/docgen/test_docgen_regenerate_disabled.py Normal file
View file

@ -0,0 +1,5 @@
from tools.docgen.docgen import REGENERATE_DOCS
def test_docgen_regenerate_disabled():
assert REGENERATE_DOCS is False

8
tests/unit/script/types/test_function.py
View file

@ -40,7 +40,7 @@ class TestFunction:
with pytest.raises(
IncompatibleFunctionArguments,
match=_incompatible_arguments_match(
expected="Map, AnyArgument, Optional[AnyArgument]",
expected="mapping: Map, key: AnyArgument, default: Optional[AnyArgument]",
recieved="%if(...)->Union[Array, Map], String",
),
):
@ -49,11 +49,11 @@ class TestFunction:
@pytest.mark.parametrize(
"function_str, expected_types, received_types",
[
("{%array_at({'a': 'dict?'}, 1)}", "Array, Integer", "Map, Integer"),
("{%array_extend('not', 'array')}", "Array, ...", "String, String"),
("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"),
("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"),
(
"{%replace('hi mom', 'mom', 'dad', 1, 0)}",
"String, String, String, Optional[Integer]",
"string: String, old: String, new: String, count: Optional[Integer]",
"String, String, String, Integer, Integer",
),
],

32
tools/docgen/docgen.py Normal file
View file

@ -0,0 +1,32 @@
from abc import abstractmethod
from pathlib import Path
REGENERATE_DOCS: bool = False
class DocGen:
"""
Home-made auto doc generation
"""
LOCATION: Path
@classmethod
@abstractmethod
def generate(cls) -> str:
"""
Generate the docs as a single string
"""
@classmethod
def generate_and_maybe_write_to_file(cls) -> str:
"""
Maybe writes the docs to their file if the global is set to True, and returns
the generated docs
"""
contents = cls.generate()
if REGENERATE_DOCS:
with open(cls.LOCATION, "w", encoding="utf-8") as out:
out.write(contents)
return contents

47
tools/docgen/entry_variables.py Normal file
View file

@ -0,0 +1,47 @@
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import camel_case_to_human
from tools.docgen.utils import get_function_docs
from tools.docgen.utils import line_section
from tools.docgen.utils import properties
from tools.docgen.utils import section
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
def _variable_class_to_name(obj: Type[Any]) -> str:
assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions"
return (
camel_case_to_human(obj.__name__)
.replace("Variable Definitions", "Variables")
.replace("Ytdl Sub", "Ytdl-Sub")
)
class EntryVariablesDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/scripting/entry_variables.rst")
@classmethod
def generate(cls) -> str:
docs = section("Entry Variables", level=0)
parent_objs: Dict[str, Type[Any]] = {
_variable_class_to_name(obj): obj for obj in VariableDefinitions.__bases__
}
for idx, name in enumerate(sorted(parent_objs.keys())):
docs += line_section(section_idx=idx)
docs += section(name, level=1)
for variable_function_name in properties(parent_objs[name]):
docs += get_function_docs(
function_name=variable_function_name,
obj=parent_objs[name],
level=2,
)
return docs

25
tools/docgen/override_variables.py Normal file
View file

@ -0,0 +1,25 @@
from pathlib import Path
from tools.docgen.docgen import DocGen
from tools.docgen.utils import get_function_docs
from tools.docgen.utils import section
from tools.docgen.utils import static_methods
from ytdl_sub.entries.variables.override_variables import OverrideVariables
class OverrideVariablesDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst")
@classmethod
def generate(cls) -> str:
docs = section("Override Variables", level=0)
for name in static_methods(OverrideVariables):
docs += get_function_docs(
function_name=name,
obj=OverrideVariables,
level=1,
)
return docs

91
tools/docgen/plugins.py Normal file
View file

@ -0,0 +1,91 @@
import inspect
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import line_section
from tools.docgen.utils import properties
from tools.docgen.utils import section
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
def should_filter_all_properties(plugin_name: str) -> bool:
return plugin_name in (
"format",
"match_filters",
"music_tags",
"filter_include",
"filter_exclude",
"embed_thumbnail",
"video_tags",
"download",
)
def should_filter_property(property_name: str) -> bool:
return property_name.startswith("_") or property_name in (
"value",
"source_variable_capture_dict",
"dict",
"keys",
"dict_with_format_strings",
"subscription_name",
"list",
)
def get_function_docs(function_name: str, obj: Any, level: int) -> str:
docs = f"\n``{function_name}``\n\n"
docs += inspect.cleandoc(getattr(obj, function_name).__doc__)
docs += "\n\n"
return docs
def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str:
docs = ""
docs += section(name, level=offset + 0)
docs += inspect.cleandoc(options.__doc__)
docs += "\n"
if should_filter_all_properties(name):
return docs
property_names = [prop for prop in properties(options) if not should_filter_property(prop)]
for property_name in sorted(property_names):
docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1)
return docs
class PluginsDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/plugins.rst")
@classmethod
def generate(cls):
options_dict: Dict[str, Type[OptionsValidator]] = {
"output_options": OutputOptions,
"ytdl_options": YTDLOptions,
"overrides": Overrides,
"download": MultiUrlValidator,
}
for plugin_name, plugin_type in PluginMapping._MAPPING.items():
if plugin_name.startswith("_"):
continue
options_dict[plugin_name] = plugin_type.plugin_options_type
docs = section("Plugins", level=0)
for idx, name in enumerate(sorted(options_dict.keys())):
docs += line_section(section_idx=idx)
docs += generate_plugin_docs(name, options_dict[name], offset=1)
return docs

82
tools/docgen/scripting_functions.py Normal file
View file

@ -0,0 +1,82 @@
import inspect
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import camel_case_to_human
from tools.docgen.utils import line_section
from tools.docgen.utils import section
from tools.docgen.utils import static_methods
from ytdl_sub.entries.script.custom_functions import CustomFunctions
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.utils.type_checking import FunctionSpec
def maybe_get_function_name(function_name: str) -> Optional[str]:
if function_name in ["register"]:
return None
if function_name.endswith("_"):
return function_name[:-1]
return function_name
def function_class_to_name(obj: Type[Any]) -> str:
assert "Functions" in obj.__name__
return camel_case_to_human(obj.__name__)
def function_type_hinting(display_function_name: str, function: Any) -> str:
spec = FunctionSpec.from_callable(function)
out = ":spec: ``"
out += display_function_name
out += spec.human_readable_input_args()
out += " -> "
out += spec.human_readable_output_type()
out += "``\n\n"
return out
def get_function_docstring(
function_name: str, function: Any, level: int, display_function_name: Optional[str] = None
) -> str:
display_function_name = display_function_name if display_function_name else function_name
docs = section(display_function_name, level=level)
docs += function_type_hinting(display_function_name=display_function_name, function=function)
docs += inspect.cleandoc(function.__doc__)
docs += "\n"
return docs
class ScriptingFunctionsDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/scripting/scripting_functions.rst")
@classmethod
def generate(cls) -> str:
docs = section("Scripting Functions", level=0)
parent_objs: Dict[str, Type[Any]] = {
function_class_to_name(obj): obj for obj in Functions.__bases__
}
parent_objs["Ytdl-Sub Functions"] = CustomFunctions
for idx, name in enumerate(sorted(parent_objs.keys())):
docs += line_section(section_idx=idx)
docs += section(name, level=1)
for function_name in static_methods(parent_objs[name]):
if display_function_name := maybe_get_function_name(function_name):
docs += get_function_docstring(
function_name=function_name,
display_function_name=display_function_name,
function=getattr(parent_objs[name], function_name),
level=2,
)
return docs

56
tools/docgen/utils.py Normal file
View file

@ -0,0 +1,56 @@
import inspect
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"}
def section(name: str, level: int, as_code: bool = False) -> str:
if as_code:
name = f"``{name}``"
return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n"
def properties(obj: Type[Any]) -> List[str]:
return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property))
def static_methods(obj: Type[Any]) -> List[str]:
return sorted(
name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod)
)
def camel_case_to_human(string: str) -> str:
output_str = string[0]
for char in string[1:]:
if char.islower():
output_str += char
else:
output_str += f" {char}"
return output_str
def get_function_docs(
function_name: str, obj: Any, level: int, display_function_name: Optional[str] = None
) -> str:
display_function_name = display_function_name if display_function_name else function_name
docs = section(display_function_name, level=level)
docs += inspect.cleandoc(getattr(obj, function_name).__doc__)
docs += "\n"
return docs
def line() -> str:
return "\n" + ("-" * 100) + "\n"
def line_section(section_idx: int) -> str:
if section_idx > 0:
return line()
return ""