[DOCS] Tooling to auto-generate custom sphinx docs (#849)

Tooling + updating docs
This commit is contained in:
Jesse Bannon 2023-12-28 22:39:59 -08:00 committed by GitHub
parent 1fcaba9274
commit 731d4e444c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
66 changed files with 2660 additions and 1967 deletions

2
.gitignore vendored
View file

@ -149,3 +149,5 @@ docker/testing/volumes
ffmpeg.exe
ffprobe.exe
tools/docgen/out

View file

View file

@ -53,6 +53,7 @@ html_theme_options = {
}
html_static_path = ["_static"]
html_css_files = ["custom.css"]
# Make sure the all autosectionlabel targets are unique

View file

@ -53,51 +53,7 @@ presets
~~~~~~~
``presets`` define a `formula` for how to format downloaded media and metadata.
download_strategy
"""""""""""""""""
Download strategies dictate what is getting downloaded from a source. Each
download strategy has its own set of parameters.
.. _url:
url
'''
.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions()
:members: url, playlist_thumbnails, source_thumbnails, download_reverse
:member-order: bysource
multi_url
'''''''''
.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions()
:members: urls, variables
-------------------------------------------------------------------------------
output_options
""""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
:members:
:member-order: bysource
:exclude-members: get_upload_date_range_to_keep, partial_validate
-------------------------------------------------------------------------------
.. _ytdl_options:
ytdl_options
""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions()
-------------------------------------------------------------------------------
.. _overrides:
overrides
"""""""""
.. autoclass:: ytdl_sub.config.overrides.Overrides()
.. _parent preset:
This section is work-in-progress!
preset
""""""

File diff suppressed because it is too large Load diff

View file

@ -1,17 +0,0 @@
==================
Config Field Types
==================
The ``config.yaml`` uses various types for its configurable fields. Below is a definition for each type.
.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator()
.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator()
.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator()

View file

@ -1,3 +1,4 @@
Entry Variables
===============
@ -111,6 +112,8 @@ webpage_url
~~~~~~~~~~~
The url to the webpage.
----------------------------------------------------------------------------------------------------
Metadata Variables
------------------
@ -130,6 +133,8 @@ source_metadata
~~~~~~~~~~~~~~~
Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry)
----------------------------------------------------------------------------------------------------
Playlist Variables
------------------
@ -210,6 +215,8 @@ playlist_webpage_url
~~~~~~~~~~~~~~~~~~~~
The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
----------------------------------------------------------------------------------------------------
Release Date Variables
----------------------
@ -285,6 +292,8 @@ release_year_truncated_reversed
The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
----------------------------------------------------------------------------------------------------
Source Variables
----------------
@ -332,6 +341,8 @@ source_webpage_url
~~~~~~~~~~~~~~~~~~
The source webpage url if it exists, otherwise returns the playlist webpage url.
----------------------------------------------------------------------------------------------------
Upload Date Variables
---------------------
@ -406,6 +417,8 @@ upload_year_truncated_reversed
The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
----------------------------------------------------------------------------------------------------
Ytdl-Sub Variables
------------------
@ -436,4 +449,4 @@ The upload_date_index padded two digits
ytdl_sub_input_url
~~~~~~~~~~~~~~~~~~
The input URL used in ytdl-sub to create this entry.
The input URL used in ytdl-sub to create this entry.

View file

@ -2,8 +2,11 @@
Scripting
=========
.. toctree::
Work in progress! Explanation of how to define scripts/variables will be added here.
.. toctree::
:maxdepth: 1
entry_variables
override_variables
scripting_functions
config_types
scripting_functions

View file

@ -1,3 +1,4 @@
Override Variables
==================
@ -41,4 +42,3 @@ For subscriptions in the form of
``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...``
and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to
``subscription_value``.

View file

@ -7,414 +7,571 @@ Array Functions
array
~~~~~
``array(maybe_array: AnyArgument) -> Array``
:spec: ``array(maybe_array: AnyArgument) -> Array``
Tries to cast an unknown variable type to an Array.
:description:
Tries to cast an unknown variable type to an Array.
array_apply
~~~~~~~~~~~
``array_apply(array: Array, lambda_function: Lambda) -> Array``
:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array``
Apply a lambda function on every element in the Array.
:description:
Apply a lambda function on every element in the Array.
:usage:
.. code-block:: python
{
%array_apply( [1, 2, 3] , %string )
}
# ["1", "2", "3"]
array_apply_fixed
~~~~~~~~~~~~~~~~~
:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array``
:description:
Apply a lambda function on every element in the Array, with ``fixed_argument``
passed as a second argument to every invocation.
array_at
~~~~~~~~
``array_at(array: Array, idx: Integer) -> AnyArgument``
:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument``
Return the element in the Array at index ``idx``.
:description:
Return the element in the Array at index ``idx``.
array_contains
~~~~~~~~~~~~~~
``array_contains(array: Array, value: AnyArgument) -> Boolean``
:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean``
Return True if the value exists in the Array. False otherwise.
:description:
Return True if the value exists in the Array. False otherwise.
array_enumerate
~~~~~~~~~~~~~~~
``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array``
:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array``
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
:description:
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
array_extend
~~~~~~~~~~~~
``array_extend(arrays: Array, ...) -> Array``
:spec: ``array_extend(arrays: Array, ...) -> Array``
Combine multiple Arrays into a single Array.
:description:
Combine multiple Arrays into a single Array.
array_first
~~~~~~~~~~~
:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument``
:description:
Returns the first element whose boolean conversion is True. Returns fallback
if all elements evaluate to False.
array_flatten
~~~~~~~~~~~~~
``array_flatten(array: Array) -> Array``
:spec: ``array_flatten(array: Array) -> Array``
Flatten any nested Arrays into a single-dimensional Array.
:description:
Flatten any nested Arrays into a single-dimensional Array.
array_index
~~~~~~~~~~~
``array_index(array: Array, value: AnyArgument) -> Integer``
:spec: ``array_index(array: Array, value: AnyArgument) -> Integer``
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
:description:
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
array_overlay
~~~~~~~~~~~~~
:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array``
:description:
Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
array_product
~~~~~~~~~~~~~
``array_product(arrays: Array, ...) -> Array``
:spec: ``array_product(arrays: Array, ...) -> Array``
Returns the Cartesian product of elements from different arrays
:description:
Returns the Cartesian product of elements from different arrays
array_reduce
~~~~~~~~~~~~
``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument``
:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument``
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
:description:
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
array_reverse
~~~~~~~~~~~~~
``array_reverse(array: Array) -> Array``
:spec: ``array_reverse(array: Array) -> Array``
Reverse an Array.
:description:
Reverse an Array.
array_size
~~~~~~~~~~
``array_size(array: Array) -> Integer``
:spec: ``array_size(array: Array) -> Integer``
Returns the size of an Array.
:description:
Returns the size of an Array.
array_slice
~~~~~~~~~~~
``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array``
:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array``
Returns the slice of the Array.
:description:
Returns the slice of the Array.
----------------------------------------------------------------------------------------------------
Boolean Functions
-----------------
and
~~~
``and(values: AnyArgument, ...) -> Boolean``
:spec: ``and(values: AnyArgument, ...) -> Boolean``
``and`` operator. Returns True if all values evaluate to True. False otherwise.
:description:
``and`` operator. Returns True if all values evaluate to True. False otherwise.
bool
~~~~
``bool(value: AnyArgument) -> Boolean``
:spec: ``bool(value: AnyArgument) -> Boolean``
Cast any type to a Boolean.
:description:
Cast any type to a Boolean.
eq
~~
``eq(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean``
``==`` operator. Returns True if left == right. False otherwise.
:description:
``==`` operator. Returns True if left == right. False otherwise.
gt
~~
``gt(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean``
``>`` operator. Returns True if left > right. False otherwise.
:description:
``>`` operator. Returns True if left > right. False otherwise.
gte
~~~
``gte(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean``
``>=`` operator. Returns True if left >= right. False otherwise.
:description:
``>=`` operator. Returns True if left >= right. False otherwise.
is_null
~~~~~~~
:spec: ``is_null(value: AnyArgument) -> Boolean``
:description:
Returns True if a value is null (i.e. an empty string). False otherwise.
lt
~~
``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
``<`` operator. Returns True if left < right. False otherwise.
:description:
``<`` operator. Returns True if left < right. False otherwise.
lte
~~~
``lte(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean``
``<=`` operator. Returns True if left <= right. False otherwise.
:description:
``<=`` operator. Returns True if left <= right. False otherwise.
ne
~~
``ne(left: AnyArgument, right: AnyArgument) -> Boolean``
:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean``
``!=`` operator. Returns True if left != right. False otherwise.
:description:
``!=`` operator. Returns True if left != right. False otherwise.
not
~~~
``not(value: Boolean) -> Boolean``
:spec: ``not(value: Boolean) -> Boolean``
``not`` operator. Returns the opposite of value.
:description:
``not`` operator. Returns the opposite of value.
or
~~
``or(values: AnyArgument, ...) -> Boolean``
:spec: ``or(values: AnyArgument, ...) -> Boolean``
``or`` operator. Returns True if any value evaluates to True. False otherwise.
:description:
``or`` operator. Returns True if any value evaluates to True. False otherwise.
xor
~~~
``xor(values: AnyArgument, ...) -> Boolean``
:spec: ``xor(values: AnyArgument, ...) -> Boolean``
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
:description:
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
----------------------------------------------------------------------------------------------------
Conditional Functions
---------------------
if
~~
``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
:description:
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
if_passthrough
~~~~~~~~~~~~~~
``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
:description:
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
----------------------------------------------------------------------------------------------------
Date Functions
--------------
datetime_strftime
~~~~~~~~~~~~~~~~~
``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String``
:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String``
Converts a posix timestamp to a date using strftime formatting.
:description:
Converts a posix timestamp to a date using strftime formatting.
----------------------------------------------------------------------------------------------------
Error Functions
---------------
assert
~~~~~~
``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument``
:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument``
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``value``.
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``value``.
assert_eq
~~~~~~~~~
:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` does not equal
``equals``. If they do equal, then return ``value``.
assert_ne
~~~~~~~~~
:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` equals
``equals``. If they do equal, then return ``value``.
assert_then
~~~~~~~~~~~
:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``ret``.
throw
~~~~~
``throw(error_message: String) -> AnyArgument``
:spec: ``throw(error_message: String) -> AnyArgument``
Explicitly throw an error with the provided error message.
:description:
Explicitly throw an error with the provided error message.
----------------------------------------------------------------------------------------------------
Json Functions
--------------
from_json
~~~~~~~~~
``from_json(argument: String) -> AnyArgument``
:spec: ``from_json(argument: String) -> AnyArgument``
Converts a JSON string into an actual type.
:description:
Converts a JSON string into an actual type.
----------------------------------------------------------------------------------------------------
Map Functions
-------------
map
~~~
``map(maybe_mapping: AnyArgument) -> Map``
:spec: ``map(maybe_mapping: AnyArgument) -> Map``
Tries to cast an unknown variable type to a Map.
:description:
Tries to cast an unknown variable type to a Map.
map_apply
~~~~~~~~~
``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array``
:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array``
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
map_contains
~~~~~~~~~~~~
``map_contains(mapping: Map, key: AnyArgument) -> Boolean``
:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean``
Returns True if the key is in the Map. False otherwise.
:description:
Returns True if the key is in the Map. False otherwise.
map_enumerate
~~~~~~~~~~~~~
``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array``
:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array``
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
map_get
~~~~~~~
``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
:description:
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
map_get_non_empty
~~~~~~~~~~~~~~~~~
``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument``
:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument``
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
:description:
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
map_size
~~~~~~~~
``map_size(mapping: Map) -> Integer``
:spec: ``map_size(mapping: Map) -> Integer``
Returns the size of a Map.
:description:
Returns the size of a Map.
----------------------------------------------------------------------------------------------------
Numeric Functions
-----------------
add
~~~
``add(values: Numeric, ...) -> Numeric``
:spec: ``add(values: Numeric, ...) -> Numeric``
``+`` operator. Returns the sum of all values.
:description:
``+`` operator. Returns the sum of all values.
div
~~~
``div(left: Numeric, right: Numeric) -> Numeric``
:spec: ``div(left: Numeric, right: Numeric) -> Numeric``
``/`` operator. Returns ``left / right``.
:description:
``/`` operator. Returns ``left / right``.
float
~~~~~
``float(value: AnyArgument) -> Float``
:spec: ``float(value: AnyArgument) -> Float``
Cast to Float.
:description:
Cast to Float.
int
~~~
``int(value: AnyArgument) -> Integer``
:spec: ``int(value: AnyArgument) -> Integer``
Cast to Integer.
:description:
Cast to Integer.
max
~~~
``max(values: Numeric, ...) -> Numeric``
:spec: ``max(values: Numeric, ...) -> Numeric``
Returns max of all values.
:description:
Returns max of all values.
min
~~~
``min(values: Numeric, ...) -> Numeric``
:spec: ``min(values: Numeric, ...) -> Numeric``
Returns min of all values.
:description:
Returns min of all values.
mod
~~~
``mod(left: Numeric, right: Numeric) -> Numeric``
:spec: ``mod(left: Numeric, right: Numeric) -> Numeric``
``%`` operator. Returns ``left % right``.
:description:
``%`` operator. Returns ``left % right``.
mul
~~~
``mul(values: Numeric, ...) -> Numeric``
:spec: ``mul(values: Numeric, ...) -> Numeric``
``*`` operator. Returns the product of all values.
:description:
``*`` operator. Returns the product of all values.
pow
~~~
``pow(base: Numeric, exponent: Numeric) -> Numeric``
:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric``
``**`` operator. Returns the exponential of the base and exponent value.
:description:
``**`` operator. Returns the exponential of the base and exponent value.
sub
~~~
``sub(values: Numeric, ...) -> Numeric``
:spec: ``sub(values: Numeric, ...) -> Numeric``
``-`` operator. Subtracts all values from left to right.
:description:
``-`` operator. Subtracts all values from left to right.
----------------------------------------------------------------------------------------------------
Regex Functions
---------------
regex_capture_groups
~~~~~~~~~~~~~~~~~~~~
:spec: ``regex_capture_groups(regex: String) -> Integer``
:description:
Returns number of capture groups in regex
regex_fullmatch
~~~~~~~~~~~~~~~
``regex_fullmatch(regex: String, string: String) -> Array``
:spec: ``regex_fullmatch(regex: String, string: String) -> Array``
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
:description:
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_match
~~~~~~~~~~~
``regex_match(regex: String, string: String) -> Array``
:spec: ``regex_match(regex: String, string: String) -> Array``
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
:description:
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_search
~~~~~~~~~~~~
``regex_search(regex: String, string: String) -> Array``
:spec: ``regex_search(regex: String, string: String) -> Array``
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
:description:
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
----------------------------------------------------------------------------------------------------
String Functions
----------------
capitalize
~~~~~~~~~~
``capitalize(string: String) -> String``
:spec: ``capitalize(string: String) -> String``
Capitalize the first character in the string.
:description:
Capitalize the first character in the string.
concat
~~~~~~
``concat(values: String, ...) -> String``
:spec: ``concat(values: String, ...) -> String``
Concatenate multiple Strings into a single String.
:description:
Concatenate multiple Strings into a single String.
contains
~~~~~~~~
:spec: ``contains(string: String, contains: String) -> Boolean``
:description:
Returns True if ``contains`` is in ``string``. False otherwise.
lower
~~~~~
``lower(string: String) -> String``
:spec: ``lower(string: String) -> String``
Lower-case the entire String.
:description:
Lower-case the entire String.
pad
~~~
``pad(string: String, length: Integer, char: String) -> String``
:spec: ``pad(string: String, length: Integer, char: String) -> String``
Pads the string to the given length
:description:
Pads the string to the given length
pad_zero
~~~~~~~~
``pad_zero(numeric: Numeric, length: Integer) -> String``
:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String``
Pads a numeric with zeros to the given length
:description:
Pads a numeric with zeros to the given length
replace
~~~~~~~
``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String``
:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String``
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
:description:
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
slice
~~~~~
``slice(string: String, start: Integer, end: Optional[Integer]) -> String``
:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String``
Returns the slice of the Array.
:description:
Returns the slice of the Array.
string
~~~~~~
``string(value: AnyArgument) -> String``
:spec: ``string(value: AnyArgument) -> String``
Cast to String.
:description:
Cast to String.
titlecase
~~~~~~~~~
``titlecase(string: String) -> String``
:spec: ``titlecase(string: String) -> String``
Capitalize each word in the string.
:description:
Capitalize each word in the string.
upper
~~~~~
``upper(string: String) -> String``
:spec: ``upper(string: String) -> String``
Upper-case the entire String.
:description:
Upper-case the entire String.
----------------------------------------------------------------------------------------------------
Ytdl-Sub Functions
------------------
legacy_bracket_safety
~~~~~~~~~~~~~~~~~~~~~
``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument``
:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument``
ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('', '') to not
interfere with its legacy variable scripting system. This function replicates that
@ -422,14 +579,14 @@ behavior.
sanitize
~~~~~~~~
``sanitize(value: AnyArgument) -> String``
:spec: ``sanitize(value: AnyArgument) -> String``
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS.
sanitize_plex_episode
~~~~~~~~~~~~~~~~~~~~~
``sanitize_plex_episode(string: String) -> String``
:spec: ``sanitize_plex_episode(string: String) -> String``
Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width
numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the
@ -437,7 +594,7 @@ season and/or episode.
to_date_metadata
~~~~~~~~~~~~~~~~
``to_date_metadata(yyyymmdd: String) -> Map``
:spec: ``to_date_metadata(yyyymmdd: String) -> Map``
Takes a date in the form of YYYYMMDD and returns a Map containing:
@ -461,14 +618,14 @@ Takes a date in the form of YYYYMMDD and returns a Map containing:
to_native_filepath
~~~~~~~~~~~~~~~~~~
``to_native_filepath(filepath: String) -> String``
:spec: ``to_native_filepath(filepath: String) -> String``
Convert any unix-based path separators ('/') with the OS's native
separator.
truncate_filepath_if_too_long
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``truncate_filepath_if_too_long(filepath: String) -> String``
:spec: ``truncate_filepath_if_too_long(filepath: String) -> String``
If a file-path is too long for the OS, this function will truncate it while preserving
the extension.

View file

@ -18,7 +18,7 @@ Jellyfin
Kodi
~~~~
* Everything that the Jellyfin version does
* Turns on :ref:`config_reference/plugins:kodi_safe`, replacing characters that would break kodi with safer characters
* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ````
Plex
~~~~

View file

@ -8,7 +8,7 @@ import mergedeep
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME
from ytdl_sub.entries.variables.override_variables import OverrideVariables
from ytdl_sub.entries.variables.override_variables import OverrideHelpers
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@ -23,9 +23,9 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
class Overrides(DictFormatterValidator, Scriptable):
"""
Optional. This section allows you to define variables that can be used in any string formatter.
For example, if you want your file and thumbnail files to match without copy-pasting a large
format string, you can define something like:
Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter.
:Usage:
.. code-block:: yaml
@ -89,7 +89,7 @@ class Overrides(DictFormatterValidator, Scriptable):
"""
Ensures the variable name does not collide with any entry variables or built-in functions.
"""
if not OverrideVariables.is_valid_name(name):
if not OverrideHelpers.is_valid_name(name):
override_type = "function" if name.startswith("%") else "variable"
raise self._validation_exception(
f"Override {override_type} with name {name} is invalid. Names must be"
@ -97,14 +97,14 @@ class Overrides(DictFormatterValidator, Scriptable):
exception_class=InvalidVariableNameException,
)
if OverrideVariables.is_entry_variable_name(name):
if OverrideHelpers.is_entry_variable_name(name):
raise self._validation_exception(
f"Override variable with name {name} cannot be used since it is a"
" built-in ytdl-sub entry variable name.",
exception_class=InvalidVariableNameException,
)
if OverrideVariables.is_function_name(name):
if OverrideHelpers.is_function_name(name):
raise self._validation_exception(
f"Override function definition with name {name} cannot be used since it is"
" a built-in ytdl-sub function name.",

View file

@ -15,12 +15,12 @@ from ytdl_sub.validators.validators import LiteralDictValidator
class YTDLOptions(LiteralDictValidator):
"""
Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader.
Allows you to add any ytdl argument to ytdl-sub's downloader.
The argument names can differ slightly from the command-line argument names. See
`this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_
for more details.
ytdl_options should be formatted like:
:Usage:
.. code-block:: yaml
@ -58,7 +58,7 @@ class OutputOptions(StrictDictValidator):
"""
Defines where to output files and thumbnails after all post-processing has completed.
Usage:
:Usage:
.. code-block:: yaml
@ -156,97 +156,119 @@ class OutputOptions(StrictDictValidator):
@property
def output_directory(self) -> OverridesStringFormatterValidator:
"""
Required. The output directory to store all media files downloaded.
:expected type: OverridesFormatter
:description:
The output directory to store all media files downloaded.
"""
return self._output_directory
@property
def file_name(self) -> StringFormatterValidator:
"""
Required. The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
:expected type: EntryFormatter
:description:
The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
"""
return self._file_name
@property
def thumbnail_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
"""
return self._thumbnail_name
@property
def info_json_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
"""
return self._info_json_name
@property
def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
:expected type: Optional[OverridesFormatter]
:description:
The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
"""
return self._download_archive_name
@property
def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this name
first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
:expected type: Optional[OverridesFormatter]
:description:
Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this
name first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
"""
return self._migrated_download_archive_name
@property
def maintain_download_archive(self) -> bool:
"""
Optional. Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
:expected type: Optional[Boolean]
:description:
Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
create a ytdl download-archive file when invoking a download on a subscription. This will
prevent ytdl from redownloading media already downloaded.
The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
create a ytdl download-archive file when invoking a download on a subscription. This will
prevent ytdl from redownloading media already downloaded.
Defaults to False.
Defaults to False.
"""
return self._maintain_download_archive.value
@property
def keep_files_before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``.
"""
return self._keep_files_before
@property
def keep_files_after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``.
"""
return self._keep_files_after
@property
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
"""
return self._keep_max_files

View file

@ -1,37 +0,0 @@
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
# TODO: Remove later - keep for docstring
class MultiUrlDownloadOptions(MultiUrlValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""

View file

@ -1,25 +0,0 @@
from ytdl_sub.downloaders.url.validators import UrlValidator
# TODO: Remove later - keep for docstring
class UrlDownloadOptions(UrlValidator):
"""
Downloads from a single URL supported by yt-dlp.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
# optional
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
download_reverse: True
"""

View file

@ -215,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]):
class MultiUrlValidator(OptionsValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Sets the URL(s) to download from. Can be used in many forms, including
:Single URL:
.. code-block:: yaml
download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
:Multi URL:
.. code-block:: yaml
download:
- "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
- "https://www.youtube.com/watch?v=3BFTio5296w"
:Thumbnails + Variables:
All variables must be defined for the top-most url. All subsequent URL variables can be either
overwritten or default to the top-most value.
If an entry is returned from more than one URL, it will use the variables in the bottom-most
URL.
.. code-block:: yaml
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""
@classmethod

File diff suppressed because it is too large Load diff

View file

@ -10,15 +10,15 @@ SUBSCRIPTION_ARRAY = "subscription_array"
class OverrideVariables:
@classmethod
def subscription_name(cls) -> str:
@staticmethod
def subscription_name() -> str:
"""
Name of the subscription
"""
return SUBSCRIPTION_NAME
@classmethod
def subscription_value(cls) -> str:
@staticmethod
def subscription_value() -> str:
"""
For subscriptions in the form of
@ -30,8 +30,8 @@ class OverrideVariables:
"""
return SUBSCRIPTION_VALUE
@classmethod
def subscription_indent_i(cls, index: int) -> str:
@staticmethod
def subscription_indent_i(index: int) -> str:
"""
For subscriptions in the form of
@ -46,8 +46,8 @@ class OverrideVariables:
"""
return f"subscription_indent_{index + 1}"
@classmethod
def subscription_value_i(cls, index: int) -> str:
@staticmethod
def subscription_value_i(index: int) -> str:
"""
For subscriptions in the form of
@ -63,6 +63,8 @@ class OverrideVariables:
"""
return f"subscription_value_{index + 1}"
class OverrideHelpers:
@classmethod
def is_entry_variable_name(cls, name: str) -> bool:
"""

View file

@ -25,15 +25,13 @@ class AudioExtractOptions(OptionsDictValidator):
"""
Extracts audio from a video file.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
audio_extract:
codec: "mp3"
quality: 128
audio_extract:
codec: "mp3"
quality: 128
"""
_required_keys = {"codec"}
@ -56,16 +54,20 @@ class AudioExtractOptions(OptionsDictValidator):
@property
def codec(self) -> str:
"""
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
opus, vorbis, wav, and best to grab the best possible format at runtime.
:expected type: String
:description:
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
opus, vorbis, wav, and best to grab the best possible format at runtime.
"""
return self._codec
@property
def quality(self) -> Optional[float]:
"""
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
:expected type: Float
:description:
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
"""
if self._quality is not None:
return self._quality.value

View file

@ -64,32 +64,29 @@ class ChaptersOptions(OptionsDictValidator):
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
chapters:
# Embedded Chapter Fields
embed_chapters: True
allow_chapters_from_comments: False
remove_chapters_regex:
- "Intro"
- "Outro"
# Sponsorblock Fields
sponsorblock_categories:
- "outro"
- "selfpromo"
- "preview"
- "interaction"
- "sponsor"
- "music_offtopic"
- "intro"
remove_sponsorblock_categories: "all"
force_key_frames: False
chapters:
# Embedded Chapter Fields
embed_chapters: True
allow_chapters_from_comments: False
remove_chapters_regex:
- "Intro"
- "Outro"
# Sponsorblock Fields
sponsorblock_categories:
- "outro"
- "selfpromo"
- "preview"
- "interaction"
- "sponsor"
- "music_offtopic"
- "intro"
remove_sponsorblock_categories: "all"
force_key_frames: False
"""
_optional_keys = {
@ -135,23 +132,29 @@ class ChaptersOptions(OptionsDictValidator):
@property
def embed_chapters(self) -> Optional[bool]:
"""
Optional. Embed chapters into the file. Defaults to True.
:expected type: Optional[Boolean]
:description:
Defaults to True. Embed chapters into the file.
"""
return self._embed_chapters
@property
def allow_chapters_from_comments(self) -> bool:
"""
Optional. If chapters do not exist in the video/description itself, attempt to scrape
comments to find the chapters. Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. If chapters do not exist in the video/description itself, attempt to
scrape comments to find the chapters.
"""
return self._allow_chapters_from_comments
@property
def remove_chapters_regex(self) -> Optional[List[re.Pattern]]:
"""
Optional. List of regex patterns to match chapter titles against and remove them from the
entry.
:expected type: Optional[List[RegexString]
:description:
List of regex patterns to match chapter titles against and remove them from the
entry.
"""
if self._remove_chapters_regex:
return [validator.compiled_regex for validator in self._remove_chapters_regex.list]
@ -160,9 +163,11 @@ class ChaptersOptions(OptionsDictValidator):
@property
def sponsorblock_categories(self) -> Optional[List[str]]:
"""
Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor",
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
"poi_highlight", or "all" to include all categories.
:expected type: Optional[List[String]]
:description:
List of SponsorBlock categories to embed as chapters. Supports "sponsor",
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
"poi_highlight", or "all" to include all categories.
"""
if self._sponsorblock_categories:
category_list = [validator.value for validator in self._sponsorblock_categories.list]
@ -174,9 +179,11 @@ class ChaptersOptions(OptionsDictValidator):
@property
def remove_sponsorblock_categories(self) -> Optional[List[str]]:
"""
Optional. List of SponsorBlock categories to remove from the output file. Can only remove
categories that are specified in ``sponsorblock_categories`` or "all", which removes
everything specified in ``sponsorblock_categories``.
:expected type: Optional[List[String]]
:description:
List of SponsorBlock categories to remove from the output file. Can only remove
categories that are specified in ``sponsorblock_categories`` or "all", which removes
everything specified in ``sponsorblock_categories``.
"""
if self._remove_sponsorblock_categories:
category_list = [
@ -190,9 +197,10 @@ class ChaptersOptions(OptionsDictValidator):
@property
def force_key_frames(self) -> bool:
"""
Optional. Force keyframes at cuts when removing sections. This is slow due to needing a
re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to
False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Force keyframes at cuts when removing sections. This is slow due to
needing a re-encode, but the resulting video may have fewer artifacts around the cuts.
"""
return self._force_key_frames

View file

@ -11,16 +11,24 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator
class DateRangeOptions(OptionsDictValidator):
"""
Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs:
Usage:
.. code-block:: Markdown
A string in the format YYYYMMDD or
(now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this.
Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest
granularity possible.
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
date_range:
before: "now"
after: "today-2weeks"
date_range:
before: "now"
after: "today-2weeks"
"""
_optional_keys = {"before", "after"}
@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator):
@property
def before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Only download videos before this datetime.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
"""
return self._before
@property
def after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Only download videos after this datetime.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
"""
return self._after

View file

@ -20,13 +20,11 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
"""
Whether to embed thumbnails to the audio/video file or not.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
embed_thumbnail: True
embed_thumbnail: True
"""

View file

@ -32,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator):
"""
Converts video files from one extension to another.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mp4"
file_convert:
convert_to: "mp4"
Supports custom ffmpeg conversions:
Also supports custom ffmpeg conversions:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
ffmpeg_post_process_args: >
-bitexact
-vcodec copy
-acodec copy
-scodec mov_text
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
ffmpeg_post_process_args: >
-bitexact
-vcodec copy
-acodec copy
-scodec mov_text
"""
_required_keys = {"convert_to"}
@ -89,35 +87,38 @@ class FileConvertOptions(OptionsDictValidator):
@property
def convert_to(self) -> str:
"""
Convert to a desired file type. Supports:
:expected type: String
:description:
Convert to a desired file type. Supports
* Video: avi, flv, mkv, mov, mp4, webm
* Audio: aac, flac, mp3, m4a, opus, vorbis, wav
- Video: avi, flv, mkv, mov, mp4, webm
- Audio: aac, flac, mp3, m4a, opus, vorbis, wav
"""
return self._convert_to
@property
def convert_with(self) -> Optional[str]:
"""
Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
:expected type: Optional[String]
:description:
Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
"""
return self._convert_with
@property
def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the
form of:
:expected type: Optional[OverridesFormatter]
:description:
ffmpeg args to post-process an entry file with. The args will be inserted in the
form of
.. code-block:: bash
``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``.
ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.
"""
return self._ffmpeg_post_process_args

View file

@ -19,15 +19,15 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
filter_exclude:
- { %contains( %lower(title), '#short' ) }
- { %contains( %lower(description), '#short' ) }
filter_exclude:
- >-
{ %contains( %lower(title), '#short' ) }
- >-
{ %contains( %lower(description), '#short' ) }
"""

View file

@ -19,24 +19,23 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
filter_include:
- {description}
- >-
{
%regex_search_any(
title,
[
"Full Episode",
"FULL",
]
)
}
filter_include:
- >-
{description}
- >-
{
%regex_search_any(
title,
[
"Full Episode",
"FULL",
]
)
}
"""

View file

@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator):
.. code-block:: yaml
presets:
my_example_preset:
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
"""
def __init__(self, name, value):

View file

@ -58,32 +58,20 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
class MatchFiltersOptions(OptionsDictValidator):
"""
Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp.
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
For logical AND's between match filters, use the ``&`` operator in a single match filter.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
match_filters:
filters: "original_url!*=/shorts/"
Supports one or multiple filters:
.. code-block:: yaml
presets:
my_example_preset:
match_filters:
filters:
- "age_limit<?18"
- "like_count>?100"
# Other common match-filters
# - "original_url!*=/shorts/ & !is_live"
# - "age_limit<?18"
# - "availability=?public"
match_filters:
filters:
- "age_limit<?18 & like_count>?100"
# Other common match-filters
# - "original_url!*=/shorts/ & !is_live"
# - "availability=?public"
"""
_optional_keys = {"filters"}

View file

@ -75,23 +75,22 @@ class MusicTagsOptions(OptionsDictValidator):
a full list of tags for various file types in MediaFile's
`source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
music_tags:
tags:
artist: "{artist}"
album: "{album}"
# Supports id3v2.4 multi-tags
genres:
- "{genre}"
- "ytdl-sub"
albumartists:
- "{artist}"
- "ytdl-sub"
artist: "{artist}"
album: "{album}"
# Supports id3v2.4 multi-tags
genres:
- "{genre}"
- "ytdl-sub"
albumartists:
- "{artist}"
- "ytdl-sub"
"""
_optional_keys = {"tags", "embed_thumbnail"}

View file

@ -60,7 +60,9 @@ class SharedNfoTagsOptions(OptionsDictValidator):
@property
def nfo_name(self) -> StringFormatterFileNameValidator:
"""
The NFO file name.
:expected type: EntryFormatter
:description:
The NFO file name.
"""
return self._nfo_name
@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator):
@property
def kodi_safe(self) -> Optional[bool]:
"""
Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some
foreign language characters. Setting this to True will replace those characters with ''.
Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Kodi does not support > 3-byte unicode characters, which include
emojis and some foreign language characters. Setting this to True will replace those
characters with ''.
"""
return self._kodi_safe
@ -190,22 +194,18 @@ class NfoTagsOptions(SharedNfoTagsOptions):
Adds an NFO file for every download file. An NFO file is simply an XML file
with a ``.nfo`` extension. You can add any values into the NFO.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
nfo_tags:
# required
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
# optional
kodi_safe: False
nfo_tags:
nfo_name: "{title_sanitized}.nfo"
nfo_root: "episodedetails"
tags:
title: "{title}"
season: "{upload_year}"
episode: "{upload_month}{upload_day_padded}"
kodi_safe: False
"""
_formatter_validator = StringFormatterValidator
@ -215,50 +215,54 @@ class NfoTagsOptions(SharedNfoTagsOptions):
@property
def nfo_root(self) -> StringFormatterValidator:
"""
The root tag of the NFO's XML. In the usage above, it would look like
:expected type: EntryFormatter
:description:
The root tag of the NFO's XML. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
</episodedetails>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
</episodedetails>
"""
return self._nfo_root
@property
def tags(self) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
:expected type: NfoTags
:description:
Tags within the nfo_root tag. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
<title>Awesome Youtube Video</title>
<season>2022</season>
<episode>502</episode>
</episodedetails>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<episodedetails>
<title>Awesome Youtube Video</title>
<season>2022</season>
<episode>502</episode>
</episodedetails>
Also supports xml attributes and duplicate keys:
Also supports xml attributes and duplicate keys:
.. code-block:: yaml
.. code-block:: yaml
tags:
season:
attributes:
name: "Best Year"
tag: "{upload_year}"
genre:
- "Comedy"
- "Drama"
tags:
season:
attributes:
name: "Best Year"
tag: "{upload_year}"
genre:
- "Comedy"
- "Drama"
Which translates to
Which translates to
.. code-block:: xml
.. code-block:: xml
<season name="Best Year">2022</season>
<genre>Comedy</genre>
<genre>Drama</genre>
<season name="Best Year">2022</season>
<genre>Comedy</genre>
<genre>Drama</genre>
"""
return self._tags

View file

@ -37,48 +37,52 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
@property
def nfo_root(self) -> StringFormatterValidator:
"""
The root tag of the NFO's XML. In the usage above, it would look like
:expected type: EntryFormatter
:description:
The root tag of the NFO's XML. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
</tvshow>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
</tvshow>
"""
return self._nfo_root
@property
def tags(self) -> NfoTagsValidator:
"""
Tags within the nfo_root tag. In the usage above, it would look like
:expected type: NfoTags
:description:
Tags within the nfo_root tag. In the usage above, it would look like
.. code-block:: xml
.. code-block:: xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
<title>Sweet youtube TV show</title>
</tvshow>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tvshow>
<title>Sweet youtube TV show</title>
</tvshow>
Also supports xml attributes and duplicate keys:
Also supports xml attributes and duplicate keys:
.. code-block:: yaml
.. code-block:: yaml
tags:
named_season:
- tag: "{source_title}"
attributes:
number: "{collection_index}"
genre:
- "Comedy"
- "Drama"
tags:
named_season:
- tag: "{source_title}"
attributes:
number: "{collection_index}"
genre:
- "Comedy"
- "Drama"
Which translates to
Which translates to
.. code-block:: xml
.. code-block:: xml
<title year="2022">Sweet youtube TV show</season>
<genre>Comedy</genre>
<genre>Drama</genre>
<title year="2022">Sweet youtube TV show</season>
<genre>Comedy</genre>
<genre>Drama</genre>
"""
return self._tags

View file

@ -122,6 +122,41 @@ class FromSourceVariablesRegex(DictValidator):
class RegexOptions(OptionsDictValidator):
r"""
.. attention::
This plugin will eventually be deprecated and replaced by scripting functions.
You can replicate the example below using the following.
.. code-block:: yaml
# Only includes videos with 'Official Video'
filter_include:
- >-
{ %contains( %lower(title), "official video" ) }
# Excludes videos with '#short' in its description
filter_exclude:
- >-
{ %contains( %lower(description), '#short' ) }
# Creates a capture array with defaults, and assigns
# each capture group to its own variable
overrides:
description_date_capture: >-
{
%regex_capture_many_with_defaults(
description,
[ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ],
[ upload_year, upload_month, upload_day ]
)
}
captured_upload_year: >-
{ %array_at(description_date_capture, 1) }
captured_upload_month: >-
{ %array_at(description_date_capture, 2) }
captured_upload_day: >-
{ %array_at(description_date_capture, 3) }
Performs regex matching on an entry's source or override variables. Regex can be used to filter
entries from proceeding with download or capture groups to create new source variables.
@ -137,51 +172,49 @@ class RegexOptions(OptionsDictValidator):
and using ``title_and_description`` can regex match/exclude from either ``title`` or
``description``.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
regex:
# By default, if any match fails and has no defaults, the entry will
# be skipped. If False, ytdl-sub will error and stop all downloads
# from proceeding.
skip_if_match_fails: True
regex:
# By default, if any match fails and has no defaults, the entry will
# be skipped. If False, ytdl-sub will error and stop all downloads
# from proceeding.
skip_if_match_fails: True
from:
# For each entry's `title` value...
title:
# Perform this regex match on it to act as a filter.
# This will only download videos with "[Official Video]" in it. Note that we
# double backslash to make YAML happy
match:
- '\\[Official Video\\]'
from:
# For each entry's `title` value...
title:
# Perform this regex match on it to act as a filter.
# This will only download videos with "[Official Video]" in it. Note that we
# double backslash to make YAML happy
match:
- '\\[Official Video\\]'
# For each entry's `description` value...
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new
# source variables
match:
- '([0-9]{4})-([0-9]{2})-([0-9]{2})'
# Exclude any entry where the description contains #short
exclude:
- '#short'
# For each entry's `description` value...
description:
# Match with capture groups and defaults.
# This tries to scrape a date from the description and produce new
# source variables
match:
- '([0-9]{4})-([0-9]{2})-([0-9]{2})'
# Exclude any entry where the description contains #short
exclude:
- '#short'
# Each capture group creates these new source variables, respectively,
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# Each capture group creates these new source variables, respectively,
# as well a sanitized version, i.e. `captured_upload_year_sanitized`
capture_group_names:
- "captured_upload_year"
- "captured_upload_month"
- "captured_upload_day"
# And if the string does not match, use these as respective default
# values for the new source variables.
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
# And if the string does not match, use these as respective default
# values for the new source variables.
capture_group_defaults:
- "{upload_year}"
- "{upload_month}"
- "{upload_day}"
"""
_required_keys = {"from"}
@ -209,8 +242,10 @@ class RegexOptions(OptionsDictValidator):
@property
def skip_if_match_fails(self) -> Optional[bool]:
"""
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
:expected type: Optional[Boolean]
:description:
Defaults to True. If True, when any match fails and has no defaults, the entry will be
skipped. If False, ytdl-sub will error and all downloads will not proceed.
"""
return self._skip_if_match_fails

View file

@ -48,23 +48,22 @@ class WhenNoChaptersValidator(StringSelectValidator):
class SplitByChaptersOptions(OptionsDictValidator):
"""
Splits a file by chapters into multiple files. Each file becomes its own entry with the
new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``,
``chapter_index_padded``, ``chapter_count``.
new variables
If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
- ``chapter_title``
- ``chapter_index``
- ``chapter_index_padded``
- ``chapter_count``
Note that when using this plugin and performing dry-run, it assumes embedded chapters are being
used with no modifications.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
split_by_chapters:
when_no_chapters: "pass"
split_by_chapters:
when_no_chapters: "pass"
"""
_required_keys = {"when_no_chapters"}
@ -100,8 +99,16 @@ class SplitByChaptersOptions(OptionsDictValidator):
@property
def when_no_chapters(self) -> str:
"""
Behavior to perform when no chapters are present. Supports "pass" (continue processing),
"drop" (exclude it from output), and "error" (stop processing for everything).
:expected type: String
:description:
Behavior to perform when no chapters are present. Supports
- "pass" (continue processing),
- "drop" (exclude it from output)
- "error" (stop processing for everything).
If a file has no chapters and is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
"""
return self._when_no_chapters

View file

@ -37,18 +37,18 @@ class SubtitleOptions(OptionsDictValidator):
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
It will set the respective language to the correct subtitle file.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
subtitles:
subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}"
subtitles_type: "srt"
embed_subtitles: False
languages: "en" # supports list of multiple languages
allow_auto_generated_subtitles: False
subtitles:
subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}"
subtitles_type: "srt"
embed_subtitles: False
languages:
- "en" # supports multiple languages
- "de"
allow_auto_generated_subtitles: False
"""
_optional_keys = {
@ -82,40 +82,50 @@ class SubtitleOptions(OptionsDictValidator):
@property
def subtitles_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's subtitles if they are present. This can include
directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and
will be placed in the output directory. ``lang`` is dynamic since you can download multiple
subtitles. It will set the respective language to the correct subtitle file.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's subtitles if they are present. This can include
directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``,
and will be placed in the output directory. ``lang`` is dynamic since you can download
multiple subtitles. It will set the respective language to the correct subtitle file.
"""
return self._subtitles_name
@property
def subtitles_type(self) -> Optional[str]:
"""
Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt"
:expected type: Optional[String]
:description:
Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc".
"""
return self._subtitles_type
@property
def embed_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to embed the subtitles into the video file. Defaults to False.
NOTE: webm files can only embed "vtt" subtitle types.
:expected type: Optional[Boolean]
:description:
Defaults to False. Whether to embed the subtitles into the video file. Note that
webm files can only embed "vtt" subtitle types.
"""
return self._embed_subtitles
@property
def languages(self) -> Optional[List[str]]:
"""
Optional. Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to "en".
:expected type: Optional[List[String]]
:description:
Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to only "en".
"""
return [lang.value for lang in self._languages]
@property
def allow_auto_generated_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to allow auto generated subtitles. Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Whether to allow auto generated subtitles.
"""
return self._allow_auto_generated_subtitles

View file

@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
range-based values, a random number will be chosen within the range to avoid sleeps looking
scripted.
Usage:
:Usage:
.. code-block:: yaml
@ -110,31 +110,39 @@ class ThrottleProtectionOptions(OptionsDictValidator):
@property
def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]:
"""
Number in seconds to sleep between each download. Does not include time it takes for
ytdl-sub to perform post-processing.
:expected type: Optional[Range]
:description:
Number in seconds to sleep between each download. Does not include time it takes for
ytdl-sub to perform post-processing.
"""
return self._sleep_per_download_s
@property
def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]:
"""
Number in seconds to sleep between each subscription.
:expected type: Optional[Range]
:description:
Number in seconds to sleep between each subscription.
"""
return self._sleep_per_subscription_s
@property
def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]:
"""
Number of downloads to perform per subscription.
:expected type: Optional[Range]
:description:
Number of downloads to perform per subscription.
"""
return self._max_downloads_per_subscription
@property
def subscription_download_probability(self) -> Optional[ProbabilityValidator]:
"""
Probability to perform any downloads, recomputed for each subscription. This is only
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
guaranteed over time to eventually download the subscription.
:expected type: Optional[Float]
:description:
Probability to perform any downloads, recomputed for each subscription. This is only
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
guaranteed over time to eventually download the subscription.
"""
return self._subscription_download_probability

View file

@ -17,16 +17,14 @@ class VideoTagsOptions(OptionsDictValidator):
"""
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
video_tags:
title: "{title}"
date: "{upload_date}"
description: "{description}"
video_tags:
title: "{title}"
date: "{upload_date}"
description: "{description}"
"""
_optional_keys = {"tags"}

View file

@ -19,12 +19,8 @@ class ArrayFunctions:
@staticmethod
def array(maybe_array: AnyArgument) -> Array:
"""
Tries to cast an unknown variable type to an Array.
Raises
------
FunctionRuntimeException
If the input type is not actually an Array.
:description:
Tries to cast an unknown variable type to an Array.
"""
if not isinstance(maybe_array, Array):
raise FunctionRuntimeException(
@ -35,14 +31,16 @@ class ArrayFunctions:
@staticmethod
def array_size(array: Array) -> Integer:
"""
Returns the size of an Array.
:description:
Returns the size of an Array.
"""
return Integer(len(array.value))
@staticmethod
def array_extend(*arrays: Array) -> Array:
"""
Combine multiple Arrays into a single Array.
:description:
Combine multiple Arrays into a single Array.
"""
output: List[Resolvable] = []
for array in arrays:
@ -55,7 +53,8 @@ class ArrayFunctions:
array: Array, overlap: Array, only_missing: Optional[Boolean] = None
) -> Array:
"""
Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
:description:
Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
"""
output: List[Resolvable] = []
output.extend(array.value)
@ -76,15 +75,17 @@ class ArrayFunctions:
@staticmethod
def array_at(array: Array, idx: Integer) -> AnyArgument:
"""
Return the element in the Array at index ``idx``.
:description:
Return the element in the Array at index ``idx``.
"""
return array.value[idx.value]
@staticmethod
def array_first(array: Array, fallback: AnyArgument) -> AnyArgument:
"""
Returns the first element whose boolean conversion is True. Returns fallback
if all elements evaluate to False.
:description:
Returns the first element whose boolean conversion is True. Returns fallback
if all elements evaluate to False.
"""
for val in array.value:
if bool(val.value):
@ -95,15 +96,17 @@ class ArrayFunctions:
@staticmethod
def array_contains(array: Array, value: AnyArgument) -> Boolean:
"""
Return True if the value exists in the Array. False otherwise.
:description:
Return True if the value exists in the Array. False otherwise.
"""
return Boolean(value in array.value)
@staticmethod
def array_index(array: Array, value: AnyArgument) -> Integer:
"""
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
:description:
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
"""
if not ArrayFunctions.array_contains(array=array, value=value):
raise ArrayValueDoesNotExist(
@ -118,7 +121,8 @@ class ArrayFunctions:
@staticmethod
def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array:
"""
Returns the slice of the Array.
:description:
Returns the slice of the Array.
"""
if end is not None:
return Array(array.value[start.value : end.value])
@ -127,7 +131,8 @@ class ArrayFunctions:
@staticmethod
def array_flatten(array: Array) -> Array:
"""
Flatten any nested Arrays into a single-dimensional Array.
:description:
Flatten any nested Arrays into a single-dimensional Array.
"""
output: List[Resolvable] = []
for elem in array.value:
@ -141,14 +146,16 @@ class ArrayFunctions:
@staticmethod
def array_reverse(array: Array) -> Array:
"""
Reverse an Array.
:description:
Reverse an Array.
"""
return Array(list(reversed(array.value)))
@staticmethod
def array_product(*arrays: Array) -> Array:
"""
Returns the Cartesian product of elements from different arrays
:description:
Returns the Cartesian product of elements from different arrays
"""
out: List[Resolvable] = []
for combo in itertools.product(*[arr.value for arr in arrays]):
@ -161,7 +168,17 @@ class ArrayFunctions:
@staticmethod
def array_apply(array: Array, lambda_function: Lambda) -> Array:
"""
Apply a lambda function on every element in the Array.
:description:
Apply a lambda function on every element in the Array.
:usage:
.. code-block:: python
{
%array_apply( [1, 2, 3] , %string )
}
# ["1", "2", "3"]
"""
return Array([Array([val]) for val in array.value])
@ -173,8 +190,9 @@ class ArrayFunctions:
reverse_args: Optional[Boolean] = None,
) -> Array:
"""
Apply a lambda function on every element in the Array, with ``fixed_argument``
passed as a second argument to every invocation.
:description:
Apply a lambda function on every element in the Array, with ``fixed_argument``
passed as a second argument to every invocation.
"""
if reverse_args and reverse_args.value:
return Array([Array([fixed_argument, val]) for val in array.value])
@ -184,15 +202,17 @@ class ArrayFunctions:
@staticmethod
def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array:
"""
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
:description:
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
"""
return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)])
@staticmethod
def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument:
"""
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
:description:
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
"""
return array

View file

@ -13,70 +13,80 @@ class BooleanFunctions:
@staticmethod
def bool(value: AnyArgument) -> Boolean:
"""
Cast any type to a Boolean.
:description:
Cast any type to a Boolean.
"""
return Boolean(bool(value.value))
@staticmethod
def eq(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``==`` operator. Returns True if left == right. False otherwise.
:description:
``==`` operator. Returns True if left == right. False otherwise.
"""
return Boolean(left.value == right.value)
@staticmethod
def ne(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``!=`` operator. Returns True if left != right. False otherwise.
:description:
``!=`` operator. Returns True if left != right. False otherwise.
"""
return Boolean(left.value != right.value)
@staticmethod
def lt(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``<`` operator. Returns True if left < right. False otherwise.
:description:
``<`` operator. Returns True if left < right. False otherwise.
"""
return Boolean(left.value < right.value)
@staticmethod
def lte(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``<=`` operator. Returns True if left <= right. False otherwise.
:description:
``<=`` operator. Returns True if left <= right. False otherwise.
"""
return Boolean(left.value <= right.value)
@staticmethod
def gt(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``>`` operator. Returns True if left > right. False otherwise.
:description:
``>`` operator. Returns True if left > right. False otherwise.
"""
return Boolean(left.value > right.value)
@staticmethod
def gte(left: AnyArgument, right: AnyArgument) -> Boolean:
"""
``>=`` operator. Returns True if left >= right. False otherwise.
:description:
``>=`` operator. Returns True if left >= right. False otherwise.
"""
return Boolean(left.value >= right.value)
@staticmethod
def and_(*values: AnyArgument) -> Boolean:
"""
``and`` operator. Returns True if all values evaluate to True. False otherwise.
:description:
``and`` operator. Returns True if all values evaluate to True. False otherwise.
"""
return Boolean(all(bool(val.value) for val in values))
@staticmethod
def or_(*values: AnyArgument) -> Boolean:
"""
``or`` operator. Returns True if any value evaluates to True. False otherwise.
:description:
``or`` operator. Returns True if any value evaluates to True. False otherwise.
"""
return Boolean(any(bool(val.value) for val in values))
@staticmethod
def xor(*values: AnyArgument) -> Boolean:
"""
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
:description:
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
"""
bit_array = [bool(val.value) for val in values]
@ -85,13 +95,15 @@ class BooleanFunctions:
@staticmethod
def not_(value: Boolean) -> Boolean:
"""
``not`` operator. Returns the opposite of value.
:description:
``not`` operator. Returns the opposite of value.
"""
return Boolean(not value.value)
@staticmethod
def is_null(value: AnyArgument) -> Boolean:
"""
Returns True if a value is null (i.e. an empty string). False otherwise.
:description:
Returns True if a value is null (i.e. an empty string). False otherwise.
"""
return Boolean(isinstance(value, String) and value.value == "")

View file

@ -11,8 +11,9 @@ class ConditionalFunctions:
condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB
) -> Union[ReturnableArgumentA, ReturnableArgumentB]:
"""
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
:description:
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
"""
if condition.value:
return true
@ -23,8 +24,9 @@ class ConditionalFunctions:
maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB
) -> Union[ReturnableArgumentA, ReturnableArgumentB]:
"""
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
:description:
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
"""
if bool(maybe_true_arg.value):
return maybe_true_arg

View file

@ -8,6 +8,7 @@ class DateFunctions:
@staticmethod
def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String:
"""
Converts a posix timestamp to a date using strftime formatting.
:description:
Converts a posix timestamp to a date using strftime formatting.
"""
return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value))

View file

@ -8,15 +8,17 @@ class ErrorFunctions:
@staticmethod
def throw(error_message: String) -> AnyArgument:
"""
Explicitly throw an error with the provided error message.
:description:
Explicitly throw an error with the provided error message.
"""
raise UserThrownRuntimeError(error_message)
@staticmethod
def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``value``.
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``value``.
"""
if not bool(value.value):
raise UserThrownRuntimeError(assert_message)
@ -27,8 +29,9 @@ class ErrorFunctions:
value: AnyArgument, ret: ReturnableArgument, assert_message: String
) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` evaluates to False.
If it evaluates to True, it will return ``ret``.
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``ret``.
"""
if not bool(value.value):
raise UserThrownRuntimeError(assert_message)
@ -39,8 +42,9 @@ class ErrorFunctions:
value: ReturnableArgument, equals: AnyArgument, assert_message: String
) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` does not equal
``equals``. If they do equal, then return ``value``.
:description:
Explicitly throw an error with the provided assert message if ``value`` does not equal
``equals``. If they do equal, then return ``value``.
"""
if not value.value == equals.value:
raise UserThrownRuntimeError(assert_message)
@ -51,8 +55,9 @@ class ErrorFunctions:
value: ReturnableArgument, equals: AnyArgument, assert_message: String
) -> ReturnableArgument:
"""
Explicitly throw an error with the provided assert message if ``value`` equals
``equals``. If they do equal, then return ``value``.
:description:
Explicitly throw an error with the provided assert message if ``value`` equals
``equals``. If they do equal, then return ``value``.
"""
if value.value == equals.value:
raise UserThrownRuntimeError(assert_message)

View file

@ -35,6 +35,7 @@ class JsonFunctions:
@staticmethod
def from_json(argument: String) -> AnyArgument:
"""
Converts a JSON string into an actual type.
:description:
Converts a JSON string into an actual type.
"""
return _from_json(json.loads(argument.value))

View file

@ -18,12 +18,8 @@ class MapFunctions:
@staticmethod
def map(maybe_mapping: AnyArgument) -> Map:
"""
Tries to cast an unknown variable type to a Map.
Raises
------
FunctionRuntimeException
If the input type is not actually a Map.
:description:
Tries to cast an unknown variable type to a Map.
"""
if not isinstance(maybe_mapping, Map):
raise FunctionRuntimeException(
@ -34,14 +30,16 @@ class MapFunctions:
@staticmethod
def map_size(mapping: Map) -> Integer:
"""
Returns the size of a Map.
:description:
Returns the size of a Map.
"""
return Integer(len(mapping.value))
@staticmethod
def map_contains(mapping: Map, key: AnyArgument) -> Boolean:
"""
Returns True if the key is in the Map. False otherwise.
:description:
Returns True if the key is in the Map. False otherwise.
"""
if not isinstance(key, Hashable):
raise KeyNotHashableRuntimeException(
@ -55,8 +53,9 @@ class MapFunctions:
mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None
) -> AnyArgument:
"""
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
:description:
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
"""
if not MapFunctions.map_contains(mapping=mapping, key=key).value:
if default is not None:
@ -70,8 +69,9 @@ class MapFunctions:
@staticmethod
def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument:
"""
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
:description:
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
"""
output = MapFunctions.map_get(mapping, key, default)
if isinstance(output, String) and output.value == "":
@ -83,16 +83,18 @@ class MapFunctions:
@staticmethod
def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array:
"""
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
"""
return Array([Array([key, value]) for key, value in mapping.value.items()])
@staticmethod
def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array:
"""
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
"""
return Array(
[

View file

@ -16,28 +16,32 @@ class NumericFunctions:
@staticmethod
def float(value: AnyArgument) -> Float:
"""
Cast to Float.
:description:
Cast to Float.
"""
return Float(value=float(value.value))
@staticmethod
def int(value: AnyArgument) -> Integer:
"""
Cast to Integer.
:description:
Cast to Integer.
"""
return Integer(value=int(value.value))
@staticmethod
def add(*values: Numeric) -> Numeric:
"""
``+`` operator. Returns the sum of all values.
:description:
``+`` operator. Returns the sum of all values.
"""
return _to_numeric(sum(val.value for val in values))
@staticmethod
def sub(*values: Numeric) -> Numeric:
"""
``-`` operator. Subtracts all values from left to right.
:description:
``-`` operator. Subtracts all values from left to right.
"""
output = values[0].value
for val in values[1:]:
@ -48,41 +52,47 @@ class NumericFunctions:
@staticmethod
def mul(*values: Numeric) -> Numeric:
"""
``*`` operator. Returns the product of all values.
:description:
``*`` operator. Returns the product of all values.
"""
return _to_numeric(math.prod([val.value for val in values]))
@staticmethod
def pow(base: Numeric, exponent: Numeric) -> Numeric:
"""
``**`` operator. Returns the exponential of the base and exponent value.
:description:
``**`` operator. Returns the exponential of the base and exponent value.
"""
return _to_numeric(math.pow(base.value, exponent.value))
@staticmethod
def div(left: Numeric, right: Numeric) -> Numeric:
"""
``/`` operator. Returns ``left / right``.
:description:
``/`` operator. Returns ``left / right``.
"""
return _to_numeric(left.value / right.value)
@staticmethod
def mod(left: Numeric, right: Numeric) -> Numeric:
"""
``%`` operator. Returns ``left % right``.
:description:
``%`` operator. Returns ``left % right``.
"""
return _to_numeric(value=left.value % right.value)
@staticmethod
def max(*values: Numeric) -> Numeric:
"""
Returns max of all values.
:description:
Returns max of all values.
"""
return _to_numeric(max(val.value for val in values))
@staticmethod
def min(*values: Numeric) -> Numeric:
"""
Returns min of all values.
:description:
Returns min of all values.
"""
return _to_numeric(min(val.value for val in values))

View file

@ -18,33 +18,37 @@ class RegexFunctions:
@staticmethod
def regex_match(regex: String, string: String) -> Array:
"""
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
:description:
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
"""
return _re_output_to_array(re.match(regex.value, string.value))
@staticmethod
def regex_search(regex: String, string: String) -> Array:
"""
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
:description:
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
"""
return _re_output_to_array(re.search(regex.value, string.value))
@staticmethod
def regex_fullmatch(regex: String, string: String) -> Array:
"""
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
:description:
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
"""
return _re_output_to_array(re.fullmatch(regex.value, string.value))
@staticmethod
def regex_capture_groups(regex: String) -> Integer:
"""
Returns number of capture groups in regex
:description:
Returns number of capture groups in regex
"""
return Integer(re.compile(regex.value).groups)

View file

@ -11,21 +11,24 @@ class StringFunctions:
@staticmethod
def string(value: AnyArgument) -> String:
"""
Cast to String.
:description:
Cast to String.
"""
return String(value=str(value.value))
@staticmethod
def contains(string: String, contains: String) -> Boolean:
"""
Returns True if ``contains`` is in ``string``. False otherwise.
:description:
Returns True if ``contains`` is in ``string``. False otherwise.
"""
return Boolean(contains.value in string.value)
@staticmethod
def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String:
"""
Returns the slice of the Array.
:description:
Returns the slice of the Array.
"""
if end is not None:
return String(string.value[start.value : end.value])
@ -34,28 +37,32 @@ class StringFunctions:
@staticmethod
def lower(string: String) -> String:
"""
Lower-case the entire String.
:description:
Lower-case the entire String.
"""
return String(string.value.lower())
@staticmethod
def upper(string: String) -> String:
"""
Upper-case the entire String.
:description:
Upper-case the entire String.
"""
return String(string.value.upper())
@staticmethod
def capitalize(string: String) -> String:
"""
Capitalize the first character in the string.
:description:
Capitalize the first character in the string.
"""
return String(string.value.capitalize())
@staticmethod
def titlecase(string: String) -> String:
"""
Capitalize each word in the string.
:description:
Capitalize each word in the string.
"""
return String(string.value.title())
@ -64,8 +71,9 @@ class StringFunctions:
string: String, old: String, new: String, count: Optional[Integer] = None
) -> String:
"""
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
:description:
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
"""
if count:
return String(string.value.replace(old.value, new.value, count.value))
@ -75,14 +83,16 @@ class StringFunctions:
@staticmethod
def concat(*values: String) -> String:
"""
Concatenate multiple Strings into a single String.
:description:
Concatenate multiple Strings into a single String.
"""
return String("".join(val.value for val in values))
@staticmethod
def pad(string: String, length: Integer, char: String) -> String:
"""
Pads the string to the given length
:description:
Pads the string to the given length
"""
output = string.value
while len(output) < length.value:
@ -93,7 +103,8 @@ class StringFunctions:
@staticmethod
def pad_zero(numeric: Numeric, length: Integer) -> String:
"""
Pads a numeric with zeros to the given length
:description:
Pads a numeric with zeros to the given length
"""
return StringFunctions.pad(
string=String(str(numeric.value)),

View file

@ -1,16 +1,11 @@
import sys
from typing import List
from typing import Type
from typing import TypeVar
from typing import Union
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import NamedType
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.type_checking import FunctionSpec
from ytdl_sub.script.utils.type_checking import get_optional_type
from ytdl_sub.script.utils.type_checking import is_optional
from ytdl_sub.script.utils.type_checking import is_union
TUserException = TypeVar("TUserException", bound=UserException)
@ -103,28 +98,10 @@ class FunctionArgumentsExceptionFormatter:
input_spec: FunctionSpec,
function_instance: BuiltInFunctionType,
):
self._args = input_spec.args
self._varargs = input_spec.varargs
self._input_spec = input_spec
self._name = function_instance.name
self._input_args = function_instance.args
@classmethod
def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str:
if is_optional(python_type):
return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]"
if is_union(python_type):
return ", ".join(
sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__)
)
return python_type.type_name()
def _expected_args_str(self) -> str:
if self._args is not None:
return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})"
if self._varargs is not None:
return f"({self._to_human_readable_name(self._varargs)}, ...)"
return "()"
def _received_args_str(self) -> str:
received_type_names: List[str] = []
for arg in self._input_args:
@ -149,5 +126,6 @@ class FunctionArgumentsExceptionFormatter:
"""
return IncompatibleFunctionArguments(
f"Incompatible arguments passed to function {self._name}.\n"
f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}"
f"Expected {self._input_spec.human_readable_input_args()}\n"
f"Received {self._received_args_str()}"
)

View file

@ -119,6 +119,7 @@ def is_type_compatible(
@dataclass(frozen=True)
class FunctionSpec:
return_type: Type[Resolvable]
arg_names: List[str]
args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None
varargs: Optional[Type[Resolvable]] = None
@ -223,6 +224,42 @@ class FunctionSpec:
return l_type
return None
@classmethod
def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str:
if is_optional(python_type):
return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]"
if is_union(python_type):
args = ", ".join(
sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__)
)
return f"Union[{args}]"
return python_type.type_name()
def human_readable_input_args(self) -> str:
"""
Returns
-------
input arg string in human-readable format
"""
if self.args is not None:
args = ", ".join(
f"{name}: {self._to_human_readable_name(type_)}"
for name, type_ in zip(self.arg_names, self.args)
)
elif self.varargs is not None:
args = f"{self.arg_names[0]}: {self._to_human_readable_name(self.varargs)}, ..."
else:
args = ""
return f"({args})"
def human_readable_output_type(self) -> str:
"""
Returns
-------
output type string in human-readable format
"""
return self._to_human_readable_name(self.return_type)
@classmethod
def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec":
"""
@ -234,10 +271,12 @@ class FunctionSpec:
if arg_spec.varargs:
return FunctionSpec(
return_type=arg_spec.annotations["return"],
arg_names=[arg_spec.varargs],
varargs=arg_spec.annotations[arg_spec.varargs],
)
return FunctionSpec(
return_type=arg_spec.annotations["return"],
arg_names=arg_spec.args,
args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args],
)

View file

@ -33,6 +33,13 @@ def get_file_extension(file_name: Path | str) -> str:
return file_name.rsplit(".", maxsplit=1)[-1]
def get_md5_hash(contents: str) -> str:
"""
Helper function to compute md5 hash
"""
return hashlib.md5(contents.encode()).hexdigest()
def get_file_md5_hash(full_file_path: Path | str) -> str:
"""
Parameters

View file

@ -1,5 +1,5 @@
{
".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/01 - Oblivion Mod Falcor p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/01 - Oblivion Mod Falcor p.1.mp3": "d53121df33ac8c4a4699ec8919196552",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
}

View file

@ -2,6 +2,6 @@
".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75",
"Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8",
"Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b",
"Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "b2d6388b4ddf8e3fbca042cb123672c3",
"Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975",
"Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530"
}

View file

@ -2,7 +2,7 @@
"Project Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584",
"Project Zombie/Season 2010/s2010.e081301 - Oblivion Mod Falcor p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
"Project Zombie/Season 2010/s2010.e081301 - Oblivion Mod Falcor p.1.info.json": "INFO_JSON",
"Project Zombie/Season 2010/s2010.e081301 - Oblivion Mod Falcor p.1.mp4": "6fb0ce965b75035079f82c84ad341e85",
"Project Zombie/Season 2010/s2010.e081301 - Oblivion Mod Falcor p.1.mp4": "246fa05b6443337785575987904848df",
"Project Zombie/Season 2010/s2010.e081301 - Oblivion Mod Falcor p.1.nfo": "a1970f06fbc4743fca6db0627de779f3",
"Project Zombie/Season 2010/s2010.e120201 - Oblivion Mod Falcor p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1",
"Project Zombie/Season 2010/s2010.e120201 - Oblivion Mod Falcor p.2.info.json": "INFO_JSON",
@ -10,7 +10,7 @@
"Project Zombie/Season 2010/s2010.e120201 - Oblivion Mod Falcor p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6",
"Project Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"Project Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON",
"Project Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "43f271ef8d3a19877f0dc9bc5040b42a",
"Project Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "ae1d5e2e3979cea3c96e6a4cfcae8073",
"Project Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee",
"Project Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"Project Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON",

View file

@ -2,7 +2,7 @@
"JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON",

View file

@ -2,7 +2,7 @@
"JMC/.ytdl-sub-JMC-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f",
"JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON",

View file

@ -1,5 +1,5 @@
{
"JMC/Oblivion Mod Falcor p.1.jpg": "fb95b510681676e81c321171fc23143e",
"JMC/Oblivion Mod Falcor p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b",
"JMC/Oblivion Mod Falcor p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6",
"JMC/Oblivion Mod Falcor p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
}

View file

@ -1,5 +1,5 @@
{
"JMC/Oblivion Mod Falcor p.1.jpg": "fb95b510681676e81c321171fc23143e",
"JMC/Oblivion Mod Falcor p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b",
"JMC/Oblivion Mod Falcor p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6",
"JMC/Oblivion Mod Falcor p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
}

View file

@ -1,4 +1,4 @@
{
"JMC/Oblivion Mod Falcor p.1.mp4": "718c187e6196c85eea73d16ebd489c91",
"JMC/Oblivion Mod Falcor p.1.mp4": "d9d2d12feee44ee97729b39ba981c542",
"JMC/Oblivion Mod Falcor p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
}

View file

View file

@ -0,0 +1,30 @@
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.entry_variables import EntryVariablesDocGen
from tools.docgen.override_variables import OverrideVariablesDocGen
from tools.docgen.plugins import PluginsDocGen
from tools.docgen.scripting_functions import ScriptingFunctionsDocGen
from ytdl_sub.utils.file_handler import get_md5_hash
def _test_doc_gen(doc_gen: Type[DocGen]) -> None:
expected_md5_hash = get_md5_hash(doc_gen.generate_and_maybe_write_to_file())
with open(doc_gen.LOCATION, "r", encoding="utf-8") as file_doc:
md5_hash = get_md5_hash(file_doc.read())
assert md5_hash == expected_md5_hash
class TestDocGen:
def test_entry_variables_generated(self):
_test_doc_gen(EntryVariablesDocGen)
def test_override_variables_generated(self):
_test_doc_gen(OverrideVariablesDocGen)
def test_scripting_functions_generated(self):
_test_doc_gen(ScriptingFunctionsDocGen)
def test_plugins_generated(self):
_test_doc_gen(PluginsDocGen)

View file

@ -0,0 +1,5 @@
from tools.docgen.docgen import REGENERATE_DOCS
def test_docgen_regenerate_disabled():
assert REGENERATE_DOCS is False

View file

@ -40,7 +40,7 @@ class TestFunction:
with pytest.raises(
IncompatibleFunctionArguments,
match=_incompatible_arguments_match(
expected="Map, AnyArgument, Optional[AnyArgument]",
expected="mapping: Map, key: AnyArgument, default: Optional[AnyArgument]",
recieved="%if(...)->Union[Array, Map], String",
),
):
@ -49,11 +49,11 @@ class TestFunction:
@pytest.mark.parametrize(
"function_str, expected_types, received_types",
[
("{%array_at({'a': 'dict?'}, 1)}", "Array, Integer", "Map, Integer"),
("{%array_extend('not', 'array')}", "Array, ...", "String, String"),
("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"),
("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"),
(
"{%replace('hi mom', 'mom', 'dad', 1, 0)}",
"String, String, String, Optional[Integer]",
"string: String, old: String, new: String, count: Optional[Integer]",
"String, String, String, Integer, Integer",
),
],

32
tools/docgen/docgen.py Normal file
View file

@ -0,0 +1,32 @@
from abc import abstractmethod
from pathlib import Path
REGENERATE_DOCS: bool = False
class DocGen:
"""
Home-made auto doc generation
"""
LOCATION: Path
@classmethod
@abstractmethod
def generate(cls) -> str:
"""
Generate the docs as a single string
"""
@classmethod
def generate_and_maybe_write_to_file(cls) -> str:
"""
Maybe writes the docs to their file if the global is set to True, and returns
the generated docs
"""
contents = cls.generate()
if REGENERATE_DOCS:
with open(cls.LOCATION, "w", encoding="utf-8") as out:
out.write(contents)
return contents

View file

@ -0,0 +1,47 @@
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import camel_case_to_human
from tools.docgen.utils import get_function_docs
from tools.docgen.utils import line_section
from tools.docgen.utils import properties
from tools.docgen.utils import section
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
def _variable_class_to_name(obj: Type[Any]) -> str:
assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions"
return (
camel_case_to_human(obj.__name__)
.replace("Variable Definitions", "Variables")
.replace("Ytdl Sub", "Ytdl-Sub")
)
class EntryVariablesDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/scripting/entry_variables.rst")
@classmethod
def generate(cls) -> str:
docs = section("Entry Variables", level=0)
parent_objs: Dict[str, Type[Any]] = {
_variable_class_to_name(obj): obj for obj in VariableDefinitions.__bases__
}
for idx, name in enumerate(sorted(parent_objs.keys())):
docs += line_section(section_idx=idx)
docs += section(name, level=1)
for variable_function_name in properties(parent_objs[name]):
docs += get_function_docs(
function_name=variable_function_name,
obj=parent_objs[name],
level=2,
)
return docs

View file

@ -0,0 +1,25 @@
from pathlib import Path
from tools.docgen.docgen import DocGen
from tools.docgen.utils import get_function_docs
from tools.docgen.utils import section
from tools.docgen.utils import static_methods
from ytdl_sub.entries.variables.override_variables import OverrideVariables
class OverrideVariablesDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst")
@classmethod
def generate(cls) -> str:
docs = section("Override Variables", level=0)
for name in static_methods(OverrideVariables):
docs += get_function_docs(
function_name=name,
obj=OverrideVariables,
level=1,
)
return docs

91
tools/docgen/plugins.py Normal file
View file

@ -0,0 +1,91 @@
import inspect
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import line_section
from tools.docgen.utils import properties
from tools.docgen.utils import section
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
def should_filter_all_properties(plugin_name: str) -> bool:
return plugin_name in (
"format",
"match_filters",
"music_tags",
"filter_include",
"filter_exclude",
"embed_thumbnail",
"video_tags",
"download",
)
def should_filter_property(property_name: str) -> bool:
return property_name.startswith("_") or property_name in (
"value",
"source_variable_capture_dict",
"dict",
"keys",
"dict_with_format_strings",
"subscription_name",
"list",
)
def get_function_docs(function_name: str, obj: Any, level: int) -> str:
docs = f"\n``{function_name}``\n\n"
docs += inspect.cleandoc(getattr(obj, function_name).__doc__)
docs += "\n\n"
return docs
def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str:
docs = ""
docs += section(name, level=offset + 0)
docs += inspect.cleandoc(options.__doc__)
docs += "\n"
if should_filter_all_properties(name):
return docs
property_names = [prop for prop in properties(options) if not should_filter_property(prop)]
for property_name in sorted(property_names):
docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1)
return docs
class PluginsDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/plugins.rst")
@classmethod
def generate(cls):
options_dict: Dict[str, Type[OptionsValidator]] = {
"output_options": OutputOptions,
"ytdl_options": YTDLOptions,
"overrides": Overrides,
"download": MultiUrlValidator,
}
for plugin_name, plugin_type in PluginMapping._MAPPING.items():
if plugin_name.startswith("_"):
continue
options_dict[plugin_name] = plugin_type.plugin_options_type
docs = section("Plugins", level=0)
for idx, name in enumerate(sorted(options_dict.keys())):
docs += line_section(section_idx=idx)
docs += generate_plugin_docs(name, options_dict[name], offset=1)
return docs

View file

@ -0,0 +1,82 @@
import inspect
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from tools.docgen.docgen import DocGen
from tools.docgen.utils import camel_case_to_human
from tools.docgen.utils import line_section
from tools.docgen.utils import section
from tools.docgen.utils import static_methods
from ytdl_sub.entries.script.custom_functions import CustomFunctions
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.utils.type_checking import FunctionSpec
def maybe_get_function_name(function_name: str) -> Optional[str]:
if function_name in ["register"]:
return None
if function_name.endswith("_"):
return function_name[:-1]
return function_name
def function_class_to_name(obj: Type[Any]) -> str:
assert "Functions" in obj.__name__
return camel_case_to_human(obj.__name__)
def function_type_hinting(display_function_name: str, function: Any) -> str:
spec = FunctionSpec.from_callable(function)
out = ":spec: ``"
out += display_function_name
out += spec.human_readable_input_args()
out += " -> "
out += spec.human_readable_output_type()
out += "``\n\n"
return out
def get_function_docstring(
function_name: str, function: Any, level: int, display_function_name: Optional[str] = None
) -> str:
display_function_name = display_function_name if display_function_name else function_name
docs = section(display_function_name, level=level)
docs += function_type_hinting(display_function_name=display_function_name, function=function)
docs += inspect.cleandoc(function.__doc__)
docs += "\n"
return docs
class ScriptingFunctionsDocGen(DocGen):
LOCATION = Path("docs/source/config_reference/scripting/scripting_functions.rst")
@classmethod
def generate(cls) -> str:
docs = section("Scripting Functions", level=0)
parent_objs: Dict[str, Type[Any]] = {
function_class_to_name(obj): obj for obj in Functions.__bases__
}
parent_objs["Ytdl-Sub Functions"] = CustomFunctions
for idx, name in enumerate(sorted(parent_objs.keys())):
docs += line_section(section_idx=idx)
docs += section(name, level=1)
for function_name in static_methods(parent_objs[name]):
if display_function_name := maybe_get_function_name(function_name):
docs += get_function_docstring(
function_name=function_name,
display_function_name=display_function_name,
function=getattr(parent_objs[name], function_name),
level=2,
)
return docs

56
tools/docgen/utils.py Normal file
View file

@ -0,0 +1,56 @@
import inspect
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"}
def section(name: str, level: int, as_code: bool = False) -> str:
if as_code:
name = f"``{name}``"
return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n"
def properties(obj: Type[Any]) -> List[str]:
return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property))
def static_methods(obj: Type[Any]) -> List[str]:
return sorted(
name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod)
)
def camel_case_to_human(string: str) -> str:
output_str = string[0]
for char in string[1:]:
if char.islower():
output_str += char
else:
output_str += f" {char}"
return output_str
def get_function_docs(
function_name: str, obj: Any, level: int, display_function_name: Optional[str] = None
) -> str:
display_function_name = display_function_name if display_function_name else function_name
docs = section(display_function_name, level=level)
docs += inspect.cleandoc(getattr(obj, function_name).__doc__)
docs += "\n"
return docs
def line() -> str:
return "\n" + ("-" * 100) + "\n"
def line_section(section_idx: int) -> str:
if section_idx > 0:
return line()
return ""