Merge remote-tracking branch 'origin' into prebuilt_presets

docs/source/config_reference/scripting/index.rst

@@ -9,7 +9,7 @@ contain reference documentation for each built-in variable and scripting functio
   :maxdepth: 1
 
   entry_variables
-  override_variables
+  static_variables
   scripting_functions
   scripting_types
 
@@ -30,7 +30,7 @@ considered *static* because it does not depend on anything from an entry.
 .. code-block:: yaml
 
    output_options:
-     output_directory: "Custom YTDL-SUB TV Show"
+     output_directory: "/path/to/tv_shows/Custom YTDL-SUB TV Show"
 
 Static Variables
 ~~~~~~~~~~~~~~~~
@@ -41,7 +41,7 @@ We can use this instead of hard-coding it above:
 .. code-block:: yaml
 
    output_options:
-     output_directory: "{subscription_name}"
+     output_directory: "/path/to/tv_shows/{subscription_name}"
 
 The syntax for variable usage is curly-braces with the variable name within it. Assuming
 our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub``
@@ -64,7 +64,7 @@ title in its name. We can do that using entry variables:
 .. code-block:: yaml
 
    output_options:
-     output_directory: "{subscription_name}"
+     output_directory: "/path/to/tv_shows/{subscription_name}"
      file_name: "{title}.{ext}"
      thumbnail_name: "{title}.{thumbnail_ext}"
 
@@ -83,7 +83,7 @@ a ``custom_file_name`` variable to use for the entry file and thumbnail fields:
 .. code-block:: yaml
 
    output_options:
-     output_directory: "{subscription_name}"
+     output_directory: "/path/to/tv_shows/{subscription_name}"
      file_name: "{custom_file_name}.{ext}"
      thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
 
@@ -104,7 +104,7 @@ are safe by using:
 .. code-block:: yaml
 
    output_options:
-     output_directory: "{subscription_name_sanitized}"
+     output_directory: "/path/to/tv_shows/{subscription_name_sanitized}"
      file_name: "{custom_file_name}.{ext}"
      thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
 
@@ -115,8 +115,9 @@ Simply add a ``_sanitized`` suffix to any variable name to make it sanitized.
 
 .. note::
 
-   Make sure you do not sanitize custom variables that intentionally create directories, otherwise
+   Make sure you do not sanitize custom variables that intentionally create directories,
-   they will... be sanitized and not resolve to directories!
+   (i.e. sanitizing ``/path/to/tv_shows/``) otherwise they will... be sanitized and not resolve to
+   directories!
 
 
 Using Scripting Functions
@@ -130,7 +131,7 @@ Let's suppose you are an avid command-line user, and like all of your file names
 .. code-block:: yaml
 
    output_options:
-     output_directory: "{subscription_name_sanitized}"
+     output_directory: "/path/to/tv_shows/{subscription_name_sanitized}"
      file_name: "{custom_file_name}.{ext}"
      thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
 
@@ -147,7 +148,7 @@ saying:
 
 - Allow a string to be multi-lined, and do not include newlines before or after it.
 
-See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=canonical_yaml>`_.
+See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=json>`_.
 Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make
 scripting function usage readable.
 
@@ -161,12 +162,41 @@ Advanced Scripting
 
 Accessing ``info.json`` Fields
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-WIP
+The entirety of an entry's ``info.json`` file resides in the
+`Map <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_types.html#map>`_
+variable
+`entry_metadata <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/entry_variables.html#entry-metadata>`_.
+
+Any field can be accessed by using the
+`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
+function like so:
+
+.. code-block:: yaml
+  :caption: Fetches the 'artist' value from the .info.json, returns null if it does not exist.
+
+   artist: >-
+     { %map_get( entry_metadata, "artist", null ) }
 
 Creating Custom Functions
 ~~~~~~~~~~~~~~~~~~~~~~~~~
-WIP
+Custom functions can be created in the overrides section using the following syntax:
 
-Parsing Maps and Arrays
+.. code-block:: yaml
-~~~~~~~~~~~~~~~~~~~~~~~
+
-WIP
+   overrides:
+     "%get_entry_metadata_field": >-
+       { %map_get( entry_metadata, $0, null ) }
+
+Custom function definitions must have ``%`` as a prefix to the function name, be surrounded by
+quotes to make YAML parsing happy, and can support arguments using ``$0``, ``$1``, ... to indicate
+their first argument, second argument, etc.
+
+Using our new custom function, we can simply the ``artist`` variable definition above to:
+
+.. code-block:: yaml
+
+   overrides:
+     "%get_entry_metadata_field": >-
+       { %map_get( entry_metadata, $0, null ) }
+     artist: >-
+       { get_entry_metadata_field("artist") }

docs/source/config_reference/scripting/scripting_types.rst

@@ -8,23 +8,22 @@ Types
 String
 ~~~~~~
 
-Strings are a series of characters surrounded by quotes and can be defined in a few ways, including:
+Strings are a series of characters surrounded by quotes.
+
+.. code-block:: yaml
+
+   string_variable: "This is a String variable"
+
+.. note::
+
+   For non-String types, they must be defined as parameters to scripting functions. This is because
+   anything in a variable definition that is not within curly-braces gets evaluated as a String.
+
+We can define Strings within curly-braces by setting them as parameters to a function:
 
 .. tab-set::
 
-  .. tab-item:: Literal
+  .. tab-item:: Multi-Line Single Quote
-
-    .. code-block:: yaml
-
-      string_variable: "This is a String variable"
-
-  .. tab-item:: In-Line
-
-    .. code-block:: yaml
-
-      string_variable: "{ %string('This is a String variable') }"
-
-  .. tab-item:: Single Quote
 
     .. code-block:: yaml
 
@@ -33,7 +32,7 @@ Strings are a series of characters surrounded by quotes and can be defined in a
           %string('This is a String variable')
         }
 
-  .. tab-item:: Double Quote
+  .. tab-item:: Multi-Line Double Quote
 
     .. code-block:: yaml
 
@@ -42,13 +41,42 @@ Strings are a series of characters surrounded by quotes and can be defined in a
           %string("This is a String variable")
         }
 
-  .. tab-item:: Triple Quote
+There are a few ways to make variables that use curly braces more compact, including:
+
+.. tab-set::
+
+  .. tab-item:: New-Line Single Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        { %string('This is a String variable') }
+
+  .. tab-item:: New-Line Double Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        { %string("This is a String variable") }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+      string_variable: "{ %string('This is a String variable') }"
+
+In the case that you want to define a string variable that contains both single and double quotes,
+triple-quotes can be used to avoid *closing* the String.
+
+.. tab-set::
+
+  .. tab-item:: Triple-Single Quote
 
     .. code-block:: yaml
 
       string_variable: >-
         {
-          %string('''This is a String variable''')
+          %string('''This has both " and ' in it.''')
         }
 
   .. tab-item:: Triple-Double Quote
@@ -57,14 +85,9 @@ Strings are a series of characters surrounded by quotes and can be defined in a
 
       string_variable: >-
         {
-          %string("""This is a String variable""")
+          %string("""This has both " and ' in it.""")
         }
 
-.. note::
-
-   For non-String types, they must be defined as parameters to scripting functions. This is because
-   anything in a variable definition that is not within curly-braces gets evaluated as a String.
-
 Integer
 ~~~~~~~
 
@@ -72,7 +95,7 @@ Integers are whole numbers with no decimal.
 
 .. tab-set::
 
-  .. tab-item:: Literal
+  .. tab-item:: Multi-Line
 
     .. code-block:: yaml
 
@@ -81,7 +104,15 @@ Integers are whole numbers with no decimal.
            %int(2022)
          }
 
-  .. tab-item:: In-Line
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       int_variable: >-
+         { %int(2022) }
+
+
+  .. tab-item:: Same-Line
 
     .. code-block:: yaml
 
@@ -94,7 +125,7 @@ Floats are floating-point decimals numbers.
 
 .. tab-set::
 
-  .. tab-item:: Literal
+  .. tab-item:: Multi-Line
 
     .. code-block:: yaml
 
@@ -103,7 +134,14 @@ Floats are floating-point decimals numbers.
            %float(3.14)
          }
 
-  .. tab-item:: In-Line
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       float_variable: >-
+         { %float(3.14) }
+
+  .. tab-item:: Same-Line
 
     .. code-block:: yaml
 
@@ -116,7 +154,7 @@ A type is considered boolean if it spells out ``True`` or ``False``, case-insens
 
 .. tab-set::
 
-  .. tab-item:: Literal
+  .. tab-item:: Multi-Line
 
     .. code-block:: yaml
 
@@ -125,7 +163,14 @@ A type is considered boolean if it spells out ``True`` or ``False``, case-insens
            %bool(True)
          }
 
-  .. tab-item:: In-Line
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       bool_variable: >-
+         { %bool(True) }
+
+  .. tab-item:: Same-Line
 
     .. code-block:: yaml
 
@@ -139,7 +184,7 @@ Arrays are defined using brackets (``[ ]``), and are accessed using zero-based i
 
 .. tab-set::
 
-  .. tab-item:: Literal
+  .. tab-item:: Multi-Line
 
     .. code-block:: yaml
 
@@ -157,7 +202,16 @@ Arrays are defined using brackets (``[ ]``), and are accessed using zero-based i
            %array_at(array_variable, 0)
          }
 
-  .. tab-item:: In-Line
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       array_variable: >-
+         { ["element with index 0", 1, 2.0, ["Nested Array 3"]] }
+       element_0: >-
+         { %array_at(array_variable, 0) }
+
+  .. tab-item:: Same-Line
 
     .. code-block:: yaml
 
@@ -168,11 +222,11 @@ Map
 ~~~
 
 A Map is a key-value store, containing mappings between keys and values.
-Maps are defined using curley-braces (``{ }``), and are accessed using their keys.
+Maps are defined using curly-braces (``{ }``), and are accessed using their keys.
 
 .. tab-set::
 
-  .. tab-item:: Literal
+  .. tab-item:: Multi-Line
 
     .. code-block:: yaml
 
@@ -189,7 +243,16 @@ Maps are defined using curley-braces (``{ }``), and are accessed using their key
            %map_get(map_variable, "string_key")
          }
 
-  .. tab-item:: In-Line
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       map_variable: >-
+         { {"string_key": "string_value", 1: "int_key", "list_value": ["elem0", 1, 2.0]} }
+       string_value: >-
+         { %map_get(map_variable, "string_key") }
+
+  .. tab-item:: Same-Line
 
     .. code-block:: yaml
 
@@ -209,7 +272,14 @@ case-insensitive.
 
        null_variable: ""
 
-  .. tab-item:: In-Line
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+      null_variable: >-
+        { %string(null) }
+
+  .. tab-item:: Same-Line
 
     .. code-block:: yaml
 
@@ -276,8 +346,9 @@ it expects the lambda function to have two input arguments. These are denoted us
 
 LambdaReduce
 ~~~~~~~~~~~~
-LambdaReduce is special type of lambda that reduces an Array to a single value by calling the
+LambdaReduce parameters are a reference to a function that will perform a *reduce* - an operation
-LabmdaReduce function repeatedly on two elements in the Array until it is reduced to a single value.
+that reduces an Array to a single value by calling the LambdaReduce function repeatedly on two
+elements in the Array until it is reduced to a single value.
 
 In this example,
 
@@ -294,11 +365,9 @@ on the input array, using
 `add <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_
 as the LambdaReduce function. This will reduce the Array to a single value by internally calling
 
-.. code-block::
+- *reduce-call 1*: ``%add(1, 2) = 3`` (first two elements)
-
+- *reduce-call 2*: ``%add(3, 3) = 6`` (output from first two and third element)
-   - %add(1, 2) = 3
+- *reduce-call 3*: ``%add(6, 4) = 10`` (output from first three elements and fourth element)
-   - %add(3, 3) = 6
-   - %add(6, 4) = 10
 
 And evaluate to ``10``.
 

docs/source/config_reference/scripting/static_variables.rst

@@ -1,9 +1,12 @@
 
-Override Variables
+Static Variables
-==================
+================
+
+Subscription Variables
+----------------------
 
 subscription_indent_i
----------------------
+~~~~~~~~~~~~~~~~~~~~~
 For subscriptions in the form of
 
 .. code-block:: yaml
@@ -16,7 +19,7 @@ For subscriptions in the form of
 ``Indent Value 1`` and ``Indent Value 2``.
 
 subscription_map
-----------------
+~~~~~~~~~~~~~~~~
 For subscriptions in the form of
 
 .. code-block:: yaml
@@ -42,11 +45,12 @@ Stores all the contents under the subscription name into the override variable
    }
 
 subscription_name
------------------
+~~~~~~~~~~~~~~~~~
-Name of the subscription
+Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``),
+the prefix and all whitespace afterwards is stripped from the subscription name.
 
 subscription_value
-------------------
+~~~~~~~~~~~~~~~~~~
 For subscriptions in the form of
 
 .. code-block:: yaml
@@ -56,7 +60,7 @@ For subscriptions in the form of
 ``subscription_value`` gets set to ``https://...``.
 
 subscription_value_i
---------------------
+~~~~~~~~~~~~~~~~~~~~
 For subscriptions in the form of
 
 .. code-block:: yaml

src/ytdl_sub/config/overrides.py

@@ -8,7 +8,7 @@ import mergedeep
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.variables.override_variables import OverrideHelpers
-from ytdl_sub.entries.variables.override_variables import OverrideVariables
+from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
 from ytdl_sub.script.parser import parse
 from ytdl_sub.script.script import Script
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@@ -135,7 +135,7 @@ class Overrides(DictFormatterValidator, Scriptable):
         """
         self.script.add(
             ScriptUtils.add_sanitized_variables(
-                {OverrideVariables.subscription_name(): subscription_name}
+                {SubscriptionVariables.subscription_name(): subscription_name}
             )
         )
         self.script.add(

src/ytdl_sub/config/validators/variable_validation.py

@@ -14,7 +14,7 @@ from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.validators.options import OptionsValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
-from ytdl_sub.entries.variables.override_variables import OverrideVariables
+from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
 from ytdl_sub.script.script import Script
 from ytdl_sub.validators.string_formatter_validators import validate_formatters
 
@@ -67,7 +67,9 @@ def _get_added_and_modified_variables(
 
 
 def _override_variables(overrides: Overrides) -> Set[str]:
-    return set(list(overrides.initial_variables().keys())) | {OverrideVariables.subscription_name()}
+    return set(list(overrides.initial_variables().keys())) | {
+        SubscriptionVariables.subscription_name()
+    }
 
 
 def _entry_variables() -> Set[str]:

src/ytdl_sub/entries/variables/override_variables.py

@@ -7,11 +7,12 @@ from ytdl_sub.script.utils.name_validation import is_valid_name
 SUBSCRIPTION_ARRAY = "subscription_array"
 
 
-class OverrideVariables:
+class SubscriptionVariables:
     @staticmethod
     def subscription_name() -> str:
         """
-        Name of the subscription
+        Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``),
+        the prefix and all whitespace afterwards is stripped from the subscription name.
         """
         return "subscription_name"
 

src/ytdl_sub/subscriptions/subscription_validators.py

@@ -9,7 +9,7 @@ from typing import final
 
 from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.overrides import Overrides
-from ytdl_sub.entries.variables.override_variables import OverrideVariables
+from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
 from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
 from ytdl_sub.validators.validators import DictValidator
@@ -32,7 +32,7 @@ class SubscriptionOutput(Validator, ABC):
         indent overrides to merge with the preset dict's overrides
         """
         return {
-            OverrideVariables.subscription_indent_i(i): self._indent_overrides[i]
+            SubscriptionVariables.subscription_indent_i(i): self._indent_overrides[i]
             for i in range(len(self._indent_overrides))
         }
 
@@ -143,7 +143,7 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator):
             presets=presets,
             indent_overrides=indent_overrides,
         )
-        self._overrides_to_add[OverrideVariables.subscription_value()] = self.value
+        self._overrides_to_add[SubscriptionVariables.subscription_value()] = self.value
 
 
 class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator):
@@ -168,10 +168,12 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid
         for idx, list_value in enumerate(self.list):
             # Write the first list value into subscription_value as well
             if idx == 0:
-                self._overrides_to_add[OverrideVariables.subscription_value()] = list_value.value
+                self._overrides_to_add[
+                    SubscriptionVariables.subscription_value()
+                ] = list_value.value
 
             self._overrides_to_add[
-                OverrideVariables.subscription_value_i(index=idx)
+                SubscriptionVariables.subscription_value_i(index=idx)
             ] = list_value.value
 
 
@@ -215,7 +217,7 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator):
             presets=presets,
             indent_overrides=indent_overrides,
         )
-        self._overrides_to_add[OverrideVariables.subscription_map()] = ScriptUtils.to_script(
+        self._overrides_to_add[SubscriptionVariables.subscription_map()] = ScriptUtils.to_script(
             self.dict
         )
 

tests/unit/docgen/test_docgen.py

@@ -2,9 +2,9 @@ from typing import Type
 
 from tools.docgen.docgen import DocGen
 from tools.docgen.entry_variables import EntryVariablesDocGen
-from tools.docgen.override_variables import OverrideVariablesDocGen
 from tools.docgen.plugins import PluginsDocGen
 from tools.docgen.scripting_functions import ScriptingFunctionsDocGen
+from tools.docgen.static_variables import StaticVariablesDocGen
 from ytdl_sub.utils.file_handler import get_md5_hash
 
 
@@ -20,8 +20,8 @@ class TestDocGen:
     def test_entry_variables_generated(self):
         _test_doc_gen(EntryVariablesDocGen)
 
-    def test_override_variables_generated(self):
+    def test_static_variables_generated(self):
-        _test_doc_gen(OverrideVariablesDocGen)
+        _test_doc_gen(StaticVariablesDocGen)
 
     def test_scripting_functions_generated(self):
         _test_doc_gen(ScriptingFunctionsDocGen)

tools/docgen/override_variables.py

@@ -1,25 +0,0 @@
-from pathlib import Path
-
-from tools.docgen.docgen import DocGen
-from tools.docgen.utils import get_function_docs
-from tools.docgen.utils import section
-from tools.docgen.utils import static_methods
-from ytdl_sub.entries.variables.override_variables import OverrideVariables
-
-
-class OverrideVariablesDocGen(DocGen):
-
-    LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst")
-
-    @classmethod
-    def generate(cls) -> str:
-        docs = section("Override Variables", level=0)
-
-        for name in static_methods(OverrideVariables):
-            docs += get_function_docs(
-                function_name=name,
-                obj=OverrideVariables,
-                level=1,
-            )
-
-        return docs

tools/docgen/static_variables.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+from tools.docgen.docgen import DocGen
+from tools.docgen.utils import get_function_docs
+from tools.docgen.utils import section
+from tools.docgen.utils import static_methods
+from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
+
+
+class StaticVariablesDocGen(DocGen):
+
+    LOCATION = Path("docs/source/config_reference/scripting/static_variables.rst")
+
+    @classmethod
+    def generate(cls) -> str:
+        docs = section("Static Variables", level=0)
+
+        docs += section("Subscription Variables", level=1)
+        for name in static_methods(SubscriptionVariables):
+            docs += get_function_docs(
+                function_name=name,
+                obj=SubscriptionVariables,
+                level=2,
+            )
+
+        return docs