Merge branch 'master' into j/music-extras

Makefile

@@ -22,7 +22,8 @@ check_lint:
 		&& black . --check  \
 		&& pylint src/
 wheel: clean
-	$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py)
+	$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"" > src/ytdl_sub/__init__.py)
+	$(shell echo "__local_version__ = \"$(LOCAL_VERSION)\"" >> src/ytdl_sub/__init__.py)
 	cat src/ytdl_sub/__init__.py
 	pip3 install build
 	python3 -m build
@@ -39,6 +40,7 @@ executable: clean
 	pyinstaller ytdl-sub.spec
 	mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX}
 docs:
+	REGENERATE_DOCS=1 pytest tests/unit/docgen/test_docgen.py
 	sphinx-build -M html docs/source/ docs/build/
 clean:
 	rm -rf \

README.md

@@ -71,13 +71,8 @@ __preset__:
     cookiefile: "/config/cookie.txt" 
 
 ###################################################################
-# Subscriptions nested under this will use the
+# TV Show Presets. Can replace Plex with Plex/Jellyfin/Kodi
-# `Plex TV Show by Date` preset.
+
-#
-# Can choose between:
-#   - Plex TV Show by Date:
-#   - Jellyfin TV Show by Date:
-#   - Kodi TV Show by Date:
 Plex TV Show by Date:
 
   # Sets genre tag to "Documentaries"
@@ -101,9 +96,18 @@ Plex TV Show by Date:
   = News | Only Recent:
     "BBC News": "https://www.youtube.com/@BBCNews"
 
+Plex TV Show Collection:
+  = Music:
+    # Prefix with ~ to set specific override variables
+    "~Beyond the Guitar":
+      s01_name: "Videos"
+      s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+      s02_name: "Covers"
+      s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
+
 ###################################################################
-# Subscriptions nested under these will use the various prebuilt
+# Music Presets. Can replace Plex with Plex/Jellyfin/Kodi
-# music presets
+
 YouTube Releases:
   = Jazz:  # Sets genre tag to "Jazz"
     "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
@@ -124,10 +128,7 @@ Bandcamp:
     "Emily Hopkins": "https://emilyharpist.bandcamp.com/"
 
 ###################################################################
-# Can choose between:
+# Music Video Presets
-#   - Plex Music Videos:
-#   - Jellyfin Music Videos:
-#   - Kodi Music Videos:
 "Plex Music Videos":
   = Pop:  # Sets genre tag to "Pop"
     "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
@@ -182,7 +183,7 @@ music_videos/
 
 ## Custom Configs
 Any part of this process is modifiable by using custom configs. See our
-[walk-through guide](https://github.com/jmbannon/ytdl-sub/wiki)
+[walk-through guide](https://ytdl-sub.readthedocs.io/en/latest/guides/index.html)
 on how to build your first config from scratch. Ready-to-use
 [example configurations](https://github.com/jmbannon/ytdl-sub/tree/master/examples)
 can be found here alongside our

docker/root/defaults/config.yaml

@@ -1,8 +1,8 @@
 # Bare-bones config. Here are some useful links to get started:
-#   Walk-through Guide:  https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction
+#   Walk-through Guide:  https://ytdl-sub.readthedocs.io/en/latest/guides/index.html
 #   Config Examples:     https://github.com/jmbannon/ytdl-sub/tree/master/examples
 #   Prebuilt Presets:    https://ytdl-sub.readthedocs.io/en/latest/presets.html
-#   Config Docs:         https://ytdl-sub.readthedocs.io/en/latest/config.html
+#   Config Reference:    https://ytdl-sub.readthedocs.io/en/latest/config_reference/index.html
 #
 # The subscriptions in `subscriptions.yaml` uses prebuilt presets which do not require
 # any additions to this config. They can be downloaded using the command:

docs/source/conf.py

@@ -7,9 +7,9 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 project = "ytdl-sub"
-copyright = "2023, Jesse Bannon"
+copyright = "2024, Jesse Bannon"
 author = "Jesse Bannon"
-release = "2023.12.15"
+release = ""
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
@@ -47,7 +47,7 @@ html_theme_options = {
             "type": "url",
         },
     ],
-    "announcement": ("Please excuse our mess as we update these documents"),
+    "announcement": "",
     "navigation_depth": 10,
     "show_toc_level": 10,
 }

docs/source/config_reference/plugins.rst

@@ -156,7 +156,7 @@ granularity possible.
 
 :expected type: Optional[OverridesFormatter]
 :description:
-  Only download videos before this datetime.
+  Only download videos after this datetime.
 
 
 ``before``
@@ -231,6 +231,8 @@ URL.
         variables:
           season_index: "2"
           season_name: "Playlist as Season"
+        ytdl_options:
+          break_on_existing: False
         playlist_thumbnails:
           - name: "season{season_index}-poster.jpg"
             uid: "latest_entry"
@@ -402,6 +404,9 @@ It supports basic tags like ``title``, ``album``, ``artist`` and ``albumartist``
 a full list of tags for various file types in MediaFile's
 `source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_.
 
+Note that the date fields ``date`` and ``original_date`` expected a standardized date in the
+form of YYYY-MM-DD. The variable ``upload_date_standardized`` returns a compatible format.
+
 :Usage:
 
 .. code-block:: yaml
@@ -418,6 +423,7 @@ a full list of tags for various file types in MediaFile's
          albumartists:
            - "{artist}"
            - "ytdl-sub"
+         date: "{upload_date_standardized}"
 
 ----------------------------------------------------------------------------------------------------
 

docs/source/config_reference/prebuilt_presets/common.rst

@@ -0,0 +1,15 @@
+=======================
+Common
+=======================
+
+.. highlight:: yaml
+
+Media Quality
+-------------
+
+.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml
+
+Only Recent Videos
+------------------
+
+.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml

docs/source/config_reference/prebuilt_presets/helpers_common.rst

@@ -1,7 +0,0 @@
-=======================
-Common Preset Reference
-=======================
-
-.. highlight:: yaml
-
-.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/common.yaml

docs/source/config_reference/prebuilt_presets/helpers_players.rst

@@ -1,7 +0,0 @@
-========================
-Players Preset Reference
-========================
-
-.. highlight:: yaml
-
-.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/players.yaml

docs/source/config_reference/prebuilt_presets/helpers_url.rst

@@ -1,7 +0,0 @@
-====================
-URL Preset Reference
-====================
-
-.. highlight:: yaml
-
-.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/url.yaml

docs/source/config_reference/prebuilt_presets/index.rst

@@ -6,7 +6,7 @@ This section contains the code for the prebuilt presets. If you just want to und
 
 
 
-.. toctree:: 
+.. toctree::
-  helpers_common
+  common
-  helpers_players
+  tv_show
-  helpers_url
+  music

docs/source/config_reference/prebuilt_presets/music.rst

@@ -0,0 +1,9 @@
+=====
+Music
+=====
+
+All audio music based presets inherit from ``_music_base``.
+
+.. highlight:: yaml
+
+.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/music/singles.yaml

docs/source/config_reference/prebuilt_presets/tv_show.rst

@@ -0,0 +1,9 @@
+========================
+TV Show
+========================
+
+All TV show based presets inherit from ``_episode_base``.
+
+.. highlight:: yaml
+
+.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml

docs/source/config_reference/scripting/entry_variables.rst

@@ -647,3 +647,15 @@ ytdl_sub_input_url
 :type: ``String``
 :description:
   The input URL used in ytdl-sub to create this entry.
+
+ytdl_sub_input_url_count
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The total number of input URLs as defined in the subscription.
+
+ytdl_sub_input_url_index
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The index of the input URL as defined in the subscription, top-most being the 0th index.

docs/source/config_reference/scripting/scripting_functions.rst

@@ -172,6 +172,41 @@ gte
 :description:
   ``>=`` operator. Returns True if left >= right. False otherwise.
 
+is_array
+~~~~~~~~
+:spec: ``is_array(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Map. False otherwise.
+
+is_bool
+~~~~~~~
+:spec: ``is_bool(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Float. False otherwise.
+
+is_float
+~~~~~~~~
+:spec: ``is_float(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Float. False otherwise.
+
+is_int
+~~~~~~
+:spec: ``is_int(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is an Integer. False otherwise.
+
+is_map
+~~~~~~
+:spec: ``is_map(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Map. False otherwise.
+
 is_null
 ~~~~~~~
 :spec: ``is_null(value: AnyArgument) -> Boolean``
@@ -179,6 +214,20 @@ is_null
 :description:
   Returns True if a value is null (i.e. an empty string). False otherwise.
 
+is_numeric
+~~~~~~~~~~
+:spec: ``is_numeric(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is either an Integer or Float. False otherwise.
+
+is_string
+~~~~~~~~~
+:spec: ``is_string(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a String. False otherwise.
+
 lt
 ~~
 :spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
@@ -366,6 +415,14 @@ map_enumerate
   Apply a lambda function on the Map, where each arg
   passed to the lambda function is ``idx, key, value`` as three separate args.
 
+map_extend
+~~~~~~~~~~
+:spec: ``map_extend(maps: Map, ...) -> Map``
+
+:description:
+  Return maps combined in the order from left-to-right. Duplicate keys will use the
+  right-most map's value.
+
 map_get
 ~~~~~~~
 :spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
@@ -503,6 +560,15 @@ regex_search
   the string as the first element of the Array. If there are capture groups, returns each
   group as a subsequent element in the Array.
 
+regex_sub
+~~~~~~~~~
+:spec: ``regex_sub(regex: String, replacement: String, string: String) -> String``
+
+:description:
+  Returns the string obtained by replacing the leftmost non-overlapping occurrences of the
+  pattern in string by the replacement string. The replacement string can reference the
+  match groups via backslash escapes. Callables as replacement argument are not supported.
+
 ----------------------------------------------------------------------------------------------------
 
 String Functions
@@ -650,7 +716,7 @@ to_native_filepath
 :spec: ``to_native_filepath(filepath: String) -> String``
 
 Convert any unix-based path separators ('/') with the OS's native
-separator.
+separator. In addition, expand ~ to absolute directories.
 
 truncate_filepath_if_too_long
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/config_reference/scripting/scripting_types.rst

@@ -88,6 +88,12 @@ triple-quotes can be used to avoid *closing* the String.
           %string("""This has both " and ' in it.""")
         }
 
+If you want a plain string that contains literal curly braces, you can escape them like so:
+
+.. code-block:: yaml
+
+   string_variable: "This contains \\{ literal curly braces \\}"
+
 Integer
 ~~~~~~~
 

docs/source/config_reference/scripting/static_variables.rst

@@ -5,6 +5,11 @@ Static Variables
 Subscription Variables
 ----------------------
 
+subscription_has_download_archive
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Returns True if the subscription has any entries recorded in a download archive. False
+otherwise.
+
 subscription_indent_i
 ~~~~~~~~~~~~~~~~~~~~~
 For subscriptions in the form of

docs/source/config_reference/subscriptions_yaml.rst

@@ -19,9 +19,7 @@ Below is an example that downloads a YouTube playlist:
 
   presets:
     playlist_preset_ex:
-      download:
+      download: "{url}"
-        download_strategy: "url"
-        url: "{url}"
       output_options:
         output_directory: "{output_directory}/{playlist_name}"
         file_name: "{playlist_name}.{title}.{ext}"

docs/source/faq/index.rst

@@ -10,6 +10,18 @@ Since ytdl-sub is relatively new to the public, there has not been many question
 How do I...
 -----------
 
+...remove the date in the video title?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default include the upload date in the ``episode_title``
+override variable. This variable is used to set the title in things like the video metadata, NFO file, etc, which is
+subsequently read by media players. This can be overwritten as you see fit by redefining it:
+
+.. code-block:: yaml
+
+   overrides:
+     episode_title: "{title}"  # Only sets the video title
+
 ...get support or reach out to contribute?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -21,7 +33,7 @@ If you need support, you can:
 
 If you would like to contribute, we're happy to accept any help, even non-coders! To find out how you can help this project, you can:
 
-* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute. 
+* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute.
 
 * If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull request <compare>` with information on what issue you're resolving and it will be reviewed as soon as possible.
 
@@ -71,17 +83,18 @@ Most likely the video has a non-English language set to its 'native' language. Y
 ...Plex is not showing my TV shows correctly
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Set the following for your ytdl-sub library that has been added to Plex.
+1. Set the following for your ytdl-sub library that has been added to Plex.
 
 .. figure:: ../../images/plex_scanner_agent.png
   :alt: The Plex library editor, under the advanced settings, showing the required options for Plex to show the TV shows correctly.
 
-**Scanner:** Plex Series Scanner
+- **Scanner:** Plex Series Scanner
+- **Agent:** Personal Media shows
+- **Visibility:** Exclude from home screen and global search
+- **Episode sorting:** Library default
+- **YES** Enable video preview thumbnails
 
-**Agent:** Personal Media shows
+2. Under **Settings** > **Agents**, confirm Plex Personal Media Shows/Movies scanner has **Local Media Assets** enabled.
 
-**Visibility:** Exclude from home screen and global search
+.. figure:: ../../images/plex_agent_sources.png
-
+  :alt: The Plex Agents settings page has Local Media Assets enabled for Personal Media Shows and Movies tabs.
-**Episode sorting:** Library default
-
-**YES** Enable video preview thumbnails

docs/source/guides/getting_started/advanced_configuration.rst

@@ -16,6 +16,41 @@ The layout of the ``config.yaml`` file is relatively straightforward:
       plugin1:
         plugin1_option1: value1
 
+This creates a preset named ``preset_name``, which contains the made-up
+:doc:`plugin </config_reference/plugins>` ``plugin1``. Under each plugin are its settings.
+A preset can contain multiple plugins.
 
 Preset Inheritance
 ------------------
+
+You can modularize your presets via preset inheritance. For example,
+
+.. code-block:: yaml
+
+  presets:
+
+    TV Show:
+      presets:
+        - "Jellyfin TV Show by Date"
+
+      overrides:
+        tv_show_directory: "/ytdl_sub_tv_shows"
+
+    TV Show Only Recent:
+      presets:
+        - "TV Show"
+        - "Only Recent"
+
+      overrides:
+        only_recent_date_range: "3weeks"
+
+This creates two presets:
+
+* ``TV Show``
+   * Inherits the :doc:`prebuilt </prebuilt_presets/index>` ``Jellyfin TV Show by Date`` preset
+   * Sets the output tv show directory
+* ``TV Show Only Recent``
+   * Inherits the ``TV Show`` preset made above it and the ``Only Recent`` prebuilt preset
+   * Sets only_recent preset to only keep the last 3 weeks worth of videos
+
+Inheritance makes it easy to extend existing presets to include logic for your specific needs.

docs/source/guides/getting_started/first_config.rst

@@ -9,6 +9,14 @@ Your first configuration will look pretty simple:
   configuration:
     working_directory: '.ytdl-sub-downloads'
 
+  presets:
+    TV Show:
+      preset:
+        - "Jellyfin TV Show by Date"
+        - "Only Recent"
+
+      overrides:
+        tv_show_directory: "/ytdl_sub_tv_shows"
 
 
 The first two lines in this ``config.yaml`` file are the ``configuration``, and define the ``working_directory``, which is described near the bottom of :ref:`this section <guides/getting_started/index:quick overview of \`\`ytdl-sub\`\`>`

docs/source/guides/getting_started/first_download.rst

@@ -1,7 +1,9 @@
 Initial Download
 ================
 
-Once you have the ``config.yaml`` and ``subscriptions.yaml`` files created and filled out, you can perform your first download. Access ``ytdl-sub``, navigate to the directory containing your ``config.yaml`` and ``subscriptions.yaml`` files, then run the below command:
+Once you have a ``subscriptions.yaml`` file created and filled out, you can perform your first
+download. Access ``ytdl-sub``, navigate to the directory containing your ``subscriptions.yaml``
+file, then run the below command:
 
 .. tab-set:: 
 
@@ -27,4 +29,4 @@ Once you have the ``config.yaml`` and ``subscriptions.yaml`` files created and f
 
     .. code-block:: shell
       
-      ytdl-sub dl --preset "My Favorite YouTube Channels" --overrides.subscription_name "Rick Astley" --overrides.subscription_value "https://www.youtube.com/@RickAstleyYT/videos"
+      ytdl-sub dl --preset "Jellyfin TV Show by Date" --overrides.subscription_name "NOVA PBS" --overrides.subscription_value "https://www.youtube.com/@novapbs" --overrides.tv_show_genre "Documentaries"

docs/source/guides/getting_started/first_sub.rst

@@ -1,7 +1,7 @@
 Initial Subscription
 ====================
 
-Your first subscription should look similar to the below:
+Your first subscription should look something like this:
 
 .. code-block:: yaml
   :linenos:
@@ -9,13 +9,124 @@ Your first subscription should look similar to the below:
   __preset__:
     overrides:
       tv_show_directory: "/tv_shows"
+      music_directory: "/music"
 
-  "My Favorite YouTube Channels":
+  # Can choose between:
-    "Rick Astley": "https://www.youtube.com/@RickAstleyYT/videos"
+  #  - Plex TV Show by Date:
+  #  - Jellyfin TV Show by Date:
+  #  - Kodi TV Show by Date:
+  #
+  Jellyfin TV Show by Date:
+    = Documentaries:
+      "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+    = Kids | = TV-Y:
+      "Jake Trains": "https://www.youtube.com/@JakeTrains"
+
+  YouTube Releases:
+    = Jazz:  # Sets genre tag to "Jazz"
+      "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
+
+  YouTube Full Albums:
+    = Lofi:
+      "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
+
+Lets break this down:
+
+.. code-block:: yaml
+  :lineno-start: 1
+
+  __preset__:
+    overrides:
+      tv_show_directory: "/tv_shows"
+      music_directory: "/music"
 
 
-The first three lines in this subscription file define where to save the downloaded files associated with all subscriptions in this file.
+The first :ref:`__preset__ <config_reference/subscriptions_yaml:File Preset>` section is where we
+can set modifications that apply to every subscription in this file.
 
-The fifth line in this subscription file is the ``preset``, which provides the "definitions" for the subscription as listed in :doc:`/guides/getting_started/first_config`.
+-------------------------------------
 
-The sixth line is the actual ``subscription``, named ``Rick Astley``, with a link to a :yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`, in this case a YouTube channel.
+.. code-block:: yaml
+  :lineno-start: 6
+
+  # Can choose between:
+  #  - Plex TV Show by Date:
+  #  - Jellyfin TV Show by Date:
+  #  - Kodi TV Show by Date:
+  #
+
+Lines 6-10 are comments that get ignored when parsing YAML since they are prefixed with ``#``.
+It is good practice to leave informative comments in your config or subscription files to remind
+yourself of various things.
+
+-------------------------------------
+
+.. code-block:: yaml
+  :lineno-start: 11
+
+  Jellyfin TV Show by Date:
+
+On line 11, we set the key to ``Jellyfin TV Show by Date``. This is a
+:ref:`prebuilt preset <prebuilt_presets/index:prebuilt presets>` that configures
+subscriptions to look like TV shows in the Jellyfin media player (can be changed to
+one of the presets outlined in the comment above). Setting it as a YAML key implies that all
+subscriptions underneath it will *inherit* this preset.
+
+-------------------------------------
+
+.. code-block:: yaml
+  :lineno-start: 11
+
+  Jellyfin TV Show by Date:
+    = Documentaries:
+
+Line 12 sets the key to ``= Documentaries``. When keys are prefixed with ``=``, it means we are
+setting the
+:ref:`subscription indent variable <config_reference/subscriptions_yaml:Beautifying Subscriptions>`.
+For TV Show presets, the first subscription indent variable maps to the TV show's genre.
+Setting subscription indent variables as a key implies all subscriptions underneath it will
+have this variable set.
+
+To better understand what variables are used in prebuilt presets, refer to the
+:ref:`prebuilt preset reference <config_reference/prebuilt_presets/index:Prebuilt Preset Reference>`.
+Here you will see the underlying variables used in prebuilt presets that can be overwritten.
+We already overwrote a few of the variables in the ``__preset__`` section above to define our
+output directory.
+
+-------------------------------------
+
+.. code-block:: yaml
+  :lineno-start: 11
+
+  Jellyfin TV Show by Date:
+    = Documentaries:
+      "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+Line 13 is where we define our first subscription. We set the subscription name to ``NOVA PBS``,
+and the subscription value to ``https://www.youtube.com/@novapbs``. Referring to the
+:ref:`TV show preset reference <config_reference/prebuilt_presets/tv_show:TV Show>`,
+we can see that ``{subscription_name}`` is used to set the ``tv_show_name`` variable.
+
+-------------------------------------
+
+.. code-block:: yaml
+  :lineno-start: 11
+
+  Jellyfin TV Show by Date:
+    = Documentaries:
+      "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+    = Kids | = TV-Y:
+      "Jake Trains": "https://www.youtube.com/@JakeTrains"
+
+Line 15 underneath ``Jellyfin TV Show by Date``, but at the same level as ``= Documentaries``.
+This means we'll inherit the TV show preset, but not the documentaries indent variable. We instead
+set the indent variables to ``= Kids | = TV-Y``. This sets two indent variables. We can set
+multiple presets and/or indent variables on the same key by using ``|`` as a separator.
+
+Referring to the
+:ref:`TV show preset reference <config_reference/prebuilt_presets/tv_show:TV Show>`, the first
+two indent variables map to the TV show genre and TV show content rating.
+
+The above info should be enough to understand the rest of the subscription file.

docs/source/guides/getting_started/index.rst

@@ -29,28 +29,44 @@ If you plan on using the headless image of ``ytdl-sub``, you:
 Additional useful (but not required) knowledge:
   ☑ Understanding how :yt-dlp:`\ ` works
 
-
+Overview
-Quick Overview of ``ytdl-sub``
+--------
-------------------------------
-
 ``ytdl-sub`` uses two types of YAML files:
 
-- ``config.yaml`` defines ``presets``, which are the "definitions" of your media. ``presets`` "define" how you want your media downloaded, which formats, naming conventions to follow when saving them, etc. These ``presets`` can also inherit other ``presets``, so that you can easily modify an existing ``preset``.
+subscriptions.yaml
-- ``subscriptions.yaml`` defines ``subscriptions``, which specify the media we want to recurrently download, like YouTube channels and playlists, SoundCloud artists, or any :yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`. ``subscriptions`` use ``presets`` to define how ``ytdl-sub`` should handle downloading, processing, and saving them.
+~~~~~~~~~~~~~~~~~~
+Defines ``subscriptions``, which specify the media we want to recurrently download, like YouTube
+channels and playlists, SoundCloud artists, or any
+:yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`. ``subscriptions`` use ``presets``
+to define how ``ytdl-sub`` should handle downloading, processing, and saving them.
 
-When ``ytdl-sub`` is run, in its most basic form:
+``ytdl-sub`` comes packaged with many
+:ref:`prebuilt presets <prebuilt_presets/index:Prebuilt Presets>`
+that will play nicely with well-known media players.
+
+config.yaml
+~~~~~~~~~~~
+To customize ``ytdl-sub`` to beyond the prebuilt presets, you will need a ``config.yaml`` file. This
+file is where custom ``presets`` can be defined to orchestrate ``ytdl-sub`` to your very specific needs.
+
+Running ytdl-sub
+~~~~~~~~~~~~~~~~
+To invoke ``ytdl-sub`` to download subscriptions, use the following command:
 
 .. tab-set-code:: 
 
     .. code-block:: shell
         
-        ytdl-sub sub
+        ytdl-sub sub subscriptions.yaml
 
     .. code-block:: powershell
 
-        ytdl-sub.exe sub
+        ytdl-sub.exe sub subscriptions.yaml
 
-``ytdl-sub`` initially downloads all files to a defined ``working_directory``. This is a temporary storage spot for metadata and media files so that errors during processing- if they occur- don't affect your existing media library. Once all file processing is complete, your media files are moved to the ``output_directory``.
+``ytdl-sub`` initially downloads all files to a defined ``working_directory``. This is a temporary
+storage spot for metadata and media files so that errors during processing- if they occur- don't
+affect your existing media library. Once all file processing is complete, your media files are
+moved to the ``output_directory``.
 
 Ready to Start?
 ---------------

docs/source/prebuilt_presets/tv_shows.rst

@@ -11,17 +11,17 @@ The following actions are taken based on the indicated player:
 
 
 Jellyfin
-~~~~~~~~
+--------
 * Places any season-specific poster art in the main show folder
 * Generates NFO tags
 
 Kodi
-~~~~
+--------
 * Everything that the Jellyfin version does
 * Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□``
 
 Plex
-~~~~
+--------
 * :ref:`Special sanitization <config_reference/scripting/entry_variables:title_sanitized_plex>` of numbers so Plex doesn't recognize numbers that are part of the title as the episode number
 * Converts all downloaded videos to the mp4 format
 * Places any season-specific poster art into the season folder
@@ -38,17 +38,39 @@ TV Show by Date
 
 TV Show by Date will organize something like a YouTube channel or playlist into a tv show, where seasons and episodes are organized using upload date.
 
-Plug and Play Presets
+Example
-~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~
-
+Must define ``tv_show_directory``. Available presets:
-You can use any of these presets in your ``subscriptions.yaml`` as a "One size fits all" solution- they should set all appropriate values. These will organize seasons by year and episodes by month, then day.
-
-Must define ``tv_show_directory``
 
 * ``"Kodi TV Show by Date"``
 * ``"Jellyfin TV Show by Date"``
 * ``"Plex TV Show by Date"``
 
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       tv_show_directory: "/tv_shows"
+
+   Plex TV Show by Date:
+
+     # Sets genre tag to "Documentaries"
+     = Documentaries:
+       "NOVA PBS": "https://www.youtube.com/@novapbs"
+       "National Geographic": "https://www.youtube.com/@NatGeo"
+       "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+     # Sets genre tag to "Kids", "TV-Y" for content rating
+     = Kids | = TV-Y:
+       "Jake Trains": "https://www.youtube.com/@JakeTrains"
+       "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
+
+     = Music:
+       # TV show subscriptions can support multiple urls and store in the same TV Show
+       "Rick Beato":
+         - "https://www.youtube.com/@RickBeato"
+         - "https://www.youtube.com/@rickbeato240"
+
 Advanced Usage
 ~~~~~~~~~~~~~~
 
@@ -71,43 +93,13 @@ And then add one of these:
   
   * Episodes are numbered by the download order. NOTE that this is fetched using the length of the download archive. Do not use if you intend to remove old videos.
 
-An example of a subscription that will be played on Kodi, organized by year with the most recent episode at the top (having a lower episode number), with a genre of "Pop":
-
-.. code-block:: yaml
-  :caption: subscriptions.yaml
-
-  __preset__:
-    overrides:
-      tv_show_directory: "/tv_shows"
-
-  kodi_tv_show_by_date:
-    season_by_year_episode_by_month_day_reversed:
-      = Pop:
-        "Rick A": "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-
-You can also choose to combine multiple URLs into one show. This will result in your videos being downloaded to the same folder, and the episode numbers being shared between them (so you won't have two episode 10's, for example). Note that you may :ytdl-sub-gh:`experience issues <issues/833>` if you use more than 20 URLs at this time.
-
-.. code-block:: yaml
-  :caption: subscriptions.yaml
-
-  __preset__:
-    overrides:
-      tv_show_directory: "/tv_shows"
-
-  kodi_tv_show_by_date:
-    season_by_year_episode_by_month_day_reversed:
-      = Pop:
-        "~Rick A": 
-          url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-          url2: "https://www.youtube.com/@just.rick_6"
-
 
 TV Show Collection
 ------------------
 
-TV Show Collections are made up of multiple URLs, where each URL is a season.
+TV Show Collections set each URL as its own season. If a video belongs to multiple URLs
-If a video belongs to multiple URLs (i.e. a channel and a channel's playlist),
+(i.e. a channel and a channel's playlist), the video will only download once and reside in
-it will resolve to the bottom-most season, as defined in the subscription.
+the higher-numbered season.
 
 Two main use cases of a collection are:
    1. Organize a YouTube channel TV show where Season 1 contains any video
@@ -116,15 +108,41 @@ Two main use cases of a collection are:
    2. Organize one or more YouTube channels/playlists, where each season
       represents a separate channel/playlist.
 
-Player Presets
+Example
+~~~~~~~
+Must define ``tv_show_directory``. Available presets:
+
+* ``"Kodi TV Show Collection"``
+* ``"Jellyfin TV Show Collection"``
+* ``"Plex TV Show Collection"``
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       tv_show_directory: "/tv_shows"
+
+   Plex TV Show Collection:
+     = Music:
+       # Prefix with ~ to set specific override variables
+       "~Beyond the Guitar":
+         s01_name: "Videos"
+         s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+         s02_name: "Covers"
+         s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
+
+Advanced Usage
 ~~~~~~~~~~~~~~
 
+If you prefer a different organization method, you can instead apply multiple presets to your subscriptions.
+
+You will need a base of one of the below:
+
 * ``kodi_tv_show_collection``
 * ``jellyfin_tv_show_collection``
 * ``plex_tv_show_collection``
 
-Episode Formatting Presets
+And then add one of these:
-~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 * ``season_by_collection__episode_by_year_month_day``
 * ``season_by_collection__episode_by_year_month_day_reversed``
@@ -132,40 +150,3 @@ Episode Formatting Presets
   
   * Only use playlist_index episode formatting for playlists that will be fully downloaded once and never again. Otherwise, indices can change.
 * ``season_by_collection__episode_by_playlist_index_reversed``
-
-Season Presets
-~~~~~~~~~~~~~~
-
-* ``collection_season_1``
-* ``collection_season_2``
-* ``collection_season_3``
-* ``collection_season_4``
-* ``...``
-* ``collection_season_40``
-
-Example
-~~~~~~~
-
-A preset/subscription requires specifying a player, episode formatting, and
-one or more season presets, with the following override variables:
-
-.. code-block:: yaml
-
-  rick_a_tv_show_collection:
-    preset:
-      - "jellyfin_tv_show_collection"
-      - "season_by_collection__episode_by_year_month_day_reversed"
-      - "collection_season_1"
-      - "collection_season_2"
-    overrides:
-      # required
-      tv_show_name: "Rick A"
-      tv_show_directory: "/path/to/youtube_shows"
-      collection_season_1_url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-      collection_season_1_name: "All Videos"
-      collection_season_2_url: "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
-      collection_season_2_name: "Official Music Videos"
-      # can be modified from their default value
-      # tv_show_genre: "ytdl-sub"
-      # episode_title: "{upload_date_standardized} - {title}"
-      # episode_description: "{webpage_url}"

examples/tv_show_subscriptions.yaml

@@ -55,4 +55,21 @@ Plex TV Show by Date:
 
   # Set "News" for genre, use `Only Recent` preset to only store videos uploaded recently
   = News | Only Recent:
-    "BBC News": "https://www.youtube.com/@BBCNews"
+    "BBC News": "https://www.youtube.com/@BBCNews"
+
+# Sets URLs to be explicit seasons. If a video resides in multiple URLs, it will
+# only appear once in the higher-numbered season. This is how you can separate a channel's
+# videos and playlists you are interested in.
+#
+# Choose the player you intend to use by setting the top-level key to be either:
+#   - Plex TV Show Collection:
+#   - Jellyfin TV Show Collection:
+#   - Kodi TV Show Collection:
+Plex TV Show Collection:
+  = Music:
+    "~Beyond the Guitar":
+      s01_name: "Videos"
+      s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+
+      s02_name: "Music Videos"
+      s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"

pyproject.toml

@@ -1,3 +1,69 @@
+[project]
+name ="ytdl-sub"
+dynamic = [ "version" ]
+authors = [ { name = "Jesse Bannon" } ]
+description = "Automate downloading metadata generation with YoutubeDL"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+classifiers = [
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Multimedia :: Video",
+    "License :: Public Domain",
+    "Environment :: Console",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+    "yt-dlp==2024.5.27",
+    "colorama~=0.4",
+    "mergedeep~=1.3",
+    "mediafile~=0.12",
+    "PyYAML~=6.0",
+]
+urls = { Homepage = "https://github.com/jmbannon/ytdl-sub" }
+
+[build-system]
+requires = [ "setuptools >= 67.0" ]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+platforms = [ "Unix" ]
+
+[tool.setuptools.dynamic]
+version = { attr = "ytdl_sub.__pypi_version__" }
+[tool.setuptools.package-dir]
+"" = "src"
+[tool.setuptools.package-data]
+"*" = ["*.yaml"]
+[tool.setuptools.packages.find]
+where = ["src"]
+
+
+[project.optional-dependencies]
+test = [
+    "coverage[toml]>=6.3,<8.0",
+    "pytest>=7.2,<9.0",
+    "pytest-rerunfailures~=14.0",
+]
+lint = [
+    "black==24.4.2",
+    "isort==5.13.2",
+    "pylint==3.2.2",
+]
+docs = [
+    "sphinx~=7.0",
+    "sphinx-rtd-theme~=2.0",
+    "sphinx-book-theme~=1.0",
+]
+build = [
+    "build~=1.2",
+    "twine~=5.0",
+    "pyinstaller~=6.5",
+]
+[project.scripts]
+ytdl-sub = "ytdl_sub.main:main"
+
 [tool.isort]
 profile = "black"
 line_length = 100
@@ -30,4 +96,4 @@ include = [
 [tool.coverage.report]
 exclude_also = [
     "raise UNREACHABLE.*",
-]
+]

setup.cfg

@@ -1,60 +0,0 @@
-[metadata]
-name = ytdl-sub
-version = attr:ytdl_sub.__pypi_version__
-author = Jesse Bannon
-description = Automate downloading and metadata generation with YoutubeDL
-long_description = file: README.md
-long_description_content_type= text/markdown
-author_email = use_github_issues@nope.com
-url = https://github.com/jmbannon/ytdl-sub
-license = GNUv3
-platforms = Unix
-classifiers =
-    Topic :: Multimedia :: Sound/Audio
-    Topic :: Multimedia :: Video
-    License :: Public Domain
-    Environment :: Console
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-
-[options.entry_points]
-console_scripts =
-    ytdl-sub = ytdl_sub.main:main
-
-[options]
-package_dir =
-    = src
-packages=find:
-
-install_requires =
-    yt-dlp==2023.12.30
-    argparse==1.4.0
-    colorama==0.4.6
-    mergedeep==1.3.4
-    mediafile==0.12.0
-    PyYAML==5.3.1
-
-[options.package_data]
-* = *.yaml
-
-[options.packages.find]
-where=src
-
-[options.extras_require]
-test =
-    coverage[toml]==6.3.2
-    pytest==7.1.1
-    pytest-rerunfailures==12.0
-lint =
-    black==22.3.0
-    isort==5.10.1
-    pylint==2.13.5
-docs =
-    sphinx==7.2.6
-    sphinx-rtd-theme==2.0.0
-    sphinx-book-theme==1.1.0
-build =
-    build
-    twine
-    pyinstaller
-

src/ytdl_sub/cli/parsers/main.py

@@ -150,6 +150,8 @@ parser.add_argument("-v", "--version", action="version", version="%(prog)s " + _
 _add_shared_arguments(parser, suppress_defaults=False)
 
 subparsers = parser.add_subparsers(dest="subparser")
+
+
 ###################################################################################################
 # SUBSCRIPTION PARSER
 class SubArguments:

src/ytdl_sub/config/config_validator.py

@@ -1,3 +1,5 @@
+import os
+import posixpath
 from typing import Any
 from typing import Dict
 from typing import Optional
@@ -147,7 +149,8 @@ class ConfigOptions(StrictDictValidator):
         The directory to temporarily store downloaded files before moving them into their final
         directory. Defaults to .ytdl-sub-working-directory
         """
-        return self._working_directory.value
+        # Expands tildas to actual paths, use native os sep
+        return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep))
 
     @property
     def umask(self) -> Optional[str]:

src/ytdl_sub/config/overrides.py

@@ -7,8 +7,8 @@ import mergedeep
 
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
+from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
 from ytdl_sub.entries.variables.override_variables import OverrideHelpers
-from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
 from ytdl_sub.script.parser import parse
 from ytdl_sub.script.script import Script
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@@ -17,11 +17,11 @@ from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.utils.scriptable import Scriptable
-from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
 
 
-class Overrides(DictFormatterValidator, Scriptable):
+class Overrides(UnstructuredDictFormatterValidator, Scriptable):
     """
     Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter.
 
@@ -51,17 +51,18 @@ class Overrides(DictFormatterValidator, Scriptable):
 
     @classmethod
     def partial_validate(cls, name: str, value: Any) -> None:
-        dict_formatter = DictFormatterValidator(name=name, value=value)
+        dict_formatter = UnstructuredDictFormatterValidator(name=name, value=value)
         _ = [parse(format_string) for format_string in dict_formatter.dict_with_format_strings]
 
     def __init__(self, name, value):
-        DictFormatterValidator.__init__(self, name, value)
+        UnstructuredDictFormatterValidator.__init__(self, name, value)
         Scriptable.__init__(self, initialize_base_script=True)
 
         for key in self._keys:
             self.ensure_variable_name_valid(key)
 
         self.unresolvable.add(VARIABLES.entry_metadata.variable_name)
+        self.unresolvable.update(REQUIRED_OVERRIDE_VARIABLE_NAMES)
 
     def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool:
         """
@@ -127,17 +128,10 @@ class Overrides(DictFormatterValidator, Scriptable):
         )
         return ScriptUtils.add_sanitized_variables(initial_variables)
 
-    def initialize_script(
+    def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides":
-        self, subscription_name: str, unresolved_variables: Set[str]
-    ) -> "Overrides":
         """
-        Initialize the override script with override variables + any unresolved variables
+        Initialize the override script with any unresolved variables
         """
-        self.script.add(
-            ScriptUtils.add_sanitized_variables(
-                {SubscriptionVariables.subscription_name(): subscription_name}
-            )
-        )
         self.script.add(
             self.initial_variables(
                 unresolved_variables={

src/ytdl_sub/config/plugin/plugin.py

@@ -9,27 +9,27 @@ from typing import Tuple
 from typing import Type
 
 from ytdl_sub.config.overrides import Overrides
+from ytdl_sub.config.validators.options import OptionsValidatorT
 from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
-from ytdl_sub.config.validators.options import TOptionsValidator
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
-# pylint: disable=no-self-use,unused-argument
+# pylint: disable=unused-argument
 
 
-class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC):
+class BasePlugin(DownloadArchiver, Generic[OptionsValidatorT], ABC):
     """
     Shared code amongst all SourcePlugins (downloaders) and Plugins (post-download modification)
     """
 
-    plugin_options_type: Type[TOptionsValidator]
+    plugin_options_type: Type[OptionsValidatorT]
 
     def __init__(
         self,
-        options: TOptionsValidator,
+        options: OptionsValidatorT,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
     ):
@@ -38,7 +38,7 @@ class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC):
         self.overrides = overrides
 
 
-class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
+class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
     """
     Class to define the new plugin functionality
     """
@@ -121,7 +121,7 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
         """
 
 
-class SplitPlugin(Plugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
+class SplitPlugin(Plugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
     """
     Plugin that splits entries into zero or more entries
     """

src/ytdl_sub/config/plugin/preset_plugins.py

@@ -5,7 +5,7 @@ from typing import Type
 
 from ytdl_sub.config.plugin.plugin import Plugin
 from ytdl_sub.config.validators.options import OptionsValidator
-from ytdl_sub.config.validators.options import TOptionsValidator
+from ytdl_sub.config.validators.options import OptionsValidatorT
 
 
 class PresetPlugins:
@@ -29,7 +29,7 @@ class PresetPlugins:
         """
         return list(zip(self.plugin_types, self.plugin_options))
 
-    def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
+    def get(self, plugin_type: Type[OptionsValidatorT]) -> Optional[OptionsValidatorT]:
         """
         Parameters
         ----------

src/ytdl_sub/config/preset.py

@@ -198,9 +198,7 @@ class Preset(_PresetShell):
             downloader_options=self.downloader_options,
             output_options=self.output_options,
             plugins=self.plugins,
-        ).initialize_overrides(
+        ).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage()
-            subscription_name=self.name, overrides=self.overrides
-        ).ensure_proper_usage()
 
     @property
     def name(self) -> str:

src/ytdl_sub/config/validators/options.py

@@ -9,7 +9,6 @@ from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
 from ytdl_sub.validators.validators import Validator
 
-# pylint: disable=no-self-use
 # pylint: disable=unused-argument
 
 
@@ -53,7 +52,7 @@ class OptionsValidator(Validator, ABC):
         return {}
 
 
-TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
+OptionsValidatorT = TypeVar("OptionsValidatorT", bound=OptionsValidator)
 
 
 class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):

src/ytdl_sub/config/validators/variable_validation.py

@@ -13,12 +13,24 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.validators.options import OptionsValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
-from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
+from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.script import _is_function
 from ytdl_sub.utils.scriptable import BASE_SCRIPT
 from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
 from ytdl_sub.validators.string_formatter_validators import validate_formatters
 
+# Entry variables to mock during validation
+_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
+    name: to_variable_dependency_format_string(
+        # pylint: disable=protected-access
+        script=BASE_SCRIPT,
+        parsed_format_string=BASE_SCRIPT._variables[name],
+        # pylint: enable=protected-access
+    )
+    for name in BASE_SCRIPT.variable_names
+}
+
 
 def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
     dummy_variables: Dict[str, str] = {}
@@ -33,6 +45,9 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
     # Have the dummy override variable contain all variable deps that it uses in the string
     dummy_overrides: Dict[str, str] = {}
     for override_name in _override_variables(overrides):
+        if _is_function(override_name):
+            continue
+
         # pylint: disable=protected-access
         dummy_overrides[override_name] = to_variable_dependency_format_string(
             script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
@@ -68,20 +83,7 @@ def _get_added_and_modified_variables(
 
 
 def _override_variables(overrides: Overrides) -> Set[str]:
-    return set(list(overrides.initial_variables().keys())) | {
+    return set(list(overrides.initial_variables().keys()))
-        SubscriptionVariables.subscription_name()
-    }
-
-
-_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
-    name: to_variable_dependency_format_string(
-        # pylint: disable=protected-access
-        script=BASE_SCRIPT,
-        parsed_format_string=BASE_SCRIPT._variables[name]
-        # pylint: enable=protected-access
-    )
-    for name in BASE_SCRIPT.variable_names
-}
 
 
 class VariableValidation:
@@ -99,13 +101,11 @@ class VariableValidation:
         self.resolved_variables: Set[str] = set()
         self.unresolved_variables: Set[str] = set()
 
-    def initialize_overrides(
+    def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation":
-        self, subscription_name: str, overrides: Overrides
-    ) -> "VariableValidation":
         """
         Do some gymnastics to initialize the Overrides script.
         """
-        override_variables = _override_variables(overrides)
+        override_variables = set(list(overrides.initial_variables().keys()))
 
         # Set resolved variables as all entry + override variables
         # at this point to generate every possible added/modified variable
@@ -141,9 +141,7 @@ class VariableValidation:
         # Initialize overrides with unresolved variables + modified variables to throw an error.
         # For modified variables, this is to prevent a resolve(update=True) to setting any
         # dependencies until it has been explicitly added
-        overrides = overrides.initialize_script(
+        overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables)
-            subscription_name=subscription_name, unresolved_variables=self.unresolved_variables
-        )
 
         # copy the script and mock entry variables
         self.script = copy.deepcopy(overrides.script)
@@ -158,7 +156,16 @@ class VariableValidation:
     def _update_script(self) -> None:
         _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
 
-    def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]:
+    def _add_subscription_override_variables(self) -> None:
+        """
+        Add dummy subscription variables for script validation
+        """
+        self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES
+
+    def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
+        """
+        Add dummy variables for script validation
+        """
         added_variables = options.added_variables(
             resolved_variables=self.resolved_variables,
             unresolved_variables=self.unresolved_variables,
@@ -171,14 +178,14 @@ class VariableValidation:
         self.resolved_variables |= resolved_variables
         self.unresolved_variables -= resolved_variables
 
-        return added_variables
-
     def ensure_proper_usage(self) -> None:
         """
         Validate variables resolve as plugins are executed, and return
         a mock script which contains actualized added variables from the plugins
         """
+
         self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
+        self._add_subscription_override_variables()
 
         # Metadata variables to be added
         for plugin_options in PluginMapping.order_options_by(

src/ytdl_sub/downloaders/info_json/info_json_downloader.py

@@ -118,6 +118,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
             )
             entries.append(entry)
 
+        # TODO: MATCH A URL TO A URL_VALIDATOR !!!
         for entry in sorted(entries, key=lambda ent: ent.get(v.download_index, int)):
             # Remove each entry from the live download archive since it will get re-added
             # unless it is filtered

src/ytdl_sub/downloaders/source_plugin.py

@@ -11,13 +11,13 @@ from typing import final
 from ytdl_sub.config.overrides import Overrides
 from ytdl_sub.config.plugin.plugin import BasePlugin
 from ytdl_sub.config.plugin.plugin import Plugin
-from ytdl_sub.config.validators.options import TOptionsValidator
+from ytdl_sub.config.validators.options import OptionsValidatorT
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 
-class SourcePluginExtension(Plugin[TOptionsValidator], ABC):
+class SourcePluginExtension(Plugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
     """
     Plugins that get added automatically by using a downloader. Downloader options
     are the plugin options.
@@ -32,12 +32,12 @@ class SourcePluginExtension(Plugin[TOptionsValidator], ABC):
         return None
 
 
-class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
+class SourcePlugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
     plugin_extensions: List[Type[SourcePluginExtension]] = []
 
     def __init__(
         self,
-        options: TOptionsValidator,
+        options: OptionsValidatorT,
         enhanced_download_archive: EnhancedDownloadArchive,
         download_ytdl_options: YTDLOptionsBuilder,
         metadata_ytdl_options: YTDLOptionsBuilder,

src/ytdl_sub/downloaders/url/downloader.py

@@ -25,6 +25,7 @@ from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
 from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.utils.thumbnail import ThumbnailTypes
 from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
 from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
@@ -41,7 +42,30 @@ class URLDownloadState:
         self.entries_downloaded = 0
 
 
-class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
+class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator]):
+    def _match_entry_to_url_validator(self, entry: Entry) -> UrlValidator:
+        """
+        Handle matching a URL to its original validator. This is for .info.json updates
+        when older entries have missing variables
+        """
+        input_url_idx = entry.get(v.ytdl_sub_input_url_index, int)
+        entry_input_url = entry.get(v.ytdl_sub_input_url, str)
+
+        if 0 <= input_url_idx < len(self.plugin_options.urls.list):
+            validator = self.plugin_options.urls.list[input_url_idx]
+            if self.overrides.apply_formatter(validator.url) == entry_input_url:
+                return validator
+
+        # Match the first validator based on the URL, if one exists
+        for validator in self.plugin_options.urls.list:
+            if self.overrides.apply_formatter(validator.url) == entry_input_url:
+                return validator
+
+        # Return the first validator if none exist
+        return self.plugin_options.urls.list[0]
+
+
+class UrlDownloaderThumbnailPlugin(UrlDownloaderBasePluginExtension):
     def __init__(
         self,
         options: MultiUrlValidator,
@@ -54,10 +78,6 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
             enhanced_download_archive=enhanced_download_archive,
         )
         self._thumbnails_downloaded: Set[str] = set()
-        self._collection_url_mapping: Dict[str, UrlValidator] = {
-            self.overrides.apply_formatter(collection_url.url): collection_url
-            for collection_url in options.urls.list
-        }
 
     def _download_parent_thumbnails(
         self,
@@ -136,15 +156,14 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
         if not self.is_dry_run:
             try_convert_download_thumbnail(entry=entry)
 
-        if (input_url := entry.get(v.ytdl_sub_input_url, str)) in self._collection_url_mapping:
+        self._download_url_thumbnails(
-            self._download_url_thumbnails(
+            collection_url=self._match_entry_to_url_validator(entry=entry),
-                collection_url=self._collection_url_mapping[input_url],
+            entry=entry,
-                entry=entry,
+        )
-            )
         return entry
 
 
-class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
+class UrlDownloaderCollectionVariablePlugin(UrlDownloaderBasePluginExtension):
     def __init__(
         self,
         options: MultiUrlValidator,
@@ -157,25 +176,12 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
             enhanced_download_archive=enhanced_download_archive,
         )
         self._thumbnails_downloaded: Set[str] = set()
-        self._collection_url_mapping: Dict[str, UrlValidator] = {
-            self.overrides.apply_formatter(collection_url.url): collection_url
-            for collection_url in options.urls.list
-        }
 
     def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
         """
         Add collection variables to the entry
         """
-        # COLLECTION_URL is a recent variable that may not exist for old entries when updating.
+        collection_url = self._match_entry_to_url_validator(entry=entry)
-        # Try to use source_webpage_url if it does not exist
-        entry_collection_url = entry.get(v.ytdl_sub_input_url, str)
-
-        # If the collection URL cannot find its mapping, use the last URL
-        collection_url = (
-            self._collection_url_mapping.get(entry_collection_url)
-            or list(self._collection_url_mapping.values())[-1]
-        )
-
         entry.add(collection_url.variables.dict_with_format_strings)
 
         return entry
@@ -232,8 +238,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         self._downloaded_entries: Set[str] = set()
         self._url_state: Optional[URLDownloadState] = None
 
-    @property
+    def download_ytdl_options(self, url_idx: Optional[int] = None) -> Dict:
-    def download_ytdl_options(self) -> Dict:
         """
         Returns
         -------
@@ -242,19 +247,28 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         return (
             self._download_ytdl_options_builder.clone()
             .add(self.ytdl_option_defaults(), before=True)
+            .add(
+                (
+                    self.plugin_options.urls.list[url_idx].ytdl_options.dict
+                    if url_idx is not None
+                    else None
+                ),
+                before=True,
+            )
             .to_dict()
         )
 
-    @property
+    def metadata_ytdl_options(self, ytdl_option_overrides: Dict) -> Dict:
-    def metadata_ytdl_options(self) -> Dict:
         """
         Returns
         -------
         YTDL options dict for fetching metadata
         """
+
         return (
             self._metadata_ytdl_options_builder.clone()
             .add(self.ytdl_option_defaults(), before=True)
+            .add(ytdl_option_overrides, before=True)
             .to_dict()
         )
 
@@ -265,7 +279,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         -------
         True if dry-run is enabled. False otherwise.
         """
-        return self.download_ytdl_options.get("skip_download", False)
+        return self.download_ytdl_options().get("skip_download", False)
 
     @property
     def is_entry_thumbnails_enabled(self) -> bool:
@@ -274,7 +288,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         -------
         True if entry thumbnails should be downloaded. False otherwise.
         """
-        return self.download_ytdl_options.get("writethumbnail", False)
+        return self.download_ytdl_options().get("writethumbnail", False)
 
     ###############################################################################################
     # DOWNLOAD FUNCTIONS
@@ -301,7 +315,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         clear_info_json_files
             Whether to delete info.json files after yield
         """
-        archive_path = self.download_ytdl_options.get("download_archive", "")
+        archive_path = self.download_ytdl_options().get("download_archive", "")
         backup_archive_path = f"{archive_path}.backup"
 
         # If archive path exists, maintain download archive is enable
@@ -336,11 +350,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
 
     def _extract_entry_info_with_retry(self, entry: Entry) -> Entry:
         download_entry_dict = YTDLP.extract_info_with_retry(
-            ytdl_options_overrides=self.download_ytdl_options,
+            ytdl_options_overrides=self.download_ytdl_options(
+                url_idx=entry.get(v.ytdl_sub_input_url_index, int)
+            ),
             is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded,
-            is_thumbnail_downloaded_fn=None
+            is_thumbnail_downloaded_fn=(
-            if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
+                None
-            else entry.is_thumbnail_downloaded_via_ytdlp,
+                if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
+                else entry.is_thumbnail_downloaded_via_ytdlp
+            ),
             url=entry.webpage_url,
         )
         return Entry(
@@ -352,45 +370,45 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         self, entries: List[Entry], download_reversed: bool
     ) -> Iterator[Entry]:
         # Iterate a list of entries, and delete the entries after yielding
-        indices = list(range(len(entries)))
+        entries_to_iter: List[Optional[Entry]] = entries
+
+        indices = list(range(len(entries_to_iter)))
         if download_reversed:
             indices = reversed(indices)
 
         for idx in indices:
             self._url_state.entries_downloaded += 1
 
-            if self._is_downloaded(entries[idx]):
+            if self._is_downloaded(entries_to_iter[idx]):
                 download_logger.info(
                     "Already downloaded entry %d/%d: %s",
                     self._url_state.entries_downloaded,
                     self._url_state.entries_total,
-                    entries[idx].title,
+                    entries_to_iter[idx].title,
                 )
-                del entries[idx]
+                entries_to_iter[idx] = None
                 continue
 
-            yield entries[idx]
+            yield entries_to_iter[idx]
-            self._mark_downloaded(entries[idx])
+            self._mark_downloaded(entries_to_iter[idx])
 
-            del entries[idx]
+            entries_to_iter[idx] = None
 
     def _iterate_parent_entry(
         self, parent: EntryParent, download_reversed: bool
     ) -> Iterator[Entry]:
-        for entry_child in self._iterate_child_entries(
+        yield from self._iterate_child_entries(
             entries=parent.entry_children(), download_reversed=download_reversed
-        ):
+        )
-            yield entry_child
 
         # Recursion the parent's parent entries
         for parent_child in reversed(parent.parent_children()):
-            for entry_child in self._iterate_parent_entry(
+            yield from self._iterate_parent_entry(
                 parent=parent_child, download_reversed=download_reversed
-            ):
+            )
-                yield entry_child
 
     def _download_url_metadata(
-        self, url: str, include_sibling_metadata: bool
+        self, url: str, include_sibling_metadata: bool, ytdl_options_overrides: Dict
     ) -> Tuple[List[EntryParent], List[Entry]]:
         """
         Downloads only info.json files and forms EntryParent trees
@@ -398,7 +416,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         with self._separate_download_archives():
             entry_dicts = YTDLP.extract_info_via_info_json(
                 working_directory=self.working_directory,
-                ytdl_options_overrides=self.metadata_ytdl_options,
+                ytdl_options_overrides=ytdl_options_overrides,
                 log_prefix_on_info_json_dl="Downloading metadata for",
                 url=url,
             )
@@ -429,44 +447,57 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         # Delete info json files afterwards so other collection URLs do not use them
         with self._separate_download_archives(clear_info_json_files=True):
             for parent in parents:
-                for entry_child in self._iterate_parent_entry(
+                yield from self._iterate_parent_entry(
                     parent=parent, download_reversed=download_reversed
-                ):
+                )
-                    yield entry_child
 
-            for orphan in self._iterate_child_entries(
+            yield from self._iterate_child_entries(
                 entries=orphans, download_reversed=download_reversed
-            ):
+            )
-                yield orphan
+
+    def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]:
+        metadata_ytdl_options = self.metadata_ytdl_options(
+            ytdl_option_overrides=validator.ytdl_options.dict
+        )
+        download_reversed = ScriptUtils.bool_formatter_output(
+            self.overrides.apply_formatter(validator.download_reverse)
+        )
+
+        parents, orphan_entries = self._download_url_metadata(
+            url=url,
+            include_sibling_metadata=validator.include_sibling_metadata,
+            ytdl_options_overrides=metadata_ytdl_options,
+        )
+
+        # TODO: Encapsulate this logic into its own class
+        self._url_state = URLDownloadState(
+            entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries)
+        )
+
+        download_logger.info("Beginning downloads for %s", url)
+        yield from self._iterate_entries(
+            parents=parents,
+            orphans=orphan_entries,
+            download_reversed=download_reversed,
+        )
 
     def download_metadata(self) -> Iterable[Entry]:
         """The function to perform the download of all media entries"""
         # download the bottom-most urls first since they are top-priority
-        for collection_url in reversed(self.collection.urls.list):
+        for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
             # URLs can be empty. If they are, then skip
-            if not (url := self.overrides.apply_formatter(collection_url.url)):
+            if not (url := self.overrides.apply_formatter(url_validator.url)):
                 continue
 
-            parents, orphan_entries = self._download_url_metadata(
+            for entry in self._download_metadata(url=url, validator=url_validator):
-                url=url, include_sibling_metadata=collection_url.include_sibling_metadata
-            )
-
-            # TODO: Encapsulate this logic into its own class
-            self._url_state = URLDownloadState(
-                entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries)
-            )
-
-            download_logger.info(
-                "Beginning downloads for %s", self.overrides.apply_formatter(collection_url.url)
-            )
-            for entry in self._iterate_entries(
-                parents=parents,
-                orphans=orphan_entries,
-                download_reversed=collection_url.download_reverse,
-            ):
                 entry.initialize_script(self.overrides).add(
-                    {v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)}
+                    {
+                        v.ytdl_sub_input_url: url,
+                        v.ytdl_sub_input_url_index: idx,
+                        v.ytdl_sub_input_url_count: len(self.collection.urls.list),
+                    }
                 )
+
                 yield entry
 
     def download(self, entry: Entry) -> Optional[Entry]:

src/ytdl_sub/downloaders/url/validators.py

@@ -4,10 +4,12 @@ from typing import Optional
 from typing import Set
 
 from ytdl_sub.config.plugin.plugin_operation import PluginOperation
+from ytdl_sub.config.preset_options import YTDLOptions
 from ytdl_sub.config.validators.options import OptionsValidator
 from ytdl_sub.script.parser import parse
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.validators import BoolValidator
@@ -49,6 +51,7 @@ class UrlValidator(StrictDictValidator):
         "source_thumbnails",
         "playlist_thumbnails",
         "download_reverse",
+        "ytdl_options",
         "include_sibling_metadata",
     }
 
@@ -77,7 +80,10 @@ class UrlValidator(StrictDictValidator):
             key="playlist_thumbnails", validator=UrlThumbnailListValidator, default=[]
         )
         self._download_reverse = self._validate_key(
-            key="download_reverse", validator=BoolValidator, default=True
+            key="download_reverse", validator=OverridesBooleanFormatterValidator, default="True"
+        )
+        self._ytdl_options = self._validate_key(
+            key="ytdl_options", validator=YTDLOptions, default={}
         )
         self._include_sibling_metadata = self._validate_key(
             key="include_sibling_metadata", validator=BoolValidator, default=False
@@ -148,12 +154,20 @@ class UrlValidator(StrictDictValidator):
         return self._source_thumbnails
 
     @property
-    def download_reverse(self) -> bool:
+    def download_reverse(self) -> OverridesBooleanFormatterValidator:
         """
         Optional. Whether to download entries in the reverse order of the metadata downloaded.
         Defaults to True.
         """
-        return self._download_reverse.value
+        return self._download_reverse
+
+    @property
+    def ytdl_options(self) -> YTDLOptions:
+        """
+        Optional. ``ytdl_options`` that only apply to this URL. These take precedence
+        over the plugin ``ytdl_options``.
+        """
+        return self._ytdl_options
 
     @property
     def include_sibling_metadata(self) -> bool:
@@ -258,6 +272,8 @@ class MultiUrlValidator(OptionsValidator):
             variables:
               season_index: "2"
               season_name: "Playlist as Season"
+            ytdl_options:
+              break_on_existing: False
             playlist_thumbnails:
               - name: "season{season_index}-poster.jpg"
                 uid: "latest_entry"

src/ytdl_sub/downloaders/ytdlp.py

@@ -248,20 +248,22 @@ class YTDLP:
                 continue
 
             cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
+            parent_dict: Optional[Dict] = None
             try:
                 parent_dict = cls.extract_info(
                     ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
                     url=uploader_url,
                 )
             except Exception:  # pylint: disable=broad-except
-                # Do not try this uploader_id again
+                pass
-                entry_ids.add(uploader_id)
-                break
 
-            if isinstance(parent_dict, dict):
+            parent_id = parent_dict.get("id") if isinstance(parent_dict, dict) else None
-                parent_id = parent_dict.get("id")
+            if parent_id and parent_id not in entry_ids:
                 parent_dicts.append(parent_dict)
-                entry_ids |= {uploader_id, parent_id}
+                entry_ids.add(parent_id)
                 cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id)
 
+            # Always add the uploader_id since it has been tried
+            entry_ids.add(uploader_id)
+
         return entry_dicts + parent_dicts

src/ytdl_sub/entries/base_entry.py

@@ -8,6 +8,7 @@ from typing import Type
 from typing import TypeVar
 from typing import final
 
+from yt_dlp.utils import LazyList
 from yt_dlp.utils import sanitize_filename
 
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
@@ -15,7 +16,7 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
 
 v: VariableDefinitions = VARIABLES
 
-TBaseEntry = TypeVar("TBaseEntry", bound="BaseEntry")
+BaseEntryT = TypeVar("BaseEntryT", bound="BaseEntry")
 
 
 class BaseEntry(ABC):
@@ -37,6 +38,12 @@ class BaseEntry(ABC):
         self._working_directory = working_directory
         self._kwargs = entry_dict
 
+        # Sometimes yt-dlp can return a LazyList which is not JSON serializable.
+        # Cast it to a native list here. (https://github.com/jmbannon/ytdl-sub/issues/910)
+        for key in self._kwargs.keys():
+            if isinstance(self._kwargs[key], LazyList):
+                self._kwargs[key] = list(self._kwargs[key])
+
     @property
     def uid(self) -> str:
         """
@@ -133,7 +140,7 @@ class BaseEntry(ABC):
         return str(Path(self.working_directory()) / self.get_download_info_json_name())
 
     @final
-    def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry:
+    def to_type(self, entry_type: Type[BaseEntryT]) -> BaseEntryT:
         """
         Returns
         -------
@@ -142,7 +149,7 @@ class BaseEntry(ABC):
         return entry_type(entry_dict=self._kwargs, working_directory=self._working_directory)
 
     @classmethod
-    def is_entry_parent(cls, entry_dict: Dict | TBaseEntry):
+    def is_entry_parent(cls, entry_dict: Dict | BaseEntryT):
         """
         Returns
         -------
@@ -157,7 +164,7 @@ class BaseEntry(ABC):
         return entry_type == "playlist"
 
     @classmethod
-    def is_entry(cls, entry_dict: Dict | TBaseEntry):
+    def is_entry(cls, entry_dict: Dict | BaseEntryT):
         """
         Returns
         -------

src/ytdl_sub/entries/entry_parent.py

@@ -4,9 +4,10 @@ from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Set
+from urllib.parse import urlparse
 
 from ytdl_sub.entries.base_entry import BaseEntry
-from ytdl_sub.entries.base_entry import TBaseEntry
+from ytdl_sub.entries.base_entry import BaseEntryT
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
@@ -20,7 +21,7 @@ v: VariableDefinitions = VARIABLES
 
 class EntryParent(BaseEntry):
     @classmethod
-    def _sort_entries(cls, entries: List[TBaseEntry]) -> List[TBaseEntry]:
+    def _sort_entries(cls, entries: List[BaseEntryT]) -> List[BaseEntryT]:
         """Try sorting by playlist_id first, then fall back to uid"""
         return sorted(
             entries,
@@ -152,7 +153,12 @@ class EntryParent(BaseEntry):
         """
 
         def _url_matches(parent: "EntryParent"):
-            return parent.webpage_url in url or url in parent.webpage_url
+            url_parsed = urlparse(url)
+            parent_parsed = urlparse(parent.webpage_url)
+            return (
+                url_parsed.hostname == parent_parsed.hostname
+                and url_parsed.path == parent_parsed.path
+            )
 
         def _uid_is_uploader_id(parent: "EntryParent"):
             return parent.uid == parent.uploader_id
@@ -168,18 +174,10 @@ class EntryParent(BaseEntry):
         if len(top_level_parents) > 1:
             top_level_parents = [parent for parent in top_level_parents if _url_matches(parent)]
 
-        match len(top_level_parents):
+        if len(top_level_parents) == 0:
-            case 0:
+            return None
-                return None
+        if len(top_level_parents) == 1:
-            case 1:
+            return top_level_parents[0]
-                return top_level_parents[0]
-            case 2:
-                # Channels can have two of the same .info.json. Handle it here
-                top0 = top_level_parents[0]
-                top1 = top_level_parents[1]
-                if top0.uploader_id == top1.uploader_id:
-                    return top0 if not top0.webpage_url.endswith("/videos") else top1
-
         raise ValueError(
             "Detected multiple top-level parents. "
             "Please file an issue on GitHub with the URLs used to produce this error"

src/ytdl_sub/entries/script/custom_functions.py

@@ -36,9 +36,9 @@ class CustomFunctions:
     def to_native_filepath(filepath: String) -> String:
         """
         Convert any unix-based path separators ('/') with the OS's native
-        separator.
+        separator. In addition, expand ~ to absolute directories.
         """
-        return String(filepath.value.replace(posixpath.sep, os.sep))
+        return String(os.path.expanduser(filepath.value.replace(posixpath.sep, os.sep)))
 
     @staticmethod
     def truncate_filepath_if_too_long(filepath: String) -> String:

src/ytdl_sub/entries/script/function_scripts.py

@@ -5,6 +5,7 @@ from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
 
 v: VariableDefinitions = VARIABLES
 
+# TODO: Make this a proper class with docstrings
 CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
     #############################################################################################
     # SIBLING GETTER

src/ytdl_sub/entries/script/variable_definitions.py

@@ -21,6 +21,7 @@ from ytdl_sub.entries.script.variable_types import Variable
 # pylint: disable=no-member
 # pylint: disable=too-many-public-methods
 # pylint: disable=too-many-lines
+# pylint: disable=method-cache-max-size-none
 
 
 class MetadataVariableDefinitions(ABC):
@@ -394,7 +395,9 @@ class UploadDateVariableDefinitions(ABC):
         :description:
           The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date.
         """
-        return StringDateMetadataVariable.from_entry(metadata_key="upload_date").as_date_variable()
+        return StringDateMetadataVariable.from_entry(
+            metadata_key="upload_date", default=self.epoch_date
+        ).as_date_variable()
 
     @cached_property
     def upload_year(self: "VariableDefinitions") -> IntegerVariable:
@@ -748,6 +751,24 @@ class YtdlSubVariableDefinitions(ABC):
         """
         return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }")
 
+    @cached_property
+    def ytdl_sub_input_url_index(self: "VariableDefinitions") -> IntegerVariable:
+        """
+        :description:
+          The index of the input URL as defined in the subscription, top-most being the 0th index.
+        """
+        # init as -1 so if prior downloaded entries are known when they do not have this value
+        # in their .info.json
+        return IntegerVariable(variable_name="ytdl_sub_input_url_index", definition="{ %int(-1) }")
+
+    @cached_property
+    def ytdl_sub_input_url_count(self: "VariableDefinitions") -> IntegerVariable:
+        """
+        :description:
+          The total number of input URLs as defined in the subscription.
+        """
+        return IntegerVariable(variable_name="ytdl_sub_input_url_count", definition="{ %int(0) }")
+
     @cached_property
     def download_index(self: "VariableDefinitions") -> IntegerVariable:
         """
@@ -1098,6 +1119,8 @@ class VariableDefinitions(
             self.chapters,
             self.sponsorblock_chapters,
             self.ytdl_sub_input_url,
+            self.ytdl_sub_input_url_index,
+            self.ytdl_sub_input_url_count,
         }
 
     @cache

src/ytdl_sub/entries/script/variable_types.py

@@ -9,6 +9,7 @@ from typing import TypeVar
 
 from ytdl_sub.script.types.array import Array
 from ytdl_sub.script.types.map import Map
+from ytdl_sub.script.types.resolvable import Boolean
 from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import String
 
@@ -16,8 +17,8 @@ ENTRY_METADATA_VARIABLE_NAME = "entry_metadata"
 PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata"
 SOURCE_METADATA_VARIABLE_NAME = "source_metadata"
 
-TMetadataVariable = TypeVar("TMetadataVariable", bound="MetadataVariable")
+MetadataVariableT = TypeVar("MetadataVariableT", bound="MetadataVariable")
-TVariable = TypeVar("TVariable", bound="Variable")
+VariableT = TypeVar("VariableT", bound="Variable")
 
 
 def _get(
@@ -25,9 +26,9 @@ def _get(
     metadata_variable_name: str,
     metadata_key: str,
     variable_name: Optional[str],
-    default: Optional[TVariable | str | int | Dict | List],
+    default: Optional[VariableT | str | int | Dict | List],
-    as_type: Type[TMetadataVariable],
+    as_type: Type[MetadataVariableT],
-) -> TMetadataVariable:
+) -> MetadataVariableT:
     if default is None:
         # TODO: assert with good error message if key DNE
         out = f"%map_get({metadata_variable_name}, '{metadata_key}')"
@@ -63,6 +64,13 @@ class Variable(ABC):
         """
 
 
+@dataclass(frozen=True)
+class BooleanVariable(Variable):
+    @classmethod
+    def human_readable_type(cls) -> str:
+        return Boolean.__name__
+
+
 @dataclass(frozen=True)
 class StringVariable(Variable):
     @classmethod

src/ytdl_sub/entries/variables/override_variables.py

@@ -1,5 +1,12 @@
+from typing import Dict
+from typing import Set
+
 from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
 from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
+from ytdl_sub.entries.script.variable_types import BooleanVariable
+from ytdl_sub.entries.script.variable_types import MapVariable
+from ytdl_sub.entries.script.variable_types import StringVariable
+from ytdl_sub.entries.script.variable_types import Variable
 from ytdl_sub.script.functions import Functions
 from ytdl_sub.script.utils.name_validation import is_valid_name
 
@@ -9,15 +16,15 @@ SUBSCRIPTION_ARRAY = "subscription_array"
 
 class SubscriptionVariables:
     @staticmethod
-    def subscription_name() -> str:
+    def subscription_name() -> StringVariable:
         """
         Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``),
         the prefix and all whitespace afterwards is stripped from the subscription name.
         """
-        return "subscription_name"
+        return StringVariable(variable_name="subscription_name", definition="{ %string('') }")
 
     @staticmethod
-    def subscription_value() -> str:
+    def subscription_value() -> StringVariable:
         """
         For subscriptions in the form of
 
@@ -27,10 +34,10 @@ class SubscriptionVariables:
 
         ``subscription_value`` gets set to ``https://...``.
         """
-        return "subscription_value"
+        return StringVariable(variable_name="subscription_value", definition="{ %string('') }")
 
     @staticmethod
-    def subscription_indent_i(index: int) -> str:
+    def subscription_indent_i(index: int) -> StringVariable:
         """
         For subscriptions in the form of
 
@@ -43,10 +50,12 @@ class SubscriptionVariables:
         ``subscription_indent_1`` and ``subscription_indent_2`` get set to
         ``Indent Value 1`` and ``Indent Value 2``.
         """
-        return f"subscription_indent_{index + 1}"
+        return StringVariable(
+            variable_name=f"subscription_indent_{index + 1}", definition="{ %string('') }"
+        )
 
     @staticmethod
-    def subscription_value_i(index: int) -> str:
+    def subscription_value_i(index: int) -> StringVariable:
         """
         For subscriptions in the form of
 
@@ -60,10 +69,12 @@ class SubscriptionVariables:
         and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to
         ``subscription_value``.
         """
-        return f"subscription_value_{index + 1}"
+        return StringVariable(
+            variable_name=f"subscription_value_{index + 1}", definition="{ %string('') }"
+        )
 
     @staticmethod
-    def subscription_map() -> str:
+    def subscription_map() -> MapVariable:
         """
         For subscriptions in the form of
 
@@ -89,7 +100,17 @@ class SubscriptionVariables:
              ]
            }
         """
-        return "subscription_map"
+        return MapVariable(variable_name="subscription_map", definition="{ {} }")
+
+    @staticmethod
+    def subscription_has_download_archive() -> BooleanVariable:
+        """
+        Returns True if the subscription has any entries recorded in a download archive. False
+        otherwise.
+        """
+        return BooleanVariable(
+            variable_name="subscription_has_download_archive", definition="{ %bool(True) }"
+        )
 
 
 class OverrideHelpers:
@@ -124,3 +145,17 @@ class OverrideHelpers:
             return is_valid_name(name=name[1:])
 
         return is_valid_name(name=name)
+
+
+REQUIRED_OVERRIDE_VARIABLES: Set[Variable] = {
+    SubscriptionVariables.subscription_name(),
+    SubscriptionVariables.subscription_has_download_archive(),
+}
+
+REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS: Dict[str, str] = {
+    var.variable_name: var.definition for var in REQUIRED_OVERRIDE_VARIABLES
+}
+
+REQUIRED_OVERRIDE_VARIABLE_NAMES: Set[str] = {
+    var.variable_name for var in REQUIRED_OVERRIDE_VARIABLES
+}

src/ytdl_sub/plugins/chapters.py

@@ -263,13 +263,13 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
                 "force_keyframes": self.plugin_options.force_key_frames,
             }
             if self.plugin_options.remove_sponsorblock_categories is not None:
-                remove_chapters_post_processor[
+                remove_chapters_post_processor["remove_sponsor_segments"] = (
-                    "remove_sponsor_segments"
+                    self.plugin_options.remove_sponsorblock_categories
-                ] = self.plugin_options.remove_sponsorblock_categories
+                )
             if self.plugin_options.remove_chapters_regex is not None:
-                remove_chapters_post_processor[
+                remove_chapters_post_processor["remove_chapters_patterns"] = (
-                    "remove_chapters_patterns"
+                    self.plugin_options.remove_chapters_regex
-                ] = self.plugin_options.remove_chapters_regex
+                )
 
             if self.plugin_options.embed_chapters:
                 builder.add(

src/ytdl_sub/plugins/date_range.py

@@ -57,7 +57,7 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
         """
         :expected type: Optional[OverridesFormatter]
         :description:
-          Only download videos before this datetime.
+          Only download videos after this datetime.
         """
         return self._after
 

src/ytdl_sub/plugins/music_tags.py

@@ -1,4 +1,6 @@
 from collections import defaultdict
+from datetime import datetime
+from typing import Any
 from typing import Dict
 from typing import List
 
@@ -9,6 +11,7 @@ from ytdl_sub.config.validators.options import OptionsDictValidator
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
+from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
@@ -36,6 +39,17 @@ def _is_multi_field(tag_name: str) -> bool:
     }
 
 
+def _is_date_field(tag_name: str) -> bool:
+    return tag_name in {
+        "date",
+        "original_date",
+    }
+
+
+def _to_datetime(tag_value: str) -> Any:
+    return datetime.strptime(tag_value, "%Y-%m-%d")
+
+
 class MusicTagsOptions(OptionsDictValidator):
     """
     Adds tags to every download audio file using
@@ -46,6 +60,9 @@ class MusicTagsOptions(OptionsDictValidator):
     a full list of tags for various file types in MediaFile's
     `source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_.
 
+    Note that the date fields ``date`` and ``original_date`` expected a standardized date in the
+    form of YYYY-MM-DD. The variable ``upload_date_standardized`` returns a compatible format.
+
     :Usage:
 
     .. code-block:: yaml
@@ -62,6 +79,7 @@ class MusicTagsOptions(OptionsDictValidator):
              albumartists:
                - "{artist}"
                - "ytdl-sub"
+             date: "{upload_date_standardized}"
     """
 
     _optional_keys = set(list(mediafile.MediaFile.sorted_fields()))
@@ -104,12 +122,26 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
                 tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
                 tags_to_write[tag_name].append(tag_value)
 
+            if _is_date_field(tag_name):
+                try:
+                    if len(tags_to_write[tag_name]) != 1:
+                        raise ValueError("caught below")
+
+                    _ = _to_datetime(tags_to_write[tag_name][0])
+                except Exception as exc:
+                    raise ValidationException(
+                        "Date-based music tags must be a single tag in the form of YYYY-MM-DD"
+                    ) from exc
+
         # write the actual tags if its not a dry run
         if not self.is_dry_run:
             audio_file = mediafile.MediaFile(entry.get_download_file_path())
             for tag_name, tag_value in tags_to_write.items():
+                # If the attribute is a date-type, set it as a datetime type
+                if _is_date_field(tag_name):
+                    setattr(audio_file, tag_name, _to_datetime(tag_value[0]))
                 # If the attribute is a multi-type, set it as the list type
-                if _is_multi_field(tag_name):
+                elif _is_multi_field(tag_name):
                     setattr(audio_file, tag_name, tag_value)
                 # Otherwise, set as single value
                 else:

src/ytdl_sub/plugins/split_by_chapters.py

@@ -89,7 +89,6 @@ class SplitByChaptersOptions(OptionsDictValidator):
         return {
             PluginOperation.MODIFY_ENTRY: {
                 "chapter_title",
-                "chapter_title_sanitized",
                 "chapter_index",
                 "chapter_index_padded",
                 "chapter_count",
@@ -160,9 +159,9 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
 
         metadata_value_dict = {}
         if self.is_dry_run:
-            metadata_value_dict[
+            metadata_value_dict["Warning"] = (
-                "Warning"
+                "Dry-run assumes embedded chapters with no modifications"
-            ] = "Dry-run assumes embedded chapters with no modifications"
+            )
 
         metadata_value_dict["Source Title"] = new_entry.title
         metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}"

src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml

@@ -1,18 +1,8 @@
 presets:
 
-  "Only Recent":
+  #############################################################################
-    preset:
+  # Only Recent Archive
-      - "Only Recent Archive"
+  #   Downloads only `date_range` amount of videos (no deletion)
-    # Only fetch videos after today minus date_range
-
-    # Only keep files uploaded after date_range
-    output_options:
-      keep_files_after: "today-{only_recent_date_range}"
-      keep_max_files: "{only_recent_max_files}"
-
-    overrides:
-      only_recent_max_files: 0
-
 
   "Only Recent Archive":
     # Only fetch videos after today minus date_range
@@ -23,9 +13,29 @@ presets:
     overrides:
       date_range: "2months"  # keep for legacy-reasons
       only_recent_date_range: "{date_range}"
-      
 
-  chunk_initial_download:
+  #############################################################################
+  # Only Recent
+  #   Downloads only `date_range` amount of videos and deletes older videos
+  #   that fall out of that range
+
+  "Only Recent":
+    preset:
+      - "Only Recent Archive"
+
+    output_options:
+      keep_files_after: "today-{only_recent_date_range}"
+      keep_max_files: "{only_recent_max_files}"
+
+    overrides:
+      only_recent_max_files: 0
+
+  #############################################################################
+  # Download in Chunks
+  #   Will only download 20 videos per invocation of ytdl-sub, starting
+  #   at the very beginning of the channel
+
+  chunk_initial_download:  # legacy preset name
     ytdl_options:
       max_downloads: 20
       playlistreverse: True

src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml

@@ -1,5 +1,9 @@
 presets:
 
+  #############################################################################
+  # Best Video Quality
+  #   Gets the best available quality
+
   best_video_quality:
     format: "bestvideo+bestaudio/best"
     ytdl_options:
@@ -9,18 +13,26 @@ presets:
     preset: 
       - best_video_quality
 
+  #############################################################################
+  # Max 2160p
 
   "Max 2160p":
     format: "(bv*[height<=2160]+bestaudio/best[height<=2160])"
     ytdl_options:
       merge_output_format: "mp4"
 
+  #############################################################################
+  # Max 1440p
+
   "Max 1440p":
     format: "(bv*[height<=1440]+bestaudio/best[height<=1440])"
     ytdl_options:
       merge_output_format: "mp4"
 
-  max_1080p:
+  #############################################################################
+  # Max 1080p
+
+  max_1080p:  # legacy name
     format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
     ytdl_options:
       merge_output_format: "mp4"
@@ -28,12 +40,18 @@ presets:
   "Max 1080p":
     preset:
       - max_1080p
-  
+
+  #############################################################################
+  # Max 720p
+
   "Max 720p":
     format: "(bv*[height<=720]+bestaudio/best[height<=720])"
     ytdl_options:
       merge_output_format: "mp4"
 
+  #############################################################################
+  # Max 480p
+
   "Max 480p":
     format: "(bv*[height<=480]+bestaudio/best[height<=480])"
     ytdl_options:

src/ytdl_sub/prebuilt_presets/helpers/url.yaml

@@ -330,3 +330,418 @@ presets:
       url98: "{subscription_value_98}"
       url99: "{subscription_value_99}"
       url100: "{subscription_value_100}"
+
+
+  # multi-url with bilateral scraping built into it via
+  # inspection of the URL and conditionally adding another
+  # ytdl-sub url download with the scraping and download reversed
+  _multi_url_bilateral_inner:
+    preset:
+      - "_url_bilateral_overrides"
+
+    download:
+      - url: "{%bilateral_url(url) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url2) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url3) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url4) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url5) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url6) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url7) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url8) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url9) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url10) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url11) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url12) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url13) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url14) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url15) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url16) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url17) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url18) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url19) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url20) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url21) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url22) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url23) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url24) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url25) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url26) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url27) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url28) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url29) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url30) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url31) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url32) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url33) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url34) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url35) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url36) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url37) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url38) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url39) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url40) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url41) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url42) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url43) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url44) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url45) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url46) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url47) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url48) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url49) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url50) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url51) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url52) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url53) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url54) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url55) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url56) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url57) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url58) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url59) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url60) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url61) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url62) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url63) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url64) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url65) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url66) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url67) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url68) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url69) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url70) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url71) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url72) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url73) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url74) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url75) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url76) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url77) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url78) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url79) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url80) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url81) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url82) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url83) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url84) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url85) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url86) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url87) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url88) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url89) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url90) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url91) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url92) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url93) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url94) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url95) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url96) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url97) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url98) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{ %bilateral_url(url99) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+      - url: "{%bilateral_url(url100) }"
+        download_reverse: False
+        ytdl_options:
+          playlist_items: "-1:0:-1"
+
+  _multi_url_bilateral:
+    preset:
+      - "_multi_url_bilateral_inner"
+      - "_multi_url"

src/ytdl_sub/prebuilt_presets/helpers/url_bilateral.yaml

@@ -0,0 +1,21 @@
+presets:
+  # multi-url with bilateral scraping built into it via
+  # inspection of the URL and conditionally adding another
+  # ytdl-sub url download with the scraping and download reversed
+  _url_bilateral_overrides:
+    overrides:
+      enable_bilateral_scraping: True
+      "%is_bilateral_url": >-
+        { %contains( $0, "youtube.com/playlist" ) }
+      "%bilateral_url": >-
+        { 
+          %if(
+            %and(
+              enable_bilateral_scraping,
+              subscription_has_download_archive,
+              %is_bilateral_url($0)
+            ),
+            $0,
+            ""
+          )
+        }

src/ytdl_sub/prebuilt_presets/music/other_websites.yaml

@@ -5,7 +5,6 @@ presets:
 
     # Download using the multi_url strategy
     download:
-      download_strategy: "multi_url"
       urls:
         # The first URL will be all the artist's tracks.
         # Treat these as singles - an album with a single track

src/ytdl_sub/prebuilt_presets/music/singles.yaml

@@ -24,6 +24,8 @@ presets:
       track: "{track_number}"
       tracktotal: "{track_total}"
       year: "{track_year}"
+      date: "{track_date}"
+      original_date: "{track_original_date}"
       # multi-tags
       artists:
         - "{track_artist}"
@@ -53,6 +55,8 @@ presets:
       track_number_padded: "01"
       track_total: "1"
       track_year: "{upload_year}"
+      track_date: "{upload_date_standardized}"
+      track_original_date: "{track_date}"
       track_genre: "{subscription_indent_1}"
 
       # Directory Overrides

src/ytdl_sub/prebuilt_presets/music_videos/music_video_base.yaml

@@ -4,6 +4,7 @@ presets:
       output_directory: "{music_video_directory}"
       file_name: "{music_video_file_name}.{ext}"
       thumbnail_name: "{music_video_file_name}.jpg"
+      info_json_name: "{music_video_file_name}.{info_json_ext}"
       maintain_download_archive: True
 
     ytdl_options:

src/ytdl_sub/prebuilt_presets/tv_show/__init__.py

@@ -52,6 +52,8 @@ class TvShowCollectionEpisodeFormattingPresets(PrebuiltPresets):
 
 
 class TvShowCollectionSeasonPresets(PrebuiltPresets):
+    """Now Deprecated"""
+
     preset_names = {
         "collection_season_1",
         "collection_season_2",

src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml

@@ -31,23 +31,6 @@ presets:
 
 ####################################################################################################
 
-  # TV show from one or more sources. Uses {url}'s avatar and banner as poster and fanart
-  _tv_show_by_date:
-    preset: "_multi_url"
-    overrides:
-      avatar_uncropped_thumbnail_file_name: "{tv_show_poster_file_name}"
-      banner_uncropped_thumbnail_file_name: "{tv_show_fanart_file_name}"
-
-####################################################################################################
-
-  # TV show from a collection. Must specify additional `tv_show_collection_season` presets in
-  # addition. Each season sets its own `collection_season_number/_padded`
-  _tv_show_collection:
-    overrides:
-      collection_season_number_padded: "{ %pad_zero(%int(collection_season_number), 2) }"
-      season_number: "{collection_season_number}"
-      season_number_padded: "{collection_season_number_padded}"
-
   _episode_video_tags:
     video_tags:
       show: "{tv_show_name}"

src/ytdl_sub/prebuilt_presets/tv_show/tv_show_by_date.yaml

@@ -31,6 +31,15 @@ presets:
       - "plex_tv_show_by_date"
       - "season_by_year__episode_by_month_day"
 
+####################################################################################################
+
+  # TV show from one or more sources. Uses {url}'s avatar and banner as poster and fanart
+  _tv_show_by_date:
+    preset: "_multi_url_bilateral"
+    overrides:
+      avatar_uncropped_thumbnail_file_name: "{tv_show_poster_file_name}"
+      banner_uncropped_thumbnail_file_name: "{tv_show_fanart_file_name}"
+
 ####################################################################################################
 
   _season_by_year:

src/ytdl_sub/script/functions/boolean_functions.py

@@ -1,5 +1,9 @@
+from ytdl_sub.script.types.array import Array
+from ytdl_sub.script.types.map import Map
 from ytdl_sub.script.types.resolvable import AnyArgument
 from ytdl_sub.script.types.resolvable import Boolean
+from ytdl_sub.script.types.resolvable import Float
+from ytdl_sub.script.types.resolvable import Integer
 from ytdl_sub.script.types.resolvable import String
 
 # pylint: disable=invalid-name
@@ -107,3 +111,59 @@ class BooleanFunctions:
           Returns True if a value is null (i.e. an empty string). False otherwise.
         """
         return Boolean(isinstance(value, String) and value.value == "")
+
+    @staticmethod
+    def is_map(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is a Map. False otherwise.
+        """
+        return Boolean(isinstance(value, Map))
+
+    @staticmethod
+    def is_array(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is a Map. False otherwise.
+        """
+        return Boolean(isinstance(value, Array))
+
+    @staticmethod
+    def is_string(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is a String. False otherwise.
+        """
+        return Boolean(isinstance(value, String))
+
+    @staticmethod
+    def is_numeric(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is either an Integer or Float. False otherwise.
+        """
+        return Boolean(isinstance(value, (Integer, Float)))
+
+    @staticmethod
+    def is_int(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is an Integer. False otherwise.
+        """
+        return Boolean(isinstance(value, Integer))
+
+    @staticmethod
+    def is_float(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is a Float. False otherwise.
+        """
+        return Boolean(isinstance(value, Float))
+
+    @staticmethod
+    def is_bool(value: AnyArgument) -> Boolean:
+        """
+        :description:
+          Returns True if a value is a Float. False otherwise.
+        """
+        return Boolean(isinstance(value, Boolean))

src/ytdl_sub/script/functions/map_functions.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from typing import Optional
 
 from ytdl_sub.script.types.array import Array
@@ -66,6 +67,19 @@ class MapFunctions:
             )
         return mapping.value[key]
 
+    @staticmethod
+    def map_extend(*maps: Map) -> Map:
+        """
+        :description:
+          Return maps combined in the order from left-to-right. Duplicate keys will use the
+          right-most map's value.
+        """
+        output_dict: Dict = {}
+        for map_i in maps:
+            output_dict |= map_i.value
+
+        return Map(output_dict)
+
     @staticmethod
     def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument:
         """

src/ytdl_sub/script/functions/regex_functions.py

@@ -52,3 +52,13 @@ class RegexFunctions:
           Returns number of capture groups in regex
         """
         return Integer(re.compile(regex.value).groups)
+
+    @staticmethod
+    def regex_sub(regex: String, replacement: String, string: String) -> String:
+        """
+        :description:
+          Returns the string obtained by replacing the leftmost non-overlapping occurrences of the
+          pattern in string by the replacement string. The replacement string can reference the
+          match groups via backslash escapes. Callables as replacement argument are not supported.
+        """
+        return String(re.sub(regex.value, replacement.value, string.value))

src/ytdl_sub/script/parser.py

@@ -73,6 +73,8 @@ CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS = InvalidSyntaxException(
 
 FUNCTION_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a function")
 
+BRACKET_INVALID_CHAR = InvalidSyntaxException("Invalid value within brackets")
+
 
 def _UNEXPECTED_CHAR_ARGUMENT(arg_type: ParsedArgType):
     return InvalidSyntaxException(f"Unexpected character when parsing {arg_type.value} arguments")
@@ -245,7 +247,7 @@ class _Parser:
         if self._read(increment_pos=False) == "-":
             numeric_string += "-"
             self._pos += 1
-        if has_decimal := (self._read(increment_pos=False) == "."):
+        if has_decimal := self._read(increment_pos=False) == ".":
             numeric_string += "."
             self._pos += 1
 
@@ -496,15 +498,23 @@ class _Parser:
                     raise MAP_KEY_WITH_NO_VALUE
                 if isinstance(key, NonHashable):
                     raise MAP_KEY_NOT_HASHABLE
+                if isinstance(key, BuiltInFunction) and issubclass(key.output_type(), NonHashable):
+                    raise MAP_KEY_NOT_HASHABLE
                 if len(value_args) > 1:
                     raise MAP_KEY_MULTIPLE_VALUES
 
                 output[key] = value_args[0]
                 key = None
             else:
-                raise UNREACHABLE
+                break
+
+        raise UNREACHABLE
 
     def _parse_main_loop(self, ch: str) -> bool:
+        if ch == "\\" and self._read(increment_pos=False) in {"{", "}"}:
+            # Escape brackets are \{ and \}, only add the second char
+            self._literal_str += self._read()
+            return True
         if ch == "}":
             if self._bracket_counter == 0:
                 raise BRACKET_NOT_CLOSED
@@ -558,9 +568,9 @@ class _Parser:
         elif self._bracket_counter == 0:
             # Only accumulate literal str if not in brackets
             self._literal_str += ch
-        else:
+        elif not ch.isspace():
-            # Should only be possible to get here if it's a space
+            self._set_highlight_position(pos=self._pos - 1)
-            assert ch.isspace()
+            raise BRACKET_INVALID_CHAR
 
         return True
 

src/ytdl_sub/script/script.py

@@ -10,6 +10,7 @@ from ytdl_sub.script.script_output import ScriptOutput
 from ytdl_sub.script.types.resolvable import Lambda
 from ytdl_sub.script.types.resolvable import Resolvable
 from ytdl_sub.script.types.syntax_tree import SyntaxTree
+from ytdl_sub.script.types.variable import FunctionArgument
 from ytdl_sub.script.types.variable import Variable
 from ytdl_sub.script.utils.exceptions import UNREACHABLE
 from ytdl_sub.script.utils.exceptions import CycleDetected
@@ -257,12 +258,23 @@ class Script:
                     f"Output filter variable contains the variable {var_dep} "
                     f"which is set as unresolvable"
                 )
+
+            # Do not recurse custom function arguments since they have no deps
+            if isinstance(var_dep, FunctionArgument):
+                continue
+
             subset_to_resolve.add(var_dep.name)
             subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
                 current_var=self._variables[var_dep.name],
                 subset_to_resolve=subset_to_resolve,
                 unresolvable=unresolvable,
             )
+        for custom_func_dep in current_var.custom_functions:
+            subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
+                current_var=self._functions[custom_func_dep.name],
+                subset_to_resolve=subset_to_resolve,
+                unresolvable=unresolvable,
+            )
 
         return subset_to_resolve
 
@@ -440,18 +452,34 @@ class Script:
             self
         """
         added_variables_to_validate: Set[str] = set()
-        for variable_name, variable_definition in variables.items():
-            self._variables[variable_name] = parse(
-                text=variable_definition,
-                name=variable_name,
-                custom_function_names=set(self._functions.keys()),
-                variable_names=set(self._variables.keys())
-                .union(variables.keys())
-                .union(unresolvable or set()),
-            )
 
-            if self._variables[variable_name].maybe_resolvable is None:
+        functions_to_add = {
-                added_variables_to_validate.add(variable_name)
+            _function_name(name): definition
+            for name, definition in variables.items()
+            if _is_function(name)
+        }
+        variables_to_add = {
+            name: definition for name, definition in variables.items() if not _is_function(name)
+        }
+
+        for definitions in [functions_to_add, variables_to_add]:
+            for name, definition in definitions.items():
+                parsed = parse(
+                    text=definition,
+                    name=name,
+                    custom_function_names=set(self._functions.keys()),
+                    variable_names=set(self._variables.keys())
+                    .union(variables.keys())
+                    .union(unresolvable or set()),
+                )
+
+                if parsed.maybe_resolvable is None:
+                    added_variables_to_validate.add(name)
+
+                if name in functions_to_add:
+                    self._functions[name] = parsed
+                else:
+                    self._variables[name] = parsed
 
         if added_variables_to_validate:
             self._validate(added_variables=added_variables_to_validate)

src/ytdl_sub/script/types/function.py

@@ -291,3 +291,6 @@ class BuiltInFunction(Function, BuiltInFunctionType):
             raise FunctionRuntimeException(
                 f"Runtime error occurred when executing the function %{self.name}: {str(exc)}"
             ) from exc
+
+    def __hash__(self):
+        return hash((self.name, *self.args))

src/ytdl_sub/script/utils/exception_formatters.py

@@ -8,11 +8,11 @@ from ytdl_sub.script.utils.exceptions import UserException
 from ytdl_sub.script.utils.type_checking import FunctionSpec
 from ytdl_sub.script.utils.type_checking import is_union
 
-TUserException = TypeVar("TUserException", bound=UserException)
+UserExceptionT = TypeVar("UserExceptionT", bound=UserException)
 
 
 class ParserExceptionFormatter:
-    def __init__(self, text: str, start: int, end: int, exception: TUserException):
+    def __init__(self, text: str, start: int, end: int, exception: UserExceptionT):
         self._text = text
         self._start = start
         self._end = end
@@ -78,7 +78,7 @@ class ParserExceptionFormatter:
 
         return "\n" + "\n".join(to_return)
 
-    def highlight(self) -> TUserException:
+    def highlight(self) -> UserExceptionT:
         """
         Returns
         -------

src/ytdl_sub/script/utils/type_checking.py

@@ -23,7 +23,7 @@ from ytdl_sub.script.types.resolvable import Resolvable
 from ytdl_sub.script.types.variable import Variable
 from ytdl_sub.script.utils.exceptions import UNREACHABLE
 
-TLambda = TypeVar("TLambda", bound=Lambda)
+LambdaT = TypeVar("LambdaT", bound=Lambda)
 
 
 def is_union(arg_type: Type) -> bool:
@@ -210,7 +210,7 @@ class FunctionSpec:
         return None
 
     @property
-    def is_lambda_like(self) -> Optional[Type[TLambda]]:
+    def is_lambda_like(self) -> Optional[Type[LambdaT]]:
         """
         Returns
         -------

src/ytdl_sub/subscriptions/base_subscription.py

@@ -9,6 +9,7 @@ from ytdl_sub.config.preset import Preset
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.preset_options import YTDLOptions
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
+from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
 from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@@ -16,6 +17,24 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
 logger = Logger.get("subscription")
 
 
+def _initialize_download_archive(
+    output_options: OutputOptions,
+    overrides: Overrides,
+    working_directory: str,
+    output_directory: str,
+) -> EnhancedDownloadArchive:
+    migrated_file_name: Optional[str] = None
+    if migrated_file_name_option := output_options.migrated_download_archive_name:
+        migrated_file_name = overrides.apply_formatter(migrated_file_name_option)
+
+    return EnhancedDownloadArchive(
+        file_name=overrides.apply_formatter(output_options.download_archive_name),
+        working_directory=working_directory,
+        output_directory=output_directory,
+        migrated_file_name=migrated_file_name,
+    ).reinitialize(dry_run=True)
+
+
 class BaseSubscription(ABC):
     """
     Subscription classes are the 'controllers' that perform...
@@ -48,20 +67,43 @@ class BaseSubscription(ABC):
         self._config_options = config_options
         self._preset_options = preset_options
 
-        migrated_file_name: Optional[str] = None
+        # Add overrides pre-archive
-        if migrated_file_name_option := self.output_options.migrated_download_archive_name:
+        self.overrides.add(
-            migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option)
+            {
+                SubscriptionVariables.subscription_name(): self.name,
+            }
+        )
 
-        # TODO: Do not include this as part of the subscription
+        self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = (
-        self._enhanced_download_archive = EnhancedDownloadArchive(
+            _initialize_download_archive(
-            file_name=self.overrides.apply_formatter(self.output_options.download_archive_name),
+                output_options=self.output_options,
-            working_directory=self.working_directory,
+                overrides=self.overrides,
-            output_directory=self.output_directory,
+                working_directory=self.working_directory,
-            migrated_file_name=migrated_file_name,
+                output_directory=self.output_directory,
+            )
+        )
+
+        # Add post-archive variables
+        self.overrides.add(
+            {
+                SubscriptionVariables.subscription_has_download_archive(): f"""{{
+                        %bool({self.download_archive.num_entries > 0})
+                    }}""",
+            }
         )
 
         self._exception: Optional[Exception] = None
 
+    @property
+    def download_archive(self) -> EnhancedDownloadArchive:
+        """
+        Returns
+        -------
+        Initialized download archive
+        """
+        assert self._enhanced_download_archive is not None
+        return self._enhanced_download_archive
+
     @property
     def downloader_options(self) -> MultiUrlValidator:
         """
@@ -141,7 +183,7 @@ class BaseSubscription(ABC):
         -------
         Number of entries added
         """
-        return self._enhanced_download_archive.num_entries_added
+        return self.download_archive.num_entries_added
 
     @property
     def num_entries_modified(self) -> int:
@@ -150,7 +192,7 @@ class BaseSubscription(ABC):
         -------
         Number of entries modified
         """
-        return self._enhanced_download_archive.num_entries_modified
+        return self.download_archive.num_entries_modified
 
     @property
     def num_entries_removed(self) -> int:
@@ -159,7 +201,7 @@ class BaseSubscription(ABC):
         -------
         Number of entries removed
         """
-        return self._enhanced_download_archive.num_entries_removed
+        return self.download_archive.num_entries_removed
 
     @property
     def num_entries(self) -> int:
@@ -168,7 +210,7 @@ class BaseSubscription(ABC):
         -------
         The number of entries
         """
-        return self._enhanced_download_archive.num_entries
+        return self.download_archive.num_entries
 
     @property
     def transaction_log(self) -> FileHandlerTransactionLog:
@@ -177,7 +219,7 @@ class BaseSubscription(ABC):
         -------
         Transaction log from the subscription
         """
-        return self._enhanced_download_archive.get_file_handler_transaction_log()
+        return self.download_archive.get_file_handler_transaction_log()
 
     @property
     def exception(self) -> Optional[Exception]:

src/ytdl_sub/subscriptions/subscription_download.py

@@ -67,7 +67,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         output_file_name = self.overrides.apply_formatter(
             formatter=self.output_options.file_name, entry=entry
         )
-        self._enhanced_download_archive.save_file_to_output_directory(
+        self.download_archive.save_file_to_output_directory(
             file_name=entry.get_download_file_name(),
             file_metadata=entry_metadata,
             output_file_name=output_file_name,
@@ -81,7 +81,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             )
 
             # Copy the thumbnails since they could be used later for other things
-            self._enhanced_download_archive.save_file_to_output_directory(
+            self.download_archive.save_file_to_output_directory(
                 file_name=entry.get_download_thumbnail_name(),
                 output_file_name=output_thumbnail_name,
                 entry=entry,
@@ -101,7 +101,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             if not dry_run:
                 entry.write_info_json()
 
-            self._enhanced_download_archive.save_file_to_output_directory(
+            self.download_archive.save_file_to_output_directory(
                 file_name=entry.get_download_info_json_name(),
                 output_file_name=output_info_json_name,
                 entry=entry,
@@ -126,8 +126,8 @@ class SubscriptionDownload(BaseSubscription, ABC):
         except Exception as exc:
             self._delete_working_directory(is_error=True)
             raise exc
-        else:
+
-            self._delete_working_directory()
+        self._delete_working_directory()
 
     @contextlib.contextmanager
     def _maintain_archive_file(self):
@@ -135,7 +135,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         Context manager to initialize the enhanced download archive
         """
         if self.maintain_download_archive:
-            self._enhanced_download_archive.prepare_download_archive()
+            self.download_archive.prepare_download_archive()
 
         yield
 
@@ -156,19 +156,19 @@ class SubscriptionDownload(BaseSubscription, ABC):
                 )
 
             if date_range_to_keep or self.output_options.keep_max_files is not None:
-                self._enhanced_download_archive.remove_stale_files(
+                self.download_archive.remove_stale_files(
                     date_range=date_range_to_keep, keep_max_files=keep_max_files
                 )
 
-            self._enhanced_download_archive.save_download_mappings()
+            self.download_archive.save_download_mappings()
-            FileHandler.delete(self._enhanced_download_archive.working_file_path)
+            FileHandler.delete(self.download_archive.working_ytdl_file_path)
 
     @contextlib.contextmanager
     def _remove_empty_directories_in_output_directory(self):
         try:
             yield
         finally:
-            if not self._enhanced_download_archive.is_dry_run:
+            if not self.download_archive.is_dry_run:
                 for root, dir_names, _ in os.walk(Path(self.output_directory), topdown=False):
                     for dir_name in dir_names:
                         dir_path = Path(root) / dir_name
@@ -194,7 +194,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             plugin_type(
                 options=plugin_options,
                 overrides=self.overrides,
-                enhanced_download_archive=self._enhanced_download_archive,
+                enhanced_download_archive=self.download_archive,
             )
             for plugin_type, plugin_options in self.plugins.zipped()
         ]
@@ -233,7 +233,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         # Re-save the download archive after each entry is moved to the output directory
         if self.maintain_download_archive:
-            self._enhanced_download_archive.save_download_mappings()
+            self.download_archive.save_download_mappings()
 
     def _process_entry(
         self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata
@@ -323,7 +323,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         for plugin in plugins:
             plugin.post_process_subscription()
 
-        return self._enhanced_download_archive.get_file_handler_transaction_log()
+        return self.download_archive.get_file_handler_transaction_log()
 
     def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
         """
@@ -336,14 +336,14 @@ class SubscriptionDownload(BaseSubscription, ABC):
             directory.
         """
         self._exception = None
-        self._enhanced_download_archive.reinitialize(dry_run=dry_run)
+        self.download_archive.reinitialize(dry_run=dry_run)
 
         plugins = self._initialize_plugins()
 
         subscription_ytdl_options = SubscriptionYTDLOptions(
             preset=self._preset_options,
             plugins=plugins,
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             overrides=self.overrides,
             working_directory=self.working_directory,
             dry_run=dry_run,
@@ -351,7 +351,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         downloader = MultiUrlDownloader(
             options=self.downloader_options,
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             download_ytdl_options=subscription_ytdl_options.download_builder(),
             metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
             overrides=self.overrides,
@@ -389,14 +389,14 @@ class SubscriptionDownload(BaseSubscription, ABC):
             If true, do not modify any video/audio files or move anything to the output directory.
         """
         self._exception = None
-        self._enhanced_download_archive.reinitialize(dry_run=dry_run)
+        self.download_archive.reinitialize(dry_run=dry_run)
 
         plugins = self._initialize_plugins()
 
         subscription_ytdl_options = SubscriptionYTDLOptions(
             preset=self._preset_options,
             plugins=plugins,
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             overrides=self.overrides,
             working_directory=self.working_directory,
             dry_run=dry_run,
@@ -406,7 +406,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         plugins.extend(
             MultiUrlDownloader(
                 options=self.downloader_options,
-                enhanced_download_archive=self._enhanced_download_archive,
+                enhanced_download_archive=self.download_archive,
                 download_ytdl_options=subscription_ytdl_options.download_builder(),
                 metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
                 overrides=self.overrides,
@@ -415,7 +415,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         downloader = InfoJsonDownloader(
             options=InfoJsonDownloaderOptions(name="no-op", value={}),
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             download_ytdl_options=YTDLOptionsBuilder(),
             metadata_ytdl_options=YTDLOptionsBuilder(),
             overrides=self.overrides,

src/ytdl_sub/subscriptions/subscription_validators.py

@@ -32,7 +32,7 @@ class SubscriptionOutput(Validator, ABC):
         indent overrides to merge with the preset dict's overrides
         """
         return {
-            SubscriptionVariables.subscription_indent_i(i): self._indent_overrides[i]
+            SubscriptionVariables.subscription_indent_i(i).variable_name: self._indent_overrides[i]
             for i in range(len(self._indent_overrides))
         }
 
@@ -143,7 +143,9 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator):
             presets=presets,
             indent_overrides=indent_overrides,
         )
-        self._overrides_to_add[SubscriptionVariables.subscription_value()] = self.value
+        self._overrides_to_add[SubscriptionVariables.subscription_value().variable_name] = (
+            self.value
+        )
 
 
 class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator):
@@ -168,12 +170,12 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid
         for idx, list_value in enumerate(self.list):
             # Write the first list value into subscription_value as well
             if idx == 0:
-                self._overrides_to_add[
+                self._overrides_to_add[SubscriptionVariables.subscription_value().variable_name] = (
-                    SubscriptionVariables.subscription_value()
+                    list_value.value
-                ] = list_value.value
+                )
 
             self._overrides_to_add[
-                SubscriptionVariables.subscription_value_i(index=idx)
+                SubscriptionVariables.subscription_value_i(index=idx).variable_name
             ] = list_value.value
 
 
@@ -217,8 +219,8 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator):
             presets=presets,
             indent_overrides=indent_overrides,
         )
-        self._overrides_to_add[SubscriptionVariables.subscription_map()] = ScriptUtils.to_script(
+        self._overrides_to_add[SubscriptionVariables.subscription_map().variable_name] = (
-            self.dict
+            ScriptUtils.to_script(self.dict)
         )
 
 

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -89,7 +89,9 @@ class SubscriptionYTDLOptions:
         ytdl_options = {}
 
         if self._preset.output_options.maintain_download_archive:
-            ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
+            ytdl_options["download_archive"] = (
+                self._enhanced_download_archive.working_ytdl_file_path
+            )
         if self._preset.output_options.keep_max_files:
             keep_max_files = int(
                 self._overrides.apply_formatter(self._preset.output_options.keep_max_files)

src/ytdl_sub/utils/file_handler.py

@@ -385,7 +385,11 @@ class FileHandler:
         dst_file_path
             Destination file
         """
-        shutil.copyfile(src=src_file_path, dst=dst_file_path)
+        # Perform the copy by first writing to a temp file, then moving it.
+        # This tries to prevent corrupted writes if the processed dies mid-write,
+        atomic_dst = f"{dst_file_path}-ytdl-sub-incomplete"
+        shutil.copyfile(src=src_file_path, dst=atomic_dst)
+        shutil.move(src=atomic_dst, dst=dst_file_path)
 
     @classmethod
     def move(cls, src_file_path: Union[str, Path], dst_file_path: Union[str, Path]):

src/ytdl_sub/utils/logger.py

@@ -213,9 +213,10 @@ class Logger:
     @classmethod
     def _append_to_error_log(cls):
         # Any time an exception occurs, dump all debug logs into the error log
-        with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open(
+        with (
-            cls.error_log_filename(), mode="a", encoding="utf-8"
+            open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs,
-        ) as error_logs:
+            open(cls.error_log_filename(), mode="a", encoding="utf-8") as error_logs,
+        ):
             error_logs.writelines(debug_logs.readlines())
 
     @classmethod

src/ytdl_sub/utils/retry.py

@@ -34,12 +34,14 @@ def retry(times: int, exceptions: Tuple[Type[Exception], ...], wait_sec: int = 5
             while attempt < times:
                 try:
                     return func(*args, **kwargs)
-                except exceptions:
+                except exceptions as exc:
                     logger.debug(
-                        "Exception thrown when attempting to run %s, attempt %d of %d",
+                        "Exception thrown when attempting to run %s, attempt %d of %d\n"
+                        "Exception:\n%s",
                         func.__name__,
                         attempt + 1,
                         times,
+                        str(exc),
                     )
                     attempt += 1
                     sleep(wait_sec)

src/ytdl_sub/utils/script.py

@@ -3,6 +3,22 @@ import re
 from typing import Any
 from typing import Dict
 
+from ytdl_sub.script.parser import parse
+from ytdl_sub.script.script import _is_function
+from ytdl_sub.script.types.array import UnresolvedArray
+from ytdl_sub.script.types.function import BuiltInFunction
+from ytdl_sub.script.types.function import Function
+from ytdl_sub.script.types.map import UnresolvedMap
+from ytdl_sub.script.types.resolvable import Argument
+from ytdl_sub.script.types.resolvable import Boolean
+from ytdl_sub.script.types.resolvable import Float
+from ytdl_sub.script.types.resolvable import Integer
+from ytdl_sub.script.types.resolvable import String
+from ytdl_sub.script.types.variable import Variable
+from ytdl_sub.script.utils.exceptions import UNREACHABLE
+
+# pylint: disable=too-many-return-statements
+
 
 class ScriptUtils:
     @classmethod
@@ -11,7 +27,9 @@ class ScriptUtils:
         Helper to add sanitized variables to a Script
         """
         sanitized_variables = {
-            f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys()
+            f"{name}_sanitized": f"{{%sanitize({name})}}"
+            for name in variables.keys()
+            if not _is_function(name)
         }
         return dict(variables, **sanitized_variables)
 
@@ -24,12 +42,12 @@ class ScriptUtils:
             out = ""
         elif isinstance(value, str):
             out = value
+        elif isinstance(value, bool):
+            out = f"{{%bool({value})}}"
         elif isinstance(value, int):
             out = f"{{%int({value})}}"
         elif isinstance(value, float):
             out = f"{{%float({value})}}"
-        elif isinstance(value, bool):
-            out = f"{{%bool({value})}}"
         else:
             dumped_json = json.dumps(value, ensure_ascii=False, sort_keys=True)
             # Remove triple-single-quotes from JSON to avoid parsing issues
@@ -39,6 +57,75 @@ class ScriptUtils:
 
         return out
 
+    @classmethod
+    def _to_script_argument(cls, value: Any) -> Argument:
+        # Handle simple types as above
+        if value is None or (isinstance(value, str) and value == ""):
+            return String("")
+        if isinstance(value, str):
+            ast = parse(text=value).ast
+            if len(ast) == 1:
+                return ast[0]
+            return BuiltInFunction(
+                name="concat", args=[BuiltInFunction(name="string", args=[arg]) for arg in ast]
+            )
+        if isinstance(value, bool):
+            return Boolean(value)
+        if isinstance(value, int):
+            return Integer(value)
+        if isinstance(value, float):
+            return Float(value)
+        if isinstance(value, list):
+            return UnresolvedArray([cls._to_script_argument(val) for val in value])
+        if isinstance(value, dict):
+            return UnresolvedMap(
+                {
+                    cls._to_script_argument(key): cls._to_script_argument(val)
+                    for key, val in value.items()
+                }
+            )
+
+        raise UNREACHABLE
+
+    @classmethod
+    def _to_script_code(cls, arg: Argument, top_level: bool = False) -> str:
+        if not top_level and isinstance(arg, (Integer, Boolean, Float)):
+            return str(arg.native)
+
+        if isinstance(arg, String):
+            if arg.native == "":
+                return "" if top_level else "''"
+            return arg.native if top_level else f"'''{arg.native}'''"
+
+        if isinstance(arg, Integer):
+            out = f"%int({arg.native})"
+        elif isinstance(arg, Boolean):
+            out = f"%bool({arg.native})"
+        elif isinstance(arg, Float):
+            out = f"%float({arg.native})"
+        elif isinstance(arg, UnresolvedArray):
+            out = f"[ {', '.join(cls._to_script_code(val) for val in arg.value)} ]"
+        elif isinstance(arg, UnresolvedMap):
+            kv_list = (
+                f"{cls._to_script_code(key)}: {cls._to_script_code(val)}"
+                for key, val in arg.value.items()
+            )
+            out = f"{{ {', '.join(kv_list)} }}"
+        elif isinstance(arg, Variable):
+            out = arg.name
+        elif isinstance(arg, Function):
+            out = f"%{arg.name}( {', '.join(cls._to_script_code(val) for val in arg.args)} )"
+        else:
+            raise UNREACHABLE
+        return f"{{ {out} }}" if top_level else out
+
+    @classmethod
+    def to_native_script(cls, value: Any) -> str:
+        """
+        Converts any JSON-compatible value into equivalent script syntax
+        """
+        return cls._to_script_code(cls._to_script_argument(value), top_level=True)
+
     @classmethod
     def bool_formatter_output(cls, output: str) -> bool:
         """

src/ytdl_sub/utils/scriptable.py

@@ -9,13 +9,16 @@ from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
 from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES
 from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
 from ytdl_sub.entries.script.variable_types import Variable
+from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS
 from ytdl_sub.script.script import Script
 from ytdl_sub.script.utils.exceptions import RuntimeException
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.script import ScriptUtils
 
 BASE_SCRIPT: Script = Script(
-    dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS)
+    ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS)
+    | ScriptUtils.add_sanitized_variables(REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS)
+    | CUSTOM_FUNCTION_SCRIPTS
 )
 
 

src/ytdl_sub/utils/thumbnail.py

@@ -72,8 +72,7 @@ def download_and_convert_url_thumbnail(
     if not thumbnail_url:
         return None
 
-    # timeout after 8 seconds
+    with urlopen(thumbnail_url, timeout=7.0) as file:
-    with urlopen(thumbnail_url, timeout=1.0) as file:
         with tempfile.NamedTemporaryFile(delete=False) as thumbnail:
             thumbnail.write(file.read())
 

src/ytdl_sub/validators/string_formatter_validators.py

@@ -11,6 +11,7 @@ from ytdl_sub.script.utils.exceptions import RuntimeException
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
 from ytdl_sub.script.utils.exceptions import UserException
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
+from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.validators.validators import DictValidator
 from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.validators.validators import LiteralDictValidator
@@ -63,8 +64,6 @@ class StringFormatterValidator(StringValidator):
         """
         return self._value
 
-    # pylint: disable=no-self-use
-
     def post_process(self, resolved: str) -> str:
         """
         Returns
@@ -73,8 +72,6 @@ class StringFormatterValidator(StringValidator):
         """
         return resolved
 
-    # pylint: enable=no-self-use
-
 
 # pylint: disable=line-too-long
 class OverridesStringFormatterValidator(StringFormatterValidator):
@@ -148,6 +145,18 @@ class OverridesDictFormatterValidator(DictFormatterValidator):
     _key_validator = OverridesStringFormatterValidator
 
 
+class UnstructuredDictFormatterValidator(DictFormatterValidator):
+    def __init__(self, name, value):
+        # Convert the unstructured-ness into a script
+        if isinstance(value, dict):
+            value = {key: ScriptUtils.to_native_script(val) for key, val in value.items()}
+        super().__init__(name, value)
+
+
+class UnstructuredOverridesDictFormatterValidator(UnstructuredDictFormatterValidator):
+    _key_validator = OverridesStringFormatterValidator
+
+
 def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str:
     """
     Create a dummy format string that contains all variable deps as a string.

src/ytdl_sub/validators/validators.py

@@ -169,6 +169,7 @@ class ListValidator(Validator, ABC, Generic[ValidatorT]):
     Validates a list of objects to validate
     """
 
+    # pylint: disable=used-before-assignment
     _expected_value_type = list
     _expected_value_type_name = "list"
 

src/ytdl_sub/ytdl_additions/enhanced_download_archive.py

@@ -506,10 +506,19 @@ class EnhancedDownloadArchive:
         """
         Returns
         -------
-        The download mapping's file path in the working directory.
+        The download mapping's file path in the working directory for ytdl usage
         """
         return str(Path(self.working_directory) / self.file_name)
 
+    @property
+    def working_ytdl_file_path(self) -> str:
+        """
+        Returns
+        -------
+        The download mapping's file path in the working directory for ytdl usage
+        """
+        return f"{self.working_file_path}-ytdl-archive"
+
     @property
     def mapping(self) -> DownloadMappings:
         """
@@ -541,7 +550,7 @@ class EnhancedDownloadArchive:
             return self
 
         # Otherwise, create a ytdl download archive file in the working directory.
-        self.mapping.to_download_archive().to_file(self.working_file_path)
+        self.mapping.to_download_archive().to_file(self.working_ytdl_file_path)
 
         return self
 
@@ -603,15 +612,18 @@ class EnhancedDownloadArchive:
         if self._migrated_file_name:
             self._download_mapping.to_file(output_json_file=self.working_file_path)
             self.save_file_to_output_directory(
-                file_name=self.file_name, output_file_name=self._migrated_file_name
+                file_name=self.file_name, output_file_name=self._migrated_file_name, copy_file=True
             )
+            FileHandler.delete(file_path=self.working_file_path)
+
             # and delete the old one if the name differs
             if self._file_name != self._migrated_file_name:
                 self.delete_file_from_output_directory(file_name=self.file_name)
         # Otherwise, only save if there are changes to the transaction log
         elif not self.get_file_handler_transaction_log().is_empty:
             self._download_mapping.to_file(output_json_file=self.working_file_path)
-            self.save_file_to_output_directory(file_name=self.file_name)
+            self.save_file_to_output_directory(file_name=self.file_name, copy_file=True)
+            FileHandler.delete(file_path=self.working_file_path)
         return self
 
     def delete_file_from_output_directory(self, file_name: str):

tests/e2e/entries/script/test_custom_functions.py

@@ -0,0 +1,22 @@
+from unit.script.conftest import single_variable_output
+
+
+class TestCustomFunctions:
+    def test_is_playlist_ordered_by_newest_true(self):
+        assert (
+            single_variable_output(
+                "{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL5BC0FC26BECA5A35')}"
+            )
+            is True
+        )
+
+    def test_is_playlist_ordered_by_newest_false(self):
+        assert (
+            single_variable_output(
+                "{%is_playlist_ordered_by_newest('https://www.youtube.com/playlist?list=PL2KvlCGf4yFftX466OnFS8wvuSosBfUgm')}"
+            )
+            is False
+        )
+
+    def test_is_playlist_ordered_by_newest_defaults_false(self):
+        assert single_variable_output("{%is_playlist_ordered_by_newest('aaaaaa')}") is False

tests/e2e/youtube/test_playlist.py

@@ -51,6 +51,49 @@ def playlist_preset_dict(output_directory):
     }
 
 
+@pytest.fixture
+def tv_show_by_date_bilateral_dict(output_directory):
+    return {
+        "preset": [
+            "Jellyfin TV Show by Date",
+        ],
+        "format": "worst[ext=mp4]",
+        "match_filters": {"filters": ["title *= Feb.1"]},
+        "overrides": {
+            "url": "https://www.youtube.com/playlist?list=PLd4Q7G88JqoekF0b30NYQcOTnTiIe9Ali",
+            "tv_show_directory": output_directory,
+        },
+        "nfo_tags": {
+            "tags": {
+                "subscription_has_download_archive": "{subscription_has_download_archive}",
+                "download_index": "{download_index}",
+            }
+        },
+    }
+
+
+@pytest.fixture
+def tv_show_collection_bilateral_dict(output_directory):
+    return {
+        "preset": [
+            "Jellyfin TV Show Collection",
+        ],
+        "format": "worst[ext=mp4]",
+        "match_filters": {"filters": ["title *= Feb.1"]},
+        "overrides": {
+            "s01_url": "https://www.youtube.com/playlist?list=PLd4Q7G88JqoekF0b30NYQcOTnTiIe9Ali",
+            "s01_name": "bilateral test",
+            "tv_show_directory": output_directory,
+        },
+        "nfo_tags": {
+            "tags": {
+                "subscription_has_download_archive": "{subscription_has_download_archive}",
+                "download_index": "{download_index}",
+            }
+        },
+    }
+
+
 class TestPlaylist:
     """
     Downloads my old minecraft youtube channel, pretends they are music videos. Ensure the above
@@ -222,3 +265,71 @@ class TestPlaylist:
 
             assert len(subscriptions) == 1
             assert subscriptions[0].transaction_log.is_empty
+
+    def test_tv_show_by_date_downloads_bilateral(
+        self,
+        tv_show_by_date_bilateral_dict: Dict,
+        output_directory: str,
+        default_config: ConfigFile,
+    ):
+        playlist_subscription = Subscription.from_dict(
+            config=default_config,
+            preset_name="bilateral_test",
+            preset_dict=tv_show_by_date_bilateral_dict,
+        )
+
+        transaction_log = playlist_subscription.download(dry_run=False)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="youtube/test_playlist_bilateral_p1.txt",
+        )
+
+        # Now that one vid is downloaded, attempt to download all and see if bilateral
+        # logic kicks in
+        del tv_show_by_date_bilateral_dict["match_filters"]
+        playlist_subscription = Subscription.from_dict(
+            config=default_config,
+            preset_name="bilateral_test",
+            preset_dict=tv_show_by_date_bilateral_dict,
+        )
+        transaction_log = playlist_subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="youtube/test_playlist_bilateral_p2.txt",
+        )
+
+    def test_tv_show_collection_downloads_bilateral(
+        self,
+        tv_show_collection_bilateral_dict: Dict,
+        output_directory: str,
+        default_config: ConfigFile,
+    ):
+        playlist_subscription = Subscription.from_dict(
+            config=default_config,
+            preset_name="bilateral_test",
+            preset_dict=tv_show_collection_bilateral_dict,
+        )
+
+        transaction_log = playlist_subscription.download(dry_run=False)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="youtube/test_playlist_bilateral_collection_p1.txt",
+        )
+
+        # Now that one vid is downloaded, attempt to download all and see if bilateral
+        # logic kicks in
+        del tv_show_collection_bilateral_dict["match_filters"]
+        playlist_subscription = Subscription.from_dict(
+            config=default_config,
+            preset_name="bilateral_test",
+            preset_dict=tv_show_collection_bilateral_dict,
+        )
+        transaction_log = playlist_subscription.download(dry_run=True)
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="youtube/test_playlist_bilateral_collection_p2.txt",
+        )

tests/e2e/youtube/test_video.py

@@ -34,6 +34,8 @@ def single_video_preset_dict(output_directory):
         "overrides": {
             "music_video_artist": "JMC",
             "music_video_directory": output_directory,
+            "test_override_map": {"{music_video_artist}": "{music_video_directory}"},
+            "test_override_map_get": "{ %map_get(test_override_map, music_video_artist) }",
         },
     }
 
@@ -120,11 +122,13 @@ class TestYoutubeVideo:
             try_convert_download_thumbnail(entry=entry)
 
         # Pretend the thumbnail did not download via returning nothing for its downloaded path
-        with patch.object(YTDLP, "_EXTRACT_ENTRY_NUM_RETRIES", 1), patch.object(
+        with (
-            Entry, "try_get_ytdlp_download_thumbnail_path"
+            patch.object(YTDLP, "_EXTRACT_ENTRY_NUM_RETRIES", 1),
-        ) as mock_ytdlp_path, patch(
+            patch.object(Entry, "try_get_ytdlp_download_thumbnail_path") as mock_ytdlp_path,
-            "ytdl_sub.downloaders.url.downloader.try_convert_download_thumbnail",
+            patch(
-            side_effect=delete_entry_thumb,
+                "ytdl_sub.downloaders.url.downloader.try_convert_download_thumbnail",
+                side_effect=delete_entry_thumb,
+            ),
         ):
             mock_ytdlp_path.return_value = None
             transaction_log = single_video_subscription.download(dry_run=False)

tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json

@@ -1,20 +1,20 @@
 {
-  ".ytdl-sub-Sithu Aye-download-archive.json": "ca37a404a860a2a6b6b0f38b659c9f17",
+  ".ytdl-sub-Sithu Aye-download-archive.json": "93ad50c4f4cca753f40c46bc6e31cabf",
-  "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/01 - Double Helix Reimagined.mp3": "56f7ee579031f4795230e68b63b15f6b",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/01 - Invent the Universe.mp3": "482e97a4f9a30aa4413f45f5f3f5dbba",
-  "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/02 - Skye Reimagined.mp3": "dfb24e0ef03e203d471bb81f854f5cd3",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/02 - Grand Unification (feat. David Maxim Micic).mp3": "34e43fd80b4c61295abbb8b794e523a7",
-  "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/03 - Baryofusion.mp3": "95bd9ab2238e5372f445c59daafd0138",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/03 - Expansion.mp3": "0ba4b8ef65e274cbf065b6f0a5f444e5",
-  "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/04 - Mandalay Reimagined.mp3": "7d2b38559b4c66a2e4841a6976f726af",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/04 - Baryogenesis.mp3": "ebde7ae4e211f10f2b25a46f2b9483c7",
-  "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/05 - Messenger EDM Remix.mp3": "971ed99fa1d80ad53dd15069459576ba",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/05 - Particles Collide (feat. Plini).mp3": "7df698f89a55fb21a2f07ffd8d2bfb75",
-  "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/folder.jpg": "bf6f70d51557a71b69fed85b2cb476f0",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/06 - Nucleosynthesis.mp3": "9a861212acf25a8ff801833f82ade55e",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/01 - Invent the Universe.mp3": "14fe5186fe68eacef265b77c4653ff73",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/07 - Recombination.mp3": "3abbfe126fbdd830195dc2cac223ea10",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/02 - Grand Unification (feat. David Maxim Micic).mp3": "839b79a68464246f19b4701c4daec3cb",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/08 - Dark Ages.mp3": "28e6a73bea1cd881e5ad38d57d1426c5",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/03 - Expansion.mp3": "f44bd1b6b2db8ab29987f47045f1acca",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/09 - Formation.mp3": "d588ed8edc324ea657abb2be96d557a1",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/04 - Baryogenesis.mp3": "a6d5a0f1026ca759c30f69a8d07d64b8",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/10 - Pale Blue Dot.mp3": "8245d285ba2403bf512f83c4a585eb81",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/05 - Particles Collide (feat. Plini).mp3": "9f41e0823c252b9c238e4ce565c78c6e",
+  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/folder.jpg": "d8cffeca026afaa619f641a95143f803",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/06 - Nucleosynthesis.mp3": "26e7c844ea45e60ff906f3f4280c991c",
+  "Sithu Aye/[2024] Kindness/01 - Run it Down.mp3": "227c366845913f3476ae83fce0b72c56",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/07 - Recombination.mp3": "8544b560ea220bed0ede54fe73e80bec",
+  "Sithu Aye/[2024] Kindness/02 - Zero Sum Groove.mp3": "0bd3b51acefc2a363309a8f94dd146d7",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/08 - Dark Ages.mp3": "8d209b11a038fb870f5a275c6304159c",
+  "Sithu Aye/[2024] Kindness/03 - Obsidian.mp3": "2a6b87e1ab5a69fc6c4503dc0dd03623",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/09 - Formation.mp3": "48b148d54eed289fd5aef6898f8b72df",
+  "Sithu Aye/[2024] Kindness/04 - Fear is the Kindness Killer.mp3": "4593d02a83f722d481846552554d13ba",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/10 - Pale Blue Dot.mp3": "f506f2d69d03430a2ab0a6890184c162",
+  "Sithu Aye/[2024] Kindness/05 - Ghost;Cleanse.mp3": "f89156a892b597be9b594ce10131ab60",
-  "Sithu Aye/[2022] Re：Invent the Universe (10th Anniversary Remaster)/folder.jpg": "d8cffeca026afaa619f641a95143f803"
+  "Sithu Aye/[2024] Kindness/folder.jpg": "8f72c9acb1a2e49fcbdc2624a46b229c"
 }

tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json

@@ -1,6 +1,7 @@
 {
-  ".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec",
+  ".ytdl-sub-chapters_from_comments-download-archive.json": "70bc3bb62af3a344b0256e1d3b98dcb0",
+  "JMC/Move 78 - Automated Improvisation [Full Album].info.json": "INFO_JSON",
   "JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6",
-  "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "068526b2d8f85fdcf914df3e23d0b1fa",
+  "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "fbe825bf5fb8ce193d47c3da3540e815",
   "JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac"
 }

tests/resources/expected_downloads_summaries/plugins/file_convert/output.json

@@ -1,6 +1,7 @@
 {
-  ".ytdl-sub-file_convert_test-download-archive.json": "f720289a704349fbe38ef5ed451af724",
+  ".ytdl-sub-file_convert_test-download-archive.json": "36ac81b6a021c1479a588477d74afd38",
+  "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "INFO_JSON",
   "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.jpg": "662fcaadf6e80d63591bac19a5fdffb0",
-  "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mp4": "24f5057254471bc6ecc3056e91af1444",
+  "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mp4": "7609c4f53565aa8f9eac50ad755b5c47",
   "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.nfo": "752e6b6eea853c8a1f62faa4b841b292"
 }

tests/resources/expected_downloads_summaries/plugins/file_convert/output_custom_ffmpeg.json

@@ -1,5 +1,6 @@
 {
-  ".ytdl-sub-file_convert_test-download-archive.json": "8fa4bf42c9686f8520ce107e48b73215",
+  ".ytdl-sub-file_convert_test-download-archive.json": "83e57376a260ab3a58978dee4a06df4a",
+  "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "INFO_JSON",
   "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.jpg": "662fcaadf6e80d63591bac19a5fdffb0",
   "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mkv": "175320a51dc3efcea84daebec3c1d7e1",
   "file_convert_test/When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.nfo": "752e6b6eea853c8a1f62faa4b841b292"

tests/resources/expected_downloads_summaries/plugins/match_filters/test_match_filters_partial.json

@@ -1,6 +1,7 @@
 {
-  ".ytdl-sub-match_filter_test-download-archive.json": "30f10646149d9eea4eb970749f352f7d",
+  ".ytdl-sub-match_filter_test-download-archive.json": "c2154694d771b47e7878622851b379b7",
+  "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
   "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
-  "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "f17a540070964a199b35f981561d94e4",
+  "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].mp4": "a0dc417dbf369c47bbac571bdb18e0e4",
   "match_filter_test/Jesse's Minecraft Server [Trailer - Mar.21].nfo": "d85f4500bb5d8a2425d734a23b5a944c"
 }

tests/resources/expected_downloads_summaries/plugins/split_by_chapters_video.json

@@ -1,15 +1,15 @@
 {
   ".ytdl-sub-Proved Records-download-archive.json": "c3fb0b4f31caaa10ac7954ea93da33c4",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "1e3583c9c1dc166b7baf98b81b4ca106",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "20fec748a00e37dc3b3868797e37fbbf",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "bb4c9085a4345515e6dffcf2c09f0f0f",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "edfbc8621ddfb163d30fee36d7ad902e",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "c6766f80d0f9993ec6051555cd37dc32",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "45a81d4ad17a4b80e916919656a7772e",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "2139e6eedc4d1bc2025f8ef78b0bd2af",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "b536cdb69653303b8eff9b2ae7352b7f",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "8194d6a4018121e40b1733a0fa7a382e",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "f8f93a483b7945feccbb48b89c608e46",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "e244183723e23c65156076b5f7ffcf56",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "4405d980fdb78a453d681054f8a5c603",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "692224c169be4735c0892e05726cb335",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "804f26a8ae8dcaa25b5882dd0051553f",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "561c339d4d1232c15ad38a5afe8d61c3",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "b0f96e5ed99e04064c3537966d332326",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "bdef7ad26848b45552891b9ea722659e",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "83d1e1acbe08b97ca30ab0c29b6473e2",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "ffff4506702e708288eb54eff8fbaca4",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "bbc6ddf2c4f0dbce33e5173d04976e6a",
-  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/11 - 11. Sunrise (Pillows) (Feat. Emmavie).mp3": "41a88d6c07047bb4215fed0e71443f17",
+  "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/11 - 11. Sunrise (Pillows) (Feat. Emmavie).mp3": "e2bdef2556deeec7a7149948c4ffcaec",
   "Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/folder.jpg": "bd3685acc53072e591bae2505ecb0648"
 }

tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475",
-  "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "b886f268a2a9b3b62f528fcf46699082",
+  "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "f1c2e04fb84768fed69ed981d9a2dbcd",
   "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
 }

tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_video.json

@@ -1,15 +1,15 @@
 {
   ".ytdl-sub-split_by_chapters_with_regex_video_preset-download-archive.json": "9798e8289742586d0efd295a97c6c906",
-  "Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "d07f90d214416a11736e3f73c0955208",
+  "Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "2311726ea7c5e6c4858810544635d8c8",
-  "Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "9057d3d7fcd98701e06d399a507c0e5b",
+  "Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "679c3893251b934830855f5f09c35d76",
-  "Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "4cc4e7066112388e83bdd9e803e8051a",
+  "Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "c7438762ed13c7768e5177aca7feced2",
-  "Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "8d0f04c04f8c2b0565954dac66ee29a3",
+  "Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "b69cbec7d90a7a28e4684d9f37c2dd01",
-  "Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "e8166110c85e6b9af8d1067b8ffbb428",
+  "Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "18772abf8868c43b4ffffaaa904ba277",
-  "Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "64e39199ddcd05b96c15c82a3c695d25",
+  "Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "e9d2cd658fd16deaca41fb491fcb28f6",
-  "Alfa Mist/[2017] Nocturne/07 - Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "9a53bee41d1985e48e771a02b052986b",
+  "Alfa Mist/[2017] Nocturne/07 - Delusions： Rumination (Interlude) (Feat. Racheal Ofori).mp3": "4db7f44cc632fd6e572f844ef7075d2d",
-  "Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "19c50e84c08eacdd1095e461306d9077",
+  "Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "9687d7dd7543a9f99c5837aea86daf25",
-  "Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "f28df20fdbadf0ec4985b5f1f30c4e5d",
+  "Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "41e4b7db91f7148b86df33f0968ce6a2",
-  "Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "d7261e48713426ad4905919915a9b103",
+  "Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "024fd4477a38f915721ebfcf81f9b1c3",
-  "Alfa Mist/[2017] Nocturne/11 - Sunrise (Pillows) (Feat. Emmavie).mp3": "5d51491d0999e1f29918d4db4ed796d3",
+  "Alfa Mist/[2017] Nocturne/11 - Sunrise (Pillows) (Feat. Emmavie).mp3": "d5459648255b4aa22d88e1b2d2411ffb",
   "Alfa Mist/[2017] Nocturne/folder.jpg": "bd3685acc53072e591bae2505ecb0648"
 }

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json

@@ -1,7 +1,7 @@
 {
   ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75",
-  "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "e7a8a94ebe9f02f086f4bbf3df3946f6",
+  "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "221dab6e9d6840a0b4b7bb24444c87ff",
-  "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "aeea4b086507fd7fde3c09c0bd868950",
+  "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "02dc8e368de9555d062bde31dcc82852",
-  "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "80200d21a46c7f521bfb23aef2f87e34",
+  "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "c808e1da2bccd419201eeeffc32c3729",
   "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530"
 }

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
-  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "829eb7dcc5dcae41240701dec4e1708d",
+  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "b85c812fc761379122a96243d98a5ccb",
   "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
 }

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-single_song_best_test-download-archive.json": "0f9484ed868dcbeef82810a3cf7b0eea",
-  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.opus": "73e4afdda9bc792807c8b07a63128e5a",
+  "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.opus": "eb32b0ef0568582cafd7fa528a5260e8",
   "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
 }