Compare commits

..

No commits in common. "master" and "j/hotfix-cron" have entirely different histories.

230 changed files with 2955 additions and 7857 deletions

3
.gitignore vendored
View file

@ -146,11 +146,8 @@ docker/testing/volumes
.local/ .local/
.ytdl-sub-working-directory .ytdl-sub-working-directory
.ytdl-sub-lock
ffmpeg.exe ffmpeg.exe
ffprobe.exe ffprobe.exe
tools/docgen/out tools/docgen/out
prof/

View file

@ -36,12 +36,12 @@ endif
all: check_lint docs docker docker_ubuntu docker_gui all: check_lint docs docker docker_ubuntu docker_gui
lint: lint:
python3 -m ruff format . python3 -m isort .
python3 -m ruff check --fix . python3 -m black .
python3 -m pylint src python3 -m pylint src
check_lint: check_lint:
ruff format --check . \ isort . --check-only --diff \
&& ruff check . \ && black . --check \
&& pylint src/ && pylint src/
wheel: clean wheel: clean
$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"" > src/ytdl_sub/__init__.py) $(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"" > src/ytdl_sub/__init__.py)

View file

@ -68,7 +68,7 @@ __preset__:
# Pass any arg directly to yt-dlp's Python API # Pass any arg directly to yt-dlp's Python API
ytdl_options: ytdl_options:
cookiefile: "/config/ytdl-sub-configs/cookie.txt" cookiefile: "/config/cookie.txt"
################################################################### ###################################################################
# TV Show Presets. Can replace Plex with Plex/Jellyfin/Emby/Kodi # TV Show Presets. Can replace Plex with Plex/Jellyfin/Emby/Kodi

View file

@ -18,9 +18,7 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
vim \ vim \
g++ \ g++ \
nano \ nano \
unzip \
make \ make \
deno \
libffi-dev \ libffi-dev \
"python3>=3.10" \ "python3>=3.10" \
py3-pip \ py3-pip \
@ -32,7 +30,6 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
"aria2>=1.36.0" && \ "aria2>=1.36.0" && \
ffmpeg -version && \ ffmpeg -version && \
aria2c --version && \ aria2c --version && \
deno --version && \
# Install phantomjs if using x86_64, ensure it is properly installed # Install phantomjs if using x86_64, ensure it is properly installed
if [[ $(uname -m) == "x86_64" ]]; then \ if [[ $(uname -m) == "x86_64" ]]; then \
echo "installing phantomjs" && \ echo "installing phantomjs" && \
@ -46,19 +43,18 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
phantomjs --version && \ phantomjs --version && \
cd -; \ cd -; \
fi && \ fi && \
# Configure pip globally echo "hi" && \
echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \ # Install ytdl-sub, ensure it is installed properly
# Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly python3 -m pip install --break-system-packages --no-cache-dir ytdl_sub-*.whl && \
python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
ytdl-sub -h && \ ytdl-sub -h && \
# Delete unneeded packages after install # Delete unneeded packages after install
rm ytdl_sub-*.whl && \ rm ytdl_sub-*.whl && \
apk del \ apk del \
g++ \ g++ \
make \ make \
libffi-dev && \ libffi-dev \
python3 -m pip --help py3-pip \
py3-setuptools
############################################################################### ###############################################################################
# CONTAINER CONFIGS # CONTAINER CONFIGS

View file

@ -23,7 +23,6 @@ RUN mkdir -p /config && \
vim \ vim \
g++ \ g++ \
nano \ nano \
unzip \
make \ make \
python3-pip \ python3-pip \
fontconfig \ fontconfig \
@ -62,13 +61,8 @@ RUN mkdir -p /config && \
echo "Phantom JS version:" && \ echo "Phantom JS version:" && \
phantomjs --version ; \ phantomjs --version ; \
fi && \ fi && \
# Install Deno, required for YouTube downloads # Install ytdl-sub, ensure it is installed properly
curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \ pip install --no-cache-dir --break-system-packages ytdl_sub-*.whl && \
deno --help && \
# Configure pip globally
echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
# Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
ytdl-sub -h && \ ytdl-sub -h && \
# Delete unneeded packages after install # Delete unneeded packages after install
rm ytdl_sub-*.whl && \ rm ytdl_sub-*.whl && \
@ -76,11 +70,11 @@ RUN mkdir -p /config && \
g++ \ g++ \
make \ make \
xz-utils \ xz-utils \
bzip2 && \ bzip2 \
python3-venv && \
apt-get autoremove -y && \ apt-get autoremove -y && \
apt-get purge -y --auto-remove && \ apt-get purge -y --auto-remove && \
rm -rf /var/lib/apt/lists/* && \ rm -rf /var/lib/apt/lists/*
python3 -m pip --help
############################################################################### ###############################################################################
# CONTAINER CONFIGS # CONTAINER CONFIGS

View file

@ -26,7 +26,6 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
vim \ vim \
g++ \ g++ \
nano \ nano \
unzip \
make \ make \
python3-pip \ python3-pip \
fontconfig \ fontconfig \
@ -65,13 +64,8 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
echo "Phantom JS version:" && \ echo "Phantom JS version:" && \
phantomjs --version ; \ phantomjs --version ; \
fi && \ fi && \
# Install Deno, required for YouTube downloads # Install ytdl-sub, ensure it is installed properly
curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \ pip install --no-cache-dir --break-system-packages ytdl_sub-*.whl && \
deno --help && \
# Configure pip globally
echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
# Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
ytdl-sub -h && \ ytdl-sub -h && \
# Delete unneeded packages after install # Delete unneeded packages after install
rm ytdl_sub-*.whl && \ rm ytdl_sub-*.whl && \
@ -79,11 +73,11 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
g++ \ g++ \
make \ make \
xz-utils \ xz-utils \
bzip2 && \ bzip2 \
python3-venv && \
apt-get autoremove -y && \ apt-get autoremove -y && \
apt-get purge -y --auto-remove && \ apt-get purge -y --auto-remove && \
rm -rf /var/lib/apt/lists/* && \ rm -rf /var/lib/apt/lists/*
python3 -m pip --help
############################################################################### ###############################################################################
# CONTAINER CONFIGS # CONTAINER CONFIGS

14
docker/root/custom-cont-init.d/defaults Executable file → Normal file
View file

@ -17,18 +17,18 @@ echo "Starting ytdl-sub..."
echo "alias ls='ls --color=auto'" > /config/.bashrc && \ echo "alias ls='ls --color=auto'" > /config/.bashrc && \
echo "cd ." >> /config/.bashrc echo "cd ." >> /config/.bashrc
# always create empty cron log file on start
echo "" > "$LOGS_TO_STDOUT"
# permissions # permissions
chown -R ${PUID:-abc}:${PGID:-abc} \ chown -R ${PUID:-abc}:${PGID:-abc} \
/config /config
# always create empty cron log file on start
echo "" > "$LOGS_TO_STDOUT"
# update command reference: # update command reference:
# https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip # https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip
if [ "$UPDATE_YT_DLP_ON_START" == "stable" ] ; then if [ "$UPDATE_YT_DLP_ON_START" == "stable" ] ; then
echo "UPDATE_YT_DLP_ON_START is set to stable, attempting to update to a new stable version of yt-dlp if it exists." echo "UPDATE_YT_DLP_ON_START is set to stable, attempting to update to a new stable version of yt-dlp if it exists."
python3 -m pip install -U "yt-dlp[default]" python3 -m pip install -U "yt-dlp[default]" --break-system-packages
elif [ "$UPDATE_YT_DLP_ON_START" == "nightly" ] ; then elif [ "$UPDATE_YT_DLP_ON_START" == "nightly" ] ; then
echo "UPDATE_YT_DLP_ON_START is set to nightly, attempting to update to the latest nightly version of yt-dlp." echo "UPDATE_YT_DLP_ON_START is set to nightly, attempting to update to the latest nightly version of yt-dlp."
python3 -m pip install -U --pre "yt-dlp[default]" python3 -m pip install -U --pre "yt-dlp[default]"
@ -37,7 +37,7 @@ elif [ "$UPDATE_YT_DLP_ON_START" == "master" ] ; then
python3 -m pip install -U pip hatchling wheel python3 -m pip install -U pip hatchling wheel
python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz" python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz"
else else
echo "UPDATE_YT_DLP_ON_START is not set, using packaged version." echo "UPDATE_YT_DLP_ON_START is not set, using packaged version."
fi fi
# set up cron # set up cron
@ -47,11 +47,9 @@ if [ "$CRON_SCHEDULE" != "" ] ; then
# create cron script wrapper # create cron script wrapper
echo '#!/bin/bash' > "$CRON_WRAPPER_SCRIPT" echo '#!/bin/bash' > "$CRON_WRAPPER_SCRIPT"
# Echo commands for easier user debugging:
echo "set -x" >> "$CRON_WRAPPER_SCRIPT"
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "$CRON_WRAPPER_SCRIPT" echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "$CRON_WRAPPER_SCRIPT"
echo "cd \"$DEFAULT_WORKSPACE\"" >> "$CRON_WRAPPER_SCRIPT" echo "cd \"$DEFAULT_WORKSPACE\"" >> "$CRON_WRAPPER_SCRIPT"
echo ". \"$CRON_SCRIPT\" >> \"$LOGS_TO_STDOUT\" 2>&1" >> "$CRON_WRAPPER_SCRIPT" echo ". \"$CRON_SCRIPT\" | tee -a \"$LOGS_TO_STDOUT\"" >> "$CRON_WRAPPER_SCRIPT"
chmod +x "$CRON_WRAPPER_SCRIPT" chmod +x "$CRON_WRAPPER_SCRIPT"
chown abc:abc "$CRON_WRAPPER_SCRIPT" chown abc:abc "$CRON_WRAPPER_SCRIPT"

View file

@ -1,15 +1,4 @@
echo "Beginning cron job..."
# Place your ytdl-sub command(s) here. # Place your ytdl-sub command(s) here.
# # This script is executed in the same relative path as this file.
# This script is executed in the same directory as this file which also contains the
# default `./config.yaml` and `./subscriptions.yaml`, so you don't need to use the
# `--config` CLI option or pass a `SUBPATH` to the `$ ytdl-sub sub` sub-command.
#
# Test your configuration and subscriptions carefully before automating downloads to
# prevent triggering throttles or bans:
#
# https://ytdl-sub.readthedocs.io/en/latest/guides/getting_started/downloading.html
#
# Once you've tested your configuration and you're ready to download entries unattended,
# remove the next line and un-comment the following line:
echo "WARNING: Read /config/ytdl-sub-configs/cron and modify to automate downloads."
# ytdl-sub sub

View file

@ -20,7 +20,7 @@ __preset__:
# Pass any arg directly to yt-dlp's Python API # Pass any arg directly to yt-dlp's Python API
# ytdl_options: # ytdl_options:
# cookiefile: "/config/ytdl-sub-configs/cookie.txt" # cookiefile: "/config/cookie.txt"
################################################################### ###################################################################
# Subscriptions nested under this will use the # Subscriptions nested under this will use the

View file

@ -9,9 +9,8 @@ services:
PUID: "1000" PUID: "1000"
PGID: "1000" PGID: "1000"
TZ: "America/Los_Angeles" TZ: "America/Los_Angeles"
CRON_SCHEDULE: '*/1 * * * *' CRON_SCHEDULE: '"*/1 * * * *"'
CRON_RUN_ON_START: "true" CRON_RUN_ON_START: "true"
UPDATE_YT_DLP_ON_START: "stable"
volumes: volumes:
- "./volumes/ytdl-sub-gui/:/config/" - "./volumes/ytdl-sub-gui/:/config/"
ports: ports:
@ -20,15 +19,14 @@ services:
ytdl-sub-headless: ytdl-sub-headless:
build: build:
context: "../" context: "../"
image: "ytdl-sub:local" image: "ytdl-sub-headless:local"
container_name: "ytdl-sub-headless" container_name: "ytdl-sub-headless"
environment: environment:
PUID: "1000" PUID: "1000"
PGID: "1000" PGID: "1000"
TZ: "America/Los_Angeles" TZ: "America/Los_Angeles"
CRON_SCHEDULE: '*/1 * * * *' CRON_SCHEDULE: '"*/1 * * * *"'
CRON_RUN_ON_START: "true" CRON_RUN_ON_START: "true"
UPDATE_YT_DLP_ON_START: "stable"
volumes: volumes:
- "./volumes/ytdl-sub-headless/:/config/" - "./volumes/ytdl-sub-headless/:/config/"
restart: "unless-stopped" restart: "unless-stopped"
@ -42,9 +40,8 @@ services:
PUID: "1000" PUID: "1000"
PGID: "1000" PGID: "1000"
TZ: "America/Los_Angeles" TZ: "America/Los_Angeles"
CRON_SCHEDULE: '*/1 * * * *' CRON_SCHEDULE: '"*/1 * * * *"'
CRON_RUN_ON_START: "true" CRON_RUN_ON_START: "true"
UPDATE_YT_DLP_ON_START: "stable"
volumes: volumes:
- "./volumes/ytdl-sub-ubuntu/:/config/" - "./volumes/ytdl-sub-ubuntu/:/config/"
restart: "unless-stopped" restart: "unless-stopped"

View file

@ -7,7 +7,7 @@
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
project = "ytdl-sub" project = "ytdl-sub"
copyright = "2026, Jesse Bannon" copyright = "2024, Jesse Bannon"
author = "Jesse Bannon" author = "Jesse Bannon"
release = "" release = ""
@ -15,8 +15,10 @@ release = ""
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [ extensions = [
"sphinx.ext.autodoc",
"sphinx.ext.autosectionlabel", "sphinx.ext.autosectionlabel",
"sphinx.ext.extlinks", "sphinx.ext.extlinks",
"sphinx.ext.napoleon",
"sphinx_copybutton", "sphinx_copybutton",
"sphinx_design", "sphinx_design",
] ]
@ -68,3 +70,19 @@ extlinks = {
"lsio-gh": ("https://github.com/linuxserver/%s", "%s image"), "lsio-gh": ("https://github.com/linuxserver/%s", "%s image"),
"ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"), "ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"),
} }
# -- Options for autodoc ----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
# Automatically extract typehints when specified and place them in
# descriptions of the relevant function/method.
autodoc_default_options = {
"autodoc_typehints_format": "short",
"autodoc_class_signature": "separated",
"add_module_names": False,
# "add_class_names": False,
}
python_use_unqualified_type_names = True
napoleon_numpy_docstring = True
napoleon_use_rtype = False

View file

@ -1,13 +1,7 @@
.. ==================
WARNING: This RST file is generated from docstrings in:
The respective function docstrings within ytdl_sub/config/config_validator.py
In order to make a change to this file, edit the respective docstring
and run `make docs`. This will automatically sync the Python RST-based
docstrings into this file. If the docstrings and RST file are out of sync,
it will fail TestDocGen tests in GitHub CI.
Configuration File Configuration File
================== ==================
ytdl-sub is configured using a ``config.yaml`` file. ytdl-sub is configured using a ``config.yaml`` file.
The ``config.yaml`` is made up of two sections: The ``config.yaml`` is made up of two sections:
@ -17,118 +11,113 @@ The ``config.yaml`` is made up of two sections:
configuration: configuration:
presets: presets:
You can jump to any section and subsection of the config using the navigation section to
the left.
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
If you prefer to use a Windows backslash, note that it must have If you wish to represent paths like Windows, you will need to
``C:\\double\\bashslash\\paths`` in order to escape the backslash character. This is due ``C:\\double\\bashslash\\paths`` in order to escape the backslash character.
to it being a YAML escape character.
configuration
-------------
The ``configuration`` section contains app-wide configs applied to all presets and
subscriptions.
.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
:members:
:member-order: bysource
:exclude-members: subscription_value, persist_logs, experimental
persist_logs
~~~~~~~~~~~~
Without this key, ``ytdl-sub`` only prints output to it's ``stdout`` and ``stderr``. If
your configuration includes the ``persist_logs:`` key, then ``ytdl-sub`` also writes log
files to disk.
.. warning::
The log files grow rapidly if ``keep_successful_logs:`` is ``true``, the default, and
may fill up disk space. Set ``keep_successful_logs: false`` or prune the log files
regularly.
For example:
.. code-block:: yaml .. code-block:: yaml
configuration: configuration:
dl_aliases:
mv: "--preset music_video"
u: "--download.url"
experimental:
enable_update_with_info_json: True
ffmpeg_path: "/usr/bin/ffmpeg"
ffprobe_path: "/usr/bin/ffprobe"
file_name_max_bytes: 255
lock_directory: "/tmp"
persist_logs: persist_logs:
keep_successful_logs: True logs_directory: "/path/to/log/directory"
logs_directory: "/var/log/ytdl-sub-logs"
umask: "022" .. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
working_directory: ".ytdl-sub-working-directory" :members:
:member-order: bysource
dl_aliases
----------
.. _dl_aliases:
Alias definitions to shorten :ref:`dl arguments <usage:Download Options>`. For example, presets
-------
Each key under ``presets:`` defines a `formula` for how to format downloaded media and
metadata. The key is the name of the preset and the value is a mapping that defines the
preset.
.. note::
The ``presets:`` key at the top of the configuration file contains multiple
user-defined presets, but *each preset* itself may include a ``preset:`` key that
defines *that preset's* base presets. For example:
.. code-block:: yaml
presets:
Foo Preset:
preset:
- "Jellyfin TV Show by Date"
- "Only Recent"
preset
~~~~~~
Presets support inheritance by defining one or more parent presets:
.. code-block:: yaml .. code-block:: yaml
configuration: presets:
dl_aliases: custom_preset:
mv: "--preset music_video" ...
u: "--download.url" parent_preset:
...
child_preset:
preset:
- "parent_preset"
Simplifies In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
Use parent presets where possible to reduce duplicate yaml definitions.
.. code-block:: bash Presets also support inheritance from multiple presets:
ytdl-sub dl --preset "Jellyfin Music Videos" --download.url "youtube.com/watch?v=a1b2c3" .. code-block:: yaml
to child_preset:
preset:
- "custom_preset"
- "parent_preset"
.. code-block:: bash In this example, ``child_preset`` will inherit all fields from ``custom_preset`` and
``parent_preset`` in that order. The bottom-most preset has the highest priority. More
specifically, presets are merged using `mergedeep`_ via `a TYPESAFE_ADDITIVE merge`_,
which means:
ytdl-sub dl --mv --u "youtube.com/watch?v=a1b2c3" - if two conflicting keys arent lists or mappings, overwrite the higher priority one
- otherwise, combine then re-evaluate
experimental If you are only inheriting from one preset, using a single string instead of a list is
------------ valid, for example ``preset: "parent_preset"``, but we recommend always using a list for
Experimental flags reside under the ``experimental`` key. consistent readability between presets.
``enable_update_with_info_json`` .. _`mergedeep`:
https://mergedeep.readthedocs.io/en/latest/
Enables modifying subscription files using info.json files using the argument .. _`a TYPESAFE_ADDITIVE merge`:
``--update-with-info-json``. This feature is still being tested and has the ability to https://mergedeep.readthedocs.io/en/latest/index.html#merge-strategies
destroy files. Ensure you have a full backup before usage. You have been warned!
ffmpeg_path
-----------
Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux,
``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows.
ffprobe_path
------------
Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux,
``./ffprobe.exe`` in the same directory as ytdl-sub for Windows.
file_name_max_bytes
-------------------
Max file name size in bytes. Most OS's typically default to 255 bytes.
lock_directory
--------------
The directory to temporarily store file locks, which prevents multiple instances
of ``ytdl-sub`` from running. Note that file locks do not work on
network-mounted directories. Ensure that this directory resides on the host
machine. Defaults to ``/tmp``.
persist_logs
------------
By default, no logs are persisted. Specifying this key will enable persisted logs. The following
options are available.
``keep_successful_logs``
Defaults to ``True``. When this key is ``False``, only write log files for failed
subscriptions.
``logs_directory``
Required field. Write log files to this directory with names like
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
umask
-----
Umask in octal format to apply to every created file. Defaults to ``022``.
working_directory
-----------------
The directory to temporarily store downloaded files before moving them into their final
directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory
that ytdl-sub is invoked from.
Presets
=======
Custom presets are defined in this section. Refer to the
:ref:`Getting Started Guide<guides/getting_started/first_config:Basic Configuration>`
on how to configure.

View file

@ -28,6 +28,7 @@ Extracts audio from a video file.
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
opus, vorbis, wav, and best to grab the best possible format at runtime. opus, vorbis, wav, and best to grab the best possible format at runtime.
``enable`` ``enable``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -36,6 +37,7 @@ Extracts audio from a video file.
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``quality`` ``quality``
:expected type: Float :expected type: Float
@ -43,6 +45,7 @@ Extracts audio from a video file.
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
chapters chapters
@ -81,12 +84,14 @@ chapters and remove specific ones. Can also remove chapters using regex.
Defaults to False. If chapters do not exist in the video/description itself, attempt to Defaults to False. If chapters do not exist in the video/description itself, attempt to
scrape comments to find the chapters. scrape comments to find the chapters.
``embed_chapters`` ``embed_chapters``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
:description: :description:
Defaults to True. Embed chapters into the file. Defaults to True. Embed chapters into the file.
``enable`` ``enable``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -95,6 +100,7 @@ chapters and remove specific ones. Can also remove chapters using regex.
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``force_key_frames`` ``force_key_frames``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -102,6 +108,7 @@ chapters and remove specific ones. Can also remove chapters using regex.
Defaults to False. Force keyframes at cuts when removing sections. This is slow due to Defaults to False. Force keyframes at cuts when removing sections. This is slow due to
needing a re-encode, but the resulting video may have fewer artifacts around the cuts. needing a re-encode, but the resulting video may have fewer artifacts around the cuts.
``remove_chapters_regex`` ``remove_chapters_regex``
:expected type: Optional[List[RegexString] :expected type: Optional[List[RegexString]
@ -109,6 +116,7 @@ chapters and remove specific ones. Can also remove chapters using regex.
List of regex patterns to match chapter titles against and remove them from the List of regex patterns to match chapter titles against and remove them from the
entry. entry.
``remove_sponsorblock_categories`` ``remove_sponsorblock_categories``
:expected type: Optional[List[String]] :expected type: Optional[List[String]]
@ -117,6 +125,7 @@ chapters and remove specific ones. Can also remove chapters using regex.
categories that are specified in ``sponsorblock_categories`` or "all", which removes categories that are specified in ``sponsorblock_categories`` or "all", which removes
everything specified in ``sponsorblock_categories``. everything specified in ``sponsorblock_categories``.
``sponsorblock_categories`` ``sponsorblock_categories``
:expected type: Optional[List[String]] :expected type: Optional[List[String]]
@ -125,6 +134,7 @@ chapters and remove specific ones. Can also remove chapters using regex.
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
"poi_highlight", or "all" to include all categories. "poi_highlight", or "all" to include all categories.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
date_range date_range
@ -159,12 +169,14 @@ intended download files.
:description: :description:
Only download videos after or on this datetime, inclusive. Only download videos after or on this datetime, inclusive.
``before`` ``before``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
:description: :description:
Only download videos only before this datetime, not inclusive. Only download videos only before this datetime, not inclusive.
``breaks`` ``breaks``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -172,6 +184,7 @@ intended download files.
Toggle to enable breaking subsequent metadata downloads if an entry's upload date Toggle to enable breaking subsequent metadata downloads if an entry's upload date
is out of range. Defaults to True. is out of range. Defaults to True.
``enable`` ``enable``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -180,6 +193,7 @@ intended download files.
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``type`` ``type``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -187,6 +201,7 @@ intended download files.
Which type of date to use. Must be either ``upload_date`` or ``release_date``. Which type of date to use. Must be either ``upload_date`` or ``release_date``.
Defaults to ``upload_date``. Defaults to ``upload_date``.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
download download
@ -290,6 +305,7 @@ Also supports custom ffmpeg conversions:
- Video: avi, flv, mkv, mov, mp4, webm - Video: avi, flv, mkv, mov, mp4, webm
- Audio: aac, flac, mp3, m4a, opus, vorbis, wav - Audio: aac, flac, mp3, m4a, opus, vorbis, wav
``convert_with`` ``convert_with``
:expected type: Optional[String] :expected type: Optional[String]
@ -298,6 +314,7 @@ Also supports custom ffmpeg conversions:
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
``enable`` ``enable``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -306,6 +323,7 @@ Also supports custom ffmpeg conversions:
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``ffmpeg_post_process_args`` ``ffmpeg_post_process_args``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -318,6 +336,7 @@ Also supports custom ffmpeg conversions:
The output file will use the extension specified in ``convert_to``. Post-processing args The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``. can still be set with ``convert_with`` set to ``yt-dlp``.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
filter_exclude filter_exclude
@ -453,6 +472,7 @@ with a ``.nfo`` extension. You can add any values into the NFO.
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: OverridesBooleanFormatterValidator :expected type: OverridesBooleanFormatterValidator
@ -461,12 +481,14 @@ with a ``.nfo`` extension. You can add any values into the NFO.
emojis and some foreign language characters. Setting this to True will replace those emojis and some foreign language characters. Setting this to True will replace those
characters with '□'. characters with '□'.
``nfo_name`` ``nfo_name``
:expected type: EntryFormatter :expected type: EntryFormatter
:description: :description:
The NFO file name. The NFO file name.
``nfo_root`` ``nfo_root``
:expected type: EntryFormatter :expected type: EntryFormatter
@ -479,6 +501,7 @@ with a ``.nfo`` extension. You can add any values into the NFO.
<episodedetails> <episodedetails>
</episodedetails> </episodedetails>
``tags`` ``tags``
:expected type: NfoTags :expected type: NfoTags
@ -515,6 +538,7 @@ with a ``.nfo`` extension. You can add any values into the NFO.
<genre>Comedy</genre> <genre>Comedy</genre>
<genre>Drama</genre> <genre>Drama</genre>
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
output_directory_nfo_tags output_directory_nfo_tags
@ -546,6 +570,7 @@ Usage:
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: OverridesBooleanFormatterValidator :expected type: OverridesBooleanFormatterValidator
@ -554,12 +579,14 @@ Usage:
emojis and some foreign language characters. Setting this to True will replace those emojis and some foreign language characters. Setting this to True will replace those
characters with '□'. characters with '□'.
``nfo_name`` ``nfo_name``
:expected type: EntryFormatter :expected type: EntryFormatter
:description: :description:
The NFO file name. The NFO file name.
``nfo_root`` ``nfo_root``
:expected type: EntryFormatter :expected type: EntryFormatter
@ -572,6 +599,7 @@ Usage:
<tvshow> <tvshow>
</tvshow> </tvshow>
``tags`` ``tags``
:expected type: NfoTags :expected type: NfoTags
@ -606,6 +634,7 @@ Usage:
<genre>Comedy</genre> <genre>Comedy</genre>
<genre>Drama</genre> <genre>Drama</genre>
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
output_options output_options
@ -640,6 +669,7 @@ Defines where to output files and thumbnails after all post-processing has compl
The file name to store a subscriptions download archive placed relative to The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
``file_name`` ``file_name``
:expected type: EntryFormatter :expected type: EntryFormatter
@ -647,6 +677,7 @@ Defines where to output files and thumbnails after all post-processing has compl
The file name for the media file. This can include directories such as The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
``info_json_name`` ``info_json_name``
:expected type: Optional[EntryFormatter] :expected type: Optional[EntryFormatter]
@ -655,6 +686,7 @@ Defines where to output files and thumbnails after all post-processing has compl
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes. directory. Can be set to empty string or `null` to disable info json writes.
``keep_files_after`` ``keep_files_after``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -666,6 +698,7 @@ Defines where to output files and thumbnails after all post-processing has compl
files after ``19000101``, which implies all files. Can be used in conjunction with files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``. ``keep_max_files``.
``keep_files_before`` ``keep_files_before``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -677,6 +710,7 @@ Defines where to output files and thumbnails after all post-processing has compl
files before ``now``, which implies all files. Can be used in conjunction with files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``. ``keep_max_files``.
``keep_files_date_eval`` ``keep_files_date_eval``
:expected type: str :expected type: str
@ -686,6 +720,7 @@ Defines where to output files and thumbnails after all post-processing has compl
perform evaluation for keep_files_before/after and keep_max_files. Defaults perform evaluation for keep_files_before/after and keep_max_files. Defaults
to the entry's upload_date_standardized variable. to the entry's upload_date_standardized variable.
``keep_max_files`` ``keep_max_files``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -695,6 +730,7 @@ Defines where to output files and thumbnails after all post-processing has compl
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
``maintain_download_archive`` ``maintain_download_archive``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -709,6 +745,7 @@ Defines where to output files and thumbnails after all post-processing has compl
Defaults to False. Defaults to False.
``migrated_download_archive_name`` ``migrated_download_archive_name``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -718,19 +755,13 @@ Defines where to output files and thumbnails after all post-processing has compl
name first, and fallback to ``download_archive_name``. It will always save to this file name first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``. and remove the original ``download_archive_name``.
``output_directory`` ``output_directory``
:expected type: OverridesFormatter :expected type: OverridesFormatter
:description: :description:
The output directory to store all media files downloaded. The output directory to store all media files downloaded.
``preserve_mtime``
:expected type: Optional[Boolean]
:description:
Preserve the video's original upload time as the file modification time.
When True, sets the file's mtime to match the video's upload_date from
yt-dlp metadata. Defaults to False.
``thumbnail_name`` ``thumbnail_name``
@ -740,6 +771,7 @@ Defines where to output files and thumbnails after all post-processing has compl
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes. directory. Can be set to empty string or `null` to disable thumbnail writes.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
overrides overrides
@ -804,6 +836,7 @@ used with no modifications.
If a file has no chapters and is set to "pass", then ``chapter_title`` is If a file has no chapters and is set to "pass", then ``chapter_title`` is
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
square_thumbnail square_thumbnail
@ -850,6 +883,7 @@ Usage:
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``kodi_safe`` ``kodi_safe``
:expected type: OverridesBooleanFormatterValidator :expected type: OverridesBooleanFormatterValidator
@ -858,12 +892,14 @@ Usage:
emojis and some foreign language characters. Setting this to True will replace those emojis and some foreign language characters. Setting this to True will replace those
characters with '□'. characters with '□'.
``nfo_name`` ``nfo_name``
:expected type: EntryFormatter :expected type: EntryFormatter
:description: :description:
The NFO file name. The NFO file name.
``nfo_root`` ``nfo_root``
:expected type: EntryFormatter :expected type: EntryFormatter
@ -876,6 +912,7 @@ Usage:
<season> <season>
</season> </season>
``tags`` ``tags``
:expected type: NfoTags :expected type: NfoTags
@ -889,6 +926,7 @@ Usage:
<title>My custom season name!</title> <title>My custom season name!</title>
</season> </season>
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
subtitles subtitles
@ -916,6 +954,7 @@ It will set the respective language to the correct subtitle file.
:description: :description:
Defaults to False. Whether to allow auto generated subtitles. Defaults to False. Whether to allow auto generated subtitles.
``embed_subtitles`` ``embed_subtitles``
:expected type: Optional[Boolean] :expected type: Optional[Boolean]
@ -923,6 +962,7 @@ It will set the respective language to the correct subtitle file.
Defaults to False. Whether to embed the subtitles into the video file. Note that Defaults to False. Whether to embed the subtitles into the video file. Note that
webm files can only embed "vtt" subtitle types. webm files can only embed "vtt" subtitle types.
``enable`` ``enable``
:expected type: Optional[OverridesFormatter] :expected type: Optional[OverridesFormatter]
@ -931,6 +971,7 @@ It will set the respective language to the correct subtitle file.
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``languages`` ``languages``
:expected type: Optional[List[String]] :expected type: Optional[List[String]]
@ -938,12 +979,6 @@ It will set the respective language to the correct subtitle file.
Language code(s) to download for subtitles. Supports a single or list of multiple Language code(s) to download for subtitles. Supports a single or list of multiple
language codes. Defaults to only "en". language codes. Defaults to only "en".
``languages_required``
:expected type: Optional[List[String]]
:description:
Language code(s) that are required to be present for downloads to continue. If missing,
ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles.
``subtitles_name`` ``subtitles_name``
@ -954,12 +989,14 @@ It will set the respective language to the correct subtitle file.
and will be placed in the output directory. ``lang`` is dynamic since you can download and will be placed in the output directory. ``lang`` is dynamic since you can download
multiple subtitles. It will set the respective language to the correct subtitle file. multiple subtitles. It will set the respective language to the correct subtitle file.
``subtitles_type`` ``subtitles_type``
:expected type: Optional[String] :expected type: Optional[String]
:description: :description:
Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc".
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
throttle_protection throttle_protection
@ -1000,12 +1037,14 @@ Range min and max values support static override variables within their definiti
this field can be set using an override variable to easily toggle whether this plugin this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean. is enabled or not via Boolean.
``max_downloads_per_subscription`` ``max_downloads_per_subscription``
:expected type: Optional[Range] :expected type: Optional[Range]
:description: :description:
Number of downloads to perform per subscription. Number of downloads to perform per subscription.
``sleep_per_download_s`` ``sleep_per_download_s``
:expected type: Optional[Range] :expected type: Optional[Range]
@ -1013,6 +1052,7 @@ Range min and max values support static override variables within their definiti
Number in seconds to sleep between each download. Does not include time it takes for Number in seconds to sleep between each download. Does not include time it takes for
ytdl-sub to perform post-processing. ytdl-sub to perform post-processing.
``sleep_per_request_s`` ``sleep_per_request_s``
:expected type: Optional[Range] :expected type: Optional[Range]
@ -1022,12 +1062,14 @@ Range min and max values support static override variables within their definiti
download for the entry. Also, yt-dlp only supports a single value at this time for this, download for the entry. Also, yt-dlp only supports a single value at this time for this,
so will always use the max value. so will always use the max value.
``sleep_per_subscription_s`` ``sleep_per_subscription_s``
:expected type: Optional[Range] :expected type: Optional[Range]
:description: :description:
Number in seconds to sleep between each subscription. Number in seconds to sleep between each subscription.
``subscription_download_probability`` ``subscription_download_probability``
:expected type: Optional[Float] :expected type: Optional[Float]
@ -1036,6 +1078,7 @@ Range min and max values support static override variables within their definiti
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
guaranteed over time to eventually download the subscription. guaranteed over time to eventually download the subscription.
---------------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------------
video_tags video_tags

View file

@ -521,13 +521,6 @@ pow
:description: :description:
``**`` operator. Returns the exponential of the base and exponent value. ``**`` operator. Returns the exponential of the base and exponent value.
range
~~~~~
:spec: ``range(end: Integer, start: Optional[Integer], step: Optional[Integer]) -> Array``
:description:
Returns the desired range of Integers in the form of an Array.
sub sub
~~~ ~~~
:spec: ``sub(values: Numeric, ...) -> Numeric`` :spec: ``sub(values: Numeric, ...) -> Numeric``
@ -837,7 +830,7 @@ behavior.
sanitize sanitize
~~~~~~~~ ~~~~~~~~
:spec: ``sanitize(value: AnyArgument, ...) -> String`` :spec: ``sanitize(value: AnyArgument) -> String``
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS. for file/directory names on any OS.

View file

@ -31,7 +31,7 @@ supply a cookies file path.
# Directly set plugin options: # Directly set plugin options:
ytdl_options: ytdl_options:
cookiefile: "/config/ytdl-sub-configs/cookie.txt" cookiefile: "/config/cookie.txt"
Layout Layout
------ ------

View file

@ -1,15 +1,6 @@
Deprecation Notices Deprecation Notices
=================== ===================
Dec 2025
--------
Override variables names can no longer be plugin names, to avoid the common pitfall of
defining a plugin underneath ``overrides``.
In the past, there was usage of a ``date_range`` override variable in a few example configs
that complimented the ``Only Recent`` preset. This overrride variable usage needs to be
replaced with ``only_recent_date_range``.
Sep 2024 Sep 2024
-------- --------

View file

@ -1,80 +1,51 @@
Automating Automating Downloads
========== ====================
Automate downloading your subscriptions by running the :ref:`'sub' sub-command :ref:`Guide for Docker and Unraid Containers <guides/getting_started/automating_downloads:docker and unraid>`
<usage:subscriptions options>` periodically. There are various tools that can run
commands on a schedule you may use any of them that work with your installation
method. Most users use `cron`_ in `Docker containers <docker and unraid_>`_.
:ref:`Guide for Linux <guides/getting_started/automating_downloads:linux>`
:ref:`Guide for Windows <guides/getting_started/automating_downloads:windows>`
.. _cron scheduling syntax: https://crontab.guru/#0_*/6_*_*_*
.. _docker-unraid-setup:
Docker and Unraid Docker and Unraid
----------------- -----------------
:doc:`The 'ytdl-sub' Docker container images <../install/docker>` provide optional cron Cron is preconfigured in every ytdl-sub docker container. Enable by adding the following
support. Enable cron support by setting `a cron schedule`_ in the ``CRON_SCHEDULE`` ENV variables to your docker setup.
environment variable:
.. code-block:: yaml .. code-block:: yaml
:caption: ./compose.yaml
:emphasize-lines: 4
services: services:
ytdl-sub: ytdl-sub:
environment: environment:
CRON_SCHEDULE: "0 */6 * * *" - CRON_SCHEDULE="0 */6 * * *"
# WARNING: See "Getting Started" -> "Automating" docs regarding throttles/bans: - CRON_RUN_ON_START=false
# CRON_RUN_ON_START: false
Then recreate the container to apply the change and start it to generate the default
``/config/ytdl-sub-configs/cron`` script. Read the comments in that script and edit as
appropriate.
The container cron wrapper script will write output from the cron job to - ``CRON_SCHEDULE`` follows the standard `cron scheduling syntax`_. The above value will
``/config/ytdl-sub-configs/.cron.log``. The default image ``ENTRYPOINT`` will ``$ tail run the script once every 6 hours.
...`` that file so you can monitor the cron job in the container's output and thus also - ``CRON_RUN_ON_START`` toggles whether to run your cron script on container start in
in the Docker logs. addition to the cron schedule.
You may also set the ``CRON_RUN_ON_START`` environment variable to ``true`` to have the The cron script will reside in the main directory with the file name ``cron``. Cron
image run your cron script whenever the container starts in addition to the cron logs should show when viewing the Docker logs.
schedule.
.. warning::
Using ``CRON_RUN_ON_START`` may cause your cron script to run too often and may
trigger throttles and bans. When enabled, your cron script will run *whenever* the
container starts including when the host reboots, when ``# dockerd`` restarts such as
when upgrading Docker itself, when a new image is pulled, when something applies
Compose changes, etc.. This may result in running ``ytdl-sub`` right before or after
the next cron scheduled run.
.. _linux-setup: .. _linux-setup:
Linux, Mac OS X, BSD, or other UNIX's Linux
------------------------------------- -----
Must configure crontab manually, like so:
For installations on systems already running ``# crond``, you can also use cron to run
``ytdl-sub`` periodically. Write a script to run ``ytdl-sub`` in the cron job. Be sure
the script changes to the same directory as your configuration and uses the full path to
``ytdl-sub``:
.. code-block:: shell .. code-block:: shell
:caption: ~/.local/bin/ytdl-sub-cron
:emphasize-lines: 2,3
#!/bin/bash crontab -e
cd "~/.config/ytdl-sub/" 0 */6 * * * /config/run_cron
~/.local/bin/ytdl-sub --dry-run sub -o '--ytdl_options.max_downloads 3' |&
tee -a "~/.local/state/ytdl-sub/.cron.log"
Then tell ``# crond`` when to run the script:
.. code-block:: console
echo "0 */6 * * * ${HOME}/.local/bin/ytdl-sub-cron" | crontab "-"
Remove the ``--dry-run`` and ``-o ...`` CLI options from your cron script when you've
tested your configuration and you're ready to download entries unattended.
.. _windows-setup: .. _windows-setup:
@ -82,51 +53,17 @@ tested your configuration and you're ready to download entries unattended.
Windows Windows
------- -------
For most Windows users, the best way to run commands periodically is `the Task To be tested (please contact code owner or join the discord server if you can test this
Scheduler`_: out for us)
.. attention:: .. code-block:: powershell
These instructions are untested. Use at your own risk. If you use them, whether they ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml
work or not, please let us know how it went in `a support post in Discord`_ or `a new
GitHub issue`_.
#. Open the Task Scheduler app.
#. Click ``Create Basic Task`` at the top of the right sidebar.
#. Set all the fields as appropriate until you get to the ``Action``...
#. For the ``Action``, select ``Start a program``...
#. Click ``Browse...`` to the installed ``ytdl-sub.exe`` executable...
#. Add CLI arguments to ``Add arguments (optional):``, for example ``--dry-run sub -o
'--ytdl_options.max_downloads 3'``...
#. Set ``Start in (optional):`` to the directory containing your configuration.
#. Finish the rest of the ``Create Basic Task`` wizard.
Next Steps Next Steps
---------- ----------
At this point, ``ytdl-sub`` should run periodically and keep your subscriptions current Once you have a significant quantity of subscriptions or have use cases not served using
in your media library without your intervention. As your :doc:`subscriptions file :doc:`YAML keys and the special characters <./subscriptions>`, it's time to start
<./subscriptions>` grows or you discover new use cases, it becomes worth while to :doc:`defining your own custom presets <./first_config>`.
simplify things by :doc:`defining your own custom presets <./first_config>`.
.. _`cron`:
https://en.wikipedia.org/wiki/Cron
.. _`a cron schedule`:
https://crontab.cronhub.io/
.. _`the Task Scheduler`:
https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-start-page
.. _`a support post in Discord`:
https://discord.com/channels/994270357957648404/1084886228266127460
.. _`a new GitHub issue`:
https://github.com/jmbannon/ytdl-sub/issues/new

View file

@ -3,33 +3,13 @@ Basic Configuration
A configuration file serves two purposes: A configuration file serves two purposes:
1. Set application-level functionality that is not specifiable in a subscription file. 1. Set advanced functionality that is not specifiable in a subscription file, such as
working directory location. These are set underneath ``configuration``.
2. Create custom presets, which can drastically simplify your subscription file. These
are defined underneath ``presets``. Presets are intended to be applicable and
reusable across multiple subscriptions.
.. note:: Below is a common configuration:
ytdl-sub does not require a configuration file. However,
certain application settings may be desirable for tweak, such as setting
``working_directory`` to make ytdl-sub perform the initial download
to an SSD drive.
2. Create custom presets.
.. note::
In the prior Initial Subscription examples, we leveraged the prebuilt preset
``Jellyfin TV Show by Date``. This preset is entirely built using the same
YAML configuration system offered to users by using a configuration file.
The following section attempts to demystify and explain how to...
- Set an application setting
- Know whether or not custom presets are actually needed
- How to create a custom preset
- How to use a custom preset on subscriptions
-------------
how this works, and show-case how
.. code-block:: yaml .. code-block:: yaml
:linenos: :linenos:
@ -242,22 +222,3 @@ Be sure to tell ytdl-sub to use your config by using the argument ``--config
If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``, If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``,
it will use it by default. it will use it by default.
Visualizing a subscription in Preset form
-----------------------------------------
Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions.
You can unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*.
.. code-block:: bash
ytdl-sub inspect --match "BBC News" /path/to/subscriptions.yaml
This can be utilized for numerous purposes including:
* Ensuring your custom preset is getting applied correctly.
* Figuring out which variables set things like file names, metadata, etc.
* Understanding how subscription syntax translates to preset representation.
The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will
present the subscription's raw layout with no fill.

View file

@ -134,7 +134,7 @@ use no preset at all and will just run ``yt-dlp`` without any customization or p
processing. The subscriptions file has special support for :ref:`overriding the presets processing. The subscriptions file has special support for :ref:`overriding the presets
of all subscriptions in the file <config_reference/subscription_yaml:file preset>`. The of all subscriptions in the file <config_reference/subscription_yaml:file preset>`. The
configuration file supports :ref:`a few special options configuration file supports :ref:`a few special options
<config_reference/config_yaml:Configuration File>` that are not about defining presets. See <config_reference/config_yaml:configuration>` that are not about defining presets. See
:doc:`the reference documentation <../../config_reference/index>` for technically :doc:`the reference documentation <../../config_reference/index>` for technically
complete details, but for almost all of the use cases served by ``ytdl-sub``, the above complete details, but for almost all of the use cases served by ``ytdl-sub``, the above
is accurate and representative. is accurate and representative.

View file

@ -38,13 +38,6 @@ For example::
$ docker compose run --rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub" ytdl-sub sub $ docker compose run --rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub" ytdl-sub sub
.. note::
In `the recommended GUI image <gui image_>`_, the ``DEFAULT_WORKSPACE`` directory is
``/config/ytdl-sub-configs/`` which is used throughout the documentation and
examples. In the headless images, that directory is just ``/config/``, so substitute
that path if using a headless image.
Install with Docker Compose Install with Docker Compose
--------------------------- ---------------------------

View file

@ -1,6 +1,3 @@
============== ==============
Helper Presets Helper Presets
============== ==============
@ -197,41 +194,3 @@ to your subscription, like so:
url: "https://youtube.com/@channel" url: "https://youtube.com/@channel"
resolution_assert_ignore_titles: resolution_assert_ignore_titles:
- "This 360p Video Title" - "This 360p Video Title"
_url
----
All prebuilt presets share the same internal ``_multi_url`` preset which comes equipped with
a few available customizations.
Sibling Metadata
~~~~~~~~~~~~~~~~
*Sibling* refers to any entry within the same *playlist*. For channel downloads, this would
imply **every** video that gets downloaded since yt-dlp treats the channel as the *playlist*.
Setting the variable ``include_sibling_metadata`` will include all sibling metadata within
each individual entry's metadata. This is used specifically for music presets. When downloading
a playlist as an album for example, it will take the max year amongst all the other sibling's metadata
to have a consistent album year that can be used in file or directory naming.
Webpage URL
~~~~~~~~~~~
``ytdl-sub`` performs downloads in two stages.
1. Metadata scrape from the original URL
2. Individual entry downloads
For step 2, ``ytdl-sub`` will use the ``webpage_url`` variable by default for the input URL to yt-dlp.
This can be modified in case it's not working as expected by using the variable ``modified_webpage_url``.
Example:
.. code-block:: yaml
:caption:
Removes yt-dlp smuggle data from the URL
overrides:
modified_webpage_url: >-
{ %regex_sub("#__youtubedl_smuggle=.*", "", webpage_url) }

View file

@ -160,8 +160,6 @@ Two main use cases of a collection are:
2. Organize one or more YouTube channels/playlists, where each season represents a 2. Organize one or more YouTube channels/playlists, where each season represents a
separate channel/playlist. separate channel/playlist.
Today, ytdl-supports up to 40 seasons with 11 URLs per season.
Example Example
~~~~~~~ ~~~~~~~
@ -187,29 +185,6 @@ Must define ``tv_show_directory``. Available presets:
s02_name: "Covers" s02_name: "Covers"
s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W" s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
Other notable features include:
* TV show poster info is pulled from the first URL in s01.
* Duplicate videos in different URLs (channel /videos vs playlist) will not download twice.
* The video will attributed to the season with the highest number.
* Individual seasons support both single and multi URL.
* s00 is supported for specials.
.. code-block:: yaml
"~Beyond the Guitar":
s00_name: "Specials"
s00_url:
- "https://www.youtube.com/watch?v=vXzguOdulAI"
- "https://www.youtube.com/watch?v=IGwYDvaGAz0"
s01_name: "Videos"
s01_url:
- "https://www.youtube.com/c/BeyondTheGuitar"
- "https://www.youtube.com/@BeyondTheGuitarAcademy"
s02_name: "Covers"
s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
Advanced Usage Advanced Usage
~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~

View file

@ -87,7 +87,8 @@ Using the command:
--overrides.tv_show_name "Rick A" \ --overrides.tv_show_name "Rick A" \
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
See how to shorten commands using :ref:`download aliases <config_reference/config_yaml:dl_aliases>`. See how to shorten commands using `download aliases
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
View Options View Options
@ -104,43 +105,3 @@ Preview the source variables for a given URL. Helpful to create new subscription
-sc, --split-chapters -sc, --split-chapters
View source variables after splitting by chapters View source variables after splitting by chapters
CLI to SUB Options
------------------
Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments.
.. code-block::
ytdl-sub cli-to-sub [YT-DLP ARGS]
Inspect
-------
Inspect a single subscription's underlying preset representation.
This can be utilized for numerous purposes including:
* Ensuring your custom preset is getting applied correctly.
* Figuring out which variables set things like file names, metadata, etc.
* Understanding how subscription syntax translates to preset representation.
Usage:
.. code-block:: bash
ytdl-sub inspect --match "Game Chops" --mock 'title=Lets Play' examples/music_subscriptions.yaml
.. code-block:: text
:caption: Additional Options
-l 0,1,2,3, --level 0,1,2,3
level of inspection to perform:
0 - original present the subscription as-is
1 - fill fill in defined values
2 - resolve resolve all possible variables (default)
3 - internal resolve all variables to their internal representation
-m MATCH [MATCH ...], --match MATCH [MATCH ...]
match subscription names to one or more substrings, and only run those subscriptions
-o DL_OVERRIDE, --dl-override DL_OVERRIDE
override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
-k VAR=VALUE, --mock VAR=VALUE
ability to mock one or more variable values, i.e. --mock 'title=Lets Play'

View file

@ -69,7 +69,7 @@ presets:
# ytdl_options lets you pass any arg into yt-dlp's Python API # ytdl_options lets you pass any arg into yt-dlp's Python API
ytdl_options: ytdl_options:
# Set the cookie file # Set the cookie file
# cookiefile: "/config/ytdl-sub-configs/youtube_cookies.txt" # cookiefile: "/config/youtube_cookies.txt"
# For YouTube, get English metadata if multiple languages are present # For YouTube, get English metadata if multiple languages are present
extractor_args: extractor_args:

View file

@ -24,6 +24,6 @@ TV Show Only Recent:
# to set only for that subscriptions # to set only for that subscriptions
"~BBC News": "~BBC News":
url: "https://www.youtube.com/@BBCNews" # use url2, url3, ... for multi-url in this form url: "https://www.youtube.com/@BBCNews" # use url2, url3, ... for multi-url in this form
only_recent_date_range: "2weeks" date_range: "2weeks"
"Frontline PBS": "https://www.youtube.com/@frontline" "Frontline PBS": "https://www.youtube.com/@frontline"
"Whitehouse": "https://www.bitchute.com/channel/zWsYVmCOu4JA/" # Supports non-YT sites "Whitehouse": "https://www.bitchute.com/channel/zWsYVmCOu4JA/" # Supports non-YT sites

View file

@ -15,7 +15,7 @@ classifiers = [
"Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.11",
] ]
dependencies = [ dependencies = [
"yt-dlp[default]==2026.6.9", "yt-dlp[default]==2025.9.26",
"colorama~=0.4", "colorama~=0.4",
"mergedeep~=1.3", "mergedeep~=1.3",
"mediafile~=0.12", "mediafile~=0.12",
@ -43,15 +43,16 @@ where = ["src"]
[project.optional-dependencies] [project.optional-dependencies]
test = [ test = [
"coverage[toml]>=6.3,<8.0", "coverage[toml]>=6.3,<8.0",
"pytest>=7.2,<10.0", "pytest>=7.2,<9.0",
"pytest-rerunfailures>=14,<17", "pytest-rerunfailures>=14,<17",
] ]
lint = [ lint = [
"pylint==4.0.5", "black==24.10.0",
"ruff==0.15.16", "isort==6.0.1",
"pylint==3.3.8",
] ]
docs = [ docs = [
"sphinx>=7,<10", "sphinx>=7,<9",
"sphinx-rtd-theme>=2,<4", "sphinx-rtd-theme>=2,<4",
"sphinx-book-theme~=1.0", "sphinx-book-theme~=1.0",
"sphinx-copybutton~=0.5", "sphinx-copybutton~=0.5",
@ -65,6 +66,15 @@ build = [
[project.scripts] [project.scripts]
ytdl-sub = "ytdl_sub.main:main" ytdl-sub = "ytdl_sub.main:main"
[tool.isort]
profile = "black"
line_length = 100
force_single_line = true
[tool.black]
line_length = 100
target-version = ["py310"]
[tool.pylint.MASTER] [tool.pylint.MASTER]
disable = [ disable = [
"C0115", # Missing class docstring "C0115", # Missing class docstring
@ -90,27 +100,3 @@ include = [
exclude_also = [ exclude_also = [
"raise UNREACHABLE.*", "raise UNREACHABLE.*",
] ]
# ruff
[tool.ruff]
line-length = 100
indent-width = 4
# Assume Python 3.10
target-version = "py310"
[tool.ruff.lint]
extend-select = ["I"]
[tool.ruff.format]
# Like Black, use double quotes for strings.
quote-style = "double"
# Like Black, indent with spaces, rather than tabs.
indent-style = "space"
# Like Black, respect magic trailing commas.
skip-magic-trailing-comma = false
# Like Black, automatically detect the appropriate line ending.
line-ending = "auto"

View file

@ -1,31 +1,28 @@
import gc import gc
import os import os
import random
import sys import sys
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional from typing import Dict
from typing import List
from typing import Optional
from yt_dlp.utils import sanitize_filename from yt_dlp.utils import sanitize_filename
from ytdl_sub.cli.output_summary import output_summary from ytdl_sub.cli.output_summary import output_summary
from ytdl_sub.cli.output_transaction_log import ( from ytdl_sub.cli.output_transaction_log import _maybe_validate_transaction_log_file
_maybe_validate_transaction_log_file, from ytdl_sub.cli.output_transaction_log import output_transaction_log
output_transaction_log,
)
from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
from ytdl_sub.cli.parsers.dl import DownloadArgsParser from ytdl_sub.cli.parsers.dl import DownloadArgsParser
from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME, parser from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME
from ytdl_sub.cli.parsers.main import parser
from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.validators.variable_validation import ResolutionLevel
from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled, ValidationException from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_lock import working_directory_lock from ytdl_sub.utils.file_lock import working_directory_lock
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
# pylint: disable=too-many-branches
logger = Logger.get() logger = Logger.get()
# View is a command to run a simple dry-run on a URL using the `_view` preset. # View is a command to run a simple dry-run on a URL using the `_view` preset.
@ -77,7 +74,6 @@ def _download_subscriptions_from_yaml_files(
subscription_override_dict: Dict, subscription_override_dict: Dict,
update_with_info_json: bool, update_with_info_json: bool,
dry_run: bool, dry_run: bool,
shuffle: bool,
) -> List[Subscription]: ) -> List[Subscription]:
""" """
Downloads all subscriptions from one or many subscription yaml files. Downloads all subscriptions from one or many subscription yaml files.
@ -94,8 +90,6 @@ def _download_subscriptions_from_yaml_files(
Whether to actually download or update using existing info json Whether to actually download or update using existing info json
dry_run dry_run
Whether to dry run or not Whether to dry run or not
shuffle
Whether to shuffle the subscription download order
Returns Returns
------- -------
@ -117,10 +111,6 @@ def _download_subscriptions_from_yaml_files(
subscription_override_dict=subscription_override_dict, subscription_override_dict=subscription_override_dict,
) )
if shuffle:
logger.info("Shuffling subscriptions")
random.shuffle(subscriptions)
for subscription in subscriptions: for subscription in subscriptions:
with subscription.exception_handling(): with subscription.exception_handling():
logger.info( logger.info(
@ -205,44 +195,6 @@ def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Su
return subscription return subscription
def _parse_inspect_mocks(mocks: Optional[List[str]]) -> Dict[str, str]:
out: Dict[str, str] = {}
for mock in mocks or []:
spl = mock.split("=", 1)
if len(spl) == 1:
raise ValidationException("inspect mock must be in the form of VAR=VALUE")
out[spl[0].strip()] = spl[1]
return out
def _inspect(
config: ConfigFile,
subscription_paths: List[str],
subscription_matches: List[str],
subscription_override_dict: Dict,
inspection_level: int,
mocks: Dict[str, str],
) -> None:
subscriptions: List[Subscription] = []
for path in subscription_paths:
subscriptions += Subscription.from_file_path(
config=config,
subscription_path=path,
subscription_matches=subscription_matches,
subscription_override_dict=subscription_override_dict,
)
if len(subscriptions) > 1:
print(
"inspect can only inspect a single subscription. Use --match to filter for a single one"
)
return
print(subscriptions[0].resolved_yaml(resolution_level=inspection_level, mocks=mocks))
def main() -> List[Subscription]: def main() -> List[Subscription]:
""" """
Entrypoint for ytdl-sub, without the error handling Entrypoint for ytdl-sub, without the error handling
@ -254,10 +206,6 @@ def main() -> List[Subscription]:
args, extra_args = parser.parse_known_args() args, extra_args = parser.parse_known_args()
if args.subparser == "cli-to-sub":
print_cli_to_sub(args=extra_args)
return []
# Load the config # Load the config
if args.config: if args.config:
config = ConfigFile.from_file_path(args.config) config = ConfigFile.from_file_path(args.config)
@ -269,23 +217,6 @@ def main() -> List[Subscription]:
subscriptions: List[Subscription] = [] subscriptions: List[Subscription] = []
if args.subparser == "inspect":
subscription_override_dict = {}
if args.dl_override:
subscription_override_dict = DownloadArgsParser.from_dl_override(
override=args.dl_override, config=config
).to_subscription_dict()
_inspect(
config=config,
subscription_paths=args.subscription_paths,
subscription_matches=args.match,
subscription_override_dict=subscription_override_dict,
inspection_level=ResolutionLevel.level_number(args.inspection_level),
mocks=_parse_inspect_mocks(args.mock),
)
return []
# If transaction log file is specified, make sure we can open it # If transaction log file is specified, make sure we can open it
_maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log) _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)
@ -317,7 +248,6 @@ def main() -> List[Subscription]:
subscription_override_dict=subscription_override_dict, subscription_override_dict=subscription_override_dict,
update_with_info_json=args.update_with_info_json, update_with_info_json=args.update_with_info_json,
dry_run=args.dry_run, dry_run=args.dry_run,
shuffle=args.shuffle,
) )
# One-off download # One-off download
@ -333,7 +263,7 @@ def main() -> List[Subscription]:
_view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters) _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters)
) )
else: else:
raise ValidationException("Must provide one of the commands: sub, dl, view, cli-to-sub") raise ValidationException("Must provide one of the commands: sub, dl, view")
if not args.suppress_transaction_log: if not args.suppress_transaction_log:
output_transaction_log( output_transaction_log(

View file

@ -1,4 +1,5 @@
from typing import List, Optional from typing import List
from typing import Optional
from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException

View file

@ -1,63 +0,0 @@
from typing import List
import yt_dlp
import yt_dlp.options
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.yaml import dump_yaml
logger = Logger.get()
# pylint: disable=missing-function-docstring
##############################################################
# --- BEGIN ----
# Copy of https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
create_parser = yt_dlp.options.create_parser
def parse_patched_options(opts):
patched_parser = create_parser()
patched_parser.defaults.update(
{
"ignoreerrors": False,
"retries": 0,
"fragment_retries": 0,
"extract_flat": False,
"concat_playlist": "never",
"update_self": False,
}
)
yt_dlp.options.create_parser = lambda: patched_parser
try:
return yt_dlp.parse_options(opts)
finally:
yt_dlp.options.create_parser = create_parser
default_opts = parse_patched_options([]).ydl_opts
def cli_to_api(opts, cli_defaults=False):
opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
diff = {k: v for k, v in opts.items() if default_opts[k] != v}
if "postprocessors" in diff:
diff["postprocessors"] = [
pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]
]
return diff
# --- END ----
##############################################################
def print_cli_to_sub(args: List[str]) -> None:
api_args = cli_to_api(args)
if not api_args:
logger.info("Does not resolve to any yt-dlp args")
return
print(dump_yaml({"ytdl_options": api_args}))

View file

@ -1,7 +1,10 @@
import hashlib import hashlib
import re import re
import shlex import shlex
from typing import Any, Dict, List, Tuple from typing import Any
from typing import Dict
from typing import List
from typing import Tuple
from mergedeep import mergedeep from mergedeep import mergedeep

View file

@ -1,6 +1,6 @@
import argparse import argparse
import dataclasses import dataclasses
from typing import Dict, List from typing import List
from ytdl_sub import __local_version__ from ytdl_sub import __local_version__
from ytdl_sub.utils.logger import LoggerLevels from ytdl_sub.utils.logger import LoggerLevels
@ -172,10 +172,6 @@ class SubArguments:
short="-o", short="-o",
long="--dl-override", long="--dl-override",
) )
SHUFFLE = CLIArgument(
short="-sh",
long="--shuffle",
)
subscription_parser = subparsers.add_parser("sub") subscription_parser = subparsers.add_parser("sub")
@ -201,13 +197,6 @@ subscription_parser.add_argument(
help="override all subscription config values using `dl` syntax, " help="override all subscription config values using `dl` syntax, "
"i.e. --dl-override='--ytdl_options.max_downloads 3'", "i.e. --dl-override='--ytdl_options.max_downloads 3'",
) )
subscription_parser.add_argument(
SubArguments.SHUFFLE.short,
SubArguments.SHUFFLE.long,
action="store_true",
help="shuffle subscription order when downloading",
default=False,
)
################################################################################################### ###################################################################################################
# DOWNLOAD PARSER # DOWNLOAD PARSER
@ -232,81 +221,3 @@ view_parser.add_argument(
help="View source variables after splitting by chapters", help="View source variables after splitting by chapters",
) )
view_parser.add_argument("url", help="URL to view source variables for") view_parser.add_argument("url", help="URL to view source variables for")
###################################################################################################
# CLI-TO-SUB PARSER
cli_to_sub_parser = subparsers.add_parser("cli-to-sub")
###################################################################################################
# INSPECT PARSER
class InspectArguments:
LEVEL = CLIArgument(
short="-l",
long="--level",
)
LevelChoices: Dict[str, str] = {
"0": "original",
"1": "fill",
"2": "resolve",
"3": "internal",
}
MOCK = CLIArgument(
short="-k",
long="--mock",
)
inspect_parser = subparsers.add_parser("inspect", formatter_class=argparse.RawTextHelpFormatter)
inspect_parser.add_argument(
InspectArguments.LEVEL.short,
InspectArguments.LEVEL.long,
metavar=",".join(str(i) for i in range(4)),
type=str,
help="""level of inspection to perform:
0 - original present the subscription as-is
1 - fill fill in defined values
2 - resolve resolve all possible variables (default)
3 - internal resolve all variables to their internal representation
""",
default="resolve",
choices=list(InspectArguments.LevelChoices.keys())
+ list(InspectArguments.LevelChoices.values()),
dest="inspection_level",
)
inspect_parser.add_argument(
MainArguments.MATCH.short,
MainArguments.MATCH.long,
dest="match",
nargs="+",
action="extend",
type=str,
help="match subscription names to one or more substrings, and only run those subscriptions",
default=[],
)
inspect_parser.add_argument(
"subscription_paths",
metavar="SUBPATH",
nargs="*",
help="path to subscription files, uses subscriptions.yaml if not provided",
default=["subscriptions.yaml"],
)
inspect_parser.add_argument(
SubArguments.OVERRIDE.short,
SubArguments.OVERRIDE.long,
type=str,
help="override all subscription config values using `dl` syntax, "
"i.e. --dl-override='--ytdl_options.max_downloads 3'",
)
inspect_parser.add_argument(
InspectArguments.MOCK.short,
InspectArguments.MOCK.long,
metavar="VAR=VALUE",
action="append",
help="ability to mock one or more variable values, i.e. --mock 'title=Lets Play'",
)

View file

@ -1,5 +1,6 @@
import os import os
from typing import Any, Dict from typing import Any
from typing import Dict
from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset import Preset

View file

@ -1,34 +1,27 @@
import os import os
import posixpath import posixpath
from typing import Any, Dict, Optional from typing import Any
from typing import Dict
from typing import Optional
from mergedeep import mergedeep from mergedeep import mergedeep
from yt_dlp.utils import datetime_from_str from yt_dlp.utils import datetime_from_str
from ytdl_sub.config.defaults import ( from ytdl_sub.config.defaults import DEFAULT_FFMPEG_PATH
DEFAULT_FFMPEG_PATH, from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH
DEFAULT_FFPROBE_PATH, from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY
DEFAULT_LOCK_DIRECTORY, from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
MAX_FILE_NAME_BYTES,
)
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
from ytdl_sub.utils.exceptions import SubscriptionPermissionError from ytdl_sub.validators.file_path_validators import FFmpegFileValidator
from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.validators.file_path_validators import FFprobeFileValidator
from ytdl_sub.validators.file_path_validators import FFmpegFileValidator, FFprobeFileValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import ( from ytdl_sub.validators.validators import BoolValidator
BoolValidator, from ytdl_sub.validators.validators import IntValidator
IntValidator, from ytdl_sub.validators.validators import LiteralDictValidator
LiteralDictValidator, from ytdl_sub.validators.validators import StringValidator
StringValidator,
)
class ExperimentalValidator(StrictDictValidator): class ExperimentalValidator(StrictDictValidator):
"""
Experimental flags reside under the ``experimental`` key.
"""
_optional_keys = {"enable_update_with_info_json"} _optional_keys = {"enable_update_with_info_json"}
_allow_extra_keys = True _allow_extra_keys = True
@ -50,11 +43,6 @@ class ExperimentalValidator(StrictDictValidator):
class PersistLogsValidator(StrictDictValidator): class PersistLogsValidator(StrictDictValidator):
"""
By default, no logs are persisted. Specifying this key will enable persisted logs. The following
options are available.
"""
_required_keys = {"logs_directory"} _required_keys = {"logs_directory"}
_optional_keys = {"keep_logs_after", "keep_successful_logs"} _optional_keys = {"keep_logs_after", "keep_successful_logs"}
@ -79,8 +67,8 @@ class PersistLogsValidator(StrictDictValidator):
@property @property
def logs_directory(self) -> str: def logs_directory(self) -> str:
""" """
Required field. Write log files to this directory with names like Write log files to this directory with names like
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. ``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. (required)
""" """
return self._logs_directory.value return self._logs_directory.value
@ -105,54 +93,15 @@ class PersistLogsValidator(StrictDictValidator):
@property @property
def keep_successful_logs(self) -> bool: def keep_successful_logs(self) -> bool:
""" """
Defaults to ``True``. When this key is ``False``, only write log files for failed If the ``persist_logs:`` key is in the configuration, then ``ytdl-sub`` *always*
subscriptions. writes log files for the subscription both for successful downloads and when it
encounters an error while downloading. When this key is ``False``, only write
log files for errors. (default ``True``)
""" """
return self._keep_successful_logs.value return self._keep_successful_logs.value
class ConfigOptions(StrictDictValidator): class ConfigOptions(StrictDictValidator):
"""
ytdl-sub is configured using a ``config.yaml`` file.
The ``config.yaml`` is made up of two sections:
.. code-block:: yaml
configuration:
presets:
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
If you prefer to use a Windows backslash, note that it must have
``C:\\\\double\\\\bashslash\\\\paths`` in order to escape the backslash character. This is due
to it being a YAML escape character.
.. code-block:: yaml
configuration:
dl_aliases:
mv: "--preset music_video"
u: "--download.url"
experimental:
enable_update_with_info_json: True
ffmpeg_path: "/usr/bin/ffmpeg"
ffprobe_path: "/usr/bin/ffprobe"
file_name_max_bytes: 255
lock_directory: "/tmp"
persist_logs:
keep_successful_logs: True
logs_directory: "/var/log/ytdl-sub-logs"
umask: "022"
working_directory: ".ytdl-sub-working-directory"
"""
_optional_keys = { _optional_keys = {
"working_directory", "working_directory",
"umask", "umask",
@ -198,18 +147,11 @@ class ConfigOptions(StrictDictValidator):
key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES
) )
if not FileHandler.is_path_writable(self.working_directory):
raise SubscriptionPermissionError(
"ytdl-sub does not have permissions to the working directory: "
f"{self.working_directory}"
)
@property @property
def working_directory(self) -> str: def working_directory(self) -> str:
""" """
The directory to temporarily store downloaded files before moving them into their final The directory to temporarily store downloaded files before moving them into their final
directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory directory. (default ``./.ytdl-sub-working-directory``)
that ytdl-sub is invoked from.
""" """
# Expands tildas to actual paths, use native os sep # Expands tildas to actual paths, use native os sep
return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep)) return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep))
@ -217,7 +159,7 @@ class ConfigOptions(StrictDictValidator):
@property @property
def umask(self) -> Optional[str]: def umask(self) -> Optional[str]:
""" """
Umask in octal format to apply to every created file. Defaults to ``022``. Umask in octal format to apply to every created file. (default ``022``)
""" """
return self._umask.value return self._umask.value
@ -226,7 +168,7 @@ class ConfigOptions(StrictDictValidator):
""" """
.. _dl_aliases: .. _dl_aliases:
Alias definitions to shorten :ref:`dl arguments <usage:Download Options>`. For example, Alias definitions to shorten ``ytdl-sub dl`` arguments. For example,
.. code-block:: yaml .. code-block:: yaml
@ -278,23 +220,23 @@ class ConfigOptions(StrictDictValidator):
The directory to temporarily store file locks, which prevents multiple instances The directory to temporarily store file locks, which prevents multiple instances
of ``ytdl-sub`` from running. Note that file locks do not work on of ``ytdl-sub`` from running. Note that file locks do not work on
network-mounted directories. Ensure that this directory resides on the host network-mounted directories. Ensure that this directory resides on the host
machine. Defaults to ``/tmp``. machine. (default ``/tmp``)
""" """
return self._lock_directory.value return self._lock_directory.value
@property @property
def ffmpeg_path(self) -> str: def ffmpeg_path(self) -> str:
""" """
Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux, Path to ffmpeg executable. (default ``/usr/bin/ffmpeg`` for Linux,
``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows. ``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows)
""" """
return self._ffmpeg_path.value return self._ffmpeg_path.value
@property @property
def ffprobe_path(self) -> str: def ffprobe_path(self) -> str:
""" """
Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux, Path to ffprobe executable. (default ``/usr/bin/ffprobe`` for Linux,
``./ffprobe.exe`` in the same directory as ytdl-sub for Windows. ``./ffprobe.exe`` in the same directory as ytdl-sub for Windows)
""" """
return self._ffprobe_path.value return self._ffprobe_path.value

View file

@ -2,8 +2,6 @@ import os
from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.utils.system import IS_WINDOWS
# pylint: disable=invalid-name
def _existing_path(*paths: str) -> str: def _existing_path(*paths: str) -> str:
""" """

View file

@ -1,30 +1,26 @@
from typing import Any, Dict, Iterable, Optional, Set, Type, TypeVar from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
import mergedeep
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.variables.override_variables import ( from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
REQUIRED_OVERRIDE_VARIABLE_NAMES, from ytdl_sub.entries.variables.override_variables import OverrideHelpers
OverrideHelpers,
)
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script from ytdl_sub.script.script import Script
from ytdl_sub.script.types.function import BuiltInFunction from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import Resolvable, String
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.utils.exceptions import ( from ytdl_sub.utils.exceptions import InvalidVariableNameException
InvalidVariableNameException, from ytdl_sub.utils.exceptions import StringFormattingException
StringFormattingException, from ytdl_sub.utils.exceptions import ValidationException
ValidationException,
)
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.string_formatter_validators import ( from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
StringFormatterValidator, from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
UnstructuredDictFormatterValidator, from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
)
ExpectedT = TypeVar("ExpectedT")
class Overrides(UnstructuredDictFormatterValidator, Scriptable): class Overrides(UnstructuredDictFormatterValidator, Scriptable):
@ -92,24 +88,6 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
return True return True
def ensure_variable_names_not_a_plugin(self, plugin_names: Iterable[str]) -> None:
"""
Throws an error if an override variable or function has the same name as a
preset key. This is to avoid confusion when accidentally defining things in
overrides that are meant to be in the preset.
"""
for name in self.keys:
if name.startswith("%"):
name = name[1:]
if name in plugin_names:
raise self._validation_exception(
f"Override variable with name {name} cannot be used since it is"
" the name of a plugin. Perhaps you meant to define it as a plugin? If so,"
" indent it left to make it at the same level as overrides.",
exception_class=InvalidVariableNameException,
)
def ensure_variable_name_valid(self, name: str) -> None: def ensure_variable_name_valid(self, name: str) -> None:
""" """
Ensures the variable name does not collide with any entry variables or built-in functions. Ensures the variable name does not collide with any entry variables or built-in functions.
@ -137,35 +115,29 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
) )
def initial_variables( def initial_variables(
self, unresolved_variables: Optional[Dict[str, SyntaxTree]] = None self, unresolved_variables: Optional[Dict[str, str]] = None
) -> Dict[str, SyntaxTree]: ) -> Dict[str, str]:
""" """
Returns Returns
------- -------
Variables and format strings for all Override variables + additional variables (Optional) Variables and format strings for all Override variables + additional variables (Optional)
""" """
initial_variables: Dict[str, SyntaxTree] = self.dict_with_parsed_format_strings initial_variables: Dict[str, str] = {}
if unresolved_variables: mergedeep.merge(
initial_variables |= unresolved_variables initial_variables,
return ScriptUtils.add_sanitized_parsed_variables(initial_variables) self.dict_with_format_strings,
unresolved_variables if unresolved_variables else {},
)
return ScriptUtils.add_sanitized_variables(initial_variables)
def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides":
""" """
Initialize the override script with any unresolved variables Initialize the override script with any unresolved variables
""" """
self.script.add_parsed( self.script.add(
self.initial_variables( self.initial_variables(
unresolved_variables={ unresolved_variables={
var_name: SyntaxTree( var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
ast=[
BuiltInFunction(
name="throw",
args=[
String(f"Plugin variable {var_name} has not been created yet")
],
)
]
)
for var_name in unresolved_variables for var_name in unresolved_variables
} }
) )
@ -186,15 +158,10 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
script = entry.script script = entry.script
unresolvable = entry.unresolvable unresolvable = entry.unresolvable
# Update the script internally so long as we are not supplying overrides
# that could alter the script with one-off state
update = function_overrides is None
try: try:
return script.resolve_once( return script.resolve_once(
dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
unresolvable=unresolvable, unresolvable=unresolvable,
update=update,
)["tmp_var"] )["tmp_var"]
except ScriptVariableNotResolved as exc: except ScriptVariableNotResolved as exc:
raise StringFormattingException( raise StringFormattingException(
@ -209,8 +176,7 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
formatter: StringFormatterValidator, formatter: StringFormatterValidator,
entry: Optional[Entry] = None, entry: Optional[Entry] = None,
function_overrides: Optional[Dict[str, str]] = None, function_overrides: Optional[Dict[str, str]] = None,
expected_type: Type[ExpectedT] = str, ) -> str:
) -> ExpectedT:
""" """
Parameters Parameters
---------- ----------
@ -220,8 +186,6 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
Optional. Entry to add source variables to the formatter Optional. Entry to add source variables to the formatter
function_overrides function_overrides
Optional. Explicit values to override the overrides themselves and source variables Optional. Explicit values to override the overrides themselves and source variables
expected_type
The expected type that should return. Defaults to string.
Returns Returns
------- -------
@ -232,15 +196,37 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
StringFormattingException StringFormattingException
If the formatter that is trying to be resolved cannot If the formatter that is trying to be resolved cannot
""" """
out = formatter.post_process( return formatter.post_process(
self._apply_to_resolvable( str(
formatter=formatter, entry=entry, function_overrides=function_overrides self._apply_to_resolvable(
).native formatter=formatter, entry=entry, function_overrides=function_overrides
)
)
) )
if not isinstance(out, expected_type): def apply_overrides_formatter_to_native(
raise StringFormattingException( self,
f"Expected type {expected_type.__name__}, but received '{out.__class__.__name__}'" formatter: OverridesStringFormatterValidator,
) ) -> Any:
"""
Parameters
----------
formatter
Overrides formatter to apply
return out Returns
-------
The native python form of the resolved variable
"""
return self._apply_to_resolvable(
formatter=formatter, entry=None, function_overrides=None
).native
def evaluate_boolean(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> bool:
"""
Apply a formatter, and evaluate it to a boolean
"""
output = self.apply_formatter(formatter=formatter, entry=entry)
return ScriptUtils.bool_formatter_output(output)

View file

@ -1,15 +1,20 @@
from abc import ABC, abstractmethod from abc import ABC
from abc import abstractmethod
from functools import cached_property from functools import cached_property
from typing import Dict, Generic, List, Optional, Tuple, Type from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import OptionsValidatorT, ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import OptionsValidatorT
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.ytdl_additions.enhanced_download_archive import ( from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
DownloadArchiver, from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
EnhancedDownloadArchive,
)
# pylint: disable=unused-argument # pylint: disable=unused-argument
@ -43,7 +48,7 @@ class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
Returns True if enabled, False if disabled. Returns True if enabled, False if disabled.
""" """
if isinstance(self.plugin_options, ToggleableOptionsDictValidator): if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return self.overrides.apply_formatter(self.plugin_options.enable, expected_type=bool) return self.overrides.evaluate_boolean(self.plugin_options.enable)
return True return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:

View file

@ -1,12 +1,15 @@
from typing import Dict, List, Optional, Tuple, Type from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin, SplitPlugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.downloader import ( from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin
UrlDownloaderCollectionVariablePlugin, from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin
UrlDownloaderThumbnailPlugin,
)
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.date_range import DateRangePlugin

View file

@ -1,7 +1,11 @@
from typing import Iterable, List, Optional, Set, Tuple, Type from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator, OptionsValidatorT from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.config.validators.options import OptionsValidatorT
class PresetPlugins: class PresetPlugins:
@ -40,34 +44,3 @@ class PresetPlugins:
if plugin_type in plugin_option_types: if plugin_type in plugin_option_types:
return self.plugin_options[plugin_option_types.index(plugin_type)] return self.plugin_options[plugin_option_types.index(plugin_type)]
return None return None
def get_added_and_modified_variables(
self, additional_options: List[OptionsValidator]
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
for plugin_options in self.plugin_options + additional_options:
added_variables: Set[str] = set()
modified_variables: Set[str] = set()
for plugin_added_variables in plugin_options.added_variables(
unresolved_variables=set(),
).values():
added_variables |= set(plugin_added_variables)
for plugin_modified_variables in plugin_options.modified_variables().values():
modified_variables = plugin_modified_variables
yield plugin_options, added_variables, modified_variables
def get_all_variables(self, additional_options: List[OptionsValidator]) -> Set[str]:
"""
Returns set of all added and modified variables' names.
"""
all_variables: Set[str] = set()
for _, added, modified in self.get_added_and_modified_variables(additional_options):
all_variables.update(added)
all_variables.update(modified)
return all_variables

View file

@ -1,5 +1,7 @@
import copy import copy
from typing import Any, Dict, List, Set from typing import Any
from typing import Dict
from typing import List
from mergedeep import mergedeep from mergedeep import mergedeep
@ -7,14 +9,18 @@ from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions, YTDLOptions from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.variable_validation import VariableValidation
from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES, PUBLISHED_PRESET_NAMES from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.utils.yaml import dump_yaml
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import StringListValidator, validation_exception from ytdl_sub.validators.validators import StringListValidator
from ytdl_sub.validators.validators import validation_exception
PRESET_KEYS = { PRESET_KEYS = {
"preset", "preset",
@ -49,12 +55,6 @@ class _PresetShell(StrictDictValidator):
class Preset(_PresetShell): class Preset(_PresetShell):
"""
Custom presets are defined in this section. Refer to the
:ref:`Getting Started Guide<guides/getting_started/first_config:Basic Configuration>`
on how to configure.
"""
@classmethod @classmethod
def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None: def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None:
""" """
@ -172,37 +172,6 @@ class Preset(_PresetShell):
mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE) mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE)
) )
def _initialize_overrides_script(self, overrides: Overrides) -> Overrides:
"""
Do some gymnastics to initialize the Overrides script.
"""
unresolved_variables: Set[str] = set()
for (
plugin_options,
added_variables,
modified_variables,
) in self.plugins.get_added_and_modified_variables(
additional_options=[self.downloader_options, self.output_options]
):
for added_variable in added_variables:
if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable):
# pylint: disable=protected-access
raise plugin_options._validation_exception(
f"Cannot use the variable name {added_variable} because it exists as a"
" built-in ytdl-sub variable name."
)
# pylint: enable=protected-access
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
unresolved_variables |= added_variables | modified_variables
# Initialize overrides with unresolved variables + modified variables to throw an error.
# For modified variables, this is to prevent a resolve(update=True) to setting any
# dependencies until it has been explicitly added
return overrides.initialize_script(unresolved_variables=unresolved_variables)
def __init__(self, config: ConfigValidator, name: str, value: Any): def __init__(self, config: ConfigValidator, name: str, value: Any):
super().__init__(name=name, value=value) super().__init__(name=name, value=value)
@ -223,10 +192,13 @@ class Preset(_PresetShell):
) )
self.plugins: PresetPlugins = self._validate_and_get_plugins() self.plugins: PresetPlugins = self._validate_and_get_plugins()
self.overrides = self._initialize_overrides_script( self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
overrides=self._validate_key(key="overrides", validator=Overrides, default={})
) VariableValidation(
self.overrides.ensure_variable_names_not_a_plugin(plugin_names=PRESET_KEYS) downloader_options=self.downloader_options,
output_options=self.output_options,
plugins=self.plugins,
).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage()
@property @property
def name(self) -> str: def name(self) -> str:
@ -255,18 +227,11 @@ class Preset(_PresetShell):
""" """
return cls(config=config, name=preset_name, value=preset_dict) return cls(config=config, name=preset_name, value=preset_dict)
def yaml(self, subscription_only: bool) -> str: @property
def yaml(self) -> str:
""" """
Parameters
----------
subscription_only:
Only include the subscription contents, not the surrounding boiler-plate.
Returns Returns
------- -------
Preset in YAML format Preset in YAML format
""" """
if subscription_only:
return dump_yaml(self._value)
return dump_yaml({"presets": {self._name: self._value}}) return dump_yaml({"presets": {self._name: self._value}})

View file

@ -1,22 +1,21 @@
from typing import Any, Dict, Optional, Set from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.utils.exceptions import SubscriptionPermissionError, ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.file_path_validators import (
OverridesStringFormatterFilePathValidator,
StringFormatterFileNameValidator,
)
from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import ( from ytdl_sub.validators.string_formatter_validators import (
OverridesIntegerFormatterValidator,
OverridesStringFormatterValidator,
StandardizedDateValidator,
StringFormatterValidator,
UnstructuredOverridesDictFormatterValidator, UnstructuredOverridesDictFormatterValidator,
) )
from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import BoolValidator
@ -58,24 +57,12 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
def to_native_dict(self, overrides: Overrides) -> Dict: def to_native_dict(self, overrides: Overrides) -> Dict:
""" """
Materializes the entire ytdl-options dict from OverrideStringFormatters into Materializes the entire ytdl-options dict from OverrideStringFormatters into
native python. native python
""" """
out = { return {
key: overrides.apply_formatter(val, expected_type=object) key: overrides.apply_overrides_formatter_to_native(val)
for key, val in self.dict.items() for key, val in self.dict.items()
} }
if "cookiefile" in out:
if not FileHandler.is_file_existent(out["cookiefile"]):
raise ValidationException(
f"Specified cookiefile {out['cookiefile']} but it does not exist as a file."
)
if not FileHandler.is_file_readable(out["cookiefile"]):
raise SubscriptionPermissionError(
f"Cannot read cookiefile {out['cookiefile']} due to permissions issue."
)
return out
# Disable for proper docstring formatting # Disable for proper docstring formatting
@ -120,7 +107,6 @@ class OutputOptions(OptionsDictValidator):
"keep_max_files", "keep_max_files",
"download_archive_standardized_date", "download_archive_standardized_date",
"keep_files_date_eval", "keep_files_date_eval",
"preserve_mtime",
} }
@classmethod @classmethod
@ -184,10 +170,6 @@ class OutputOptions(OptionsDictValidator):
default=f"{{{v.upload_date_standardized.variable_name}}}", default=f"{{{v.upload_date_standardized.variable_name}}}",
) )
self._preserve_mtime = self._validate_key_if_present(
key="preserve_mtime", validator=BoolValidator, default=False
)
if ( if (
self._keep_files_before or self._keep_files_after or self._keep_max_files self._keep_files_before or self._keep_files_after or self._keep_max_files
) and not self.maintain_download_archive: ) and not self.maintain_download_archive:
@ -327,17 +309,6 @@ class OutputOptions(OptionsDictValidator):
""" """
return self._keep_max_files return self._keep_max_files
@property
def preserve_mtime(self) -> bool:
"""
:expected type: Optional[Boolean]
:description:
Preserve the video's original upload time as the file modification time.
When True, sets the file's mtime to match the video's upload_date from
yt-dlp metadata. Defaults to False.
"""
return self._preserve_mtime.value
def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]: def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]:
return { return {
# PluginOperation.MODIFY_ENTRY_METADATA: { # PluginOperation.MODIFY_ENTRY_METADATA: {

View file

@ -1,5 +1,7 @@
from abc import ABC from abc import ABC
from typing import Dict, Set, TypeVar from typing import Dict
from typing import Set
from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
@ -59,9 +61,9 @@ class ToggleableOptionsDictValidator(OptionsDictValidator):
_optional_keys = {"enable"} _optional_keys = {"enable"}
def __init__(self, name, value): def __init__(self, name, value):
assert "enable" in self._optional_keys, ( assert (
f"{self.__class__.__name__} does not have enable as an optional field" "enable" in self._optional_keys
) ), f"{self.__class__.__name__} does not have enable as an optional field"
super().__init__(name, value) super().__init__(name, value)
self._enable = self._validate_key( self._enable = self._validate_key(

View file

@ -1,4 +1,10 @@
from typing import Dict, List, Optional, Set import copy
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
@ -7,212 +13,203 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES, VARIABLES from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
from ytdl_sub.script.script import Script from ytdl_sub.script.script import Script
from ytdl_sub.script.utils.name_validation import is_function from ytdl_sub.script.script import _is_function
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.scriptable import BASE_SCRIPT
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
from ytdl_sub.validators.string_formatter_validators import validate_formatters from ytdl_sub.validators.string_formatter_validators import validate_formatters
# Entry variables to mock during validation
_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
name: to_variable_dependency_format_string(
# pylint: disable=protected-access
script=BASE_SCRIPT,
parsed_format_string=BASE_SCRIPT._variables[name],
# pylint: enable=protected-access
)
for name in BASE_SCRIPT.variable_names
}
class ResolutionLevel:
ORIGINAL = 0
FILL = 1
RESOLVE = 2
INTERNAL = 3
@classmethod def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
def name_of(cls, resolution_level: int) -> str: dummy_variables: Dict[str, str] = {}
""" for var in variables:
Name of the resolution level. dummy_variables[var] = ""
""" dummy_variables[f"{var}_sanitized"] = ""
if resolution_level == cls.ORIGINAL:
return "original"
if resolution_level == cls.FILL:
return "fill"
if resolution_level == cls.RESOLVE:
return "resolve"
if resolution_level == cls.INTERNAL:
return "internal"
raise ValueError("Invalid resolution level")
@classmethod return dummy_variables
def level_number(cls, resolution_arg: str) -> int:
"""
Numeric resolution level
"""
if resolution_arg in ("0", "original"):
return 0
if resolution_arg in ("1", "fill"):
return 1
if resolution_arg in ("2", "resolve"):
return 2
if resolution_arg in ("3", "internal"):
return 3
raise ValueError("Invalid resolution level")
@classmethod
def all(cls) -> List[int]: def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
""" # Have the dummy override variable contain all variable deps that it uses in the string
All possible resolution levels. dummy_overrides: Dict[str, str] = {}
""" for override_name in _override_variables(overrides):
return [cls.ORIGINAL, cls.FILL, cls.RESOLVE, cls.INTERNAL] if _is_function(override_name):
continue
# pylint: disable=protected-access
dummy_overrides[override_name] = to_variable_dependency_format_string(
script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
)
# pylint: enable=protected-access
return dummy_overrides
def _get_added_and_modified_variables(
plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
options.append(output_options)
for plugin_options in options:
added_variables: Set[str] = set()
modified_variables: Set[str] = set()
for plugin_added_variables in plugin_options.added_variables(
unresolved_variables=set(),
).values():
added_variables |= set(plugin_added_variables)
for plugin_modified_variables in plugin_options.modified_variables().values():
modified_variables = plugin_modified_variables
yield plugin_options, added_variables, modified_variables
def _override_variables(overrides: Overrides) -> Set[str]:
return set(list(overrides.initial_variables().keys()))
class VariableValidation: class VariableValidation:
def _get_resolve_partial_filter(self) -> Set[str]:
# Exclude sanitized variables from partial validation. This lessens the work
# and prevents double-evaluation, which can lead to bad behavior like double-prints.
return {
name
for name in self.script.variable_names
if name not in self.unresolved_variables and not name.endswith("_sanitized")
}
def _apply_resolution_level(self, mocks: Optional[Dict[str, str]]) -> None:
if self._resolution_level == ResolutionLevel.FILL:
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
# Only partial resolve definitions that are already resolved
self.unresolved_variables |= {
name
for name in self.overrides.keys
if not is_function(name) and not self.script.definition_of(name).maybe_resolvable
}
elif self._resolution_level == ResolutionLevel.RESOLVE:
# Partial resolve everything, but not including internal variables
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
elif self._resolution_level == ResolutionLevel.INTERNAL:
# Partial resolve everything including internal variables
pass
else:
raise ValueError("Invalid resolution level for validation")
if mocks is not None:
for mock_name in mocks.keys():
if mock_name in self.unresolved_variables:
self.unresolved_variables.remove(mock_name)
self.script.add(
variables=mocks,
unresolvable=self.unresolved_variables,
)
self.script = self.script.resolve_partial(
unresolvable=self.unresolved_variables,
output_filter=self._get_resolve_partial_filter(),
)
def __init__( def __init__(
self, self,
overrides: Overrides,
downloader_options: MultiUrlValidator, downloader_options: MultiUrlValidator,
output_options: OutputOptions, output_options: OutputOptions,
plugins: PresetPlugins, plugins: PresetPlugins,
resolution_level: int = ResolutionLevel.RESOLVE,
mocks: Optional[Dict[str, str]] = None,
): ):
self.overrides = overrides
self.downloader_options = downloader_options self.downloader_options = downloader_options
self.output_options = output_options self.output_options = output_options
self.plugins = plugins self.plugins = plugins
self.script: Script = self.overrides.script self.script: Optional[Script] = None
self.unresolved_variables = ( self.resolved_variables: Set[str] = set()
self.plugins.get_all_variables( self.unresolved_variables: Set[str] = set()
additional_options=[self.output_options, self.downloader_options]
)
| UNRESOLVED_VARIABLES
)
self.unresolved_runtime_variables = self.plugins.get_all_variables(
additional_options=[self.output_options, self.downloader_options]
)
self._resolution_level = resolution_level
self._apply_resolution_level(mocks=mocks)
def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None: def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation":
"""
Do some gymnastics to initialize the Overrides script.
"""
override_variables = set(list(overrides.initial_variables().keys()))
# Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable
self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables
plugin_variables: Set[str] = set()
for (
plugin_options,
added_variables,
modified_variables,
) in _get_added_and_modified_variables(
plugins=self.plugins,
downloader_options=self.downloader_options,
output_options=self.output_options,
):
for added_variable in added_variables:
if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable):
# pylint: disable=protected-access
raise plugin_options._validation_exception(
f"Cannot use the variable name {added_variable} because it exists as a"
" built-in ytdl-sub variable name."
)
# pylint: enable=protected-access
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables
plugin_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables
# Initialize overrides with unresolved variables + modified variables to throw an error.
# For modified variables, this is to prevent a resolve(update=True) to setting any
# dependencies until it has been explicitly added
overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables)
# copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script)
self.script.add(
variables=_add_dummy_overrides(overrides=overrides)
| _add_dummy_variables(variables=plugin_variables)
| _DUMMY_ENTRY_VARIABLES
)
return self
def _update_script(self) -> None:
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
def _add_subscription_override_variables(self) -> None:
"""
Add dummy subscription variables for script validation
"""
self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
""" """
Add dummy variables for script validation Add dummy variables for script validation
""" """
added_variables = options.added_variables( added_variables = options.added_variables(
unresolved_variables=self.unresolved_runtime_variables, unresolved_variables=self.unresolved_variables,
).get(plugin_op, set()) ).get(plugin_op, set())
modified_variables = options.modified_variables().get(plugin_op, set()) modified_variables = options.modified_variables().get(plugin_op, set())
self.unresolved_runtime_variables -= added_variables | modified_variables resolved_variables = added_variables | modified_variables
def _output_override_variables(self) -> Dict: self.resolved_variables |= resolved_variables
output = {} self.unresolved_variables -= resolved_variables
for name in self.overrides.keys:
value = self.script.definition_of(name)
if name in self.script.function_names:
# Keep custom functions as-is
output[name] = self.overrides.dict_with_format_strings[name]
elif resolved := value.maybe_resolvable:
output[name] = resolved.native
else:
output[name] = ScriptUtils.to_native_script(value)
return output def ensure_proper_usage(self) -> None:
def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict:
""" """
Validate variables resolve as plugins are executed, and return Validate variables resolve as plugins are executed, and return
a mock script which contains actualized added variables from the plugins a mock script which contains actualized added variables from the plugins
""" """
resolved_subscription: Dict = {}
self._add_runtime_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
self._add_subscription_override_variables()
# Always add output options first # Always add output options first
self._add_runtime_variables( self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options)
PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options
)
# Metadata variables to be added # Metadata variables to be added
for plugin_options in PluginMapping.order_options_by( for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
): ):
self._add_runtime_variables( self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options
)
for plugin_options in PluginMapping.order_options_by( for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
): ):
self._add_runtime_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
# Validate that any formatter in the plugin options can resolve # Validate that any formatter in the plugin options can resolve
resolved_subscription |= validate_formatters( validate_formatters(
script=self.script, script=self.script,
unresolved_variables=self.unresolved_variables, unresolved_variables=self.unresolved_variables,
unresolved_runtime_variables=self.unresolved_runtime_variables,
validator=plugin_options, validator=plugin_options,
partial_resolve_formatters=partial_resolve_formatters,
) )
resolved_subscription |= validate_formatters( validate_formatters(
script=self.script, script=self.script,
unresolved_variables=self.unresolved_variables, unresolved_variables=self.unresolved_variables,
unresolved_runtime_variables=self.unresolved_runtime_variables,
validator=self.output_options, validator=self.output_options,
partial_resolve_formatters=partial_resolve_formatters,
) )
# TODO: make this a function assert not self.unresolved_variables
raw_download_output = validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
unresolved_runtime_variables=self.unresolved_runtime_variables,
validator=self.downloader_options.urls,
partial_resolve_formatters=partial_resolve_formatters,
)
resolved_subscription["download"] = []
for url_output in raw_download_output["download"]:
if isinstance(url_output["url"], list):
url_output["url"] = [url for url in url_output["url"] if bool(url)]
if url_output["url"]:
resolved_subscription["download"].append(url_output)
resolved_subscription["overrides"] = self._output_override_variables()
return resolved_subscription

View file

@ -1,24 +1,24 @@
import copy import copy
import json import json
from pathlib import Path from pathlib import Path
from typing import Dict, Iterable, List, Optional from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import ( from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
VARIABLE_SCRIPTS, from ytdl_sub.entries.script.variable_definitions import VARIABLES
VARIABLES, from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
VariableDefinitions,
)
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler, get_file_extension from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.ytdl_additions.enhanced_download_archive import ( from ytdl_sub.utils.file_handler import get_file_extension
DownloadMapping, from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping
EnhancedDownloadArchive, from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
)
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -1,9 +1,16 @@
import abc import abc
from abc import ABC from abc import ABC
from typing import Dict, Generic, Iterable, List, Optional, Type, final from typing import Dict
from typing import Generic
from typing import Iterable
from typing import List
from typing import Optional
from typing import Type
from typing import final
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import BasePlugin, Plugin from ytdl_sub.config.plugin.plugin import BasePlugin
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidatorT from ytdl_sub.config.validators.options import OptionsValidatorT
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry

View file

@ -1,29 +1,33 @@
import contextlib import contextlib
import os import os
from pathlib import Path from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple from typing import Dict
from typing import Iterable
from typing import Iterator
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from yt_dlp.utils import RejectedVideoReached from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.downloaders.source_plugin import SourcePlugin, SourcePluginExtension from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.url.validators import ( from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
MultiUrlValidator, from ytdl_sub.downloaders.url.validators import MultiUrlValidator
UrlThumbnailListValidator, from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator
UrlValidator, from ytdl_sub.downloaders.url.validators import UrlValidator
)
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ( from ytdl_sub.utils.thumbnail import ThumbnailTypes
ThumbnailTypes, from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
download_and_convert_url_thumbnail, from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
try_convert_download_thumbnail,
)
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES
@ -48,12 +52,12 @@ class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator])
if 0 <= input_url_idx < len(self.plugin_options.urls.list): if 0 <= input_url_idx < len(self.plugin_options.urls.list):
validator = self.plugin_options.urls.list[input_url_idx] validator = self.plugin_options.urls.list[input_url_idx]
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list): if self.overrides.apply_formatter(validator.url) == entry_input_url:
return validator return validator
# Match the first validator based on the URL, if one exists # Match the first validator based on the URL, if one exists
for validator in self.plugin_options.urls.list: for validator in self.plugin_options.urls.list:
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list): if self.overrides.apply_formatter(validator.url) == entry_input_url:
return validator return validator
# Return the first validator if none exist # Return the first validator if none exist
@ -93,6 +97,7 @@ class UrlDownloaderThumbnailPlugin(UrlDownloaderBasePluginExtension):
# If latest entry, always update the thumbnail on each entry # If latest entry, always update the thumbnail on each entry
if thumbnail_id == ThumbnailTypes.LATEST_ENTRY: if thumbnail_id == ThumbnailTypes.LATEST_ENTRY:
# always save in dry-run even if it doesn't exist... # always save in dry-run even if it doesn't exist...
if self.is_dry_run or entry.is_thumbnail_downloaded(): if self.is_dry_run or entry.is_thumbnail_downloaded():
self.save_file( self.save_file(
@ -252,16 +257,6 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
.to_dict() .to_dict()
) )
def webpage_url(self, entry: Entry) -> str:
"""
Returns
-------
The webpage_url to use for the actual download
"""
url_idx = entry.get(v.ytdl_sub_input_url_index, int)
webpage_url_formatter = self.plugin_options.urls.list[url_idx].webpage_url
return self.overrides.apply_formatter(webpage_url_formatter, entry=entry)
def metadata_ytdl_options(self, ytdl_option_overrides: Dict) -> Dict: def metadata_ytdl_options(self, ytdl_option_overrides: Dict) -> Dict:
""" """
Returns Returns
@ -363,7 +358,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
if (self.is_dry_run or not self.is_entry_thumbnails_enabled) if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
else entry.is_thumbnail_downloaded_via_ytdlp else entry.is_thumbnail_downloaded_via_ytdlp
), ),
url=self.webpage_url(entry=entry), url=entry.webpage_url,
) )
return Entry( return Entry(
download_entry_dict, download_entry_dict,
@ -371,13 +366,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
) )
def _iterate_child_entries( def _iterate_child_entries(
self, entries: List[Entry], validator: UrlValidator self, entries: List[Entry], download_reversed: bool
) -> Iterator[Entry]: ) -> Iterator[Entry]:
# Iterate a list of entries, and delete the entries after yielding # Iterate a list of entries, and delete the entries after yielding
entries_to_iter: List[Optional[Entry]] = entries entries_to_iter: List[Optional[Entry]] = entries
indices = list(range(len(entries_to_iter))) indices = list(range(len(entries_to_iter)))
if self.overrides.apply_formatter(validator.download_reverse, expected_type=bool): if download_reversed:
indices = reversed(indices) indices = reversed(indices)
for idx in indices: for idx in indices:
@ -399,13 +394,17 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
entries_to_iter[idx] = None entries_to_iter[idx] = None
def _iterate_parent_entry( def _iterate_parent_entry(
self, parent: EntryParent, validator: UrlValidator self, parent: EntryParent, download_reversed: bool
) -> Iterator[Entry]: ) -> Iterator[Entry]:
yield from self._iterate_child_entries(entries=parent.entry_children(), validator=validator) yield from self._iterate_child_entries(
entries=parent.entry_children(), download_reversed=download_reversed
)
# Recursion the parent's parent entries # Recursion the parent's parent entries
for parent_child in reversed(parent.parent_children()): for parent_child in reversed(parent.parent_children()):
yield from self._iterate_parent_entry(parent=parent_child, validator=validator) yield from self._iterate_parent_entry(
parent=parent_child, download_reversed=download_reversed
)
def _download_url_metadata( def _download_url_metadata(
self, url: str, include_sibling_metadata: bool, ytdl_options_overrides: Dict self, url: str, include_sibling_metadata: bool, ytdl_options_overrides: Dict
@ -439,7 +438,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
self, self,
parents: List[EntryParent], parents: List[EntryParent],
orphans: List[Entry], orphans: List[Entry],
validator: UrlValidator, download_reversed: bool,
) -> Iterator[Entry]: ) -> Iterator[Entry]:
""" """
Downloads the leaf entries from EntryParent trees Downloads the leaf entries from EntryParent trees
@ -447,17 +446,21 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
# Delete info json files afterwards so other collection URLs do not use them # Delete info json files afterwards so other collection URLs do not use them
with self._separate_download_archives(clear_info_json_files=True): with self._separate_download_archives(clear_info_json_files=True):
for parent in parents: for parent in parents:
yield from self._iterate_parent_entry(parent=parent, validator=validator) yield from self._iterate_parent_entry(
parent=parent, download_reversed=download_reversed
)
yield from self._iterate_child_entries(entries=orphans, validator=validator) yield from self._iterate_child_entries(
entries=orphans, download_reversed=download_reversed
)
def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]: def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]:
metadata_ytdl_options = self.metadata_ytdl_options( metadata_ytdl_options = self.metadata_ytdl_options(
ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides) ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides)
) )
download_reversed = self.overrides.evaluate_boolean(validator.download_reverse)
include_sibling_metadata = self.overrides.apply_formatter( include_sibling_metadata = self.overrides.evaluate_boolean(
validator.include_sibling_metadata, expected_type=bool validator.include_sibling_metadata
) )
parents, orphan_entries = self._download_url_metadata( parents, orphan_entries = self._download_url_metadata(
@ -466,15 +469,16 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
ytdl_options_overrides=metadata_ytdl_options, ytdl_options_overrides=metadata_ytdl_options,
) )
# TODO: Encapsulate this logic into its own class
self._url_state = URLDownloadState( self._url_state = URLDownloadState(
entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries), entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries)
) )
download_logger.info("Beginning downloads for %s", url) download_logger.info("Beginning downloads for %s", url)
yield from self._iterate_entries( yield from self._iterate_entries(
parents=parents, parents=parents,
orphans=orphan_entries, orphans=orphan_entries,
validator=validator, download_reversed=download_reversed,
) )
def download_metadata(self) -> Iterable[Entry]: def download_metadata(self) -> Iterable[Entry]:
@ -482,25 +486,19 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
# download the bottom-most urls first since they are top-priority # download the bottom-most urls first since they are top-priority
for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))): for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
# URLs can be empty. If they are, then skip # URLs can be empty. If they are, then skip
if not (urls := self.overrides.apply_formatter(url_validator.url, expected_type=list)): if not (url := self.overrides.apply_formatter(url_validator.url)):
continue continue
for url in reversed(urls): for entry in self._download_metadata(url=url, validator=url_validator):
assert isinstance(url, str) entry.initialize_script(self.overrides).add(
{
v.ytdl_sub_input_url: url,
v.ytdl_sub_input_url_index: idx,
v.ytdl_sub_input_url_count: len(self.collection.urls.list),
}
)
if not url: yield entry
continue
for entry in self._download_metadata(url=url, validator=url_validator):
entry.initialize_script(self.overrides).add(
{
v.ytdl_sub_input_url: url,
v.ytdl_sub_input_url_index: idx,
v.ytdl_sub_input_url_count: len(self.collection.urls.list),
}
)
yield entry
def download(self, entry: Entry) -> Optional[Entry]: def download(self, entry: Entry) -> Optional[Entry]:
""" """

View file

@ -1,16 +1,17 @@
from typing import Any, Dict, List, Optional, Set from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ( from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
DictFormatterValidator, from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
OverridesBooleanFormatterValidator, from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
OverridesStringFormatterValidator, from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
StringFormatterValidator,
)
from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import ListValidator
@ -20,7 +21,7 @@ class UrlThumbnailValidator(StrictDictValidator):
def __init__(self, name, value): def __init__(self, name, value):
super().__init__(name, value) super().__init__(name, value)
self._thumb_name = self._validate_key(key="name", validator=StringFormatterValidator) self._name = self._validate_key(key="name", validator=StringFormatterValidator)
self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator) self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator)
@property @property
@ -28,7 +29,7 @@ class UrlThumbnailValidator(StrictDictValidator):
""" """
File name for the thumbnail File name for the thumbnail
""" """
return self._thumb_name return self._name
@property @property
def uid(self) -> OverridesStringFormatterValidator: def uid(self) -> OverridesStringFormatterValidator:
@ -42,19 +43,6 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
_inner_list_type = UrlThumbnailValidator _inner_list_type = UrlThumbnailValidator
class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator):
def post_process(self, resolved: Any) -> List[str]:
if isinstance(resolved, str):
return [resolved]
if isinstance(resolved, list):
for value in resolved:
if not isinstance(value, str):
raise self._validation_exception("Must be a string or an array of strings.")
return resolved
raise self._validation_exception("Must be a string or an array of strings.")
class UrlValidator(StrictDictValidator): class UrlValidator(StrictDictValidator):
_required_keys = {"url"} _required_keys = {"url"}
_optional_keys = { _optional_keys = {
@ -64,7 +52,6 @@ class UrlValidator(StrictDictValidator):
"download_reverse", "download_reverse",
"ytdl_options", "ytdl_options",
"include_sibling_metadata", "include_sibling_metadata",
"webpage_url",
} }
@classmethod @classmethod
@ -80,7 +67,7 @@ class UrlValidator(StrictDictValidator):
super().__init__(name, value) super().__init__(name, value)
# TODO: url validate using yt-dlp IE # TODO: url validate using yt-dlp IE
self._url = self._validate_key(key="url", validator=OverridesOneOrManyUrlValidator) self._url = self._validate_key(key="url", validator=OverridesStringFormatterValidator)
self._variables = self._validate_key_if_present( self._variables = self._validate_key_if_present(
key="variables", validator=DictFormatterValidator, default={} key="variables", validator=DictFormatterValidator, default={}
) )
@ -102,9 +89,6 @@ class UrlValidator(StrictDictValidator):
validator=OverridesBooleanFormatterValidator, validator=OverridesBooleanFormatterValidator,
default="False", default="False",
) )
self._webpage_url = self._validate_key(
key="webpage_url", validator=StringFormatterValidator, default="{webpage_url}"
)
@property @property
def url(self) -> OverridesStringFormatterValidator: def url(self) -> OverridesStringFormatterValidator:
@ -196,19 +180,6 @@ class UrlValidator(StrictDictValidator):
""" """
return self._include_sibling_metadata return self._include_sibling_metadata
@property
def webpage_url(self) -> StringFormatterValidator:
"""
Optional. After ytdl-sub performs the metadata download, it will inspect each
entry's .info.json file and perform the actual download from yt-dlp using
`webpage_url <config_reference/scripting/entry_variables:webpage_url>`. This
can be overwritten by supplying parameter with a modification to ``webpage_url`` in the
form of an override variable.
Defaults to ``{webpage_url}``.
"""
return self._webpage_url
class UrlStringOrDictValidator(UrlValidator): class UrlStringOrDictValidator(UrlValidator):
""" """

View file

@ -1,5 +1,6 @@
import copy import copy
from typing import Dict, Optional from typing import Dict
from typing import Optional
import mergedeep import mergedeep

View file

@ -5,10 +5,15 @@ import os
import time import time
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from typing import Callable, Dict, List, Optional from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
import yt_dlp as ytdl import yt_dlp as ytdl
from yt_dlp.utils import ExistingVideoReached, MaxDownloadsReached, RejectedVideoReached from yt_dlp.utils import ExistingVideoReached
from yt_dlp.utils import MaxDownloadsReached
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener
from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.exceptions import FileNotDownloadedException

View file

@ -1,11 +1,18 @@
# pylint: disable=protected-access # pylint: disable=protected-access
from abc import ABC from abc import ABC
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Optional, Type, TypeVar, final from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from yt_dlp.utils import LazyList, sanitize_filename from yt_dlp.utils import LazyList
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -3,15 +3,24 @@ import copy
import json import json
import os import os
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Optional, Type, TypeVar, final from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_types import ArrayVariable, StringVariable, Variable from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_types import ArrayVariable
from ytdl_sub.entries.script.variable_types import StringVariable
from ytdl_sub.entries.script.variable_types import Variable
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS, VIDEO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -1,10 +1,16 @@
import math import math
from typing import Any, Dict, List, Optional, Set from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from urllib.parse import urlparse from urllib.parse import urlparse
from ytdl_sub.entries.base_entry import BaseEntry, BaseEntryT from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import BaseEntryT
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_types import MetadataVariable from ytdl_sub.entries.script.variable_types import MetadataVariable
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -5,7 +5,10 @@ from yt_dlp.utils import sanitize_filename
from ytdl_sub.script.functions import Functions from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.map import Map from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import AnyArgument, Integer, ReturnableArgument, String from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.file_path import FilePathTruncater from ytdl_sub.utils.file_path import FilePathTruncater
@ -46,12 +49,12 @@ class CustomFunctions:
return String(FilePathTruncater.maybe_truncate_file_path(filepath.value)) return String(FilePathTruncater.maybe_truncate_file_path(filepath.value))
@staticmethod @staticmethod
def sanitize(*value: AnyArgument) -> String: def sanitize(value: AnyArgument) -> String:
""" """
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS. for file/directory names on any OS.
""" """
return String("".join(sanitize_filename(str(val)) for val in value)) return String(sanitize_filename(str(value)))
@staticmethod @staticmethod
def sanitize_plex_episode(string: String) -> String: def sanitize_plex_episode(string: String) -> String:

View file

@ -1,6 +1,7 @@
from typing import Dict from typing import Dict
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -1,21 +1,21 @@
from abc import ABC from abc import ABC
from functools import cache, cached_property from functools import cache
from typing import Dict, Optional, Set from functools import cached_property
from typing import Dict
from typing import Set
from ytdl_sub.entries.script.custom_functions import CustomFunctions from ytdl_sub.entries.script.custom_functions import CustomFunctions
from ytdl_sub.entries.script.variable_types import ( from ytdl_sub.entries.script.variable_types import ArrayMetadataVariable
ArrayMetadataVariable, from ytdl_sub.entries.script.variable_types import IntegerMetadataVariable
IntegerMetadataVariable, from ytdl_sub.entries.script.variable_types import IntegerVariable
IntegerVariable, from ytdl_sub.entries.script.variable_types import MapMetadataVariable
MapMetadataVariable, from ytdl_sub.entries.script.variable_types import MapVariable
MapVariable, from ytdl_sub.entries.script.variable_types import MetadataVariable
MetadataVariable, from ytdl_sub.entries.script.variable_types import StringDateMetadataVariable
StringDateMetadataVariable, from ytdl_sub.entries.script.variable_types import StringDateVariable
StringDateVariable, from ytdl_sub.entries.script.variable_types import StringMetadataVariable
StringMetadataVariable, from ytdl_sub.entries.script.variable_types import StringVariable
StringVariable, from ytdl_sub.entries.script.variable_types import Variable
Variable,
)
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion # This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
# pylint: disable=no-member # pylint: disable=no-member
@ -1135,16 +1135,6 @@ class VariableDefinitions(
] ]
} }
@cache
def variable_names(self, include_sanitized: bool):
"""
Returns all variable names, and can include sanitized.
"""
var_names: Set[str] = self.scripts().keys()
if include_sanitized:
var_names |= {f"{name}_sanitized" for name in var_names}
return var_names
@cache @cache
def injected_variables(self) -> Set[MetadataVariable]: def injected_variables(self) -> Set[MetadataVariable]:
""" """
@ -1172,7 +1162,6 @@ class VariableDefinitions(
""" """
return { return {
self.uid, self.uid,
self.extractor,
self.extractor_key, self.extractor_key,
self.epoch, self.epoch,
self.webpage_url, self.webpage_url,
@ -1215,14 +1204,6 @@ class VariableDefinitions(
VARIABLES.entry_metadata, VARIABLES.entry_metadata,
} | self.injected_variables() } | self.injected_variables()
def get(self, name: str) -> Optional[Variable]:
"""
Returns the variable attribute if it exists. None otherwise.
"""
if not hasattr(self, name):
return None
return getattr(self, name)
# Singletons to use externally # Singletons to use externally
VARIABLES: VariableDefinitions = VariableDefinitions() VARIABLES: VariableDefinitions = VariableDefinitions()

View file

@ -1,10 +1,17 @@
from abc import ABC, abstractmethod from abc import ABC
from abc import abstractmethod
from dataclasses import dataclass from dataclasses import dataclass
from typing import Dict, List, Optional, Type, TypeVar from typing import Dict
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import Boolean, Integer, String from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
ENTRY_METADATA_VARIABLE_NAME = "entry_metadata" ENTRY_METADATA_VARIABLE_NAME = "entry_metadata"
PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata" PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata"
@ -15,6 +22,7 @@ VariableT = TypeVar("VariableT", bound="Variable")
def _get( def _get(
cast: str,
metadata_variable_name: str, metadata_variable_name: str,
metadata_key: str, metadata_key: str,
variable_name: Optional[str], variable_name: Optional[str],
@ -39,7 +47,7 @@ def _get(
return as_type( return as_type(
variable_name=variable_name or metadata_key, variable_name=variable_name or metadata_key,
metadata_key=metadata_key, metadata_key=metadata_key,
definition=f"{{ {out} }}", definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}",
) )
@ -174,6 +182,7 @@ class MapMetadataVariable(MetadataVariable, MapVariable):
Creates a map variable from entry metadata Creates a map variable from entry metadata
""" """
return _get( return _get(
"map",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,
@ -195,6 +204,7 @@ class ArrayMetadataVariable(MetadataVariable, ArrayVariable):
Creates an array variable from entry metadata Creates an array variable from entry metadata
""" """
return _get( return _get(
"array",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,
@ -216,6 +226,7 @@ class StringMetadataVariable(MetadataVariable, StringVariable):
Creates a string variable from entry metadata Creates a string variable from entry metadata
""" """
return _get( return _get(
"string",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,
@ -234,6 +245,7 @@ class StringMetadataVariable(MetadataVariable, StringVariable):
Creates a string variable from playlist metadata Creates a string variable from playlist metadata
""" """
return _get( return _get(
"string",
metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,
@ -252,6 +264,7 @@ class StringMetadataVariable(MetadataVariable, StringVariable):
Creates a string variable from source metadata Creates a string variable from source metadata
""" """
return _get( return _get(
"string",
metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME, metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,
@ -288,6 +301,7 @@ class IntegerMetadataVariable(MetadataVariable, IntegerVariable):
Creates an int variable from entry metadata Creates an int variable from entry metadata
""" """
return _get( return _get(
"int",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,
@ -306,6 +320,7 @@ class IntegerMetadataVariable(MetadataVariable, IntegerVariable):
Creates an int variable from playlist metadata Creates an int variable from playlist metadata
""" """
return _get( return _get(
"int",
metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME,
metadata_key=metadata_key, metadata_key=metadata_key,
variable_name=variable_name, variable_name=variable_name,

View file

@ -1,17 +1,19 @@
from typing import Dict, Set from typing import Dict
from typing import Set
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.entries.script.variable_types import ( from ytdl_sub.entries.script.variable_types import ArrayVariable
ArrayVariable, from ytdl_sub.entries.script.variable_types import BooleanVariable
BooleanVariable, from ytdl_sub.entries.script.variable_types import MapVariable
MapVariable, from ytdl_sub.entries.script.variable_types import StringVariable
StringVariable, from ytdl_sub.entries.script.variable_types import Variable
Variable,
)
from ytdl_sub.script.functions import Functions from ytdl_sub.script.functions import Functions
from ytdl_sub.script.utils.name_validation import is_valid_name from ytdl_sub.script.utils.name_validation import is_valid_name
# TODO: use this
SUBSCRIPTION_ARRAY = "subscription_array"
class SubscriptionVariables: class SubscriptionVariables:
@staticmethod @staticmethod
@ -161,7 +163,7 @@ class OverrideHelpers:
True if the override name itself is valid. False otherwise. True if the override name itself is valid. False otherwise.
""" """
if name.startswith("%"): if name.startswith("%"):
name = name[1:] return is_valid_name(name=name[1:])
return is_valid_name(name=name) return is_valid_name(name=name)

View file

@ -1,7 +1,7 @@
import sys import sys
from ytdl_sub.cli.parsers.main import parser from ytdl_sub.cli.parsers.main import parser
from ytdl_sub.utils.logger import Logger, LoggerLevels from ytdl_sub.utils.logger import Logger
def _main() -> int: def _main() -> int:
@ -10,11 +10,6 @@ def _main() -> int:
args, _ = parser.parse_known_args() args, _ = parser.parse_known_args()
Logger.set_log_level(log_level_name=args.ytdl_sub_log_level) Logger.set_log_level(log_level_name=args.ytdl_sub_log_level)
# Suppress all logs during inspection since the output of the subcommand itself
# is all that is necessary
if args.subparser == "inspect":
Logger.set_log_level(log_level_name=LoggerLevels.QUIET.name)
# pylint: disable=import-outside-toplevel # pylint: disable=import-outside-toplevel
import ytdl_sub.cli.entrypoint import ytdl_sub.cli.entrypoint

View file

@ -1,19 +1,21 @@
import os.path import os.path
from typing import Any, Dict, Optional, Set from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.audo_codec_validator import ( from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
AUDIO_CODEC_EXTS, from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_TYPES_EXTENSION_MAPPING
AUDIO_CODEC_TYPES_EXTENSION_MAPPING, from ytdl_sub.validators.audo_codec_validator import AudioTypeValidator
AudioTypeValidator,
)
from ytdl_sub.validators.validators import FloatValidator from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -1,23 +1,26 @@
import collections import collections
import re import re
from typing import Dict, List, Optional, Set from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import ( from ytdl_sub.entries.entry import Entry
Entry, from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
ytdl_sub_chapters_from_comments, from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
ytdl_sub_split_by_chapters_parent_uid, from ytdl_sub.entries.script.variable_definitions import VARIABLES
) from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.regex_validator import RegexListValidator from ytdl_sub.validators.regex_validator import RegexListValidator
from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator, ListValidator from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import ListValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -1,4 +1,7 @@
from typing import List, Optional, Set, Tuple from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
@ -113,7 +116,7 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
date_validator=self.plugin_options.after, overrides=self.overrides date_validator=self.plugin_options.after, overrides=self.overrides
) )
after_filter = f"{date_type} >= {after_str}" after_filter = f"{date_type} >= {after_str}"
if self.overrides.apply_formatter(self.plugin_options.breaks, expected_type=bool): if self.overrides.evaluate_boolean(self.plugin_options.breaks):
breaking_match_filters.append(after_filter) breaking_match_filters.append(after_filter)
else: else:
match_filters.append(after_filter) match_filters.append(after_filter)

View file

@ -1,4 +1,5 @@
from typing import List, Optional from typing import List
from typing import Optional
import mediafile import mediafile
@ -6,7 +7,8 @@ from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
@ -31,7 +33,7 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
@property @property
def _embed_thumbnail(self) -> bool: def _embed_thumbnail(self) -> bool:
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool) return self.overrides.evaluate_boolean(self.plugin_options)
@classmethod @classmethod
def _embed_video_thumbnail(cls, entry: Entry) -> None: def _embed_video_thumbnail(cls, entry: Entry) -> None:

View file

@ -1,16 +1,22 @@
import os import os
from subprocess import CalledProcessError from subprocess import CalledProcessError
from typing import Any, Dict, Optional, Set from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.utils.exceptions import FileNotDownloadedException, ValidationException from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.audo_codec_validator import FileTypeValidator from ytdl_sub.validators.audo_codec_validator import FileTypeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator

View file

@ -1,4 +1,5 @@
from typing import Dict, Optional from typing import Dict
from typing import Optional
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
@ -6,14 +7,13 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("filter-exclude") logger = Logger.get("filter-exclude")
class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator): class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
""" """
Applies a conditional OR on any number of filters comprised of either variables or scripts. Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded. If any filter evaluates to True, the entry will be excluded.
@ -29,8 +29,6 @@ class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsVali
{ %contains( %lower(description), '#short' ) } { %contains( %lower(description), '#short' ) }
""" """
_inner_list_type = BooleanFormatterValidator
class FilterExcludePlugin(Plugin[FilterExcludeOptions]): class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
plugin_options_type = FilterExcludeOptions plugin_options_type = FilterExcludeOptions
@ -54,9 +52,7 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry return entry
for formatter in self.plugin_options.list: for formatter in self.plugin_options.list:
should_exclude = self.overrides.apply_formatter( should_exclude = self.overrides.evaluate_boolean(formatter=formatter, entry=entry)
formatter=formatter, entry=entry, expected_type=bool
)
if should_exclude: if should_exclude:
logger.info( logger.info(

View file

@ -1,4 +1,5 @@
from typing import Dict, Optional from typing import Dict
from typing import Optional
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
@ -6,14 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("filter-include") logger = Logger.get("filter-include")
class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator): class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
""" """
Applies a conditional AND on any number of filters comprised of either variables or scripts. Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included. If all filters evaluate to True, the entry will be included.
@ -37,8 +38,6 @@ class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsVali
} }
""" """
_inner_list_type = BooleanFormatterValidator
class FilterIncludePlugin(Plugin[FilterIncludeOptions]): class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
plugin_options_type = FilterIncludeOptions plugin_options_type = FilterIncludeOptions
@ -62,8 +61,8 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry return entry
for formatter in self.plugin_options.list: for formatter in self.plugin_options.list:
should_exclude = self.overrides.apply_formatter( should_exclude = ScriptUtils.bool_formatter_output(
formatter=formatter, entry=entry, expected_type=bool self.overrides.apply_formatter(formatter=formatter, entry=entry)
) )
if not should_exclude: if not should_exclude:
logger.info( logger.info(

View file

@ -1,4 +1,5 @@
from typing import Dict, Optional from typing import Dict
from typing import Optional
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator

View file

@ -1,5 +1,7 @@
import copy import copy
from typing import Any, List, Tuple from typing import Any
from typing import List
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator

View file

@ -1,21 +1,22 @@
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List from typing import Any
from typing import Dict
from typing import List
import mediafile import mediafile
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.string_formatter_validators import ( from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
ListFormatterValidator, from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
StringFormatterValidator,
)
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES
@ -146,7 +147,8 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
else: else:
if len(tag_value) > 1: if len(tag_value) > 1:
logger.warning( logger.warning(
"Music tag '%s' does not support lists. Only setting the first element", "Music tag '%s' does not support lists. "
"Only setting the first element",
tag_name, tag_name,
) )
setattr(audio_file, tag_name, tag_value[0]) setattr(audio_file, tag_name, tag_value[0])

View file

@ -2,25 +2,24 @@ import os
from abc import ABC from abc import ABC
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List from typing import Any
from typing import Dict
from typing import List
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.xml import ( from ytdl_sub.utils.file_handler import FileMetadata
XmlElement, from ytdl_sub.utils.xml import XmlElement
to_max_3_byte_utf8_dict, from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
to_max_3_byte_utf8_string, from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
to_xml, from ytdl_sub.utils.xml import to_xml
)
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.nfo_validators import NfoTagsValidator from ytdl_sub.validators.nfo_validators import NfoTagsValidator
from ytdl_sub.validators.string_formatter_validators import ( from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
DictFormatterValidator, from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
OverridesBooleanFormatterValidator, from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
StringFormatterValidator,
)
class SharedNfoTagsOptions(ToggleableOptionsDictValidator): class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
@ -141,7 +140,7 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC):
if not nfo_tags: if not nfo_tags:
return return
if self.overrides.apply_formatter(self.plugin_options.kodi_safe, expected_type=bool): if self.overrides.evaluate_boolean(self.plugin_options.kodi_safe):
nfo_root = to_max_3_byte_utf8_string(nfo_root) nfo_root = to_max_3_byte_utf8_string(nfo_root)
nfo_tags = { nfo_tags = {
to_max_3_byte_utf8_string(key): [ to_max_3_byte_utf8_string(key): [

View file

@ -1,6 +1,8 @@
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator, SharedNfoTagsOptions, SharedNfoTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive

View file

@ -1,14 +1,23 @@
from typing import Any, Dict, List, Optional, Set, Tuple from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.plugin.plugin import SplitPlugin from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry, ytdl_sub_split_by_chapters_parent_uid from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.utils.chapters import Chapters, Timestamp from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES

View file

@ -1,10 +1,12 @@
from typing import List, Optional from typing import List
from typing import Optional
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
@ -29,7 +31,7 @@ class SquareThumbnailPlugin(Plugin[SquareThumbnailOptions]):
@property @property
def _square_thumbnail(self) -> bool: def _square_thumbnail(self) -> bool:
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool) return self.overrides.evaluate_boolean(self.plugin_options)
@classmethod @classmethod
def _convert_to_square_thumbnail(cls, entry: Entry) -> None: def _convert_to_square_thumbnail(cls, entry: Entry) -> None:

View file

@ -1,5 +1,7 @@
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator, SharedNfoTagsOptions, SharedNfoTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator

View file

@ -1,21 +1,25 @@
import os
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator, StringListValidator from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import StringListValidator
v: VariableDefinitions = VARIABLES v: VariableDefinitions = VARIABLES
@ -53,7 +57,6 @@ class SubtitleOptions(ToggleableOptionsDictValidator):
"subtitles_type", "subtitles_type",
"embed_subtitles", "embed_subtitles",
"languages", "languages",
"languages_required",
"allow_auto_generated_subtitles", "allow_auto_generated_subtitles",
} }
@ -73,9 +76,6 @@ class SubtitleOptions(ToggleableOptionsDictValidator):
self._languages = self._validate_key_if_present( self._languages = self._validate_key_if_present(
key="languages", validator=StringListValidator, default=["en"] key="languages", validator=StringListValidator, default=["en"]
).list ).list
self._languages_required = self._validate_key_if_present(
key="languages_required", validator=StringListValidator, default=[]
).list
self._allow_auto_generated_subtitles = self._validate_key_if_present( self._allow_auto_generated_subtitles = self._validate_key_if_present(
key="allow_auto_generated_subtitles", validator=BoolValidator, default=False key="allow_auto_generated_subtitles", validator=BoolValidator, default=False
).value ).value
@ -121,16 +121,6 @@ class SubtitleOptions(ToggleableOptionsDictValidator):
""" """
return [lang.value for lang in self._languages] return [lang.value for lang in self._languages]
@property
def languages_required(self) -> Optional[List[str]]:
"""
:expected type: Optional[List[String]]
:description:
Language code(s) that are required to be present for downloads to continue. If missing,
ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles.
"""
return [lang.value for lang in self._languages_required]
@property @property
def allow_auto_generated_subtitles(self) -> Optional[bool]: def allow_auto_generated_subtitles(self) -> Optional[bool]:
""" """
@ -221,21 +211,7 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
if self.plugin_options.embed_subtitles: if self.plugin_options.embed_subtitles:
file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}") file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}")
if self.plugin_options.subtitles_name: if self.plugin_options.subtitles_name:
download_subtitle_lang_file_names: List[Tuple[str, str]] = []
for lang in langs: for lang in langs:
download_subtitle_file_name = entry.base_filename(
ext=f"{lang}.{self.plugin_options.subtitles_type}"
)
if os.path.isfile(Path(self.working_directory) / download_subtitle_file_name):
download_subtitle_lang_file_names.append((lang, download_subtitle_file_name))
elif lang in self.plugin_options.languages_required:
raise UserThrownRuntimeError(
f"Required the subtitle lang {lang}, but the file could not be found for "
f"the entry {entry.title}."
)
for lang, file_name in download_subtitle_lang_file_names:
output_subtitle_file_name = self.overrides.apply_formatter( output_subtitle_file_name = self.overrides.apply_formatter(
formatter=self.plugin_options.subtitles_name, formatter=self.plugin_options.subtitles_name,
entry=entry, entry=entry,
@ -243,7 +219,9 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
) )
self.save_file( self.save_file(
file_name=file_name, file_name=entry.base_filename(
ext=f"{lang}.{self.plugin_options.subtitles_type}"
),
output_file_name=output_subtitle_file_name, output_file_name=output_subtitle_file_name,
entry=entry, entry=entry,
) )

View file

@ -1,7 +1,10 @@
import random import random
import time import time
from abc import ABC from abc import ABC
from typing import Dict, Optional, Type, TypeVar from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import Plugin
@ -10,10 +13,8 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import ( from ytdl_sub.validators.string_formatter_validators import FloatFormatterValidator
FloatFormatterValidator, from ytdl_sub.validators.string_formatter_validators import OverridesFloatFormatterValidator
OverridesFloatFormatterValidator,
)
from ytdl_sub.validators.validators import ProbabilityValidator from ytdl_sub.validators.validators import ProbabilityValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -41,8 +42,8 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC):
) )
def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float: def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
actualized_min = overrides.apply_formatter(self._min, entry=entry, expected_type=float) actualized_min = float(overrides.apply_formatter(self._min, entry=entry))
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float) actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
if actualized_min < 0: if actualized_min < 0:
raise self._validation_exception( raise self._validation_exception(
@ -69,7 +70,7 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC):
------- -------
Max possible value Max possible value
""" """
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float) actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
if actualized_max < 0: if actualized_max < 0:
raise self._validation_exception( raise self._validation_exception(
f"max must be greater than zero, received {actualized_max}" f"max must be greater than zero, received {actualized_max}"

View file

@ -34,7 +34,7 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]):
Tags the entry's audio file using values defined in the metadata options Tags the entry's audio file using values defined in the metadata options
""" """
tags_to_write: Dict[str, str] = {} tags_to_write: Dict[str, str] = {}
for tag_name, tag_formatter in sorted(self.plugin_options.dict.items()): for tag_name, tag_formatter in self.plugin_options.dict.items():
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
tags_to_write[tag_name] = tag_value tags_to_write[tag_name] = tag_value

View file

@ -1,6 +1,8 @@
import os import os
import pathlib import pathlib
from typing import Any, Dict, Set from typing import Any
from typing import Dict
from typing import Set
import mergedeep import mergedeep

View file

@ -11,7 +11,8 @@ presets:
# Set the default date_range to 2 months # Set the default date_range to 2 months
overrides: overrides:
only_recent_date_range: "2months" date_range: "2months" # keep for legacy-reasons
only_recent_date_range: "{date_range}"
############################################################################# #############################################################################
# Only Recent # Only Recent

View file

@ -14,7 +14,7 @@ presets:
# #
_multi_url: _multi_url:
download: download:
- url: "{ %array_at(urls, 0) }" - url: "{url}"
playlist_thumbnails: playlist_thumbnails:
- name: "{avatar_uncropped_thumbnail_file_name}" - name: "{avatar_uncropped_thumbnail_file_name}"
uid: "avatar_uncropped" uid: "avatar_uncropped"
@ -26,135 +26,411 @@ presets:
- name: "{banner_uncropped_thumbnail_file_name}" - name: "{banner_uncropped_thumbnail_file_name}"
uid: "banner_uncropped" uid: "banner_uncropped"
include_sibling_metadata: "{include_sibling_metadata}" include_sibling_metadata: "{include_sibling_metadata}"
webpage_url: "{modified_webpage_url}" - url: "{url2}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{ %array_slice(urls, 1) }" - url: "{url3}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url4}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url5}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url6}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url7}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url8}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url9}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url10}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url11}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url12}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url13}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url14}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url15}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url16}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url17}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url18}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url19}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url20}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url21}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url22}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url23}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url24}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url25}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url26}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url27}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url28}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url29}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url30}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url31}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url32}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url33}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url34}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url35}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url36}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url37}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url38}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url39}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url40}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url41}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url42}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url43}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url44}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url45}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url46}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url47}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url48}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url49}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url50}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url51}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url52}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url53}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url54}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url55}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url56}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url57}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url58}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url59}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url60}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url61}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url62}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url63}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url64}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url65}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url66}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url67}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url68}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url69}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url70}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url71}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url72}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url73}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url74}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url75}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url76}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url77}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url78}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url79}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url80}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url81}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url82}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url83}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url84}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url85}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url86}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url87}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url88}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url89}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url90}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url91}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url92}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url93}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url94}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url95}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url96}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url97}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url98}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url99}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url100}"
include_sibling_metadata: "{include_sibling_metadata}" include_sibling_metadata: "{include_sibling_metadata}"
webpage_url: "{modified_webpage_url}"
overrides: overrides:
avatar_uncropped_thumbnail_file_name: "" avatar_uncropped_thumbnail_file_name: ""
banner_uncropped_thumbnail_file_name: "" banner_uncropped_thumbnail_file_name: ""
include_sibling_metadata: False include_sibling_metadata: False
modified_webpage_url: "{webpage_url}"
subscription_array: >-
{
[
url, url2, url3, url4, url5, url6, url7, url8, url9, url10,
url11, url12, url13, url14, url15, url16, url17, url18, url19, url20,
url21, url22, url23, url24, url25, url26, url27, url28, url29, url30,
url31, url32, url33, url34, url35, url36, url37, url38, url39, url40,
url41, url42, url43, url44, url45, url46, url47, url48, url49, url50,
url51, url52, url53, url54, url55, url56, url57, url58, url59, url60,
url61, url62, url63, url64, url65, url66, url67, url68, url69, url70,
url71, url72, url73, url74, url75, url76, url77, url78, url79, url80,
url81, url82, url83, url84, url85, url86, url87, url88, url89, url90,
url91, url92, url93, url94, url95, url96, url97, url98, url99, url100
]
}
urls: "{subscription_array}"
subscription_array: "{ [] }"
subscription_value: "" subscription_value: ""
subscription_value_2: ""
subscription_value_3: ""
subscription_value_4: ""
subscription_value_5: ""
subscription_value_6: ""
subscription_value_7: ""
subscription_value_8: ""
subscription_value_9: ""
subscription_value_10: ""
subscription_value_11: ""
subscription_value_12: ""
subscription_value_13: ""
subscription_value_14: ""
subscription_value_15: ""
subscription_value_16: ""
subscription_value_17: ""
subscription_value_18: ""
subscription_value_19: ""
subscription_value_20: ""
subscription_value_21: ""
subscription_value_22: ""
subscription_value_23: ""
subscription_value_24: ""
subscription_value_25: ""
subscription_value_26: ""
subscription_value_27: ""
subscription_value_28: ""
subscription_value_29: ""
subscription_value_30: ""
subscription_value_31: ""
subscription_value_32: ""
subscription_value_33: ""
subscription_value_34: ""
subscription_value_35: ""
subscription_value_36: ""
subscription_value_37: ""
subscription_value_38: ""
subscription_value_39: ""
subscription_value_40: ""
subscription_value_41: ""
subscription_value_42: ""
subscription_value_43: ""
subscription_value_44: ""
subscription_value_45: ""
subscription_value_46: ""
subscription_value_47: ""
subscription_value_48: ""
subscription_value_49: ""
subscription_value_50: ""
subscription_value_51: ""
subscription_value_52: ""
subscription_value_53: ""
subscription_value_54: ""
subscription_value_55: ""
subscription_value_56: ""
subscription_value_57: ""
subscription_value_58: ""
subscription_value_59: ""
subscription_value_60: ""
subscription_value_61: ""
subscription_value_62: ""
subscription_value_63: ""
subscription_value_64: ""
subscription_value_65: ""
subscription_value_66: ""
subscription_value_67: ""
subscription_value_68: ""
subscription_value_69: ""
subscription_value_70: ""
subscription_value_71: ""
subscription_value_72: ""
subscription_value_73: ""
subscription_value_74: ""
subscription_value_75: ""
subscription_value_76: ""
subscription_value_77: ""
subscription_value_78: ""
subscription_value_79: ""
subscription_value_80: ""
subscription_value_81: ""
subscription_value_82: ""
subscription_value_83: ""
subscription_value_84: ""
subscription_value_85: ""
subscription_value_86: ""
subscription_value_87: ""
subscription_value_88: ""
subscription_value_89: ""
subscription_value_90: ""
subscription_value_91: ""
subscription_value_92: ""
subscription_value_93: ""
subscription_value_94: ""
subscription_value_95: ""
subscription_value_96: ""
subscription_value_97: ""
subscription_value_98: ""
subscription_value_99: ""
subscription_value_100: ""
url: "{subscription_value}" url: "{subscription_value}"
url2: "" url2: "{subscription_value_2}"
url3: "" url3: "{subscription_value_3}"
url4: "" url4: "{subscription_value_4}"
url5: "" url5: "{subscription_value_5}"
url6: "" url6: "{subscription_value_6}"
url7: "" url7: "{subscription_value_7}"
url8: "" url8: "{subscription_value_8}"
url9: "" url9: "{subscription_value_9}"
url10: "" url10: "{subscription_value_10}"
url11: "" url11: "{subscription_value_11}"
url12: "" url12: "{subscription_value_12}"
url13: "" url13: "{subscription_value_13}"
url14: "" url14: "{subscription_value_14}"
url15: "" url15: "{subscription_value_15}"
url16: "" url16: "{subscription_value_16}"
url17: "" url17: "{subscription_value_17}"
url18: "" url18: "{subscription_value_18}"
url19: "" url19: "{subscription_value_19}"
url20: "" url20: "{subscription_value_20}"
url21: "" url21: "{subscription_value_21}"
url22: "" url22: "{subscription_value_22}"
url23: "" url23: "{subscription_value_23}"
url24: "" url24: "{subscription_value_24}"
url25: "" url25: "{subscription_value_25}"
url26: "" url26: "{subscription_value_26}"
url27: "" url27: "{subscription_value_27}"
url28: "" url28: "{subscription_value_28}"
url29: "" url29: "{subscription_value_29}"
url30: "" url30: "{subscription_value_30}"
url31: "" url31: "{subscription_value_31}"
url32: "" url32: "{subscription_value_32}"
url33: "" url33: "{subscription_value_33}"
url34: "" url34: "{subscription_value_34}"
url35: "" url35: "{subscription_value_35}"
url36: "" url36: "{subscription_value_36}"
url37: "" url37: "{subscription_value_37}"
url38: "" url38: "{subscription_value_38}"
url39: "" url39: "{subscription_value_39}"
url40: "" url40: "{subscription_value_40}"
url41: "" url41: "{subscription_value_41}"
url42: "" url42: "{subscription_value_42}"
url43: "" url43: "{subscription_value_43}"
url44: "" url44: "{subscription_value_44}"
url45: "" url45: "{subscription_value_45}"
url46: "" url46: "{subscription_value_46}"
url47: "" url47: "{subscription_value_47}"
url48: "" url48: "{subscription_value_48}"
url49: "" url49: "{subscription_value_49}"
url50: "" url50: "{subscription_value_50}"
url51: "" url51: "{subscription_value_51}"
url52: "" url52: "{subscription_value_52}"
url53: "" url53: "{subscription_value_53}"
url54: "" url54: "{subscription_value_54}"
url55: "" url55: "{subscription_value_55}"
url56: "" url56: "{subscription_value_56}"
url57: "" url57: "{subscription_value_57}"
url58: "" url58: "{subscription_value_58}"
url59: "" url59: "{subscription_value_59}"
url60: "" url60: "{subscription_value_60}"
url61: "" url61: "{subscription_value_61}"
url62: "" url62: "{subscription_value_62}"
url63: "" url63: "{subscription_value_63}"
url64: "" url64: "{subscription_value_64}"
url65: "" url65: "{subscription_value_65}"
url66: "" url66: "{subscription_value_66}"
url67: "" url67: "{subscription_value_67}"
url68: "" url68: "{subscription_value_68}"
url69: "" url69: "{subscription_value_69}"
url70: "" url70: "{subscription_value_70}"
url71: "" url71: "{subscription_value_71}"
url72: "" url72: "{subscription_value_72}"
url73: "" url73: "{subscription_value_73}"
url74: "" url74: "{subscription_value_74}"
url75: "" url75: "{subscription_value_75}"
url76: "" url76: "{subscription_value_76}"
url77: "" url77: "{subscription_value_77}"
url78: "" url78: "{subscription_value_78}"
url79: "" url79: "{subscription_value_79}"
url80: "" url80: "{subscription_value_80}"
url81: "" url81: "{subscription_value_81}"
url82: "" url82: "{subscription_value_82}"
url83: "" url83: "{subscription_value_83}"
url84: "" url84: "{subscription_value_84}"
url85: "" url85: "{subscription_value_85}"
url86: "" url86: "{subscription_value_86}"
url87: "" url87: "{subscription_value_87}"
url88: "" url88: "{subscription_value_88}"
url89: "" url89: "{subscription_value_89}"
url90: "" url90: "{subscription_value_90}"
url91: "" url91: "{subscription_value_91}"
url92: "" url92: "{subscription_value_92}"
url93: "" url93: "{subscription_value_93}"
url94: "" url94: "{subscription_value_94}"
url95: "" url95: "{subscription_value_95}"
url96: "" url96: "{subscription_value_96}"
url97: "" url97: "{subscription_value_97}"
url98: "" url98: "{subscription_value_98}"
url99: "" url99: "{subscription_value_99}"
url100: "" url100: "{subscription_value_100}"
# multi-url with bilateral scraping built into it via # multi-url with bilateral scraping built into it via
@ -165,11 +441,406 @@ presets:
- "_url_bilateral_overrides" - "_url_bilateral_overrides"
download: download:
- url: "{ %array_apply(urls, %bilateral_url) }" - url: "{%bilateral_url(url) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url2) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url3) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url4) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url5) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url6) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url7) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url8) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url9) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url10) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url11) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url12) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url13) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url14) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url15) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url16) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url17) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url18) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url19) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url20) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url21) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url22) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url23) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url24) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url25) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url26) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url27) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url28) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url29) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url30) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url31) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url32) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url33) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url34) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url35) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url36) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url37) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url38) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url39) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url40) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url41) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url42) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url43) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url44) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url45) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url46) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url47) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url48) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url49) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url50) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url51) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url52) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url53) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url54) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url55) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url56) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url57) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url58) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url59) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url60) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url61) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url62) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url63) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url64) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url65) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url66) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url67) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url68) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url69) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url70) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url71) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url72) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url73) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url74) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url75) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url76) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url77) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url78) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url79) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url80) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url81) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url82) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url83) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url84) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url85) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url86) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url87) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url88) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url89) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url90) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url91) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url92) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url93) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url94) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url95) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url96) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url97) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url98) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url99) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{%bilateral_url(url100) }"
download_reverse: False download_reverse: False
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
webpage_url: "{modified_webpage_url}"
_multi_url_bilateral: _multi_url_bilateral:
preset: preset:

View file

@ -108,4 +108,103 @@ presets:
} }
subscription_map: "{ {} }" subscription_map: "{ {} }"
urls: "{ %array_apply(%range(100, 1), %get_url_i) }" url: "{ %get_url_i(1) }"
url2: "{ %get_url_i(2) }"
url3: "{ %get_url_i(3) }"
url4: "{ %get_url_i(4) }"
url5: "{ %get_url_i(5) }"
url6: "{ %get_url_i(6) }"
url7: "{ %get_url_i(7) }"
url8: "{ %get_url_i(8) }"
url9: "{ %get_url_i(9) }"
url10: "{ %get_url_i(10) }"
url11: "{ %get_url_i(11) }"
url12: "{ %get_url_i(12) }"
url13: "{ %get_url_i(13) }"
url14: "{ %get_url_i(14) }"
url15: "{ %get_url_i(15) }"
url16: "{ %get_url_i(16) }"
url17: "{ %get_url_i(17) }"
url18: "{ %get_url_i(18) }"
url19: "{ %get_url_i(19) }"
url20: "{ %get_url_i(20) }"
url21: "{ %get_url_i(21) }"
url22: "{ %get_url_i(22) }"
url23: "{ %get_url_i(23) }"
url24: "{ %get_url_i(24) }"
url25: "{ %get_url_i(25) }"
url26: "{ %get_url_i(26) }"
url27: "{ %get_url_i(27) }"
url28: "{ %get_url_i(28) }"
url29: "{ %get_url_i(29) }"
url30: "{ %get_url_i(30) }"
url31: "{ %get_url_i(31) }"
url32: "{ %get_url_i(32) }"
url33: "{ %get_url_i(33) }"
url34: "{ %get_url_i(34) }"
url35: "{ %get_url_i(35) }"
url36: "{ %get_url_i(36) }"
url37: "{ %get_url_i(37) }"
url38: "{ %get_url_i(38) }"
url39: "{ %get_url_i(39) }"
url40: "{ %get_url_i(40) }"
url41: "{ %get_url_i(41) }"
url42: "{ %get_url_i(42) }"
url43: "{ %get_url_i(43) }"
url44: "{ %get_url_i(44) }"
url45: "{ %get_url_i(45) }"
url46: "{ %get_url_i(46) }"
url47: "{ %get_url_i(47) }"
url48: "{ %get_url_i(48) }"
url49: "{ %get_url_i(49) }"
url50: "{ %get_url_i(50) }"
url51: "{ %get_url_i(51) }"
url52: "{ %get_url_i(52) }"
url53: "{ %get_url_i(53) }"
url54: "{ %get_url_i(54) }"
url55: "{ %get_url_i(55) }"
url56: "{ %get_url_i(56) }"
url57: "{ %get_url_i(57) }"
url58: "{ %get_url_i(58) }"
url59: "{ %get_url_i(59) }"
url60: "{ %get_url_i(60) }"
url61: "{ %get_url_i(61) }"
url62: "{ %get_url_i(62) }"
url63: "{ %get_url_i(63) }"
url64: "{ %get_url_i(64) }"
url65: "{ %get_url_i(65) }"
url66: "{ %get_url_i(66) }"
url67: "{ %get_url_i(67) }"
url68: "{ %get_url_i(68) }"
url69: "{ %get_url_i(69) }"
url70: "{ %get_url_i(70) }"
url71: "{ %get_url_i(71) }"
url72: "{ %get_url_i(72) }"
url73: "{ %get_url_i(73) }"
url74: "{ %get_url_i(74) }"
url75: "{ %get_url_i(75) }"
url76: "{ %get_url_i(76) }"
url77: "{ %get_url_i(77) }"
url78: "{ %get_url_i(78) }"
url79: "{ %get_url_i(79) }"
url80: "{ %get_url_i(80) }"
url81: "{ %get_url_i(81) }"
url82: "{ %get_url_i(82) }"
url83: "{ %get_url_i(83) }"
url84: "{ %get_url_i(84) }"
url85: "{ %get_url_i(85) }"
url86: "{ %get_url_i(86) }"
url87: "{ %get_url_i(87) }"
url88: "{ %get_url_i(88) }"
url89: "{ %get_url_i(89) }"
url90: "{ %get_url_i(90) }"
url91: "{ %get_url_i(91) }"
url92: "{ %get_url_i(92) }"
url93: "{ %get_url_i(93) }"
url94: "{ %get_url_i(94) }"
url95: "{ %get_url_i(95) }"
url96: "{ %get_url_i(96) }"
url97: "{ %get_url_i(97) }"
url98: "{ %get_url_i(98) }"
url99: "{ %get_url_i(99) }"
url100: "{ %get_url_i(100) }"

View file

@ -1,4 +1,5 @@
from ytdl_sub.prebuilt_presets import PrebuiltPresets, get_prebuilt_preset_package_name from ytdl_sub.prebuilt_presets import PrebuiltPresets
from ytdl_sub.prebuilt_presets import get_prebuilt_preset_package_name
PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__) PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__)

View file

@ -8,7 +8,7 @@ presets:
urls: urls:
# The first URL will be all the artist's tracks. # The first URL will be all the artist's tracks.
# Treat these as singles - an album with a single track # Treat these as singles - an album with a single track
- url: "{url}" - url: "{url}/tracks"
include_sibling_metadata: False include_sibling_metadata: False
variables: variables:
sc_track_album: "{title}" sc_track_album: "{title}"

View file

@ -1,4 +1,5 @@
from ytdl_sub.prebuilt_presets import PrebuiltPresets, get_prebuilt_preset_package_name from ytdl_sub.prebuilt_presets import PrebuiltPresets
from ytdl_sub.prebuilt_presets import get_prebuilt_preset_package_name
PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__) PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__)

View file

@ -1,4 +1,5 @@
from ytdl_sub.prebuilt_presets import PrebuiltPresets, get_prebuilt_preset_package_name from ytdl_sub.prebuilt_presets import PrebuiltPresets
from ytdl_sub.prebuilt_presets import get_prebuilt_preset_package_name
PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__) PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__)

View file

@ -124,7 +124,7 @@ presets:
- "_tv_show_collection_asserts" - "_tv_show_collection_asserts"
download: download:
- url: "{ %array_at( %get_season_urls(collection_season_1_url), 0) }" - url: "{collection_season_1_url}"
variables: variables:
collection_season_number: "1" collection_season_number: "1"
collection_season_name: "{collection_season_1_name}" collection_season_name: "{collection_season_1_name}"
@ -142,493 +142,318 @@ presets:
uid: "avatar_uncropped" uid: "avatar_uncropped"
- name: "{tv_show_fanart_file_name}" - name: "{tv_show_fanart_file_name}"
uid: "banner_uncropped" uid: "banner_uncropped"
- url: "{ %array_slice( %get_season_urls(collection_season_1_url), 1) }"
variables:
collection_season_number: "1"
collection_season_name: "{collection_season_1_name}"
- url: "{ %array_at( %get_season_urls(collection_season_2_url), 0) }" - url: "{collection_season_2_url}"
variables: variables:
collection_season_number: "2" collection_season_number: "2"
collection_season_name: "{collection_season_2_name}" collection_season_name: "{collection_season_2_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_2_url), 1) }"
variables:
collection_season_number: "2"
collection_season_name: "{collection_season_2_name}"
- url: "{ %array_at( %get_season_urls(collection_season_3_url), 0) }" - url: "{collection_season_3_url}"
variables: variables:
collection_season_number: "3" collection_season_number: "3"
collection_season_name: "{collection_season_3_name}" collection_season_name: "{collection_season_3_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_3_url), 1) }"
variables:
collection_season_number: "3"
collection_season_name: "{collection_season_3_name}"
- url: "{ %array_at( %get_season_urls(collection_season_4_url), 0) }" - url: "{collection_season_4_url}"
variables: variables:
collection_season_number: "4" collection_season_number: "4"
collection_season_name: "{collection_season_4_name}" collection_season_name: "{collection_season_4_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_4_url), 1) }"
variables:
collection_season_number: "4"
collection_season_name: "{collection_season_4_name}"
- url: "{ %array_at( %get_season_urls(collection_season_5_url), 0) }" - url: "{collection_season_5_url}"
variables: variables:
collection_season_number: "5" collection_season_number: "5"
collection_season_name: "{collection_season_5_name}" collection_season_name: "{collection_season_5_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_5_url), 1) }"
variables:
collection_season_number: "5"
collection_season_name: "{collection_season_5_name}"
- url: "{ %array_at( %get_season_urls(collection_season_6_url), 0) }" - url: "{collection_season_6_url}"
variables: variables:
collection_season_number: "6" collection_season_number: "6"
collection_season_name: "{collection_season_6_name}" collection_season_name: "{collection_season_6_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_6_url), 1) }"
variables:
collection_season_number: "6"
collection_season_name: "{collection_season_6_name}"
- url: "{ %array_at( %get_season_urls(collection_season_7_url), 0) }" - url: "{collection_season_7_url}"
variables: variables:
collection_season_number: "7" collection_season_number: "7"
collection_season_name: "{collection_season_7_name}" collection_season_name: "{collection_season_7_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_7_url), 1) }"
variables:
collection_season_number: "7"
collection_season_name: "{collection_season_7_name}"
- url: "{ %array_at( %get_season_urls(collection_season_8_url), 0) }" - url: "{collection_season_8_url}"
variables: variables:
collection_season_number: "8" collection_season_number: "8"
collection_season_name: "{collection_season_8_name}" collection_season_name: "{collection_season_8_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_8_url), 1) }"
variables:
collection_season_number: "8"
collection_season_name: "{collection_season_8_name}"
- url: "{ %array_at( %get_season_urls(collection_season_9_url), 0) }" - url: "{collection_season_9_url}"
variables: variables:
collection_season_number: "9" collection_season_number: "9"
collection_season_name: "{collection_season_9_name}" collection_season_name: "{collection_season_9_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_9_url), 1) }"
variables:
collection_season_number: "9"
collection_season_name: "{collection_season_9_name}"
- url: "{ %array_at( %get_season_urls(collection_season_10_url), 0) }" - url: "{collection_season_10_url}"
variables: variables:
collection_season_number: "10" collection_season_number: "10"
collection_season_name: "{collection_season_10_name}" collection_season_name: "{collection_season_10_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_10_url), 1) }"
variables:
collection_season_number: "10"
collection_season_name: "{collection_season_10_name}"
- url: "{ %array_at( %get_season_urls(collection_season_11_url), 0) }" - url: "{collection_season_11_url}"
variables: variables:
collection_season_number: "11" collection_season_number: "11"
collection_season_name: "{collection_season_11_name}" collection_season_name: "{collection_season_11_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_11_url), 1) }"
variables:
collection_season_number: "11"
collection_season_name: "{collection_season_11_name}"
- url: "{ %array_at( %get_season_urls(collection_season_12_url), 0) }" - url: "{collection_season_12_url}"
variables: variables:
collection_season_number: "12" collection_season_number: "12"
collection_season_name: "{collection_season_12_name}" collection_season_name: "{collection_season_12_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_12_url), 1) }"
variables:
collection_season_number: "12"
collection_season_name: "{collection_season_12_name}"
- url: "{ %array_at( %get_season_urls(collection_season_13_url), 0) }" - url: "{collection_season_13_url}"
variables: variables:
collection_season_number: "13" collection_season_number: "13"
collection_season_name: "{collection_season_13_name}" collection_season_name: "{collection_season_13_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_13_url), 1) }"
variables:
collection_season_number: "13"
collection_season_name: "{collection_season_13_name}"
- url: "{ %array_at( %get_season_urls(collection_season_14_url), 0) }" - url: "{collection_season_14_url}"
variables: variables:
collection_season_number: "14" collection_season_number: "14"
collection_season_name: "{collection_season_14_name}" collection_season_name: "{collection_season_14_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_14_url), 1) }"
variables:
collection_season_number: "14"
collection_season_name: "{collection_season_14_name}"
- url: "{ %array_at( %get_season_urls(collection_season_15_url), 0) }" - url: "{collection_season_15_url}"
variables: variables:
collection_season_number: "15" collection_season_number: "15"
collection_season_name: "{collection_season_15_name}" collection_season_name: "{collection_season_15_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_15_url), 1) }"
variables:
collection_season_number: "15"
collection_season_name: "{collection_season_15_name}"
- url: "{ %array_at( %get_season_urls(collection_season_16_url), 0) }" - url: "{collection_season_16_url}"
variables: variables:
collection_season_number: "16" collection_season_number: "16"
collection_season_name: "{collection_season_16_name}" collection_season_name: "{collection_season_16_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_16_url), 1) }"
variables:
collection_season_number: "16"
collection_season_name: "{collection_season_16_name}"
- url: "{ %array_at( %get_season_urls(collection_season_17_url), 0) }" - url: "{collection_season_17_url}"
variables: variables:
collection_season_number: "17" collection_season_number: "17"
collection_season_name: "{collection_season_17_name}" collection_season_name: "{collection_season_17_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_17_url), 1) }"
variables:
collection_season_number: "17"
collection_season_name: "{collection_season_17_name}"
- url: "{ %array_at( %get_season_urls(collection_season_18_url), 0) }" - url: "{collection_season_18_url}"
variables: variables:
collection_season_number: "18" collection_season_number: "18"
collection_season_name: "{collection_season_18_name}" collection_season_name: "{collection_season_18_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_18_url), 1) }"
variables:
collection_season_number: "18"
collection_season_name: "{collection_season_18_name}"
- url: "{ %array_at( %get_season_urls(collection_season_19_url), 0) }" - url: "{collection_season_19_url}"
variables: variables:
collection_season_number: "19" collection_season_number: "19"
collection_season_name: "{collection_season_19_name}" collection_season_name: "{collection_season_19_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_19_url), 1) }"
variables:
collection_season_number: "19"
collection_season_name: "{collection_season_19_name}"
- url: "{ %array_at( %get_season_urls(collection_season_20_url), 0) }" - url: "{collection_season_20_url}"
variables: variables:
collection_season_number: "20" collection_season_number: "20"
collection_season_name: "{collection_season_20_name}" collection_season_name: "{collection_season_20_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_20_url), 1) }"
variables:
collection_season_number: "20"
collection_season_name: "{collection_season_20_name}"
- url: "{ %array_at( %get_season_urls(collection_season_21_url), 0) }" - url: "{collection_season_21_url}"
variables: variables:
collection_season_number: "21" collection_season_number: "21"
collection_season_name: "{collection_season_21_name}" collection_season_name: "{collection_season_21_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_21_url), 1) }"
variables:
collection_season_number: "21"
collection_season_name: "{collection_season_21_name}"
- url: "{ %array_at( %get_season_urls(collection_season_22_url), 0) }" - url: "{collection_season_22_url}"
variables: variables:
collection_season_number: "22" collection_season_number: "22"
collection_season_name: "{collection_season_22_name}" collection_season_name: "{collection_season_22_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_22_url), 1) }"
variables:
collection_season_number: "22"
collection_season_name: "{collection_season_22_name}"
- url: "{ %array_at( %get_season_urls(collection_season_23_url), 0) }" - url: "{collection_season_23_url}"
variables: variables:
collection_season_number: "23" collection_season_number: "23"
collection_season_name: "{collection_season_23_name}" collection_season_name: "{collection_season_23_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_23_url), 1) }"
variables:
collection_season_number: "23"
collection_season_name: "{collection_season_23_name}"
- url: "{ %array_at( %get_season_urls(collection_season_24_url), 0) }" - url: "{collection_season_24_url}"
variables: variables:
collection_season_number: "24" collection_season_number: "24"
collection_season_name: "{collection_season_24_name}" collection_season_name: "{collection_season_24_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_24_url), 1) }"
variables:
collection_season_number: "24"
collection_season_name: "{collection_season_24_name}"
- url: "{ %array_at( %get_season_urls(collection_season_25_url), 0) }" - url: "{collection_season_25_url}"
variables: variables:
collection_season_number: "25" collection_season_number: "25"
collection_season_name: "{collection_season_25_name}" collection_season_name: "{collection_season_25_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_25_url), 1) }"
variables:
collection_season_number: "25"
collection_season_name: "{collection_season_25_name}"
- url: "{ %array_at( %get_season_urls(collection_season_26_url), 0) }" - url: "{collection_season_26_url}"
variables: variables:
collection_season_number: "26" collection_season_number: "26"
collection_season_name: "{collection_season_26_name}" collection_season_name: "{collection_season_26_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_26_url), 1) }"
variables:
collection_season_number: "26"
collection_season_name: "{collection_season_26_name}"
- url: "{ %array_at( %get_season_urls(collection_season_27_url), 0) }" - url: "{collection_season_27_url}"
variables: variables:
collection_season_number: "27" collection_season_number: "27"
collection_season_name: "{collection_season_27_name}" collection_season_name: "{collection_season_27_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_27_url), 1) }"
variables:
collection_season_number: "27"
collection_season_name: "{collection_season_27_name}"
- url: "{ %array_at( %get_season_urls(collection_season_28_url), 0) }" - url: "{collection_season_28_url}"
variables: variables:
collection_season_number: "28" collection_season_number: "28"
collection_season_name: "{collection_season_28_name}" collection_season_name: "{collection_season_28_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_28_url), 1) }"
variables:
collection_season_number: "28"
collection_season_name: "{collection_season_28_name}"
- url: "{ %array_at( %get_season_urls(collection_season_29_url), 0) }" - url: "{collection_season_29_url}"
variables: variables:
collection_season_number: "29" collection_season_number: "29"
collection_season_name: "{collection_season_29_name}" collection_season_name: "{collection_season_29_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_29_url), 1) }"
variables:
collection_season_number: "29"
collection_season_name: "{collection_season_29_name}"
- url: "{ %array_at( %get_season_urls(collection_season_30_url), 0) }" - url: "{collection_season_30_url}"
variables: variables:
collection_season_number: "30" collection_season_number: "30"
collection_season_name: "{collection_season_30_name}" collection_season_name: "{collection_season_30_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_30_url), 1) }"
variables:
collection_season_number: "30"
collection_season_name: "{collection_season_30_name}"
- url: "{ %array_at( %get_season_urls(collection_season_31_url), 0) }" - url: "{collection_season_31_url}"
variables: variables:
collection_season_number: "31" collection_season_number: "31"
collection_season_name: "{collection_season_31_name}" collection_season_name: "{collection_season_31_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_31_url), 1) }"
variables:
collection_season_number: "31"
collection_season_name: "{collection_season_31_name}"
- url: "{ %array_at( %get_season_urls(collection_season_32_url), 0) }" - url: "{collection_season_32_url}"
variables: variables:
collection_season_number: "32" collection_season_number: "32"
collection_season_name: "{collection_season_32_name}" collection_season_name: "{collection_season_32_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_32_url), 1) }"
variables:
collection_season_number: "32"
collection_season_name: "{collection_season_32_name}"
- url: "{ %array_at( %get_season_urls(collection_season_33_url), 0) }" - url: "{collection_season_33_url}"
variables: variables:
collection_season_number: "33" collection_season_number: "33"
collection_season_name: "{collection_season_33_name}" collection_season_name: "{collection_season_33_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_33_url), 1) }"
variables:
collection_season_number: "33"
collection_season_name: "{collection_season_33_name}"
- url: "{ %array_at( %get_season_urls(collection_season_34_url), 0) }" - url: "{collection_season_34_url}"
variables: variables:
collection_season_number: "34" collection_season_number: "34"
collection_season_name: "{collection_season_34_name}" collection_season_name: "{collection_season_34_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_34_url), 1) }"
variables:
collection_season_number: "34"
collection_season_name: "{collection_season_34_name}"
- url: "{ %array_at( %get_season_urls(collection_season_35_url), 0) }" - url: "{collection_season_35_url}"
variables: variables:
collection_season_number: "35" collection_season_number: "35"
collection_season_name: "{collection_season_35_name}" collection_season_name: "{collection_season_35_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_35_url), 1) }"
variables:
collection_season_number: "35"
collection_season_name: "{collection_season_35_name}"
- url: "{ %array_at( %get_season_urls(collection_season_36_url), 0) }" - url: "{collection_season_36_url}"
variables: variables:
collection_season_number: "36" collection_season_number: "36"
collection_season_name: "{collection_season_36_name}" collection_season_name: "{collection_season_36_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_36_url), 1) }"
variables:
collection_season_number: "36"
collection_season_name: "{collection_season_36_name}"
- url: "{ %array_at( %get_season_urls(collection_season_37_url), 0) }" - url: "{collection_season_37_url}"
variables: variables:
collection_season_number: "37" collection_season_number: "37"
collection_season_name: "{collection_season_37_name}" collection_season_name: "{collection_season_37_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_37_url), 1) }"
variables:
collection_season_number: "37"
collection_season_name: "{collection_season_37_name}"
- url: "{ %array_at( %get_season_urls(collection_season_38_url), 0) }" - url: "{collection_season_38_url}"
variables: variables:
collection_season_number: "38" collection_season_number: "38"
collection_season_name: "{collection_season_38_name}" collection_season_name: "{collection_season_38_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_38_url), 1) }"
variables:
collection_season_number: "38"
collection_season_name: "{collection_season_38_name}"
- url: "{ %array_at( %get_season_urls(collection_season_39_url), 0) }" - url: "{collection_season_39_url}"
variables: variables:
collection_season_number: "39" collection_season_number: "39"
collection_season_name: "{collection_season_39_name}" collection_season_name: "{collection_season_39_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_39_url), 1) }"
variables:
collection_season_number: "39"
collection_season_name: "{collection_season_39_name}"
- url: "{ %array_at( %get_season_urls(collection_season_40_url), 0) }" - url: "{collection_season_40_url}"
variables: variables:
collection_season_number: "40" collection_season_number: "40"
collection_season_name: "{collection_season_40_name}" collection_season_name: "{collection_season_40_name}"
playlist_thumbnails: playlist_thumbnails:
- name: "{season_poster_file_name}" - name: "{season_poster_file_name}"
uid: "latest_entry" uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_40_url), 1) }"
variables:
collection_season_number: "40"
collection_season_name: "{collection_season_40_name}"
# Place season 0 at end
- url: "{ %array_at( %get_season_urls(collection_season_0_url), 0) }"
variables:
collection_season_number: "0"
collection_season_name: "{collection_season_0_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_0_url), 1) }"
variables:
collection_season_number: "0"
collection_season_name: "{collection_season_0_name}"
output_directory_nfo_tags: output_directory_nfo_tags:
tags: tags:
@ -800,7 +625,6 @@ presets:
collection_season_38_name: "{s38_name}" collection_season_38_name: "{s38_name}"
collection_season_39_name: "{s39_name}" collection_season_39_name: "{s39_name}"
collection_season_40_name: "{s40_name}" collection_season_40_name: "{s40_name}"
collection_season_0_name: "{s00_name}"
# Legacy url variable # Legacy url variable
collection_season_1_url: "{s01_url}" collection_season_1_url: "{s01_url}"
@ -843,7 +667,6 @@ presets:
collection_season_38_url: "{s38_url}" collection_season_38_url: "{s38_url}"
collection_season_39_url: "{s39_url}" collection_season_39_url: "{s39_url}"
collection_season_40_url: "{s40_url}" collection_season_40_url: "{s40_url}"
collection_season_0_url: "{s00_url}"
s01_name: "" s01_name: ""
s02_name: "" s02_name: ""
@ -885,7 +708,6 @@ presets:
s38_name: "" s38_name: ""
s39_name: "" s39_name: ""
s40_name: "" s40_name: ""
s00_name: ""
s01_url: "" s01_url: ""
s02_url: "" s02_url: ""
@ -927,22 +749,13 @@ presets:
s38_url: "" s38_url: ""
s39_url: "" s39_url: ""
s40_url: "" s40_url: ""
s00_url: ""
"%get_season_urls": >-
{ %if( %is_array( $0 ), $0, [ $0 ] ) }
# $0 - season url variable
# $1 - get the i'th url from the array
"%get_season_url": >-
{ %array_at( %get_season_urls($0), $1, "" ) }
_tv_show_collection_bilateral: _tv_show_collection_bilateral:
preset: preset:
- "_url_bilateral_overrides" - "_url_bilateral_overrides"
download: download:
- url: "{ %array_apply( %get_season_urls(collection_season_1_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_1_url) }"
variables: variables:
collection_season_number: "1" collection_season_number: "1"
collection_season_name: "{collection_season_1_name}" collection_season_name: "{collection_season_1_name}"
@ -950,7 +763,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_2_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_2_url) }"
variables: variables:
collection_season_number: "2" collection_season_number: "2"
collection_season_name: "{collection_season_2_name}" collection_season_name: "{collection_season_2_name}"
@ -958,7 +771,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_3_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_3_url) }"
variables: variables:
collection_season_number: "3" collection_season_number: "3"
collection_season_name: "{collection_season_3_name}" collection_season_name: "{collection_season_3_name}"
@ -966,7 +779,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_4_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_4_url) }"
variables: variables:
collection_season_number: "4" collection_season_number: "4"
collection_season_name: "{collection_season_4_name}" collection_season_name: "{collection_season_4_name}"
@ -974,7 +787,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_5_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_5_url) }"
variables: variables:
collection_season_number: "5" collection_season_number: "5"
collection_season_name: "{collection_season_5_name}" collection_season_name: "{collection_season_5_name}"
@ -982,7 +795,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_6_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_6_url) }"
variables: variables:
collection_season_number: "6" collection_season_number: "6"
collection_season_name: "{collection_season_6_name}" collection_season_name: "{collection_season_6_name}"
@ -990,7 +803,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_7_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_7_url) }"
variables: variables:
collection_season_number: "7" collection_season_number: "7"
collection_season_name: "{collection_season_7_name}" collection_season_name: "{collection_season_7_name}"
@ -998,7 +811,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_8_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_8_url) }"
variables: variables:
collection_season_number: "8" collection_season_number: "8"
collection_season_name: "{collection_season_8_name}" collection_season_name: "{collection_season_8_name}"
@ -1006,7 +819,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_9_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_9_url) }"
variables: variables:
collection_season_number: "9" collection_season_number: "9"
collection_season_name: "{collection_season_9_name}" collection_season_name: "{collection_season_9_name}"
@ -1014,7 +827,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_10_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_10_url) }"
variables: variables:
collection_season_number: "10" collection_season_number: "10"
collection_season_name: "{collection_season_10_name}" collection_season_name: "{collection_season_10_name}"
@ -1022,7 +835,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_11_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_11_url) }"
variables: variables:
collection_season_number: "11" collection_season_number: "11"
collection_season_name: "{collection_season_11_name}" collection_season_name: "{collection_season_11_name}"
@ -1030,7 +843,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_12_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_12_url) }"
variables: variables:
collection_season_number: "12" collection_season_number: "12"
collection_season_name: "{collection_season_12_name}" collection_season_name: "{collection_season_12_name}"
@ -1038,7 +851,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_13_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_13_url) }"
variables: variables:
collection_season_number: "13" collection_season_number: "13"
collection_season_name: "{collection_season_13_name}" collection_season_name: "{collection_season_13_name}"
@ -1046,7 +859,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_14_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_14_url) }"
variables: variables:
collection_season_number: "14" collection_season_number: "14"
collection_season_name: "{collection_season_14_name}" collection_season_name: "{collection_season_14_name}"
@ -1054,7 +867,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_15_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_15_url) }"
variables: variables:
collection_season_number: "15" collection_season_number: "15"
collection_season_name: "{collection_season_15_name}" collection_season_name: "{collection_season_15_name}"
@ -1062,7 +875,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_16_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_16_url) }"
variables: variables:
collection_season_number: "16" collection_season_number: "16"
collection_season_name: "{collection_season_16_name}" collection_season_name: "{collection_season_16_name}"
@ -1070,7 +883,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_17_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_17_url) }"
variables: variables:
collection_season_number: "17" collection_season_number: "17"
collection_season_name: "{collection_season_17_name}" collection_season_name: "{collection_season_17_name}"
@ -1078,7 +891,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_18_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_18_url) }"
variables: variables:
collection_season_number: "18" collection_season_number: "18"
collection_season_name: "{collection_season_18_name}" collection_season_name: "{collection_season_18_name}"
@ -1086,7 +899,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_19_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_19_url) }"
variables: variables:
collection_season_number: "19" collection_season_number: "19"
collection_season_name: "{collection_season_19_name}" collection_season_name: "{collection_season_19_name}"
@ -1094,7 +907,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_20_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_20_url) }"
variables: variables:
collection_season_number: "20" collection_season_number: "20"
collection_season_name: "{collection_season_20_name}" collection_season_name: "{collection_season_20_name}"
@ -1102,7 +915,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_21_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_21_url) }"
variables: variables:
collection_season_number: "21" collection_season_number: "21"
collection_season_name: "{collection_season_21_name}" collection_season_name: "{collection_season_21_name}"
@ -1110,7 +923,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_22_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_22_url) }"
variables: variables:
collection_season_number: "22" collection_season_number: "22"
collection_season_name: "{collection_season_22_name}" collection_season_name: "{collection_season_22_name}"
@ -1118,7 +931,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_23_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_23_url) }"
variables: variables:
collection_season_number: "23" collection_season_number: "23"
collection_season_name: "{collection_season_23_name}" collection_season_name: "{collection_season_23_name}"
@ -1126,7 +939,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_24_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_24_url) }"
variables: variables:
collection_season_number: "24" collection_season_number: "24"
collection_season_name: "{collection_season_24_name}" collection_season_name: "{collection_season_24_name}"
@ -1134,7 +947,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_25_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_25_url) }"
variables: variables:
collection_season_number: "25" collection_season_number: "25"
collection_season_name: "{collection_season_25_name}" collection_season_name: "{collection_season_25_name}"
@ -1142,7 +955,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_26_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_26_url) }"
variables: variables:
collection_season_number: "26" collection_season_number: "26"
collection_season_name: "{collection_season_26_name}" collection_season_name: "{collection_season_26_name}"
@ -1150,7 +963,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_27_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_27_url) }"
variables: variables:
collection_season_number: "27" collection_season_number: "27"
collection_season_name: "{collection_season_27_name}" collection_season_name: "{collection_season_27_name}"
@ -1158,7 +971,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_28_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_28_url) }"
variables: variables:
collection_season_number: "28" collection_season_number: "28"
collection_season_name: "{collection_season_28_name}" collection_season_name: "{collection_season_28_name}"
@ -1166,7 +979,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_29_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_29_url) }"
variables: variables:
collection_season_number: "29" collection_season_number: "29"
collection_season_name: "{collection_season_29_name}" collection_season_name: "{collection_season_29_name}"
@ -1174,7 +987,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_30_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_30_url) }"
variables: variables:
collection_season_number: "30" collection_season_number: "30"
collection_season_name: "{collection_season_30_name}" collection_season_name: "{collection_season_30_name}"
@ -1182,7 +995,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_31_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_31_url) }"
variables: variables:
collection_season_number: "31" collection_season_number: "31"
collection_season_name: "{collection_season_31_name}" collection_season_name: "{collection_season_31_name}"
@ -1190,7 +1003,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_32_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_32_url) }"
variables: variables:
collection_season_number: "32" collection_season_number: "32"
collection_season_name: "{collection_season_32_name}" collection_season_name: "{collection_season_32_name}"
@ -1198,7 +1011,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_33_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_33_url) }"
variables: variables:
collection_season_number: "33" collection_season_number: "33"
collection_season_name: "{collection_season_33_name}" collection_season_name: "{collection_season_33_name}"
@ -1206,7 +1019,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_34_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_34_url) }"
variables: variables:
collection_season_number: "34" collection_season_number: "34"
collection_season_name: "{collection_season_34_name}" collection_season_name: "{collection_season_34_name}"
@ -1214,7 +1027,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_35_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_35_url) }"
variables: variables:
collection_season_number: "35" collection_season_number: "35"
collection_season_name: "{collection_season_35_name}" collection_season_name: "{collection_season_35_name}"
@ -1222,7 +1035,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_36_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_36_url) }"
variables: variables:
collection_season_number: "36" collection_season_number: "36"
collection_season_name: "{collection_season_36_name}" collection_season_name: "{collection_season_36_name}"
@ -1230,7 +1043,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_37_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_37_url) }"
variables: variables:
collection_season_number: "37" collection_season_number: "37"
collection_season_name: "{collection_season_37_name}" collection_season_name: "{collection_season_37_name}"
@ -1238,7 +1051,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_38_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_38_url) }"
variables: variables:
collection_season_number: "38" collection_season_number: "38"
collection_season_name: "{collection_season_38_name}" collection_season_name: "{collection_season_38_name}"
@ -1246,7 +1059,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_39_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_39_url) }"
variables: variables:
collection_season_number: "39" collection_season_number: "39"
collection_season_name: "{collection_season_39_name}" collection_season_name: "{collection_season_39_name}"
@ -1254,7 +1067,7 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
- url: "{ %array_apply( %get_season_urls(collection_season_40_url), %bilateral_url) }" - url: "{ %bilateral_url(collection_season_40_url) }"
variables: variables:
collection_season_number: "40" collection_season_number: "40"
collection_season_name: "{collection_season_40_name}" collection_season_name: "{collection_season_40_name}"
@ -1262,15 +1075,6 @@ presets:
ytdl_options: ytdl_options:
playlist_items: "-1:0:-1" playlist_items: "-1:0:-1"
# Season 0 at end (to download first)
- url: "{ %array_apply( %get_season_urls(collection_season_0_url), %bilateral_url) }"
variables:
collection_season_number: "0"
collection_season_name: "{collection_season_0_name}"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
_tv_show_collection_asserts: _tv_show_collection_asserts:
overrides: overrides:
url: "" url: ""

View file

@ -1,4 +1,5 @@
from typing import Callable, Dict from typing import Callable
from typing import Dict
from ytdl_sub.script.functions.array_functions import ArrayFunctions from ytdl_sub.script.functions.array_functions import ArrayFunctions
from ytdl_sub.script.functions.boolean_functions import BooleanFunctions from ytdl_sub.script.functions.boolean_functions import BooleanFunctions

View file

@ -1,21 +1,18 @@
import itertools import itertools
from typing import List, Optional from typing import List
from typing import Optional
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import ( from ytdl_sub.script.types.resolvable import AnyArgument
AnyArgument, from ytdl_sub.script.types.resolvable import Boolean
Boolean, from ytdl_sub.script.types.resolvable import Integer
Integer, from ytdl_sub.script.types.resolvable import Lambda
Lambda, from ytdl_sub.script.types.resolvable import LambdaReduce
LambdaReduce, from ytdl_sub.script.types.resolvable import LambdaTwo
LambdaTwo, from ytdl_sub.script.types.resolvable import Resolvable
Resolvable, from ytdl_sub.script.utils.exceptions import UNREACHABLE
) from ytdl_sub.script.utils.exceptions import ArrayValueDoesNotExist
from ytdl_sub.script.utils.exceptions import ( from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
UNREACHABLE,
ArrayValueDoesNotExist,
FunctionRuntimeException,
)
class ArrayFunctions: class ArrayFunctions:

View file

@ -1,6 +1,10 @@
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import AnyArgument, Boolean, Float, Integer, String from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
# pylint: disable=invalid-name # pylint: disable=invalid-name

View file

@ -1,11 +1,9 @@
from typing import Union from typing import Union
from ytdl_sub.script.types.resolvable import ( from ytdl_sub.script.types.resolvable import AnyArgument
AnyArgument, from ytdl_sub.script.types.resolvable import Boolean
Boolean, from ytdl_sub.script.types.resolvable import ReturnableArgumentA
ReturnableArgumentA, from ytdl_sub.script.types.resolvable import ReturnableArgumentB
ReturnableArgumentB,
)
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException from ytdl_sub.script.utils.exceptions import FunctionRuntimeException

View file

@ -1,6 +1,7 @@
from datetime import datetime from datetime import datetime
from ytdl_sub.script.types.resolvable import Integer, String from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
class DateFunctions: class DateFunctions:

View file

@ -1,4 +1,6 @@
from ytdl_sub.script.types.resolvable import AnyArgument, ReturnableArgument, String from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError

View file

@ -3,14 +3,12 @@ from typing import Any
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import ( from ytdl_sub.script.types.resolvable import AnyArgument
AnyArgument, from ytdl_sub.script.types.resolvable import Boolean
Boolean, from ytdl_sub.script.types.resolvable import Float
Float, from ytdl_sub.script.types.resolvable import Integer
Integer, from ytdl_sub.script.types.resolvable import Resolvable
Resolvable, from ytdl_sub.script.types.resolvable import String
String,
)
from ytdl_sub.script.utils.exceptions import UNREACHABLE from ytdl_sub.script.utils.exceptions import UNREACHABLE

View file

@ -1,21 +1,18 @@
from typing import Dict, Optional from typing import Dict
from typing import Optional
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import ( from ytdl_sub.script.types.resolvable import AnyArgument
AnyArgument, from ytdl_sub.script.types.resolvable import Boolean
Boolean, from ytdl_sub.script.types.resolvable import Hashable
Hashable, from ytdl_sub.script.types.resolvable import Integer
Integer, from ytdl_sub.script.types.resolvable import LambdaThree
LambdaThree, from ytdl_sub.script.types.resolvable import LambdaTwo
LambdaTwo, from ytdl_sub.script.types.resolvable import String
String, from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
) from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException
from ytdl_sub.script.utils.exceptions import ( from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException
FunctionRuntimeException,
KeyDoesNotExistRuntimeException,
KeyNotHashableRuntimeException,
)
class MapFunctions: class MapFunctions:

View file

@ -1,8 +1,9 @@
import math import math
from typing import Optional
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import AnyArgument, Float, Integer, Numeric from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Numeric
def _to_numeric(value: int | float) -> Numeric: def _to_numeric(value: int | float) -> Numeric:
@ -95,18 +96,3 @@ class NumericFunctions:
Returns min of all values. Returns min of all values.
""" """
return _to_numeric(min(val.value for val in values)) return _to_numeric(min(val.value for val in values))
@staticmethod
def range(
end: Integer, start: Optional[Integer] = None, step: Optional[Integer] = None
) -> Array:
"""
:description:
Returns the desired range of Integers in the form of an Array.
"""
if start is None:
start = Integer(0)
if step is None:
step = Integer(1)
return Array(value=[Integer(idx) for idx in range(start.value, end.value, step.value)])

View file

@ -1,6 +1,8 @@
from typing import Optional from typing import Optional
from ytdl_sub.script.types.resolvable import AnyArgument, Integer, ReturnableArgument from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.logger import Logger
logger = Logger.get(name="preset") logger = Logger.get(name="preset")

View file

@ -1,9 +1,15 @@
import re import re
from typing import AnyStr, List, Match, Optional from typing import AnyStr
from typing import List
from typing import Match
from typing import Optional
from ytdl_sub.script.functions.array_functions import ArrayFunctions from ytdl_sub.script.functions.array_functions import ArrayFunctions
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import Boolean, Float, Integer, String from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException from ytdl_sub.script.utils.exceptions import FunctionRuntimeException

View file

@ -1,7 +1,12 @@
from typing import Optional from typing import Optional
from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import AnyArgument, Boolean, Float, Integer, Numeric, String from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Numeric
from ytdl_sub.script.types.resolvable import String
class StringFunctions: class StringFunctions:

View file

@ -1,31 +1,37 @@
import json import json
from enum import Enum from enum import Enum
from typing import Dict, List, Optional, Set from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.script.functions import Functions from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.array import UnresolvedArray from ytdl_sub.script.types.array import UnresolvedArray
from ytdl_sub.script.types.function import Argument, BuiltInFunction, CustomFunction, Function from ytdl_sub.script.types.function import Argument
from ytdl_sub.script.types.function import BuiltInFunction
from ytdl_sub.script.types.function import CustomFunction
from ytdl_sub.script.types.function import Function
from ytdl_sub.script.types.map import UnresolvedMap from ytdl_sub.script.types.map import UnresolvedMap
from ytdl_sub.script.types.resolvable import Boolean, Float, Integer, Lambda, NonHashable, String from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import NonHashable
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.types.variable import FunctionArgument, Variable from ytdl_sub.script.types.variable import FunctionArgument
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.utils.exception_formatters import ParserExceptionFormatter from ytdl_sub.script.utils.exception_formatters import ParserExceptionFormatter
from ytdl_sub.script.utils.exceptions import ( from ytdl_sub.script.utils.exceptions import UNREACHABLE
UNREACHABLE, from ytdl_sub.script.utils.exceptions import CycleDetected
CycleDetected, from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist
FunctionDoesNotExist, from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
IncompatibleFunctionArguments, from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName
InvalidCustomFunctionArgumentName, from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
InvalidSyntaxException, from ytdl_sub.script.utils.exceptions import InvalidVariableName
InvalidVariableName, from ytdl_sub.script.utils.exceptions import UserException
UserException, from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
VariableDoesNotExist, from ytdl_sub.script.utils.name_validation import validate_variable_name
)
from ytdl_sub.script.utils.name_validation import (
is_function,
to_function_name,
validate_variable_name,
)
# pylint: disable=invalid-name # pylint: disable=invalid-name
# pylint: disable=too-many-branches # pylint: disable=too-many-branches
@ -138,9 +144,6 @@ class _Parser:
): ):
self._text = text self._text = text
self._name = name self._name = name
if name and is_function(name):
self._name = to_function_name(name)
self._custom_function_names = custom_function_names self._custom_function_names = custom_function_names
self._variable_names = variable_names self._variable_names = variable_names
self._pos = 0 self._pos = 0
@ -572,6 +575,7 @@ class _Parser:
return True return True
def _parse(self) -> SyntaxTree: def _parse(self) -> SyntaxTree:
while ch := self._read(): while ch := self._read():
continue_parse = self._parse_main_loop(ch) continue_parse = self._parse_main_loop(ch)
if not continue_parse: if not continue_parse:
@ -601,7 +605,7 @@ def parse(
name=name, name=name,
custom_function_names=custom_function_names, custom_function_names=custom_function_names,
variable_names=variable_names, variable_names=variable_names,
).ast.maybe_resolvable_casted() ).ast
# pylint: enable=invalid-name # pylint: enable=invalid-name

View file

@ -1,31 +1,46 @@
# pylint: disable=missing-raises-doc # pylint: disable=missing-raises-doc
from collections import defaultdict from typing import Dict
from typing import Dict, List, Optional, Set from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.script.functions import Functions from ytdl_sub.script.functions import Functions
from ytdl_sub.script.parser import parse from ytdl_sub.script.parser import parse
from ytdl_sub.script.script_output import ScriptOutput from ytdl_sub.script.script_output import ScriptOutput
from ytdl_sub.script.types.resolvable import Argument, BuiltInFunctionType, Lambda, Resolvable from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree, SyntaxTree from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.variable import FunctionArgument, Variable from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.variable_dependency import VariableDependency from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.exceptions import ( from ytdl_sub.script.types.variable import FunctionArgument
UNREACHABLE, from ytdl_sub.script.types.variable import Variable
CycleDetected, from ytdl_sub.script.utils.exceptions import UNREACHABLE
IncompatibleFunctionArguments, from ytdl_sub.script.utils.exceptions import CycleDetected
InvalidCustomFunctionArguments, from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
RuntimeException, from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments
ScriptVariableNotResolved, from ytdl_sub.script.utils.exceptions import RuntimeException
) from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.script.utils.name_validation import ( from ytdl_sub.script.utils.name_validation import validate_variable_name
is_function,
to_function_definition_name,
to_function_name,
validate_variable_name,
)
from ytdl_sub.script.utils.type_checking import FunctionSpec from ytdl_sub.script.utils.type_checking import FunctionSpec
def _is_function(override_name: str):
return override_name.startswith("%")
def _function_name(function_key: str) -> str:
"""
Drop the % in %custom_function
"""
return function_key[1:]
def _to_function_definition_name(function_key: str) -> str:
"""
Add % in %custom_function
"""
return f"%{function_key}"
class Script: class Script:
""" """
Takes a dictionary of both Takes a dictionary of both
@ -34,71 +49,43 @@ class Script:
``{ %custom_function: syntax }`` ``{ %custom_function: syntax }``
""" """
def _throw_cycle_error( def _ensure_no_cycle(
self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree] self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree]
): ):
type_name, pre = ( if dep not in definitions:
("custom functions", "%") if definitions is self._functions else ("variables", "") return # does not exist, will throw downstream in parser
)
cycle_deps = [name] + deps + [dep]
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}") if name in deps + [dep]:
type_name, pre = (
("custom functions", "%") if definitions is self._functions else ("variables", "")
)
cycle_deps = [name] + deps + [dep]
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}")
def _traverse_variable_dependencies( def _traverse_variable_dependencies(
self, self,
variable_name: str, variable_name: str,
variable_dependency: SyntaxTree, variable_dependency: SyntaxTree,
deps: List[str], deps: List[str],
ensured: Dict[str, Set[str]],
) -> None: ) -> None:
for dep in variable_dependency.variables: for dep in variable_dependency.variables:
if variable_name == dep.name: self._ensure_no_cycle(
self._throw_cycle_error( name=variable_name, dep=dep.name, deps=deps, definitions=self._variables
name=variable_name, dep=dep.name, deps=deps, definitions=self._variables )
)
if dep.name in ensured[variable_name]:
continue
self._traverse_variable_dependencies( self._traverse_variable_dependencies(
variable_name=variable_name, variable_name=variable_name,
variable_dependency=self._variables[dep.name], variable_dependency=self._variables[dep.name],
deps=deps + [dep.name], deps=deps + [dep.name],
ensured=ensured,
) )
ensured[variable_name].add(dep.name)
for custom_func in variable_dependency.custom_function_dependencies(
custom_function_definitions=self._functions
):
for dep in self._functions[custom_func.name].variables:
if variable_name == dep.name:
self._throw_cycle_error(
name=variable_name,
dep=dep.name,
deps=deps + [custom_func.definition_name()],
definitions=self._variables,
)
if dep.name in ensured[variable_name]:
continue
self._traverse_variable_dependencies(
variable_name=variable_name,
variable_dependency=self._variables[dep.name],
deps=deps + [custom_func.definition_name(), dep.name],
ensured=ensured,
)
ensured[variable_name].add(dep.name)
def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]): def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]):
ensured: Dict[str, Set[str]] = defaultdict(set)
for variable_name, variable_definition in variables.items(): for variable_name, variable_definition in variables.items():
self._traverse_variable_dependencies( self._traverse_variable_dependencies(
variable_name=variable_name, variable_name=variable_name,
variable_dependency=variable_definition, variable_dependency=variable_definition,
deps=[], deps=[],
ensured=ensured,
) )
def _traverse_custom_function_dependencies( def _traverse_custom_function_dependencies(
@ -108,10 +95,9 @@ class Script:
deps: List[str], deps: List[str],
) -> None: ) -> None:
for dep in custom_function_dependency.custom_functions: for dep in custom_function_dependency.custom_functions:
if custom_function_name == dep.name: self._ensure_no_cycle(
self._throw_cycle_error( name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions
name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions )
)
self._traverse_custom_function_dependencies( self._traverse_custom_function_dependencies(
custom_function_name=custom_function_name, custom_function_name=custom_function_name,
custom_function_dependency=self._functions[dep.name], custom_function_dependency=self._functions[dep.name],
@ -254,23 +240,23 @@ class Script:
def __init__(self, script: Dict[str, str]): def __init__(self, script: Dict[str, str]):
function_names: Set[str] = { function_names: Set[str] = {
to_function_name(name) for name in script.keys() if is_function(name) _function_name(name) for name in script.keys() if _is_function(name)
} }
variable_names: Set[str] = { variable_names: Set[str] = {
validate_variable_name(name) for name in script.keys() if not is_function(name) validate_variable_name(name) for name in script.keys() if not _is_function(name)
} }
self._functions: Dict[str, SyntaxTree] = { self._functions: Dict[str, SyntaxTree] = {
# custom_function_name must be passed to properly type custom function # custom_function_name must be passed to properly type custom function
# arguments uniquely if they're nested (i.e. $0 to $custom_func___0) # arguments uniquely if they're nested (i.e. $0 to $custom_func___0)
to_function_name(function_key): parse( _function_name(function_key): parse(
text=function_value, text=function_value,
name=to_function_name(function_key), name=_function_name(function_key),
custom_function_names=function_names, custom_function_names=function_names,
variable_names=variable_names, variable_names=variable_names,
) )
for function_key, function_value in script.items() for function_key, function_value in script.items()
if is_function(function_key) if _is_function(function_key)
} }
self._variables: Dict[str, SyntaxTree] = { self._variables: Dict[str, SyntaxTree] = {
@ -281,13 +267,13 @@ class Script:
variable_names=variable_names, variable_names=variable_names,
) )
for variable_key, variable_value in script.items() for variable_key, variable_value in script.items()
if not is_function(variable_key) if not _is_function(variable_key)
} }
self._validate() self._validate()
def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None: def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None:
for variable_name, resolved in resolved_variables.items(): for variable_name, resolved in resolved_variables.items():
self._variables[variable_name] = ResolvedSyntaxTree(ast=[resolved]) self._variables[variable_name] = SyntaxTree(ast=[resolved])
def _recursive_get_unresolved_output_filter_variables( def _recursive_get_unresolved_output_filter_variables(
self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable] self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
@ -316,21 +302,14 @@ class Script:
unresolvable=unresolvable, unresolvable=unresolvable,
) )
for lambda_func in current_var.lambdas:
if lambda_func.value in self._functions:
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._functions[lambda_func.value],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return subset_to_resolve return subset_to_resolve
def _get_unresolved_output_filter( def _get_unresolved_output_filter(
self, self,
unresolved: Dict[Variable, SyntaxTree],
output_filter: Set[str], output_filter: Set[str],
unresolvable: Set[Variable], unresolvable: Set[Variable],
) -> Set[str]: ) -> Dict[Variable, SyntaxTree]:
""" """
When an output filter is applied, only a subset of variables that the filter When an output filter is applied, only a subset of variables that the filter
depends on need to be resolved. depends on need to be resolved.
@ -351,7 +330,7 @@ class Script:
unresolvable=unresolvable, unresolvable=unresolvable,
) )
return subset_to_resolve return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve}
def _resolve( def _resolve(
self, self,
@ -387,21 +366,18 @@ class Script:
unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})} unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})}
unresolved_filter = set(resolved.keys()).union(unresolvable) unresolved_filter = set(resolved.keys()).union(unresolvable)
unresolved: Dict[Variable, SyntaxTree] = {
Variable(name): ast
for name, ast in self._variables.items()
if Variable(name) not in unresolved_filter
}
if output_filter: if output_filter:
unresolved = { unresolved = self._get_unresolved_output_filter(
Variable(name): self._variables[name] unresolved=unresolved,
for name in self._get_unresolved_output_filter( output_filter=output_filter,
output_filter=output_filter, unresolvable=unresolvable,
unresolvable=unresolvable, )
)
}
else:
unresolved = {
Variable(name): ast
for name, ast in self._variables.items()
if Variable(name) not in unresolved_filter
}
while unresolved: while unresolved:
unresolved_count: int = len(unresolved) unresolved_count: int = len(unresolved)
@ -422,8 +398,8 @@ class Script:
# Otherwise, if it has dependencies that are all resolved, then # Otherwise, if it has dependencies that are all resolved, then
# resolve the definition # resolve the definition
elif definition.is_subset_of( elif not definition.is_subset_of(
variables=resolved, custom_function_definitions=self._functions variables=resolved.keys(), custom_function_definitions=self._functions
): ):
resolved[variable] = unresolved[variable].resolve( resolved[variable] = unresolved[variable].resolve(
resolved_variables=resolved, resolved_variables=resolved,
@ -506,12 +482,12 @@ class Script:
added_variables_to_validate: Set[str] = set() added_variables_to_validate: Set[str] = set()
functions_to_add = { functions_to_add = {
to_function_name(name): definition _function_name(name): definition
for name, definition in variables.items() for name, definition in variables.items()
if is_function(name) if _is_function(name)
} }
variables_to_add = { variables_to_add = {
name: definition for name, definition in variables.items() if not is_function(name) name: definition for name, definition in variables.items() if not _is_function(name)
} }
custom_function_names = set(self._functions.keys()) | functions_to_add.keys() custom_function_names = set(self._functions.keys()) | functions_to_add.keys()
@ -541,52 +517,11 @@ class Script:
return self return self
def add_parsed(self, variables: Dict[str, SyntaxTree]) -> "Script":
"""
Adds already parsed, new variables to the script.
Parameters
----------
variables
Mapping containing variable name to definition.
Returns
-------
Script
self
"""
added_variables_to_validate: Set[str] = set()
functions_to_add = {
to_function_name(name): definition
for name, definition in variables.items()
if is_function(name)
}
variables_to_add = {
name: definition for name, definition in variables.items() if not is_function(name)
}
for definitions in [functions_to_add, variables_to_add]:
for name, parsed in definitions.items():
if parsed.maybe_resolvable is None:
added_variables_to_validate.add(name)
if name in functions_to_add:
self._functions[name] = parsed
else:
self._variables[name] = parsed
if added_variables_to_validate:
self._validate(added_variables=added_variables_to_validate)
return self
def resolve_once( def resolve_once(
self, self,
variable_definitions: Dict[str, str], variable_definitions: Dict[str, str],
resolved: Optional[Dict[str, Resolvable]] = None, resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None, unresolvable: Optional[Set[str]] = None,
update: bool = False,
) -> Dict[str, Resolvable]: ) -> Dict[str, Resolvable]:
""" """
Given a new set of variable definitions, resolve them using the Script, but do not Given a new set of variable definitions, resolve them using the Script, but do not
@ -601,8 +536,6 @@ class Script:
unresolvable unresolvable
Optional. Unresolvable variables that will be ignored in resolution, including all Optional. Unresolvable variables that will be ignored in resolution, including all
variables with a dependency to them. variables with a dependency to them.
update
Whether to update the script's state with resolved variables. Defaults to False.
Returns Returns
------- -------
@ -615,51 +548,11 @@ class Script:
pre_resolved=resolved, pre_resolved=resolved,
unresolvable=unresolvable, unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())), output_filter=set(list(variable_definitions.keys())),
update=update,
).output ).output
finally: finally:
for name in variable_definitions.keys(): for name in variable_definitions.keys():
self._variables.pop(name, None) if name in self._variables:
del self._variables[name]
def resolve_once_parsed(
self,
variable_definitions: Dict[str, SyntaxTree],
resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None,
update: bool = False,
) -> Dict[str, Resolvable]:
"""
Given a new set of variable definitions, resolve them using the Script, but do not
add them to the Script itself.
Parameters
----------
variable_definitions
Variables to resolve, but not store in the Script
resolved
Optional. Pre-resolved variables that should be used instead of what is in the script.
unresolvable
Optional. Unresolvable variables that will be ignored in resolution, including all
variables with a dependency to them.
update
Whether to update the script's state with resolved variables. Defaults to False.
Returns
-------
Dict[str, Resolvable]
Dict containing the variable names to their resolved values.
"""
try:
self.add_parsed(variable_definitions)
return self._resolve(
pre_resolved=resolved,
unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())),
update=update,
).output
finally:
for name in variable_definitions.keys():
self._variables.pop(name, None)
def get(self, variable_name: str) -> Resolvable: def get(self, variable_name: str) -> Resolvable:
""" """
@ -688,18 +581,6 @@ class Script:
raise RuntimeException(f"Tried to get unresolved variable {variable_name}") raise RuntimeException(f"Tried to get unresolved variable {variable_name}")
def definition_of(self, name: str) -> SyntaxTree:
"""
Returns
-------
The definition of the variable or function.
"""
if name.startswith("%") and name[1:] in self._functions:
return self._functions[name[1:]]
if name in self._variables:
return self._variables[name]
raise RuntimeException(f"Tried to get non-existent definition with name {name}")
@property @property
def variable_names(self) -> Set[str]: def variable_names(self) -> Set[str]:
""" """
@ -718,120 +599,4 @@ class Script:
Set[str] Set[str]
Names of all functions within the Script. Names of all functions within the Script.
""" """
return set(to_function_definition_name(name) for name in self._functions.keys()) return set(_to_function_definition_name(name) for name in self._functions.keys())
def _resolve_partial_loop(
self,
output_filter: Optional[Set[str]],
resolved: Dict[Variable, Resolvable],
unresolved: Dict[Variable, Argument],
unresolvable: Optional[Set[str]],
):
to_partially_resolve: Set[Variable] = (
{Variable(name) for name in output_filter} if output_filter else set(unresolved.keys())
)
partially_resolved = True
while partially_resolved:
partially_resolved = False
for variable in list(to_partially_resolve):
definition = unresolved[variable]
maybe_resolved = definition
if isinstance(definition, Variable) and definition.name not in unresolvable:
if definition in resolved:
maybe_resolved = resolved[definition]
elif definition in unresolved:
maybe_resolved = unresolved[definition]
else:
raise UNREACHABLE
elif isinstance(definition, VariableDependency):
maybe_resolved = definition.partial_resolve(
resolved_variables=resolved,
unresolved_variables=unresolved,
custom_functions=self._functions,
)
if isinstance(maybe_resolved, Resolvable):
resolved[variable] = maybe_resolved
del unresolved[variable]
to_partially_resolve.remove(variable)
partially_resolved = True
else:
unresolved[variable] = maybe_resolved
# If the definition changed, then the script changed
# which means we can iterate again
partially_resolved |= definition != maybe_resolved
def _resolve_partial(
self,
unresolvable: Optional[Set[str]] = None,
output_filter: Optional[Set[str]] = None,
) -> Dict[str, SyntaxTree]:
"""
Returns
-------
New (deep-copied) script that resolves inner variables as much
as possible.
"""
unresolvable: Set[str] = unresolvable or {}
resolved: Dict[Variable, Resolvable] = {}
unresolved: Dict[Variable, Argument] = {
Variable(name): definition
for name, definition in self._variables.items()
if name not in unresolvable
}
self._resolve_partial_loop(
output_filter=output_filter,
resolved=resolved,
unresolved=unresolved,
unresolvable=unresolvable,
)
if output_filter:
out: Dict[str, SyntaxTree] = {}
for name in output_filter:
variable_name = Variable(name)
if variable_name in resolved:
out[name] = ResolvedSyntaxTree(ast=[resolved[variable_name]])
else:
out[name] = SyntaxTree(ast=[unresolved[variable_name]])
return out
return {
var.name: ResolvedSyntaxTree(ast=[definition]) for var, definition in resolved.items()
} | {var.name: SyntaxTree(ast=[definition]) for var, definition in unresolved.items()}
def resolve_partial(
self,
unresolvable: Optional[Set[str]] = None,
output_filter: Optional[Set[str]] = None,
) -> "Script":
"""
Updates the internal script to resolve as much as possible.
"""
out = self._resolve_partial(unresolvable=unresolvable, output_filter=output_filter)
for var_name, definition in out.items():
self._variables[var_name] = definition
return self
def resolve_partial_once(
self, variable_definitions: Dict[str, SyntaxTree], unresolvable: Optional[Set[str]] = None
) -> Dict[str, SyntaxTree]:
"""
Partially resolves the input variable definitions as much as possible.
"""
try:
self.add_parsed(variable_definitions)
return self._resolve_partial(
unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())),
)
finally:
for name in variable_definitions.keys():
self._variables.pop(name, None)

View file

@ -1,5 +1,6 @@
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Dict from typing import Any
from typing import Dict
from ytdl_sub.script.types.resolvable import Resolvable from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved

View file

@ -1,14 +1,15 @@
from abc import ABC from abc import ABC
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Dict, List, Type from typing import Any
from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.script.types.resolvable import ( from ytdl_sub.script.types.resolvable import Argument
Argument, from ytdl_sub.script.types.resolvable import FutureResolvable
FutureResolvable, from ytdl_sub.script.types.resolvable import NonHashable
NonHashable, from ytdl_sub.script.types.resolvable import Resolvable
Resolvable, from ytdl_sub.script.types.resolvable import ResolvableToJson
ResolvableToJson,
)
from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.types.variable_dependency import VariableDependency from ytdl_sub.script.types.variable_dependency import VariableDependency
@ -27,7 +28,7 @@ class UnresolvedArray(_Array, VariableDependency, FutureResolvable):
value: List[Argument] value: List[Argument]
@property @property
def iterable_arguments(self) -> List[Argument]: def _iterable_arguments(self) -> List[Argument]:
return self.value return self.value
def resolve( def resolve(
@ -46,27 +47,6 @@ class UnresolvedArray(_Array, VariableDependency, FutureResolvable):
] ]
) )
def partial_resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, VariableDependency],
) -> Argument | Resolvable:
maybe_resolvable_values, is_resolvable = VariableDependency.try_partial_resolve(
args=self.value,
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
out = UnresolvedArray(value=maybe_resolvable_values)
if is_resolvable:
return out.resolve(
resolved_variables=resolved_variables, custom_functions=custom_functions
)
return out
def future_resolvable_type(self) -> Type[Resolvable]: def future_resolvable_type(self) -> Type[Resolvable]:
return Array return Array

Some files were not shown because too many files have changed in this diff Show more