danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into j/camp/shuffle

Browse source
This commit is contained in:
Jesse Bannon 2026-02-02 10:09:41 -08:00
commit ce931ef82d
162 changed files with 8755 additions and 2594 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -146,8 +146,11 @@ docker/testing/volumes
.local/
.ytdl-sub-working-directory
.ytdl-sub-lock
ffmpeg.exe
ffprobe.exe
tools/docgen/out
tools/docgen/out
prof/

3
.readthedocs.yaml
View file

@ -7,6 +7,7 @@ build:
sphinx:
configuration: docs/source/conf.py
fail_on_warning: true
python:
install:
@ -14,4 +15,4 @@ python:
- method: pip
path: .
extra_requirements:
- docs
- docs

31
Makefile
View file

@ -1,7 +1,20 @@
# Defensive settings for make:
# https://tech.davis-hansson.com/p/make/
SHELL:=bash
.ONESHELL:
.SHELLFLAGS:=-eu -o pipefail -c
.SILENT:
.DELETE_ON_ERROR:
MAKEFLAGS+=--warn-undefined-variables
MAKEFLAGS+=--no-builtin-rules
export PS1?=$$
# Prefix echoed recipe commands with the recipe line number for debugging:
export PS4?=:$$LINENO+
# Get version related variables
export DATE=$(shell date +'%Y.%m.%d')
export DATE_COMMIT_COUNT=$(shell git rev-list --count HEAD --since="$(DATE) 00:00:00")
export COMMIT_HASH=$(shell git rev-parse --short HEAD)
export DATE:=$(shell date +'%Y.%m.%d')
export DATE_COMMIT_COUNT:=$(shell git rev-list --count HEAD --since="$(DATE) 00:00:00")
export COMMIT_HASH:=$(shell git rev-parse --short HEAD)
# Set Local version to YYYY.MM.DD-<hash>
export LOCAL_VERSION="$(DATE)+$(COMMIT_HASH)"
@ -13,6 +26,15 @@ else
export PYPI_VERSION="$(DATE).post$(DATE_COMMIT_COUNT)"
endif
# Finished with `$(shell)`, echo recipe commands going forward
.SHELLFLAGS+= -x
### Top-level targets:
.PHONY: all
all: check_lint docs docker docker_ubuntu docker_gui
lint:
python3 -m isort .
python3 -m black .
@ -41,7 +63,8 @@ executable: clean
mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX}
docs:
REGENERATE_DOCS=1 pytest tests/unit/docgen/test_docgen.py
sphinx-build -M html docs/source/ docs/build/
sphinx-build --write-all --fail-on-warning --nitpicky -b html \
"./docs/source/" "./docs/build/"
clean:
rm -rf \
.pytest_cache/ \

32
docker/Dockerfile
View file

@ -7,16 +7,20 @@ FROM ghcr.io/linuxserver/baseimage-alpine:edge
ENV OPENSSL_CONF="/etc/ssl"
# For downloading thumbnails
ENV SSL_CERT_DIR="/etc/ssl/certs/"
# Working directory used at both build and run times:
ENV DEFAULT_WORKSPACE="/config"
COPY root/ /
RUN mkdir -p /config && \
RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
apk update --no-cache && \
apk upgrade --no-cache && \
apk add --no-cache --repository=http://dl-3.alpinelinux.org/alpine/edge/main/ \
vim \
g++ \
nano \
unzip \
make \
deno \
libffi-dev \
"python3>=3.10" \
py3-pip \
@ -28,6 +32,7 @@ RUN mkdir -p /config && \
"aria2>=1.36.0" && \
ffmpeg -version && \
aria2c --version && \
deno --version && \
# Install phantomjs if using x86_64, ensure it is properly installed
if [[ $(uname -m) == "x86_64" ]]; then \
echo "installing phantomjs" && \
@ -41,29 +46,30 @@ RUN mkdir -p /config && \
phantomjs --version && \
cd -; \
fi && \
echo "hi" && \
# Install ytdl-sub, ensure it is installed properly
python3 -m pip install --break-system-packages --no-cache-dir ytdl_sub-*.whl && \
# Configure pip globally
echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
# Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
ytdl-sub -h && \
# Delete unneeded packages after install
rm ytdl_sub-*.whl && \
apk del \
g++ \
make \
libffi-dev \
py3-pip \
py3-setuptools
libffi-dev && \
python3 -m pip --help
###############################################################################
# CONTAINER CONFIGS
ENV EDITOR="nano" \
HOME="/config" \
HOME="${DEFAULT_WORKSPACE}" \
DOCKER_MODS=linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-cron \
DEFAULT_WORKSPACE=/config \
CRON_SCRIPT="/config/cron" \
CRON_WRAPPER_SCRIPT="/config/.cron_wrapper" \
LOGS_TO_STDOUT=/config/.cron.log \
CRON_SCRIPT="${DEFAULT_WORKSPACE}/cron" \
CRON_WRAPPER_SCRIPT="${DEFAULT_WORKSPACE}/.cron_wrapper" \
LOGS_TO_STDOUT="${DEFAULT_WORKSPACE}/.cron.log" \
LSIO_FIRST_PARTY=false
VOLUME /config
VOLUME "${DEFAULT_WORKSPACE}"
WORKDIR "${DEFAULT_WORKSPACE}"

24
docker/Dockerfile.gui
View file

@ -4,6 +4,8 @@ FROM lscr.io/linuxserver/code-server:4.98.2
ENV OPENSSL_CONF="/etc/ssl"
# For downloading thumbnails
ENV SSL_CERT_DIR="/etc/ssl/certs/"
# Working directory used at both build and run times:
ENV DEFAULT_WORKSPACE="/config/ytdl-sub-configs"
###############################################################################
# YTDL-SUB INSTALL
@ -21,6 +23,7 @@ RUN mkdir -p /config && \
vim \
g++ \
nano \
unzip \
make \
python3-pip \
fontconfig \
@ -59,8 +62,13 @@ RUN mkdir -p /config && \
echo "Phantom JS version:" && \
phantomjs --version ; \
fi && \
# Install ytdl-sub, ensure it is installed properly
pip install --no-cache-dir --break-system-packages ytdl_sub-*.whl && \
# Install Deno, required for YouTube downloads
curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \
deno --help && \
# Configure pip globally
echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
# Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
ytdl-sub -h && \
# Delete unneeded packages after install
rm ytdl_sub-*.whl && \
@ -68,11 +76,11 @@ RUN mkdir -p /config && \
g++ \
make \
xz-utils \
bzip2 \
python3-venv && \
bzip2 && \
apt-get autoremove -y && \
apt-get purge -y --auto-remove && \
rm -rf /var/lib/apt/lists/*
rm -rf /var/lib/apt/lists/* && \
python3 -m pip --help
###############################################################################
# CONTAINER CONFIGS
@ -80,10 +88,10 @@ RUN mkdir -p /config && \
ENV EDITOR="nano" \
HOME="/config" \
DOCKER_MODS=linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-cron \
DEFAULT_WORKSPACE=/config/ytdl-sub-configs \
CRON_SCRIPT="/config/ytdl-sub-configs/cron" \
CRON_SCRIPT="${DEFAULT_WORKSPACE}/cron" \
CRON_WRAPPER_SCRIPT="/config/.cron_wrapper" \
LOGS_TO_STDOUT=/config/.cron.log \
LSIO_FIRST_PARTY=false
VOLUME /config
VOLUME /config
WORKDIR "${DEFAULT_WORKSPACE}"

1
docker/Dockerfile.headless Symbolic link
View file

@ -0,0 +1 @@
Dockerfile

32
docker/Dockerfile.ubuntu
View file

@ -7,13 +7,15 @@ ARG DEBIAN_FRONTEND=noninteractive
ENV OPENSSL_CONF="/etc/ssl"
# For downloading thumbnails
ENV SSL_CERT_DIR="/etc/ssl/certs/"
# Working directory used at both build and run times:
ENV DEFAULT_WORKSPACE="/config"
###############################################################################
# YTDL-SUB INSTALL
SHELL ["/bin/bash", "-c"]
COPY root/ /
RUN mkdir -p /config && \
RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
apt-get -y update && \
apt-get -y upgrade && \
apt-get install --no-install-recommends -y \
@ -24,6 +26,7 @@ RUN mkdir -p /config && \
vim \
g++ \
nano \
unzip \
make \
python3-pip \
fontconfig \
@ -62,8 +65,13 @@ RUN mkdir -p /config && \
echo "Phantom JS version:" && \
phantomjs --version ; \
fi && \
# Install ytdl-sub, ensure it is installed properly
pip install --no-cache-dir --break-system-packages ytdl_sub-*.whl && \
# Install Deno, required for YouTube downloads
curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \
deno --help && \
# Configure pip globally
echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
# Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
ytdl-sub -h && \
# Delete unneeded packages after install
rm ytdl_sub-*.whl && \
@ -71,22 +79,22 @@ RUN mkdir -p /config && \
g++ \
make \
xz-utils \
bzip2 \
python3-venv && \
bzip2 && \
apt-get autoremove -y && \
apt-get purge -y --auto-remove && \
rm -rf /var/lib/apt/lists/*
rm -rf /var/lib/apt/lists/* && \
python3 -m pip --help
###############################################################################
# CONTAINER CONFIGS
ENV EDITOR="nano" \
HOME="/config" \
HOME="${DEFAULT_WORKSPACE}" \
DOCKER_MODS=linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-cron \
DEFAULT_WORKSPACE=/config \
CRON_SCRIPT="/config/cron" \
CRON_WRAPPER_SCRIPT="/config/.cron_wrapper" \
LOGS_TO_STDOUT=/config/.cron.log \
CRON_SCRIPT="${DEFAULT_WORKSPACE}/cron" \
CRON_WRAPPER_SCRIPT="${DEFAULT_WORKSPACE}/.cron_wrapper" \
LOGS_TO_STDOUT="${DEFAULT_WORKSPACE}/.cron.log" \
LSIO_FIRST_PARTY=false
VOLUME /config
VOLUME "${DEFAULT_WORKSPACE}"
WORKDIR "${DEFAULT_WORKSPACE}"

22
docker/root/custom-cont-init.d/defaults Normal file → Executable file
View file

@ -17,12 +17,28 @@ echo "Starting ytdl-sub..."
echo "alias ls='ls --color=auto'" > /config/.bashrc && \
echo "cd ." >> /config/.bashrc
# always create empty cron log file on start
echo "" > "$LOGS_TO_STDOUT"
# permissions
chown -R ${PUID:-abc}:${PGID:-abc} \
/config
# always create empty cron log file on start
echo "" > "$LOGS_TO_STDOUT"
# update command reference:
# https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip
if [ "$UPDATE_YT_DLP_ON_START" == "stable" ] ; then
echo "UPDATE_YT_DLP_ON_START is set to stable, attempting to update to a new stable version of yt-dlp if it exists."
python3 -m pip install -U "yt-dlp[default]"
elif [ "$UPDATE_YT_DLP_ON_START" == "nightly" ] ; then
echo "UPDATE_YT_DLP_ON_START is set to nightly, attempting to update to the latest nightly version of yt-dlp."
python3 -m pip install -U --pre "yt-dlp[default]"
elif [ "$UPDATE_YT_DLP_ON_START" == "master" ] ; then
echo "UPDATE_YT_DLP_ON_START is set to master, pulling yt-dlp's latest commit for install."
python3 -m pip install -U pip hatchling wheel
python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz"
else
echo "UPDATE_YT_DLP_ON_START is not set, using packaged version."
fi
# set up cron
if [ "$CRON_SCHEDULE" != "" ] ; then
@ -33,7 +49,7 @@ if [ "$CRON_SCHEDULE" != "" ] ; then
echo '#!/bin/bash' > "$CRON_WRAPPER_SCRIPT"
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "$CRON_WRAPPER_SCRIPT"
echo "cd \"$DEFAULT_WORKSPACE\"" >> "$CRON_WRAPPER_SCRIPT"
echo ". \"$CRON_SCRIPT\" | tee -a \"$LOGS_TO_STDOUT\"" >> "$CRON_WRAPPER_SCRIPT"
echo ". \"$CRON_SCRIPT\" >> \"$LOGS_TO_STDOUT\" 2>&1" >> "$CRON_WRAPPER_SCRIPT"
chmod +x "$CRON_WRAPPER_SCRIPT"
chown abc:abc "$CRON_WRAPPER_SCRIPT"

3
docker/root/defaults/config.yaml
View file

@ -16,4 +16,7 @@
# See the documentation above on how to build your own custom presets.
#
configuration:
# Avoid unnecessarily long large file renames, set this to a path on the same
# filesystem as the destination for downloaded files in the `overrides: /
# *_directory:` paths:
working_directory: ".ytdl-sub-working-directory"

72
docker/root/defaults/subscriptions.yaml
View file

@ -15,12 +15,12 @@ __preset__:
music_video_directory: "/music_videos"
# For 'Only Recent' preset, only keep vids within this range and limit
only_recent_date_range: "2months"
only_recent_max_files: 30
# only_recent_date_range: "2months"
# only_recent_max_files: 30
# Pass any arg directly to yt-dlp's Python API
ytdl_options:
cookiefile: "/config/cookie.txt"
# ytdl_options:
# cookiefile: "/config/cookie.txt"
###################################################################
# Subscriptions nested under this will use the
@ -35,52 +35,54 @@ Plex TV Show by Date:
# Sets genre tag to "Documentaries"
= Documentaries:
"NOVA PBS": "https://www.youtube.com/@novapbs"
"National Geographic": "https://www.youtube.com/@NatGeo"
"Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
# "National Geographic": "https://www.youtube.com/@NatGeo"
# "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
# Sets genre tag to "Kids", "TV-Y" for content rating
= Kids | = TV-Y:
"Jake Trains": "https://www.youtube.com/@JakeTrains"
"Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
# = Kids | = TV-Y:
# "Jake Trains": "https://www.youtube.com/@JakeTrains"
# "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
= Music:
# TV show subscriptions can support multiple urls and store in the same TV Show
"Rick Beato":
- "https://www.youtube.com/@RickBeato"
- "https://www.youtube.com/@rickbeato240"
# = Music:
# # TV show subscriptions can support multiple urls and store in the same TV Show
# "Rick Beato":
# - "https://www.youtube.com/@RickBeato"
# - "https://www.youtube.com/@rickbeato240"
# Set genre tag to "News", use `Only Recent` preset to only store videos uploaded recently
= News | Only Recent:
"BBC News": "https://www.youtube.com/@BBCNews"
# = News | Only Recent:
# "BBC News": "https://www.youtube.com/@BBCNews"
###################################################################
# Subscriptions nested under these will use the various prebuilt
# music presets
YouTube Releases:
= Jazz: # Sets genre tag to "Jazz"
"Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
YouTube Full Albums:
= Lofi:
"Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
# YouTube Releases:
# = Jazz: # Sets genre tag to "Jazz"
# "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
SoundCloud Discography:
= Chill Hop:
"UKNOWY": "https://soundcloud.com/uknowymunich"
= Synthwave:
"Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
"Earmake": "https://soundcloud.com/earmake"
# YouTube Full Albums:
# = Lofi:
# "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
Bandcamp:
= Lofi:
"Emily Hopkins": "https://emilyharpist.bandcamp.com/"
# SoundCloud Discography:
# = Chill Hop:
# "UKNOWY": "https://soundcloud.com/uknowymunich"
# = Synthwave:
# "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
# "Earmake": "https://soundcloud.com/earmake"
# Bandcamp:
# = Lofi:
# "Emily Hopkins": "https://emilyharpist.bandcamp.com/"
###################################################################
# Can choose between:
# - Plex Music Videos:
# - Jellyfin Music Videos:
# - Kodi Music Videos:
"Plex Music Videos":
= Pop: # Sets genre tag to "Pop"
"Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
"Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"
# "Plex Music Videos":
# = Pop: # Sets genre tag to "Pop"
# "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
# "Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"

46
docker/testing/Makefile Normal file
View file

@ -0,0 +1,46 @@
# Local building and testing of the Docker image variants.
# Defensive settings for make:
# https://tech.davis-hansson.com/p/make/
SHELL:=bash
.ONESHELL:
.SHELLFLAGS:=-eu -o pipefail -c
.SILENT:
.DELETE_ON_ERROR:
MAKEFLAGS+=--warn-undefined-variables
MAKEFLAGS+=--no-builtin-rules
export PS1?=$$
# Prefix echoed recipe commands with the recipe line number for debugging:
export PS4?=:$$LINENO+
VARIANTS=headless gui ubuntu
ROOT_PREREQS:=$(shell find ../root -type f)
# Finished with `$(shell)`, echo recipe commands going forward
.SHELLFLAGS+= -x
### Top-level targets:
.PHONY: all
all: build
.PHONY: build
build: $(VARIANTS:%=./build/ytdl-sub-%.log)
.PHONY: run
run: build $(VARIANTS:%=./volumes/ytdl-sub-%/)
docker compose up
### Real targets:
# Re-build the local images when changes require it.
./build/ytdl-sub-%.log: ../Dockerfile.% $(ROOT_PREREQS)
mkdir -pv "$(dir $(@))"
docker compose build "$(@:build/ytdl-sub-%.log=ytdl-sub-%)" |&
tee -a "$(@)"
# Ensure volumes are owned by the developer's normal user:
./volumes/ytdl-sub-%/:
mkdir -pv "$(@)"

69
docker/testing/docker-compose.yml
View file

@ -1,39 +1,50 @@
services:
ytdl-sub-gui:
image: ytdl-sub-gui:local
container_name: ytdl-sub-gui
build:
context: "../"
dockerfile: "./Dockerfile.gui"
image: "ytdl-sub-gui:local"
container_name: "ytdl-sub-gui"
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- CRON_SCHEDULE="*/1 * * * *"
- CRON_RUN_ON_START=true
PUID: "1000"
PGID: "1000"
TZ: "America/Los_Angeles"
CRON_SCHEDULE: '*/1 * * * *'
CRON_RUN_ON_START: "true"
UPDATE_YT_DLP_ON_START: "stable"
volumes:
- ./volumes/ytdl-sub-gui:/config
- "./volumes/ytdl-sub-gui/:/config/"
ports:
- 8443:8443
restart: unless-stopped
ytdl-sub:
image: ytdl-sub:local
container_name: ytdl-sub
- "8443:8443"
restart: "unless-stopped"
ytdl-sub-headless:
build:
context: "../"
image: "ytdl-sub:local"
container_name: "ytdl-sub-headless"
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- CRON_SCHEDULE="*/1 * * * *"
- CRON_RUN_ON_START=true
PUID: "1000"
PGID: "1000"
TZ: "America/Los_Angeles"
CRON_SCHEDULE: '*/1 * * * *'
CRON_RUN_ON_START: "true"
UPDATE_YT_DLP_ON_START: "stable"
volumes:
- ./volumes/ytdl-sub:/config
restart: unless-stopped
- "./volumes/ytdl-sub-headless/:/config/"
restart: "unless-stopped"
ytdl-sub-ubuntu:
image: ytdl-sub-ubuntu:local
container_name: ytdl-sub-ubuntu
build:
context: "../"
dockerfile: "./Dockerfile.ubuntu"
image: "ytdl-sub-ubuntu:local"
container_name: "ytdl-sub-ubuntu"
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- CRON_SCHEDULE="*/1 * * * *"
- CRON_RUN_ON_START=true
PUID: "1000"
PGID: "1000"
TZ: "America/Los_Angeles"
CRON_SCHEDULE: '*/1 * * * *'
CRON_RUN_ON_START: "true"
UPDATE_YT_DLP_ON_START: "stable"
volumes:
- ./volumes/ytdl-sub:/config
restart: unless-stopped
- "./volumes/ytdl-sub-ubuntu/:/config/"
restart: "unless-stopped"

93
docs/source/config_reference/config_yaml.rst
View file

@ -1,9 +1,6 @@
==================
Configuration File
==================
-----------
config.yaml
-----------
ytdl-sub is configured using a ``config.yaml`` file.
@ -14,17 +11,19 @@ The ``config.yaml`` is made up of two sections:
configuration:
presets:
You can jump to any section and subsection of the config using the navigation
section to the left.
You can jump to any section and subsection of the config using the navigation section to
the left.
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
If you wish to represent paths like Windows, you will need to ``C:\\double\\bashslash\\paths``
in order to escape the backslash character.
If you wish to represent paths like Windows, you will need to
``C:\\double\\bashslash\\paths`` in order to escape the backslash character.
configuration
~~~~~~~~~~~~~
The ``configuration`` section contains app-wide configs applied to all presets
and subscriptions.
-------------
The ``configuration`` section contains app-wide configs applied to all presets and
subscriptions.
.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
:members:
@ -32,9 +31,19 @@ and subscriptions.
:exclude-members: subscription_value, persist_logs, experimental
persist_logs
""""""""""""
Within ``configuration``, define whether logs from subscription downloads
should be persisted.
~~~~~~~~~~~~
Without this key, ``ytdl-sub`` only prints output to it's ``stdout`` and ``stderr``. If
your configuration includes the ``persist_logs:`` key, then ``ytdl-sub`` also writes log
files to disk.
.. warning::
The log files grow rapidly if ``keep_successful_logs:`` is ``true``, the default, and
may fill up disk space. Set ``keep_successful_logs: false`` or prune the log files
regularly.
For example:
.. code-block:: yaml
@ -42,22 +51,36 @@ should be persisted.
persist_logs:
logs_directory: "/path/to/log/directory"
Log files are stored as
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
:members:
:member-order: bysource
presets
~~~~~~~
``presets`` define a `formula` for how to format downloaded media and metadata.
This section is work-in-progress!
presets
-------
Each key under ``presets:`` defines a `formula` for how to format downloaded media and
metadata. The key is the name of the preset and the value is a mapping that defines the
preset.
.. note::
The ``presets:`` key at the top of the configuration file contains multiple
user-defined presets, but *each preset* itself may include a ``preset:`` key that
defines *that preset's* base presets. For example:
.. code-block:: yaml
presets:
Foo Preset:
preset:
- "Jellyfin TV Show by Date"
- "Only Recent"
preset
""""""
Presets support inheritance by defining a parent preset:
~~~~~~
Presets support inheritance by defining one or more parent presets:
.. code-block:: yaml
@ -67,11 +90,11 @@ Presets support inheritance by defining a parent preset:
parent_preset:
...
child_preset:
preset: "parent_preset"
preset:
- "parent_preset"
In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
It is advantageous to use parent presets where possible to reduce duplicate yaml
definitions.
Use parent presets where possible to reduce duplicate yaml definitions.
Presets also support inheritance from multiple presets:
@ -82,9 +105,19 @@ Presets also support inheritance from multiple presets:
- "custom_preset"
- "parent_preset"
In this example, ``child_preset`` will inherit all fields from ``custom_preset``
and ``parent_preset`` in that order. The bottom-most preset has the highest
priority.
In this example, ``child_preset`` will inherit all fields from ``custom_preset`` and
``parent_preset`` in that order. The bottom-most preset has the highest priority. More
specifically, presets are merged using `mergedeep`_ via `a TYPESAFE_ADDITIVE merge`_,
which means:
If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is
valid YAML. Inheriting from multiple presets require use of a list.
- if two conflicting keys arent lists or mappings, overwrite the higher priority one
- otherwise, combine then re-evaluate
If you are only inheriting from one preset, using a single string instead of a list is
valid, for example ``preset: "parent_preset"``, but we recommend always using a list for
consistent readability between presets.
.. _`mergedeep`:
https://mergedeep.readthedocs.io/en/latest/
.. _`a TYPESAFE_ADDITIVE merge`:
https://mergedeep.readthedocs.io/en/latest/index.html#merge-strategies

43
docs/source/config_reference/index.rst
View file

@ -2,11 +2,50 @@
Reference
=========
This section contains direct references to the code of ``ytdl-sub`` and information on how it functions.
This section contains direct references to the code of ``ytdl-sub`` and information on
how it functions.
Terminology
-----------
Must-know terminology:
- ``subscription``: URL(s) that you want to download with specific metadata
requirements.
- ``preset``: A media profile comprised of YAML configuration that can specify anything
from metadata layout, media quality, or any feature of ytdl-sub, to apply to
subscriptions. A preset can inherit other presets.
- ``prebuilt preset``: Presets that are included in ytdl-sub. These do most of the work
defining plugins, overrides, etc in order to make downloads ready for player
consumption.
- ``override``: Verb describing the act of overriding something in a preset. For
example, the TV Show presets practically expect you to *override* the URL variable to
tell ytdl-sub where to download from.
- ``override variables``: User-defined variables that are intended to *override*
something.
- ``subscription file``: The file to specify all of your subscriptions and some override
variables.
Intermediate terminology:
- ``plugin``: Modular logic to apply to a subscription. To use a plugin, it must be
defined in a preset.
- ``config file``: An optional file where you can define custom presets and other
advanced configuration.
- ``yt-dlp``: The underlying application that handles downloading for ytdl-sub.
Advanced terminology:
- ``entry variables``: Variables that derive from a downloaded yt-dlp entry (media).
- ``static variables``: Variables that do not have a dependency to entry variables.
- ``scripting``: Syntax that allows the use of entry variables, static variables, and
functions in override variables.
.. toctree::
config_yaml
subscription_yaml
plugins
scripting/index
prebuilt_presets/index
prebuilt_presets/index

48
docs/source/config_reference/plugins.rst
View file

@ -1,3 +1,10 @@
..
WARNING: This RST file is generated from docstrings in:
The respective plugin files under src/ytdl_sub/plugins/
In order to make a change to this file, edit the respective docstring
and run `make docs`. This will automatically sync the Python RST-based
docstrings into this file. If the docstrings and RST file are out of sync,
it will fail TestDocGen tests in GitHub CI.
Plugins
=======
@ -140,9 +147,11 @@ Dates must adhere to a yt-dlp datetime. From their docs:
A string in the format YYYYMMDD or
(now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this.
Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest
granularity possible.
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in
this. Note that yt-dlp will round times to the closest day, meaning that `day` is
the lowest granularity possible. Also note that, considering time zones, it's best
to include a margin of an extra day on either side to be sure it includes the
intended download files.
:Usage:
@ -158,14 +167,14 @@ granularity possible.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos after this datetime.
Only download videos after or on this datetime, inclusive.
``before``
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
Only download videos only before this datetime, not inclusive.
``breaks``
@ -754,6 +763,15 @@ Defines where to output files and thumbnails after all post-processing has compl
The output directory to store all media files downloaded.
``preserve_mtime``
:expected type: Optional[Boolean]
:description:
Preserve the video's original upload time as the file modification time.
When True, sets the file's mtime to match the video's upload_date from
yt-dlp metadata. Defaults to False.
``thumbnail_name``
:expected type: Optional[EntryFormatter]
@ -845,10 +863,11 @@ for representing audio albums.
static_nfo_tags
---------------
Adds an NFO file for every entry, but does not link it to an entry in the download archive.
This is intended to produce ``season.nfo``s in each season directory. Each entry within a
season will overwrite this file with its season name. If the entry gets deleted from ytdl-sub,
this file will remain since it's not linked.
Adds an NFO file for every entry, but does not link it to an entry in the download
archive. This is intended to produce ``season.nfo`` files in each season
directory. Each entry within a season will overwrite this file with its season
name. If the entry gets deleted from ytdl-sub, this file will remain since it's not
linked.
Usage:
@ -970,6 +989,14 @@ It will set the respective language to the correct subtitle file.
language codes. Defaults to only "en".
``languages_required``
:expected type: Optional[List[String]]
:description:
Language code(s) that are required to be present for downloads to continue. If missing,
ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles.
``subtitles_name``
:expected type: Optional[EntryFormatter]
@ -995,6 +1022,9 @@ Provides options to make ytdl-sub look more 'human-like' to protect from throttl
range-based values, a random number will be chosen within the range to avoid sleeps looking
scripted.
Range min and max values support static override variables within their definitions.
``sleep_per_download_s`` supports both static and override variables.
:Usage:
.. code-block:: yaml

23
docs/source/config_reference/prebuilt_presets/common.rst
View file

@ -4,17 +4,30 @@ Common
.. highlight:: yaml
Filtering
-------------
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml
Filter Keywords
---------------
.. literalinclude::
/../../src/ytdl_sub/prebuilt_presets/helpers/filter_keywords.yaml
Filter Duration
---------------
.. literalinclude::
/../../src/ytdl_sub/prebuilt_presets/helpers/filter_duration.yaml
Media Quality
-------------
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml
.. literalinclude::
/../../src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml
Only Recent Videos
------------------
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml
.. literalinclude::
/../../src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml

7
docs/source/config_reference/prebuilt_presets/index.rst
View file

@ -2,11 +2,10 @@
Prebuilt Preset Reference
=========================
This section contains the code for the prebuilt presets. If you just want to understand how to use the presets, check :doc:`this section instead</prebuilt_presets/index>`.
This section contains the code for the prebuilt presets. If you just want to understand
how to use the presets, check :doc:`this section instead</prebuilt_presets/index>`.
.. toctree::
common
tv_show
music
music

3
docs/source/config_reference/prebuilt_presets/music.rst
View file

@ -6,4 +6,5 @@ All audio music based presets inherit from ``_music_base``.
.. highlight:: yaml
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/music/singles.yaml
.. literalinclude::
/../../src/ytdl_sub/prebuilt_presets/music/singles.yaml

3
docs/source/config_reference/prebuilt_presets/tv_show.rst
View file

@ -6,4 +6,5 @@ All TV show based presets inherit from ``_episode_base``.
.. highlight:: yaml
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml
.. literalinclude::
/../../src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml

7
docs/source/config_reference/scripting/entry_variables.rst
View file

@ -1,3 +1,10 @@
..
WARNING: This RST file is generated from docstrings in:
src/ytdl_sub/entries/script/variable_definitions.py
In order to make a change to this file, edit the respective docstring
and run `make docs`. This will automatically sync the Python RST-based
docstrings into this file. If the docstrings and RST file are out of sync,
it will fail TestDocGen tests in GitHub CI.
Entry Variables
===============

82
docs/source/config_reference/scripting/index.rst
View file

@ -2,8 +2,9 @@
Scripting
=========
``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The links below
contain reference documentation for each built-in variable and scripting function.
``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The
links below contain reference documentation for each built-in variable and scripting
function.
.. toctree::
:maxdepth: 1
@ -13,6 +14,7 @@ contain reference documentation for each built-in variable and scripting functio
scripting_functions
scripting_types
How it Works
------------
@ -44,22 +46,22 @@ We can use this instead of hard-coding it above:
output_directory: "/path/to/tv_shows/{subscription_name}"
The syntax for variable usage is curly-braces with the variable name within it. Assuming
our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub``
will actually write to that directory.
our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub`` will
actually write to that directory.
Entry Variables
~~~~~~~~~~~~~~~
For context, an *entry* is a video or audio file downloaded from ``yt-dlp``.
*Entry variables* are variables that are derived from an entry's ``info.json`` file. This file
For context, an *entry* is a video or audio file downloaded from ``yt-dlp``. *Entry
variables* are variables that are derived from an entry's ``info.json`` file. This file
comes from ``yt-dlp`` and contains every piece of metadata that it scraped.
These variables are not considered static since they change per entry download. There are a
few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For others,
we are free to use values that derive from an entry.
These variables are not considered static since they change per entry download. There
are a few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For
others, we are free to use values that derive from an entry.
Suppose we want to customize the name of an entry's output file and thumbnail to include its
title in its name. We can do that using entry variables:
Suppose we want to customize the name of an entry's output file and thumbnail to include
its title in its name. We can do that using entry variables:
.. code-block:: yaml
@ -74,8 +76,8 @@ Creating Custom Variables
Suppose we want to include the date in our file names. This means we'd need to update
both the ``file_name`` and ``thumbnail_name`` fields to include it.
Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method
for creating and overriding custom variables.
Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method for
creating and overriding custom variables.
These are created in the ``overrides`` section. Let's take our above example and create
a ``custom_file_name`` variable to use for the entry file and thumbnail fields:
@ -97,9 +99,9 @@ For experienced ``yt-dlp`` scrapers, you may be thinking:
- What if the title has characters that do not play nice with my operating system?
``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic characters
with safe alternatives that can be used in file names. We can ensure our file names and directories
are safe by using:
``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic
characters with safe alternatives that can be used in file names. We can ensure our file
names and directories are safe by using:
.. code-block:: yaml
@ -116,16 +118,16 @@ Simply add a ``_sanitized`` suffix to any variable name to make it sanitized.
.. note::
Make sure you do not sanitize custom variables that intentionally create directories,
(i.e. sanitizing ``/path/to/tv_shows/``) otherwise they will... be sanitized and not resolve to
directories!
(i.e. sanitizing ``/path/to/tv_shows/``) otherwise they will... be sanitized and not
resolve to directories!
Using Scripting Functions
~~~~~~~~~~~~~~~~~~~~~~~~~
Let's suppose you are an avid command-line user, and like all of your file names to be
``snake_cased_with_no_spaces``. We can use the
`replace <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_
``snake_cased_with_no_spaces``. We can use the `replace
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_
*scripting function* to create and use a snake-cased title.
.. code-block:: yaml
@ -143,42 +145,48 @@ Let's suppose you are an avid command-line user, and like all of your file names
custom_file_name: "{upload_date_standardized}_{snake_cased_title_sanitized}"
Scripting functions are similar to variables - they must be used within curly-braces.
It is good practice to use ``>-`` when defining variables that use functions. It is YAML's way of
saying:
It is good practice to use ``>-`` when defining variables that use functions. It is
YAML's way of saying:
- Allow a string to be multi-lined, and do not include newlines before or after it.
See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=json>`_.
Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make
scripting function usage readable.
See for yourself `here
<https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=json>`_.
Any whitespace within curly-braces is okay since it will be parsed out. This is needed
to make scripting function usage readable.
.. important::
It is important to use ``>-`` over other YAML new-line directives like ``>`` because they
add newlines before or after curly-braces, and will be included in your variable's output string.
It is important to use ``>-`` over other YAML new-line directives like ``>`` because
they add newlines before or after curly-braces, and will be included in your
variable's output string.
Advanced Scripting
------------------
Accessing ``info.json`` Fields
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The entirety of an entry's ``info.json`` file resides in the
`Map <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_types.html#map>`_
variable
`entry_metadata <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/entry_variables.html#entry-metadata>`_.
Any field can be accessed by using the
`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
The entirety of an entry's ``info.json`` file resides in the `Map
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_types.html#map>`_
variable `entry_metadata
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/entry_variables.html#entry-metadata>`_.
Any field can be accessed by using the `map_get
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
function like so:
.. code-block:: yaml
:caption: Fetches the 'artist' value from the .info.json, returns null if it does not exist.
:caption:
Fetches the 'artist' value from the .info.json, returns null if it does not exist.
artist: >-
{ %map_get( entry_metadata, "artist", null ) }
Creating Custom Functions
~~~~~~~~~~~~~~~~~~~~~~~~~
Custom functions can be created in the overrides section using the following syntax:
.. code-block:: yaml
@ -187,9 +195,9 @@ Custom functions can be created in the overrides section using the following syn
"%get_entry_metadata_field": >-
{ %map_get( entry_metadata, $0, null ) }
Custom function definitions must have ``%`` as a prefix to the function name, be surrounded by
quotes to make YAML parsing happy, and can support arguments using ``$0``, ``$1``, ... to indicate
their first argument, second argument, etc.
Custom function definitions must have ``%`` as a prefix to the function name, be
surrounded by quotes to make YAML parsing happy, and can support arguments using ``$0``,
``$1``, ... to indicate their first argument, second argument, etc.
Using our new custom function, we can simply the ``artist`` variable definition above to:

31
docs/source/config_reference/scripting/scripting_functions.rst
View file

@ -1,3 +1,10 @@
..
WARNING: This RST file is generated from docstrings in:
The respective function files under src/ytdl_sub/script/functions/
In order to make a change to this file, edit the respective docstring
and run `make docs`. This will automatically sync the Python RST-based
docstrings into this file. If the docstrings and RST file are out of sync,
it will fail TestDocGen tests in GitHub CI.
Scripting Functions
===================
@ -514,6 +521,13 @@ pow
:description:
``**`` operator. Returns the exponential of the base and exponent value.
range
~~~~~
:spec: ``range(end: Integer, start: Optional[Integer], step: Optional[Integer]) -> Array``
:description:
Returns the desired range of Integers in the form of an Array.
sub
~~~
:spec: ``sub(values: Numeric, ...) -> Numeric``
@ -531,27 +545,26 @@ print
:spec: ``print(message: AnyArgument, passthrough: ReturnableArgument, level: Optional[Integer]) -> ReturnableArgument``
:description:
Print the ``message`` and return ``passthrough``.
Optionally can pass level, where < 0 is debug, 0 is info, 1 is warning, > 1 is error.
Defaults to info.
Log the ``message`` and return ``passthrough``. Optionally can pass level,
where < 0 is debug, 0 is info, 1 is warning, > 1 is error. (default ``0``)
print_if_false
~~~~~~~~~~~~~~
:spec: ``print_if_false(message: AnyArgument, passthrough: ReturnableArgument, level: Optional[Integer]) -> ReturnableArgument``
:description:
Print the ``message`` if ``passthrough`` evaluates to ``false``. Return ``passthrough``.
Optionally can pass level, where < 0 is debug, 0 is info, 1 is warning, > 1 is error.
Defaults to info.
Log the ``message`` if ``passthrough`` evaluates to ``false``. Return
``passthrough``. Optionally can pass level, where < 0 is debug, 0 is info, 1
is warning, > 1 is error. (default ``0``)
print_if_true
~~~~~~~~~~~~~
:spec: ``print_if_true(message: AnyArgument, passthrough: ReturnableArgument, level: Optional[Integer]) -> ReturnableArgument``
:description:
Print the ``message`` if ``passthrough`` evaluates to ``true``. Return ``passthrough``.
Optionally can pass level, where < 0 is debug, 0 is info, 1 is warning, > 1 is error.
Defaults to info.
Log the ``message`` if ``passthrough`` evaluates to ``true``. Return
``passthrough``. Optionally can pass level, where < 0 is debug, 0 is info, 1
is warning, > 1 is error. (default ``0``)
----------------------------------------------------------------------------------------------------

80
docs/source/config_reference/scripting/scripting_types.rst
View file

@ -1,7 +1,8 @@
===============
Scripting Types
===============
Types
-----
@ -16,8 +17,9 @@ Strings are a series of characters surrounded by quotes.
.. note::
For non-String types, they must be defined as parameters to scripting functions. This is because
anything in a variable definition that is not within curly-braces gets evaluated as a String.
For non-String types, they must be defined as parameters to scripting functions. This
is because anything in a variable definition that is not within curly-braces gets
evaluated as a String.
We can define Strings within curly-braces by setting them as parameters to a function:
@ -65,8 +67,8 @@ There are a few ways to make variables that use curly braces more compact, inclu
string_variable: "{ %string('This is a String variable') }"
In the case that you want to define a string variable that contains both single and double quotes,
triple-quotes can be used to avoid *closing* the String.
In the case that you want to define a string variable that contains both single and
double quotes, triple-quotes can be used to avoid *closing* the String.
.. tab-set::
@ -88,7 +90,8 @@ triple-quotes can be used to avoid *closing* the String.
%string("""This has both " and ' in it.""")
}
If you want a plain string that contains literal curly braces, you can escape them like so:
If you want a plain string that contains literal curly braces, you can escape them like
so:
.. code-block:: yaml
@ -185,8 +188,8 @@ A type is considered boolean if it spells out ``True`` or ``False``, case-insens
Array
~~~~~
An Array contains multiple types of any kind, including nested Arrays and Maps.
Arrays are defined using brackets (``[ ]``), and are accessed using zero-based indexing.
An Array contains multiple types of any kind, including nested Arrays and Maps. Arrays
are defined using brackets (``[ ]``), and are accessed using zero-based indexing.
.. tab-set::
@ -227,8 +230,8 @@ Arrays are defined using brackets (``[ ]``), and are accessed using zero-based i
Map
~~~
A Map is a key-value store, containing mappings between keys and values.
Maps are defined using curly-braces (``{ }``), and are accessed using their keys.
A Map is a key-value store, containing mappings between keys and values. Maps are
defined using curly-braces (``{ }``), and are accessed using their keys.
.. tab-set::
@ -267,6 +270,7 @@ Maps are defined using curly-braces (``{ }``), and are accessed using their keys
Null
~~~~
Null is represented by an empty String, and can be conveyed by spelling out ``null``,
case-insensitive.
@ -297,7 +301,9 @@ Function Type-Hints
AnyArgument
~~~~~~~~~~~
AnyArgument means any of the above Types are valid as input or output to a scripting function.
AnyArgument means any of the above Types are valid as input or output to a scripting
function.
.. note::
@ -306,13 +312,15 @@ AnyArgument means any of the above Types are valid as input or output to a scrip
Numeric
~~~~~~~
Numeric refers to either an Integer or Float.
Optional
~~~~~~~~
Optional means a particular scripting function argument can be either provided or not included.
For example, the function
`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
Optional means a particular scripting function argument can be either provided or not
included. For example, the function `map_get
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
has an optional default value. Both of these usages are valid:
.. tab-set::
@ -331,8 +339,9 @@ has an optional default value. Both of these usages are valid:
Lambda
~~~~~~
Lambda parameters are a reference to a function, and will call that lambda function
on the input. In this example,
Lambda parameters are a reference to a function, and will call that lambda function on
the input. In this example,
.. code-block:: yaml
@ -341,20 +350,23 @@ on the input. In this example,
%array_apply( [ 1, 2, 3, 4], %string )
}
We apply ``%string`` as a lambda function to
`array_apply <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_,
which is called on every element in the input array. The output becomes ``["1", "2", "3", "4"]``.
We apply ``%string`` as a lambda function to `array_apply
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_,
which is called on every element in the input array. The output becomes ``["1", "2",
"3", "4"]``.
This example has one input-argument being passed into the lambda. For other lambda-based functions
like `array_enumerate <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_,
This example has one input-argument being passed into the lambda. For other lambda-based
functions like `array_enumerate
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_,
it expects the lambda function to have two input arguments. These are denoted using
``LambdaTwo``, ``LambdaThree``, etc within the function spec.
LambdaReduce
~~~~~~~~~~~~
LambdaReduce parameters are a reference to a function that will perform a *reduce* - an operation
that reduces an Array to a single value by calling the LambdaReduce function repeatedly on two
elements in the Array until it is reduced to a single value.
LambdaReduce parameters are a reference to a function that will perform a *reduce* - an
operation that reduces an Array to a single value by calling the LambdaReduce function
repeatedly on two elements in the Array until it is reduced to a single value.
In this example,
@ -365,11 +377,12 @@ In this example,
%array_reduce( [ 1, 2, 3, 4], %add )
}
We call
`array_reduce <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_
on the input array, using
`add <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_
as the LambdaReduce function. This will reduce the Array to a single value by internally calling
We call `array_reduce
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_
on the input array, using `add
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_
as the LambdaReduce function. This will reduce the Array to a single value by internally
calling
- *reduce-call 1*: ``%add(1, 2) = 3`` (first two elements)
- *reduce-call 2*: ``%add(3, 3) = 6`` (output from first two and third element)
@ -380,9 +393,10 @@ And evaluate to ``10``.
ReturnableArguments
~~~~~~~~~~~~~~~~~~~
Returnable arguments are used in conditional functions like
`if <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_,
which implies the argument passed into the function is the function's output. For example,
Returnable arguments are used in conditional functions like `if
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_,
which implies the argument passed into the function is the function's output. For
example,
.. code-block:: yaml
@ -391,4 +405,4 @@ which implies the argument passed into the function is the function's output. Fo
%if( True, "Return this if True", "Return this if False" )
}
is going to return ``"Return this if True"`` since the condition parameter is ``True``.
is going to return ``"Return this if True"`` since the condition parameter is ``True``.

22
docs/source/config_reference/scripting/static_variables.rst
View file

@ -1,3 +1,10 @@
..
WARNING: This RST file is generated from docstrings in:
src/ytdl_sub/entries/variables/override_variables.py
In order to make a change to this file, edit the respective docstring
and run `make docs`. This will automatically sync the Python RST-based
docstrings into this file. If the docstrings and RST file are out of sync,
it will fail TestDocGen tests in GitHub CI.
Static Variables
================
@ -24,16 +31,21 @@ otherwise.
subscription_indent_i
~~~~~~~~~~~~~~~~~~~~~
For subscriptions in the form of
For subscriptions where the ancestor keys contain the ``= ...`` prefix, the
variables ``subscription_indent_1``, ``subscription_indent_2``, and so on get
set to each subsequent value. For example, given the following subscriptions
file snippet:
.. code-block:: yaml
Preset | = Indent Value 1:
= Indent Value 2:
Preset 1 | = Indent Value 1 | Preset 2:
Preset 3 | = Indent Value 2 | Preset 4:
"Subscription Name": "https://..."
``subscription_indent_1`` and ``subscription_indent_2`` get set to
``Indent Value 1`` and ``Indent Value 2``.
The ``{subscription_indent_1}`` variable will be ``Indent Value 1`` and
``{subscription_indent_2}`` will be ``Indent Value 2``. The most common use of
these variables is to :doc:`set the genre and rating for subscriptions from the
YAML keys <../prebuilt_presets/tv_show>`.
subscription_map
~~~~~~~~~~~~~~~~

75
docs/source/config_reference/subscription_yaml.rst
View file

@ -2,19 +2,22 @@
Subscription File
==================
A subscription file is designed to both define and organize many things
to download in condensed YAML.
A subscription file is designed to both define and organize many things to download in
condensed YAML.
.. hint::
Read the :ref:`getting started guide <guides/getting_started/index:Getting Started>`
first before reviewing this section.
File Preset
-----------
Many examples show ``__preset__`` at the top. This is known as the *subscription file preset*.
It is where a single :ref:`preset <guides/getting_started/first_config:Custom Preset Definition>`
can be defined that gets applied to each subscription within the file.
Many examples show ``__preset__`` at the top. This is known as the *subscription file
preset*. It is where a single :ref:`preset <guides/getting_started/first_config:Custom
Preset Definition>` can be defined that gets applied to each subscription within the
file.
This is a good place to apply file-wide variables such as ``tv_show_directory`` or
supply a cookies file path.
@ -22,15 +25,17 @@ supply a cookies file path.
.. code-block:: yaml
__preset__:
# Variables that override defaults from `overrides:` for presets in YAML keys:
overrides:
tv_show_directory: "/tv_shows"
# Directly set plugin options:
ytdl_options:
cookiefile: "/config/cookie.txt"
Layout
------
A subscription file is comprised of YAML keys and values. Keys can be either
- a preset
@ -52,32 +57,33 @@ All three types of keys are used for the following:
- ``= News`` - an override value for genre
- ``Breaking News``, ``BBC News`` - The subscription names
The bottom-most keys, or leaf keys, should always be the subscription name.
It is good practice to put subscription names in quotes to differentiate
between preset names and subscription names.
The lowest level, most indented keys should always be the subscription name. It is good
practice to put subscription names in quotes to differentiate between preset names and
subscription names.
Values should always be the subscription itself. The simplest form is
just the URL. Further sections will show more exotic examples that go beyond
a single URL.
Values should always be the subscription itself. The simplest form is just the
URL. Further sections will show more exotic examples that go beyond a single URL.
Inheritance
-----------
A subscription inherits every key above it. In the above example,
both ``Breaking News`` and ``BBC News`` inherits the ``Jellyfin TV Show by Date``
preset and the ``= News`` override value.
A subscription inherits every key above it. In the above example, both ``Breaking News``
and ``BBC News`` inherits the ``Jellyfin TV Show by Date`` preset and the ``= News``
override value.
.. note::
There are no limits or boundaries on how one structures
their presets. This flexibility is intended for subscription authors
to organize their downloads as they see fit.
There are no limits or boundaries on how one structures their presets. This
flexibility is intended for subscription authors to organize their downloads as they
see fit.
Multi Keys
----------
Subscription keys support pipe syntax, or ``|``, which allows multiple
keys to be defined on a single line. The following is equivalent to the above
example:
Subscription keys support pipe syntax, or ``|``, which allows multiple keys to be
defined on a single line. The following is equivalent to the above example:
.. code-block:: yaml
@ -85,16 +91,16 @@ example:
"Breaking News": "https://www.youtube.com/@SomeBreakingNews"
"BBC News": "https://www.youtube.com/@BBCNews"
Override Mode
-------------
Often times, it is convenient to set multiple override values for
a single subscription. We can put a preset in *override mode* by
using tilda syntax, or ``~``.
Often times, it is convenient to set multiple override values for a single
subscription. We can put a preset in *override mode* by using tilda syntax, or ``~``.
Suppose we want to apply the :ref:`Only Recent <prebuilt_presets/helpers:Only Recent>`
preset to the above examples. But for ``BBC News`` specifically, we want to
set the date range to be different than the default ``2months`` value to
``2weeks``.
preset to the above examples. But for ``BBC News`` specifically, we want to set the date
range to be different than the default ``2months`` value to ``2weeks``.
We can change it as follows:
@ -109,13 +115,14 @@ We can change it as follows:
.. important::
When using override mode, we need to set the ``url``
variable since we are no longer using the simplified
*subscription_value*. For more info on how this works,
read about :ref:`subscription variables <config_reference/scripting/static_variables:Subscription Variables>`.
When using override mode, we need to set the ``url`` variable since we are no longer
using the simplified *subscription_value*. For more info on how this works, read about
:ref:`subscription variables <config_reference/scripting/static_variables:Subscription
Variables>`.
Map Mode
--------
Map mode is for highly advanced presets that benefit
from a more complex subscription definition. TODO: Show music video
example here.
Map mode is for highly advanced presets that benefit from a more complex subscription
definition. TODO: Show music video example here.

78
docs/source/debugging.rst Normal file
View file

@ -0,0 +1,78 @@
Debugging
=========
Run with ``--log-level debug`` to show all log messages, often too much information for
normal operation but useful when investigating a specific problem.
:ref:`ytdl-sub builds on yt-dlp <introduction:motivation>`, which is in itself a complex
tool. It performs an intricate and fragile task, web scraping, which in turn :ref:`is
subject to the whims of external services <guides/getting_started/index:minimize the
work to only what's necessary>` outside its control. Finally, because :ref:`ytdl-sub is
a lower-level tool <guides/getting_started/index:prerequisite knowledge>`, many users,
if not most, will have problems getting their configuration working and it can be
difficult to determine when the root cause is their configuration, just a limit imposed
by the services, or, least likely, a bug in one of the tools involved.
To expedite resolution and conserve the limited resources of both yourself and
volunteers, do as much investigation yourself as you can:
#. Start by assuming the issue is your configuration:
Review :doc:`the guides <./guides/index>` to confirm your understanding. Increase
output using the ``--log-level`` CLI option and read the output carefully for hints
and clues. Use those clues to `search the docs`_. Read :doc:`the reference docs
<./config_reference/index>` of the involved ``ytdl-sub`` components.
#. Try to determine if the issue is happening in ``yt-dlp`` or ``ytdl-sub``:
The user's configuration tells ``ytdl-sub`` how to run ``yt-dlp``. ``yt-dlp`` handles
all the web scraping and downloading. ``ytdl-sub`` then assembles the files and metadata
produced by ``yt-dlp`` and places them in your library.
If the issue is happening while scraping or downloading from the external service,
then it's happening in the running of ``yt-dlp``. Look for output showing failed
downloads, ``403`` errors, or signs of throttles. That doesn't mean it's a bug in
``yt-dlp``, it could be in how your configuration tells ``ytdl-sub`` to run
``yt-dlp`` or limits imposed by the service that are constantly changing, but you may
be able to find answers from other ``yt-dlp`` users running into similar issues.
See `the yt-dlp known issues`_ and `search their issues`_ for clues and hints. Read
the comments for more understanding, workarounds, and maybe even fixes. If you still
don't understand the cause after reading everything you can find there, try to find
help in `the yt-dlp Discord`_.
#. If the issue is happening in ``ytdl-sub``, reach out for help:
Once you've done everything you can to get your configuration working and you've
determined that the issue isn't happening in ``yt-dlp``, look for answers in
``ytdl-sub``:
#. Cut your configuration and subscriptions down to the minimum that reproduces the
issue.
#. Run with the ``--log-level debug`` CLI option and copy the full output.
#. `Search the ytdl-sub issues`_ using clues and hints from the output.
#. `Open a support post in Discord`_ with those details and all other relevant
details.
#. If someone from the Discord discussion directs you to, then `open a new issue`_
with those same details.
.. _`the yt-dlp known issues`:
https://github.com/yt-dlp/yt-dlp/wiki/FAQ#known-issues
.. _`search their issues`:
https://github.com/yt-dlp/yt-dlp/issues
.. _`the yt-dlp Discord`:
https://discord.gg/H5MNcFW63r
.. _`search the docs`:
https://ytdl-sub.readthedocs.io/en/latest/search.html
.. _`search the ytdl-sub issues`:
https://github.com/jmbannon/ytdl-sub/issues
.. _`open a support post in Discord`:
https://discord.com/channels/994270357957648404/1084886228266127460
.. _`open a new issue`:
https://github.com/jmbannon/ytdl-sub/issues/new

29
docs/source/deprecation_notices.rst
View file

@ -1,14 +1,25 @@
Deprecation Notices
===================
Dec 2025
--------
Override variables names can no longer be plugin names, to avoid the common pitfall of
defining a plugin underneath ``overrides``.
In the past, there was usage of a ``date_range`` override variable in a few example configs
that complimented the ``Only Recent`` preset. This overrride variable usage needs to be
replaced with ``only_recent_date_range``.
Sep 2024
--------
regex plugin
~~~~~~~~~~~~
Regex plugin has been removed in favor of scripting. The function
:ref:`config_reference/scripting/scripting_functions:regex_capture_many`
has been created to replicate the plugin's behavior. See the following converted example:
:ref:`config_reference/scripting/scripting_functions:regex_capture_many` has been
created to replicate the plugin's behavior. See the following converted example:
.. code-block:: yaml
:caption: regex plugin
@ -40,13 +51,16 @@ has been created to replicate the plugin's behavior. See the following converted
}
track_title: "{%array_at(captured_track_title, 1)}"
Oct 2023
--------
subscription preset and value
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The use of ``__value__`` will go away in Dec 2023 in favor of the method found in
:ref:`config_reference/subscription_yaml:Subscription File`. ``__preset__`` will still be supported for the time being.
:ref:`config_reference/subscription_yaml:Subscription File`. ``__preset__`` will still
be supported for the time being.
July 2023
---------
@ -54,8 +68,9 @@ July 2023
music_tags
~~~~~~~~~~
Music tags are getting simplified. ``tags`` will now reside directly under music_tags, and
``embed_thumbnail`` is getting moved to its own plugin (supports video files as well). Convert from:
Music tags are getting simplified. ``tags`` will now reside directly under music_tags,
and ``embed_thumbnail`` is getting moved to its own plugin (supports video files as
well). Convert from:
.. code-block:: yaml
@ -79,8 +94,8 @@ The old format will be removed in October 2023.
video_tags
~~~~~~~~~~
Video tags are getting simplified as well. ``tags`` will now reside directly under video_tags.
Convert from:
Video tags are getting simplified as well. ``tags`` will now reside directly under
video_tags. Convert from:
.. code-block:: yaml

191
docs/source/faq/index.rst
View file

@ -2,45 +2,36 @@
FAQ
===
Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as more questions get asked.
Since ytdl-sub is relatively new to the public, there has not been many question asked
yet. We will update this as more questions get asked.
.. contents:: Frequently Asked Questions
:depth: 3
How do I...
-----------
...remove the date in the video title?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default include the upload date in the ``episode_title``
override variable. This variable is used to set the title in things like the video metadata, NFO file, etc, which is
subsequently read by media players. This can be overwritten as you see fit by redefining it:
The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default include
the upload date in the ``episode_title`` override variable. This variable is used to set
the title in things like the video metadata, NFO file, etc, which is subsequently read
by media players. This can be overwritten as you see fit by redefining it:
.. code-block:: yaml
overrides:
episode_title: "{title}" # Only sets the video title
...get support or reach out to contribute?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If you need support, you can:
* :ytdl-sub-gh:`Open an issue on GitHub <issues/new>`
* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_
If you would like to contribute, we're happy to accept any help, even non-coders! To find out how you can help this project, you can:
* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute.
* If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull request <compare>` with information on what issue you're resolving and it will be reviewed as soon as possible.
...download age-restricted YouTube videos?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
See `yt-dl's recommended way <https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_ to download your YouTube cookie, then add it to your :ref:`ytdl options <config_reference/plugins:ytdl_options>` section of your config:
See `yt-dl's recommended way
<https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_ to
download your YouTube cookie, then add it to your :ref:`ytdl options
<config_reference/plugins:ytdl_options>` section of your config:
.. code-block:: yaml
@ -50,7 +41,8 @@ See `yt-dl's recommended way <https://github.com/ytdl-org/youtube-dl#how-do-i-pa
...automate my downloads?
~~~~~~~~~~~~~~~~~~~~~~~~~
:doc:`This page </guides/getting_started/automating_downloads>` shows how to set up ``ytdl-sub`` to run automatically on various platforms.
:doc:`This page </guides/getting_started/automating_downloads>` shows how to set up
``ytdl-sub`` to run automatically on various platforms.
...download large channels?
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -65,7 +57,8 @@ See the prebuilt preset :doc:`Filter Keywords </prebuilt_presets/helpers>`.
...prevent creation of NFO file
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Creation of NFO files is done by the NFO tags plugin. It, as any other plugin, can be disabled:
Creation of NFO files is done by the NFO tags plugin. It, as any other plugin, can be
disabled:
.. code-block:: yaml
@ -75,8 +68,9 @@ Creation of NFO files is done by the NFO tags plugin. It, as any other plugin, c
...prevent download of images
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default downloads images corresponding to show and each episode.
This can be prevented by overriding following variables:
The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default
downloads images corresponding to show and each episode. This can be prevented by
overriding following variables:
.. code-block:: yaml
@ -88,10 +82,11 @@ This can be prevented by overriding following variables:
...use only part of the media's title
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ytdl-sub offers a range of functions that can be used to parse a subset of a title
for use in your media player. Consider the example:
ytdl-sub offers a range of functions that can be used to parse a subset of a title for
use in your media player. Consider the example:
* I want to remove "NOVA PBS - " from the title ``NOVA PBS - Hidden Cities All Around Us``.
* I want to remove "NOVA PBS - " from the title ``NOVA PBS - Hidden Cities All Around
Us``.
There are several solutions using ytdl-sub's scripting capabilities to override
``episode_title`` by manipulating the original media's ``title``.
@ -117,7 +112,9 @@ There are several solutions using ytdl-sub's scripting capabilities to override
}
.. code-block:: yaml
:caption: Regex capture. Supports multiple capture strings and default values if captures are unsuccessful.
:caption:
Regex capture. Supports multiple capture strings and default values if captures
are unsuccessful.
"~Nova PBS":
url: "https://www.youtube.com/@novapbs"
@ -132,44 +129,132 @@ There are several solutions using ytdl-sub's scripting capabilities to override
episode_title: >-
{ %array_at( captured_episode_title, 1 ) }
There is no single solution to this problem - it will vary case-by-case. See
our full suite of
:ref:`scripting functions <config_reference/scripting/scripting_functions:Scripting Functions>`
to create your own clever scraping mechanisms.
There is no single solution to this problem - it will vary case-by-case. See our full
suite of :ref:`scripting functions
<config_reference/scripting/scripting_functions:Scripting Functions>` to create your own
clever scraping mechanisms.
...force ytdl-sub to re-download a file
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Sometimes users may wish to replace a file already in the archive, for example, if the
current file is a lower resolution than desired, missing subtitles, corrupt, etc..
``ytdl-sub`` decides what files have already been downloaded by entries in :ref:`the
download archive file <config_reference/plugins:output_options>`,
``./.ytdl-sub-...-download-archive.json``, at the top of the subscription/series/show
:ref:`output directory <config_reference/plugins:output_options>` in the appropriate
``overrides: / ..._directory:`` library path, *and* the presence of the corresponding
downloaded files under the same path. To force ``ytdl-sub`` to re-download an entry both
need to be removed:
- Move aside the downloaded files:
Rename or move the downloaded files, including the associated files with the same
base/stem name, such as ``./*.nfo``, ``./*.info-json``, etc..
- Ensure ``ytdl-sub`` is not running and won't run, such as by cron:
``ytdl-sub`` loads the ``./.ytdl-sub-...-download-archive.json`` file early, keeps it
in memory, and writes it back out late. If it's running or starts running while you're
modifying that file, then your changes will be overwritten when it exits.
- Remove the ``./.ytdl-sub-...-download-archive.json`` JSON array item:
Search for the stem name, the basename without any extension or suffix, common to all
the downloaded files in this file and delete that whole entry, from the YouTube ID
string to the closing curly braces. Be ware of JSON traling commas.
- Run ``$ ytdl-sub sub`` again with the appropriate CLI plugin options:
In normal operation, :ref:`yt-dlp minimizes requests and the files considered for
download <guides/getting_started/index:minimize the work to only what's
necessary>`. To re-download, those options must be disabled or modified. Disable
:ref:`the 'break_on_existing' option <config_reference/plugins:ytdl_options>`, set
:ref:`the 'date_range:' plugin <config_reference/plugins:date_range>`, and :ref:`limit
the subscriptions <guides/getting_started/downloading:preview>` to
download only the files that you've renamed in the steps above.
Set the appropriate dates, :ref:`including a sufficient margin
<config_reference/plugins:date_range>`, and subscription name to include only the
files you've renamed, and re-run. For example, if you've renamed all the files from
2024 in the ``NOVA PBS`` subscription:
.. code-block:: shell
ytdl-sub --match="NOVA PBS" sub -o "\
--ytdl_options.break_on_existing False \
--date_range.after 20240101 \
--date_range.before 20250101 \
"
...download a file missing from the archive
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The root causes are unknown, but sometimes even after successful, complete runs, some
files will be missing from the archive. To attempt to download those missing files,
use `the same CLI options as re-downloading a file`_
.. _`the same CLI options as re-downloading a file`:
`...force ytdl-sub to re-download a file`_
...get support?
~~~~~~~~~~~~~~~
See :doc:`the debugging documentation <../debugging>`.
...reach out to contribute?
~~~~~~~~~~~~~~~~~~~~~~~~~~~
If you would like to contribute, we're happy to accept any help, including from
non-coders! To find out how you can help this project, you can:
- `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in
#development with where you think you can assist or what skills you would like to
contribute.
- If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull
request <compare>` with information on what issue you're resolving and it will be
reviewed as soon as possible.
There is a bug where...
-----------------------
..ytdl-sub is not downloading
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
...ytdl-sub is not downloading
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Run with ``--log-level verbose`` to see all yt-dlp logs, to rule out whether it is a yt-dlp or ytdl-sub issue.
...ytdl-sub is downloading at 360p or other lower quality
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Any logs showing failed downloads, 403 errors, signs of throttles, etc, are a yt-dlp issue.
A good strategy is to see if your same issue has been reported in
`yt-dlp's GitHub issues <https://github.com/yt-dlp/yt-dlp/issues>`_, and search to see if there is a comment including
a fix or workaround.
...ytdl-sub downloads 2-4 videos and then fails
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If it looks like a ytdl-sub issue, run with ``--log-level debug`` and make a
`GitHub issue in ytdl-sub <https://github.com/jmbannon/ytdl-sub/issues>`_
containing these logs and other relevant info.
These are often just limits imposed by the external services that are not bugs. There
may be little that can be done about them, but see :ref:`the '_throttle_protection'
preset <prebuilt_presets/helpers:_throttle_protection>` for more information.
...date_range is not downloading older videos after I changed the range
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether.
Your preset most likely has ``break_on_existing`` set to True, which will stop
downloading additional metadata/videos if the video exists in your download archive. Set
the following in your config to skip downloading videos that exist instead of stopping
altogether.
.. code-block:: yaml
ytdl_options:
break_on_existing: False
After you download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads.
After you download your new date_range duration, re-enable ``break_on_existing`` to
speed up successive downloads.
...it is downloading non-English title and description metadata
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using.
Most likely the video has a non-English language set to its 'native' language. You can
tell yt-dlp to explicitly download English metadata using.
.. code-block:: yaml
@ -185,7 +270,9 @@ Most likely the video has a non-English language set to its 'native' language. Y
1. Set the following for your ytdl-sub library that has been added to Plex.
.. figure:: ../../images/plex_scanner_agent.png
:alt: The Plex library editor, under the advanced settings, showing the required options for Plex to show the TV shows correctly.
:alt:
The Plex library editor, under the advanced settings, showing the required options
for Plex to show the TV shows correctly.
- **Scanner:** Plex Series Scanner
- **Agent:** Personal Media shows
@ -193,7 +280,15 @@ Most likely the video has a non-English language set to its 'native' language. Y
- **Episode sorting:** Library default
- **YES** Enable video preview thumbnails
2. Under **Settings** > **Agents**, confirm Plex Personal Media Shows/Movies scanner has **Local Media Assets** enabled.
2. Under **Settings** > **Agents**, confirm Plex Personal Media Shows/Movies scanner has
**Local Media Assets** enabled.
.. figure:: ../../images/plex_agent_sources.png
:alt: The Plex Agents settings page has Local Media Assets enabled for Personal Media Shows and Movies tabs.
:alt:
The Plex Agents settings page has Local Media Assets enabled for Personal Media
Shows and Movies tabs.
...ytdl-sub errors when downloading a 360p video with resolution assert
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:ref:`See how to either ignore this specific video or disable resolution assertion entirely here. <resolution assert handling>`

64
docs/source/guides/development/index.rst
View file

@ -1,8 +1,10 @@
Development and Contributing
============================
Requirements
------------
- python >= 3.10
- ffmpeg/ffprobe 4.4.5 (test checksums rely on this version)
- make
@ -20,15 +22,18 @@ Local Install
pip install -e .\[test,lint,docs\]
Linter
------
All source code contributed must be formatted to our linter specification.
Run the following to auto-format and check for any issues with your code:
All source code contributed must be formatted to our linter specification. Run the
following to auto-format and check for any issues with your code:
.. code-block:: shell
make lint
Adding Documentation
--------------------
@ -36,18 +41,21 @@ Docs can be found in ``ytdl-sub/docs/source/``, and are built using the command:
.. code-block:: shell
:caption: Viewable at http://localhost:63342/ytdl-sub/docs/build/html/index.html once built
:caption:
Viewable at http://localhost:63342/ytdl-sub/docs/build/html/index.html once built
make docs
Some of the documentation is built using doc-strings from the python source code. The above
command will rebuild those as well.
Some of the documentation is built using doc-strings from the python source code. The
above command will rebuild those as well.
Testing
-------
Tests are written using pytest. Many of them evaluate checksums of output files to ensure no unintended
changes are introduced to the way ``ytdl-sub`` produces files. This checksum can be inaccurate for
end-to-end tests, but are reliable for integration tests.
Tests are written using pytest. Many of them evaluate checksums of output files to
ensure no unintended changes are introduced to the way ``ytdl-sub`` produces files. This
checksum can be inaccurate for end-to-end tests, but are reliable for integration tests.
If integration tests are failing, ensure...
@ -55,26 +63,36 @@ If integration tests are failing, ensure...
- you are developing on Linux or Mac (have not tested windows yet)
- your local ``ytdl-sub`` dependencies are up-to-date
Docker
------
Test changes to the Docker image variants locally:
.. code-block:: shell
cd ./docker/testing/
make -j run
See ``./docker/testing/docker-compose.yml`` for the Compose services for each image
variant.
IDE Setup
---------
PyCharm is our preferred IDE. The codebase is simple enough to where it's not required, but
is highly recommended.
PyCharm is our preferred IDE. The codebase is simple enough to where it's not required,
but is highly recommended.
TODO: screenshots of configuration
Debugging
---------
Debug Logs
^^^^^^^^^^^^^^^
Run with ``--log-level debug`` to show all debug logs when running ytdl-sub.
Reproducing a Failing Subscription
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Subscriptions will dump their entire *compiled* yaml at the beginning of exeuction
when using ``--log-level debug``. This can be copy-pasted into the file
``resources/file_fixtures/repro.yaml``.
----------------------------------
Running the test ``e2e.test_debug_repro.TestReproduce.test_debug_log_repro``
will fully reproduce that subscription in order to debug it.
Subscriptions will dump their entire *compiled* yaml at the beginning of exeuction
:doc:`when using '--log-level debug' <../../debugging>`. This can be copy-pasted into
the file ``resources/file_fixtures/repro.yaml``.
Running the test ``e2e.test_debug_repro.TestReproduce.test_debug_log_repro`` will fully
reproduce that subscription in order to debug it.

25
docs/source/guides/getting_started/automating_downloads.rst
View file

@ -9,6 +9,7 @@ Automating Downloads
.. _cron scheduling syntax: https://crontab.guru/#0_*/6_*_*_*
.. _docker-unraid-setup:
Docker and Unraid
@ -26,11 +27,13 @@ ENV variables to your docker setup.
- CRON_RUN_ON_START=false
- ``CRON_SCHEDULE`` follows the standard `cron scheduling syntax`_. The above value will run the script once every 6 hours.
- ``CRON_RUN_ON_START`` toggles whether to run your cron script on container start in addition to the cron schedule.
- ``CRON_SCHEDULE`` follows the standard `cron scheduling syntax`_. The above value will
run the script once every 6 hours.
- ``CRON_RUN_ON_START`` toggles whether to run your cron script on container start in
addition to the cron schedule.
The cron script will reside in the main directory with the file name ``cron``.
Cron logs should show when viewing the Docker logs.
The cron script will reside in the main directory with the file name ``cron``. Cron
logs should show when viewing the Docker logs.
.. _linux-setup:
@ -45,14 +48,22 @@ Must configure crontab manually, like so:
0 */6 * * * /config/run_cron
.. _windows-setup:
Windows
-------
To be tested (please contact code owner or join the discord server if you can test this out for us)
To be tested (please contact code owner or join the discord server if you can test this
out for us)
.. code-block:: powershell
ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml
Next Steps
----------
Once you have a significant quantity of subscriptions or have use cases not served using
:doc:`YAML keys and the special characters <./subscriptions>`, it's time to start
:doc:`defining your own custom presets <./first_config>`.

69
docs/source/guides/getting_started/downloading.rst Normal file
View file

@ -0,0 +1,69 @@
Downloading
===========
Once you've :doc:`defined your subscriptions <./subscriptions>`, it's time to test your
configuration and try your first download. As a web scraping tool, :ref:`it's important
to minimize the requests sent to external services
<guides/getting_started/index:minimize the work to only what's necessary>` to avoid
triggering throttling or bans. Further, a full download of even one subscription can
take significant time. Test each change to your subscriptions carefully and quickly as
follows.
Preview
-------
Preview what ``ytdl-sub`` would do for this subscription. Run the :ref:`'sub'
sub-command <usage:subscriptions options>` with CLI options to restrict requests as much
as possible:
- Pull metadata and *simulate* a download without actually downloading any media files
using the ``--dry-run`` CLI option.
- Limit requests by narrowing the run to one subscription by giving a
subscription name to the ``--match`` CLI option.
- Stop after just a few downloads to further minimize requests and make testing faster
using the ``max_downloads`` setting from ``yt-dlp``.
Change to the directory containing your ``./subscriptions.yaml`` file and run with those
options:
.. code-block:: console
cd "/config/ytdl-sub-configs/"
ytdl-sub --dry-run --match="NOVA PBS" sub -o '--ytdl_options.max_downloads 3'
Examine the output carefully, investigate anything that doesn't look right and repeat
this step until everything looks right.
Review
------
Review the results of real downloads. Run it again without the ``--dry-run`` option to
actually download media and place the files in your library:
.. code-block:: console
ytdl-sub --match="NOVA PBS" sub -o '--ytdl_options.max_downloads 3'
Examine the output carefully again. Then examine how the resulting downloads work in
your library. Repeat with a larger value for ``max_downloads`` and examine the output
and downloads again.
Next Steps
----------
Once you're `previewed <preview_>`_ and `reviewed <review_>`_ successful downloads of
each of your subscriptions, you're ready to run a full download of all your
subscriptions. Run the sub-command without the CLI options you used to limit what
``ytdl-sub`` does while testing:
.. code-block:: console
ytdl-sub sub
If you're ready to let ``ytdl-sub`` run unattended, it's time to :doc:`automate
downloads <./automating_downloads>`.

105
docs/source/guides/getting_started/first_config.rst
View file

@ -3,10 +3,11 @@ Basic Configuration
A configuration file serves two purposes:
1. Set advanced functionality that is not specifiable in a subscription file, such as working directory location. These
are set underneath ``configuration``.
2. Create custom presets, which can drastically simplify your subscription file. These are defined underneath ``presets``.
Presets are intended to be applicable and reusable across multiple subscriptions.
1. Set advanced functionality that is not specifiable in a subscription file, such as
working directory location. These are set underneath ``configuration``.
2. Create custom presets, which can drastically simplify your subscription file. These
are defined underneath ``presets``. Presets are intended to be applicable and
reusable across multiple subscriptions.
Below is a common configuration:
@ -14,7 +15,7 @@ Below is a common configuration:
:linenos:
configuration:
working_directory: '/mnt/ssd/.ytdl-sub-downloads'
working_directory: ".ytdl-sub-working-directory"
presets:
TV Show:
@ -36,28 +37,36 @@ Below is a common configuration:
max: 36
overrides:
tv_show_directory: "/ytdl_sub_tv_shows"
tv_show_directory: "/tv_shows"
TV Show Only Recent:
preset:
- "TV Show"
- "Only Recent"
Configuration Section
---------------------
The :ref:`configuration <config_reference/config_yaml:Configuration File>` section sets options for ytdl-sub execution.
The :ref:`configuration <config_reference/config_yaml:Configuration File>` section sets
options for ytdl-sub execution. Most users should set the path where ``ytdl-sub``
temporarily stores downloaded data before assembling it and moving it into your
library. To avoid unnecessarily long large file renames, use a path on the same
filesystem as your library in the ``overrides: / *_directory:`` paths:
.. code-block:: yaml
:lineno-start: 1
configuration:
working_directory: '/mnt/ssd/.ytdl-sub-downloads'
working_directory: ".ytdl-sub-working-directory"
Preset Section
--------------
Underneath ``presets``, we define two custom presets with the names ``TV Show`` and ``TV Show Only Recent``.
Underneath ``presets``, we define two custom presets with the names ``TV Show`` and ``TV
Show Only Recent``.
.. code-block:: yaml
@ -69,6 +78,7 @@ Underneath ``presets``, we define two custom presets with the names ``TV Show``
The indentation example above shows how to define multiple presets.
Custom Preset Definition
------------------------
@ -88,12 +98,15 @@ Before we break down the above ``TV Show`` preset, lets first outline a preset l
Presets can contain three important things:
1. ``preset`` section, which can inherit :ref:`prebuilt presets <config_reference/prebuilt_presets/index:Prebuilt Preset Reference>`
or other presets defined in your config.
1. ``preset`` section, which can inherit :ref:`prebuilt presets
<config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` or other presets
defined in your config.
2. :ref:`Plugin definitions <config_reference/plugins:Plugins>`
3. :ref:`overrides <config_reference/plugins:overrides>`, which can override inherited preset variables
3. :ref:`overrides <config_reference/plugins:overrides>`, which can override inherited
preset variables
Presets do not have to define all of these, as we'll see in the ``TV Show Only Recent`` preset.
Presets do not have to define all of these, as we'll see in the ``TV Show Only Recent``
preset.
Inheriting Presets
~~~~~~~~~~~~~~~~~~
@ -106,14 +119,16 @@ Inheriting Presets
- "Jellyfin TV Show by Date"
- "Max 1080p"
The following snippet shows that the ``TV Show`` preset will inherit all properties
of the prebuilt presets ``Jellyfin TV Show by Date`` and ``Max 1080p`` in that order.
The following snippet shows that the ``TV Show`` preset will inherit all properties of
the prebuilt presets ``Jellyfin TV Show by Date`` and ``Max 1080p`` in that order.
Order matters for preset inheritance. Bottom-most presets will override ones above them.
It is highly advisable to use :ref:`prebuilt presets <config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` as
a starting point for custom preset building, as they do the work of preset building to ensure things show as expected
in their respective media players. Read on to see how to override prebuilt preset specifics such as title.
It is highly advisable to use :ref:`prebuilt presets
<config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` as a starting point
for custom preset building, as they do the work of preset building to ensure things show
as expected in their respective media players. Read on to see how to override prebuilt
preset specifics such as title.
Defining Plugins
~~~~~~~~~~~~~~~~
@ -134,17 +149,18 @@ Defining Plugins
min: 10
max: 36
Our ``TV Show`` sets two plugins, :ref:`throttle_protection <config_reference/plugins:throttle_protection>` and
:ref:`embed_thumbnail <config_reference/plugins:embed_thumbnail>`. Each plugin's documentation shows the respective
fields that they support.
Our ``TV Show`` sets two plugins, :ref:`throttle_protection
<config_reference/plugins:throttle_protection>` and :ref:`embed_thumbnail
<config_reference/plugins:embed_thumbnail>`. Each plugin's documentation shows the
respective fields that they support.
If an inherited preset defines the same plugin, the custom preset will use 'merge-and-append' strategy to
combine their definitions. What this means is:
1. If the field is a map (i.e. has sub-params like ``sleep_per_download_s`` above) or array, it will try to merge them
2. If both the inherited preset and custom preset set the same exact field and value (i.e. ``embed_thumbnail``)
the custom preset will overwrite it
If an inherited preset defines the same plugin, the custom preset will use
'merge-and-append' strategy to combine their definitions. What this means is:
1. If the field is a map (i.e. has sub-params like ``sleep_per_download_s`` above) or
array, it will try to merge them
2. If both the inherited preset and custom preset set the same exact field and value
(i.e. ``embed_thumbnail``) the custom preset will overwrite it
Setting Override Variables
~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -155,27 +171,31 @@ Setting Override Variables
overrides:
tv_show_directory: "/ytdl_sub_tv_shows"
All override variables reside underneath the :ref:`overrides <config_reference/plugins:overrides>` section.
All override variables reside underneath the :ref:`overrides
<config_reference/plugins:overrides>` section.
It is important to remember that individual subscriptions can override specific override variables.
When defining variables in a preset, it is best practice to define them with the intention that
It is important to remember that individual subscriptions can override specific override
variables. When defining variables in a preset, it is best practice to define them with
the intention that
1. All subscriptions will use its value them
2. Use them as placeholders to perform other logic, then have subscriptions or child presets
define their specific value
2. Use them as placeholders to perform other logic, then have subscriptions or child
presets define their specific value
For simplicity, we'll focus on (1) for now. The above snippet sets the ``tv_show_directory``
variable to a file path. This variable name is specific to the prebuilt TV show presets.
For simplicity, we'll focus on (1) for now. The above snippet sets the
``tv_show_directory`` variable to a file path. This variable name is specific to the
prebuilt TV show presets.
See the :ref:`prebuilt preset reference <config_reference/prebuilt_presets/index:Prebuilt Preset Reference>`
to see all available variables that are overridable.
See the :ref:`prebuilt preset reference
<config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` to see all
available variables that are overridable.
Using Custom Presets in Subscriptions
--------------------------------------
Subscription files can use custom presets just like any other prebuilt preset.
Below shows a complete subscription file using the above two custom presets.
Subscription files can use custom presets just like any other prebuilt preset. Below
shows a complete subscription file using the above two custom presets.
.. code-block:: yaml
@ -193,11 +213,12 @@ Below shows a complete subscription file using the above two custom presets.
Notice how we do not need to define ``tv_show_directory`` in the ``__preset__`` section
like in prior examples. This is because our custom presets do the work of defining it.
Reference Custom Config in the CLI
----------------------------------
Be sure to tell ytdl-sub to use your config by using the argument
``--config /path/to/config.yaml``.
Be sure to tell ytdl-sub to use your config by using the argument ``--config
/path/to/config.yaml``.
If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``, it will
use it by default.
If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``,
it will use it by default.

50
docs/source/guides/getting_started/first_download.rst
View file

@ -1,50 +0,0 @@
Initial Download
================
Once you have a ``subscriptions.yaml`` file created, you can perform your first
download. Access ``ytdl-sub``, navigate to the directory containing your ``subscriptions.yaml``
file.
Dry Run
-------
Performing a dry run is important when applying any change to your subscriptions to
ensure output looks as expected. Dry runs will pull metadata to *simulate* a download
without actually downloading the media file.
.. code-block:: shell
ytdl-sub --dry-run sub subscriptions.yaml
Faster Iteration Cycle
----------------------
Testing subscriptions can take quite some time to perform a full download.
This can be speed up by applying an override via command-line to set max number
of downloads.
.. code-block:: shell
ytdl-sub --dry-run sub subscriptions.yaml -o '--ytdl_options.max_downloads 3'
Having many subscriptions could still make this dry run take a while. A subset of
subscriptions can be dry ran using a match.
.. code-block:: shell
:caption: Only run subscriptions that have PBS in their names
ytdl-sub --dry-run sub subscriptions.yaml -o '--ytdl_options.max_downloads 3' --match PBS
Downloading
-----------
Once the subscriptions file is validated, a download can be performed by omitting the dry run argument.
.. code-block:: shell
ytdl-sub sub subscriptions.yaml
Multiple subscription file names can be provided to perform a download on all of them. A single file
named ``subscriptions.yaml`` does not require a file name specification since it will
look for that file name by default, making the following command valid.
.. code-block:: shell
ytdl-sub sub

141
docs/source/guides/getting_started/first_sub.rst
View file

@ -1,141 +0,0 @@
Initial Subscription
====================
Your first subscription file should look something like this:
.. code-block:: yaml
:linenos:
__preset__:
overrides:
tv_show_directory: "/tv_shows"
music_directory: "/music"
# Can choose between:
# - Plex TV Show by Date:
# - Jellyfin TV Show by Date:
# - Kodi TV Show by Date:
#
Jellyfin TV Show by Date:
= Documentaries:
"NOVA PBS": "https://www.youtube.com/@novapbs"
= Kids | = TV-Y:
"Jake Trains": "https://www.youtube.com/@JakeTrains"
YouTube Releases:
= Jazz: # Sets genre tag to "Jazz"
"Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
YouTube Full Albums:
= Lofi:
"Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
Let's break this down:
.. code-block:: yaml
:lineno-start: 1
__preset__:
overrides:
tv_show_directory: "/tv_shows"
music_directory: "/music"
The first :ref:`__preset__ <config_reference/subscription_yaml:File Preset>` section is where we
can set modifications that apply to every subscription in this file.
This snippet specifically adds two :ref:`override <config_reference/plugins:Overrides>` variables,
which are used by the presets below.
.. note::
It is tempting to put any override underneath ``overrides``. Keep in mind that this section
is solely for variable defining. Other :ref:`plugins <config_reference/plugins:Plugins>` need to be
set at the same indentation level as ``overrides``, not within it.
-------------------------------------
.. code-block:: yaml
:lineno-start: 6
# Can choose between:
# - Plex TV Show by Date:
# - Jellyfin TV Show by Date:
# - Kodi TV Show by Date:
#
Lines 6-10 are comments that get ignored when parsing YAML since they are prefixed with ``#``.
It is good practice to leave informative comments in your config or subscription files to remind
yourself of various things.
-------------------------------------
.. code-block:: yaml
:lineno-start: 11
Jellyfin TV Show by Date:
On line 11, we set the key to ``Jellyfin TV Show by Date``. This is a
:ref:`prebuilt preset <prebuilt_presets/index:prebuilt presets>` that configures
subscriptions to look like TV shows in the Jellyfin media player (can be changed to
one of the presets outlined in the comment above). Setting it as a YAML key implies that all
subscriptions underneath it will *inherit* this preset.
This preset expects the variable ``tv_show_directory`` to be set, which we do above.
-------------------------------------
.. code-block:: yaml
:lineno-start: 11
Jellyfin TV Show by Date:
= Documentaries:
Line 12 sets the key to ``= Documentaries``. When keys are prefixed with ``=``, it means we are
setting the genre. This value will get written to the respective metadata tags for both TV show
and music presets.
Behind the scenes, this sets the override variable ``subscription_indent_1``. Further documentation
can be found here for
:ref:`subscription syntax <config_reference/subscription_yaml:Subscription File>` and
:ref:`subscription variables <config_reference/scripting/static_variables:Subscription Variables>`.
-------------------------------------
.. code-block:: yaml
:lineno-start: 11
Jellyfin TV Show by Date:
= Documentaries:
"NOVA PBS": "https://www.youtube.com/@novapbs"
Line 13 is where we define our first subscription. We set the subscription name to ``NOVA PBS``,
and the subscription value to ``https://www.youtube.com/@novapbs``.
To see how presets ingest subscription definitions, refer to the
:ref:`preset references <config_reference/prebuilt_presets/tv_show:TV Show>`,
we can see that ``{subscription_name}`` is used to set the ``tv_show_name`` variable.
-------------------------------------
.. code-block:: yaml
:lineno-start: 11
Jellyfin TV Show by Date:
= Documentaries:
"NOVA PBS": "https://www.youtube.com/@novapbs"
= Kids | = TV-Y:
"Jake Trains": "https://www.youtube.com/@JakeTrains"
Line 15 underneath ``Jellyfin TV Show by Date``, but at the same level as ``= Documentaries``.
This means we'll inherit the TV show preset, but not the documentaries indent variable. We instead
set the indent variables to ``= Kids | = TV-Y``. This sets two indent variables. We can set
multiple presets and/or indent variables on the same key by using ``|`` as a separator.
Referring to the
:ref:`TV show preset reference <config_reference/prebuilt_presets/tv_show:TV Show>`, the first
two indent variables map to the TV show genre and TV show content rating.
The above info should be enough to understand the rest of the subscription file.

166
docs/source/guides/getting_started/index.rst
View file

@ -1,57 +1,157 @@
Getting Started
===============
Prerequisite Knowledge
----------------------
In order to use ``ytdl-sub`` in any of the forms listed in these docs, you will need some basic knowledge.
Using ``ytdl-sub`` requires some technical knowledge. You must be able to:
Be sure that you:
☑ Can navigate directories in a command line interface (or CLI)
- do `basic CLI shell navigation`_
- read and write `YAML text files`_
☑ Have a basic understanding of YAML syntax
If you plan on using a :ref:`Docker headless image variant
<guides/install/docker:headless image>` of ``ytdl-sub``, you can:
If you plan on using the headless image of ``ytdl-sub``, you:
☑ Can use ``nano`` or ``vim`` to edit OR
- use ``$ nano /config/...`` to edit configuration files inside the container
- or bind mount ``/config/`` as a Docker volume and use the editor of your choice from
the host
☑ Can mount the config directory somewhere you can open it using gui text editors
Soon, it's time to start configuring ``ytdl-sub``. We provide a :doc:`./quick_start`
with rigid, rote instructions on how to get a minimal configuration up and running, but
if that serves all your needs, then you're probably better off with :ref:`one of the
more user-friendly yt-dlp wrappers available <introduction:motivation>`. As a lower
level tool with no GUI, most ``ytdl-sub`` users will need to understand at least some of
how ``ytdl-sub`` works, how it "thinks". So before you start configuring ``ytdl-sub``,
`read on <architecture>`_ to learn how ``ytdl-sub`` works.
Additional useful (but not required) knowledge:
☑ Understanding how :yt-dlp:`\ ` works
.. _`basic CLI shell navigation`:
https://developer.mozilla.org/en-US/docs/Learn_web_development/Getting_started/Environment_setup/Command_line
.. _`YAML text files`: http://thomasloven.com/blog/2018/08/YAML-For-Nonprogrammers/
Terminology
-----------
Must-know terminology:
- ``subscription``: URL(s) that you want to download with specific metadata requirements.
- ``preset``: A media profile comprised of YAML configuration that can specify anything from metadata layout, media quality, or any feature of ytdl-sub, to apply to subscriptions. A preset can inherit other presets.
- ``prebuilt preset``: Presets that are included in ytdl-sub. These do most of the work defining plugins, overrides, etc in order to make downloads ready for player consumption.
- ``override``: Verb describing the act of overriding something in a preset. For example, the TV Show presets practically expect you to *override* the URL variable to tell ytdl-sub where to download from.
- ``override variables``: User-defined variables that are intended to *override* something.
- ``subscription file``: The file to specify all of your subscriptions and some override variables.
Architecture
------------
Intermediate terminology:
For most users, ``ytdl-sub`` works as follows:
- ``plugin``: Modular logic to apply to a subscription. To use a plugin, it must be defined in a preset.
- ``config file``: An optional file where you can define custom presets and other advanced configuration.
- ``yt-dlp``: The underlying application that handles downloading for ytdl-sub.
Subscriptions use presets
~~~~~~~~~~~~~~~~~~~~~~~~~
Advanced terminology:
Run ``$ ytdl-sub sub`` to read :doc:`a subscription file <./subscriptions>` that defines
what subscriptions to download and place into your media library. Each subscription
selects which :doc:`presets <../../prebuilt_presets/index>` to apply. Those presets
configure how each subscription is downloaded and placed in the media library.
- ``entry variables``: Variables that derive from a downloaded yt-dlp entry (media).
- ``static variables``: Variables that do not have a dependency to entry variables.
- ``scripting``: Syntax that allows the use of entry variables, static variables, and functions in override variables.
Presets configure plugins
~~~~~~~~~~~~~~~~~~~~~~~~~
:doc:`A preset <../../prebuilt_presets/index>` is effectively a set of plugin
configurations. Specifically, a preset consists of:
- base presets that it inherits from and extends
- plugin configurations
When a preset has multiple base presets and more than one of those base presets
configures the same keys for a plugin, the later/lower base preset overrides the plugin
key configurations of earlier/higher base presets. Similarly, when the preset configures
the same keys for a plugin that one of its base plugins configures, the preset
configuration overrides the base presets.
Plugins do the work
~~~~~~~~~~~~~~~~~~~
``ytdl-sub`` applies the plugins that the presets configure when it downloads a
subscription. :doc:`The plugins <../../config_reference/plugins>` control how to run
``yt-dlp``, which media in the subscription to download, how to collect and format
metadata for those media, how to place the resulting files into your media library, and
more.
Presets and subscriptions accept overrides
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Presets accept override keys and values and the preset uses those overrides to modify
their plugin configurations. Similarly, individual subscriptions can supply overrides of
their presets for just that subscription.
Subscriptions are grouped by indentation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Most subscriptions have more in common with each other than not. Thus, defining the
presets and overrides for each subscription would result in mostly repetition and would
multiply the burden of management for the user. The more subscriptions the more work.
To avoid this redundant work, and so that the subscription configurations describe the
intent of the user, subscriptions are nested/indented under parent/ancestor keys that
define their shared configuration. To support this, ``ytdl-sub`` uses special handling
of the ancestor YAML keys above each subscription. A subscription is the most
nested/indented/descendant key that specifies the URLs for that subscription. The
ancestor keys above that subscription describe the shared presets of that subscription
and all the other descendant subscriptions under them.
Genres are also more often shared between subscriptions than not. To accommodate that
reality, the ancestor keys of subscriptions may also use :ref:`the special '= ...'
prefix to pass specific overrides
<config_reference/scripting/static_variables:subscription_indent_i>` supported by the
preset. By convention in the pre-built media type presets, the first ``= ...`` value
specifies the genre for all descendant subscriptions.
Finally, ancestor keys may use :ref:`the '... | ...' special character
<config_reference/subscription_yaml:multi keys>` to combine multiple presets and/or
genres for the descendant subscriptions beneath.
The configuration file extends pre-defined presets
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Users define additional presets in :doc:`their configuration file <./first_config>` that
they then use in most of their subscriptions. Most user-defined presets extend the
:doc:`../../prebuilt_presets/index` provided by ``ytdl-sub``.
Minimize the work to only what's necessary
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Throttling and bans are a core problem for any web scraping tool, perhaps even more so
for ``yt-dlp``, and no good actor *wants* to be an onerous burden on a
service. Similarly, many web scraping use cases involve very large sets of data that are
too big to process as a whole for performance. It's important to narrow the amount of
data considered and minimize requests.
To these ends, most presets tell ``yt-dlp`` not to consider files before the most
recently downloaded file using :ref:`the 'break_on_existing' option
<config_reference/plugins:ytdl_options>`. Similarly, and particularly for huge channels
or playlists, most users should use either :ref:`an 'Only Recent' preset
<prebuilt_presets/helpers:only recent>` and/or :ref:`the 'Chunk Downloads' preset
<prebuilt_presets/helpers:chunk downloads>` to restrict the number of downloads
considered.
Caveats
~~~~~~~
Some of these descriptions are not technically complete. For example, a subscription may
use no preset at all and will just run ``yt-dlp`` without any customization or post
processing. The subscriptions file has special support for :ref:`overriding the presets
of all subscriptions in the file <config_reference/subscription_yaml:file preset>`. The
configuration file supports :ref:`a few special options
<config_reference/config_yaml:configuration>` that are not about defining presets. See
:doc:`the reference documentation <../../config_reference/index>` for technically
complete details, but for almost all of the use cases served by ``ytdl-sub``, the above
is accurate and representative.
Next Steps
----------
With ``ytdl-sub`` installed and the above understood, the next step is to :doc:`start
adding subscriptions <./subscriptions>`.
Ready to Start?
---------------
Now that you've completed your install of ``ytdl-sub``, it's time to get started.
It is recommended to go through the below sections in order to fully grasp ytdl-sub.
.. toctree::
:maxdepth: 2
:hidden:
first_sub
first_download
subscriptions
downloading
automating_downloads
first_config
quick_start

54
docs/source/guides/getting_started/quick_start.rst Normal file
View file

@ -0,0 +1,54 @@
Quick Start
===========
:ref:`Again <guides/getting_started/index:prerequisite knowledge>`, if the following
serves all your needs, then you're probably better off with :ref:`one of the more
user-friendly yt-dlp wrappers available <introduction:motivation>`. If you still want to
get ``ytdl-sub`` up and running quickly and without understanding, then follow these
instructions to the letter.
#. Install using :ref:`the official Docker GUI image variant <guides/install/docker:gui
image>`.
#. Update the paths for your media library:
Edit :ref:`the subscriptions file <guides/install/docker:configuration>`. Near the
top, under ``__preset__:`` and then ``overrides:``, update the values under the
``*_directory:`` keys with the correct paths for your media library *as they appear
inside the container*.
#. Select your media library software:
Change the ``Plex TV Show by Date:`` *key itself* to the preset for your media
library software. See the comment above for the available options.
#. Select the genre:
Under the library software preset key from the previous step, change the ``=
Documentaries`` *key itself* to the genre for this subscription prefixed with ``=
...``. When adding other subscriptions that have the same genre, place them under the
same key.
#. Update the subscription name and URL:
Under the genre key from the previous step, update the ``"NOVA PBS":`` key to the
directory name the downloaded files should be placed beneath. This directory will be
created under the ``tv_show_directory:`` from step #2. Then update the
``"https://www.youtube.com/@novapbs"`` value to the URL of the channel or playlist
for this subscription.
#. :ref:`Preview <guides/getting_started/downloading:preview>` and :ref:`Review
<guides/getting_started/downloading:review>` the subscription.
#. Add the rest of your subscriptions:
Repeat steps #3-6 for each of your subscriptions. Be sure to repeat the preview and
review steps for each subscription. In general, move slowly and carefully review
everything. It's best to catch issues early :ref:`to avoid repeating downloads and to
minimize requests <guides/getting_started/index:minimize the work to only what's
necessary>`.
#. Automate downloads:
:ref:`Set up ytdl-sub to run periodically
<guides/getting_started/automating_downloads:docker and unraid>`.

167
docs/source/guides/getting_started/subscriptions.rst Normal file
View file

@ -0,0 +1,167 @@
Subscriptions
=============
Once you understand :ref:`how ytdl-sub works
<guides/getting_started/index:architecture>`, it's time to start writing your
:doc:`../../config_reference/subscription_yaml`.
Media library paths
-------------------
Everyone's media library may use different paths so ``ytdl-sub`` can't provide
defaults. Tell ``ytdl-sub`` where to put your media using :ref:`overrides
<guides/getting_started/index:presets and subscriptions accept overrides>`:
.. code-block:: yaml
:caption: subscriptions.yaml
:emphasize-lines: 3-
__preset__:
overrides:
tv_show_directory: "/tv_shows"
music_directory: "/music"
music_video_directory: "/music_videos"
See the reference documentation for details about :ref:`the '__preset__:' special key
<config_reference/subscription_yaml:file preset>`.
Media library software and media types
--------------------------------------
Different media library software, such as `Jellyfin`_, `Kodi`_, Plex, or Emby, have
different requirements for where media files are placed, how those files are named, how
metadata is formatted, and more. Those software also have different requirements for
different types of media, such as shows/series, music, music videos, etc.. Use
:doc:`prebuilt presets <../../prebuilt_presets/index>` in :ref:`YAML keys
<guides/getting_started/index:subscriptions are grouped by indentation>` to tell
``ytdl-sub`` which media library software and media type to process downloaded files
for.
The actual subscription is defined in the lowest indentation level YAML keys. The
example below defines a subscription named ``NOVA PBS`` to archive downloads from the
entries in the ``https://www.youtube.com/@novapbs`` URL.
.. code-block:: yaml
:caption: subscriptions.yaml
:emphasize-lines: 1,4
Jellyfin TV Show by Date:
"NOVA PBS": "https://www.youtube.com/@novapbs"
Bandcamp:
"Emily Hopkins": "https://emilyharpist.bandcamp.com/"
.. _`Jellyfin`:
https://jellyfin.org/
.. _`Kodi`:
https://kodi.tv/
Which entries
-------------
The :doc:`helper presets <../../prebuilt_presets/helpers>` also provide support for
controlling which entries are downloaded and archived. These presets are intended to be
combined with the library software and media type presets.
Combine presets using :ref:`the '.. | ...' special character
<guides/getting_started/index:subscriptions are grouped by indentation>` in the YAML
keys:
.. code-block:: yaml
:caption: subscriptions.yaml
:emphasize-lines: 2,6
# Only download entries whose upload date is within the past 2 months:
Kodi TV Show by Date | Only Recent:
"NOVA PBS": "https://www.youtube.com/@novapbs"
# Only download 20 entries per run:
Soundcloud Discography | Chunk Downloads:
"UKNOWY": "https://soundcloud.com/uknowymunich"
What format, quality, or resolution
-----------------------------------
The :doc:`media quality presets <../../prebuilt_presets/media_quality>` provide support
for controlling which ``yt-dlp`` media "format" to download, such as ``1080p`` video
resolution or ``320k`` audio bitrate.
Users may also group and combine presets :ref:`using the YAML hierarchy
<guides/getting_started/index:subscriptions are grouped by indentation>`. Subscriptions
merge all the presets from their ancestor YAML keys. The hierarchy indentation depth may
be as deep as needed to group your subscriptions for easy maintenance:
.. code-block:: yaml
:caption: subscriptions.yaml
:emphasize-lines: 3,7,12
Jellyfin TV Show by Date | Only Recent:
# Download the highest resolution available:
Max Video Quality:
"NOVA PBS": "https://www.youtube.com/@novapbs"
"National Geographic": "https://www.youtube.com/@NatGeo"
# Download the highest resolution available that is 720p or less:
Max 720p:
"Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
Soundcloud Discography | Chunk Downloads:
# Only download audio using the Opus codec, not MP3 or other codecs:
Max Opus Quality:
"UKNOWY": "https://soundcloud.com/uknowymunich"
Genre and rating metadata
-------------------------
Presets may also support using arbitrary values from :ref:`YAML keys prefixed with '=
...' <guides/getting_started/index:subscriptions are grouped by indentation>`. The ``=
...`` prefix may be used at any indentation depth and may also be combined with presets
and other ``= ...`` values using the ``... | ...`` special character to best group your
subscriptions.
:ref:`By convention <config_reference/scripting/static_variables:subscription_indent_i>`
in the built-in library software and media type presets, the first ``= ...`` value
specifies the genre for all descendant subscriptions. For the ``TV Show ...`` presets,
the second ``= ...`` value specifies the rating for all descendant subscriptions:
.. code-block:: yaml
:caption: subscriptions.yaml
:emphasize-lines: 1,3
= Kids:
Jellyfin TV Show by Date | = TV-Y:
"Jake Trains": "https://www.youtube.com/@JakeTrains"
"Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
Soundcloud Discography:
"Foo Kids Band": "https://soundcloud.com/foo-kids-band"
Override variables for one subscription
---------------------------------------
Most variable overrides aren't actually specific to just one subscription and should be
set in :doc:`your own custom presets <./first_config>`. But use :ref:`the override mode
'~...' prefix <config_reference/subscription_yaml:override mode>` when an override is
specific to only one subscription and will never be shared with another:
.. code-block:: yaml
:caption: subscriptions.yaml
:emphasize-lines: 2-
Jellyfin TV Show by Date:
"~NOVA PBS":
url: "https://www.youtube.com/@novapbs"
tv_show_directory: "/media/Unique/Series/Path"
Next Steps
----------
Once you've defined your subscriptions, it's time to :doc:`test your configuration and
try your first download <./downloading>`.

22
docs/source/guides/install/agnostic.rst
View file

@ -2,13 +2,15 @@
Environment Agnostic
====================
The PIP install method is not recommended; use of this method may cause unintended requirement conflicts if you have other locally installed apps that depend on ffmpeg.
The PIP install method is not recommended; use of this method may cause unintended
requirement conflicts if you have other locally installed apps that depend on ffmpeg.
PIP Install
--------------
You can install our
`PyPI package <https://pypi.org/project/ytdl-sub/>`_.
Both ffmpeg and Python 3.10 or greater are required.
You can install our `PyPI package <https://pypi.org/project/ytdl-sub/>`_. Both ffmpeg
and Python 3.10 or greater are required.
.. code-block:: bash
@ -17,10 +19,13 @@ Both ffmpeg and Python 3.10 or greater are required.
Install for Development
=======================
These environment-agnostic methods of installing ``ytdl-sub`` are meant for local development of ``ytdl-sub``. If you want to contribute your changes, please read :doc:`/guides/development/index`.
These environment-agnostic methods of installing ``ytdl-sub`` are meant for local
development of ``ytdl-sub``. If you want to contribute your changes, please read
:doc:`/guides/development/index`.
Local Install
--------------
With a Python 3.10 virtual environment, you can clone and install the repo.
.. code-block:: bash
@ -32,12 +37,13 @@ With a Python 3.10 virtual environment, you can clone and install the repo.
Local Docker Build
-------------------
Run ``make docker`` in the root directory of this repo to build the image. This
will build the python wheel and install it in the Dockerfile.
Run ``make docker`` in the root directory of this repo to build the image. This will
build the python wheel and install it in the Dockerfile.
.. code-block:: bash
git clone https://github.com/jmbannon/ytdl-sub.git
cd ytdl-sub
make docker
make docker

209
docs/source/guides/install/docker.rst
View file

@ -2,141 +2,98 @@
Docker
======
For automating ``subscriptions.yaml`` downloads to pull new media, see :ref:`this page <guides/getting_started/automating_downloads:docker and unraid>` on how to set up a cron job in any of the docker containers.
The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install ytdl-sub
on top. There are two flavors or variants to choose from. For a more user-friendly
experience editing the `configuration`_, we recommend the `GUI image`_
variant. :ref:`Docker Compose <guides/install/docker:install with docker compose>` is
the recommended way of managing a ``ytdl-sub`` docker container. See :ref:`Automating
Downloads <guides/getting_started/automating_downloads:docker and unraid>` for how to
automate running ``ytdl-sub`` in a container running either variant.
The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install ytdl-sub on top. There are two flavors to choose from.
.. margin::
.. tip::
The recommended docker image is the GUI image.
:ref:`Docker Compose <guides/install/docker:install with docker compose>` is the recommended way of setting up a ``ytdl-sub`` docker container.
GUI Image
---------
The GUI image uses LSIO's :lsio-gh:`docker-code-server image <\ >` for its base image. More info on other code-server environment variables can be found within its documentation.
The GUI image is based on LSIO's :lsio-gh:`docker-code-server` to provide you full
management of ``ytdl-sub``, such as file editing and terminal access, all within your
browser using the VS Code web UI. See its documentation regarding environment variables
and other details. Once running, open `the web UI`_ to edit the `configuration`_ and run
``ytdl-sub``.
.. _`the web UI`: http://localhost:8443
After starting, the code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``.
Headless Image
--------------
The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image <\ >` for its base image. Execute the following command to access and interact with ``ytdl-sub``:
The headless image is based on LSIO's :lsio-gh:`docker-baseimage-alpine`. Once running,
the default command just starts services including cron for :ref:`Automating Downloads
<guides/getting_started/automating_downloads:docker and unraid>` but otherwise doesn't
run ``ytdl-sub``. You may run arbitrary ``ytdl-sub`` commands using the
``--rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub"`` options to either ``$ docker
run`` or ``$ docker compose run``. Overriding the image's ``ENTRYPOINT`` is important so
that cron doesn't run ``ytdl-sub`` while you're running it manually.
.. code-block:: bash
For example::
$ docker compose run --rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub" ytdl-sub sub
docker exec -u abc -it ytdl-sub /bin/bash
Install with Docker Compose
---------------------------
Docker Compose is an easy "set it and forget it" install method. Follow the instructions below to create a ``compose.yaml`` file for your chosen ``ytdl-sub`` image.
.. margin::
.. important::
Set the PUID and PGID to the UID and GID associated with the user you want to own the downloaded files. Setting these values to root UID and GID may create issues with your media managers.
.. tab-set::
.. tab-item:: GUI Image
.. code-block:: yaml
:caption: compose.yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
ports:
- 8443:8443
restart: unless-stopped
.. tab-item:: Headless Image
.. code-block:: yaml
:caption: compose.yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
restart: unless-stopped
Device Passthrough
~~~~~~~~~~~~~~~~~~~
For CPU or GPU passthrough, you must use either the GUI image or the headless Ubuntu image
``ghcr.io/jmbannon/ytdl-sub:ubuntu-latest``.
The docker-compose examples use the GUI image.
CPU Passthrough
^^^^^^^^^^^^^^^
Docker Compose provides a declarative way to configure and orchestrate containers which
makes them easier to manage and re-use. Create a ``compose.yaml`` file in your project
directory such as:
.. code-block:: yaml
:emphasize-lines: 5-6
:caption: compose.yaml
:caption: compose.yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
devices:
- /dev/dri:/dev/dri # CPU passthrough
restart: unless-stopped
services:
ytdl-sub:
# The GUI image variant:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
# Or use the headless image variant:
# image: ghcr.io/jmbannon/ytdl-sub:latest
# For CPU/GPU passthrough, use the GUI image above or the headless Ubuntu image:
# image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest
container_name: ytdl-sub
restart: unless-stopped
environment:
- TZ=America/Los_Angeles
# Set these as appropriate so your users can access the downloaded files in
# your library:
- PUID=1000
- PGID=1000
# Optionally passthrough your NVidia GPU:
# - NVIDIA_DRIVER_CAPABILITIES=all
# - NVIDIA_VISIBLE_DEVICES=all
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
# Not necessary for the headless image variant:
ports:
- 8443:8443
# Optionally passthrough the CPU for hardware acceleration:
# devices:
# - /dev/dri:/dev/dri
# Optionally passthrough the GPU:
# deploy:
# resources:
# reservations:
# devices:
# - capabilities: ["gpu"]
GPU Passthrough
^^^^^^^^^^^^^^^
.. Awe
.. code-block:: yaml
:caption: compose.yaml
:emphasize-lines: 5-13
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- ..
- NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args
- NVIDIA_VISIBLE_DEVICES=all
deploy:
resources:
reservations:
devices:
- capabilities: ["gpu"] # GPU passthrough
restart: unless-stopped
Docker CLI
----------
If you prefer to only run the container once, you can use the CLI command instead. The following command is for the gui image, and will not restart if it comes down for any reason. See `the Docker reference <https://docs.docker.com/engine/reference/run/>`_ for further information on the parameters and other options you can use.
You can run the container on an ad-hoc basis without Docker Compose using the Docker CLI
instead. It will not restart if stopped for any reason, including rebooting the
host. The following command is for the gui image:
.. code-block:: bash
@ -151,4 +108,32 @@ If you prefer to only run the container once, you can use the CLI command instea
-v <OPTIONAL/path/to/movies>:/movies \
-v <OPTIONAL/path/to/music_videos>:/music_videos \
-v <OPTIONAL/path/to/music>:/music \
ghcr.io/jmbannon/ytdl-sub-gui:latest
ghcr.io/jmbannon/ytdl-sub-gui:latest
See `the Docker reference <https://docs.docker.com/engine/reference/run/>`_ for further
details.
Environment Variables
---------------------
``ytdl-sub`` docker images support the following environment variables.
.. csv-table:: Docker Environment Variables
:header: "Name", "Supported Values", "Description"
:widths: 15, 10, 60
"``PUID``", "integer", "User ID"
"``PGID``", "integer", "Group ID"
"``TZ``", "timezone", "Optional. Timezone to use in the logs. For supported values, see this `list <https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List>`_. "
"``CRON_SCHEDULE``", "cron schedule `format <https://crontab.guru/#0_*/6_*_*_*>`_", "Optional. Schedule to run the ``cron`` file in ytdl-sub's container. More info :ref:`here <guides/getting_started/automating_downloads:docker and unraid>`."
"``CRON_RUN_ON_START``", "true/false", "Optional. Whether to run the cron script on container start."
"``UPDATE_YT_DLP_ON_START``", "stable/nightly/master", "Optional. Whether to update yt-dlp to the latest configured version on container start."
For the GUI image, you can set LSIO's underlying code-server `env variables <https://docs.linuxserver.io/images/docker-code-server/#environment-variables-e>`_ as well."
Configuration
-------------
In these examples, the configuration files will be at
``<path/to/ytdl-sub/config>/config.yaml`` and
``<path/to/ytdl-sub/config>/subscriptions.yaml``. Start the container the first time to
populate those files with default examples.

9
docs/source/guides/install/index.rst
View file

@ -1,5 +1,6 @@
Install by Platform
===================
``ytdl-sub`` can be installed on the following platforms.
All installations require a 64-bit CPU. 32-bit is not supported.
@ -8,7 +9,8 @@ All installations require a 64-bit CPU. 32-bit is not supported.
.. tip::
The recommended install method of ``ytdl-sub`` is one of our :doc:`docker containers </guides/install/docker>`. For install on Unraid, check out our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>`.
The recommended install method of ``ytdl-sub`` is one of our :doc:`docker containers
</guides/install/docker>`.
:doc:`/guides/install/docker`
@ -20,9 +22,8 @@ All installations require a 64-bit CPU. 32-bit is not supported.
:doc:`/guides/install/agnostic`
Once you've completed your installation, please refer to the :doc:`../getting_started/index` guide for next steps
Once you've completed your installation, please refer to the
:doc:`../getting_started/index` guide for next steps
.. toctree::
:hidden:

10
docs/source/guides/install/linux.rst
View file

@ -2,8 +2,8 @@
Linux
=====
``ytdl-sub`` should be installable using any Linux package manager, and requires ffmpeg to be installed.
``ytdl-sub`` should be installable using any Linux package manager, and requires ffmpeg
to be installed.
.. tab-set::
@ -15,7 +15,8 @@ Linux
chmod +x ytdl-sub
./ytdl-sub -h
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to
date:
.. code-block:: bash
@ -36,7 +37,8 @@ Linux
chmod +x ytdl-sub
./ytdl-sub -h
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to
date:
.. code-block:: bash

31
docs/source/guides/install/unraid.rst
View file

@ -1,16 +1,29 @@
======
Unraid
--------------
You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>` through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_.
======
You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>`
through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_.
If you installed the ``ytdl-sub-gui`` app, the code-server will be running at http://localhost:8443 (replace ``localhost`` with the IP of the computer running Unraid if you aren't trying to access ``ytdl-sub`` on that computer). Open this page in a browser to access and interact with ``ytdl-sub``.
If you installed the ``ytdl-sub-gui`` app, the code-server will be running at
http://localhost:8443 (replace ``localhost`` with the IP of the computer running Unraid
if you aren't trying to access ``ytdl-sub`` on that computer). Open this page in a
browser to access and interact with ``ytdl-sub``.
If you installed the ``ytdl-sub`` app (headless), open the normal app-specific console to access and interact with ``ytdl-sub``. Once open, you must first run ``su abc -s /bin/bash`` to change to the non-root user. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``.
If you installed the ``ytdl-sub`` app (headless), open the normal app-specific console
to access and interact with ``ytdl-sub``. Once open, you must first run ``su abc -s
/bin/bash`` to change to the non-root user. You can confirm that this command worked by
running ``whoami`` and verifying that the result is ``abc``.
.. warning::
.. warning::
If you use the below option to access the ``ytdl-sub`` console, be sure to run ``su
abc -s /bin/bash`` first thing. You can confirm that this command worked by running
``whoami`` and verifying that the result is ``abc``. Do **NOT** run ``ytdl-sub`` as
the root user! Running as root will set the owner of all modified files to root,
which prevents most media managers and players from accessing the files.
If you use the below option to access the ``ytdl-sub`` console, be sure to run ``su abc -s /bin/bash`` first thing. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``. Do **NOT** run ``ytdl-sub`` as the root user! Running as root will set the owner of all modified files to root, which prevents most media managers and players from accessing the files.
.. figure:: ../../../images/unraid_badconsole.png
:alt: The Unraid community app plugin GUI, with an arrow pointing at the "Console" option in the dropdown after selecting ytdl-sub-gui
.. figure:: ../../../images/unraid_badconsole.png
:alt:
The Unraid community app plugin GUI, with an arrow pointing at the "Console"
option in the dropdown after selecting ytdl-sub-gui

6
docs/source/guides/install/windows.rst
View file

@ -1,5 +1,7 @@
=======
Windows
--------------
=======
From powershell, run:
.. code-block:: powershell
@ -12,4 +14,4 @@ From powershell, run:
# Download ytdl-sub
curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe
ytdl-sub.exe -h
ytdl-sub.exe -h

3
docs/source/index.rst
View file

@ -10,7 +10,6 @@ ytdl-sub User Guide
prebuilt_presets/index
usage
config_reference/index
debugging
faq/index
deprecation_notices
.. note:: The docs are heavily work-in-progress. Please bear with us while we're under construction!

69
docs/source/introduction.rst
View file

@ -8,10 +8,23 @@ What is ytdl-sub?
.. _plex: https://github.com/plexinc/pms-docker
.. _emby: https://github.com/plexinc/pms-docker
``ytdl-sub`` is a command-line tool that downloads media via `yt-dlp`_ and prepares it for your favorite media player (`Kodi`_, `Jellyfin`_, `Plex`_, `Emby`_, modern music players).
``ytdl-sub`` is a command-line tool that builds on and orchestrates `yt-dlp`_ to
download media from YouTube and/or other online services. It provides a declarative,
expressive YAML configuration system that allows you to describe which media to download
and how it should appear in your media library servers and applications such as
`Jellyfin`_, `Plex`_, `Emby`_, `Kodi`_, modern music players, etc..
Visual examples
===============
To these ends, ``ytdl-sub``:
- wraps and runs `yt-dlp`_, per your configuration to:
- download the media, remux and/or optionally transcode it
- prepares additional metadata both embedded and in external files
- renames the resulting files
- places them in your library
.. figure:: https://user-images.githubusercontent.com/10107080/182677243-b4184e51-9780-4094-bd40-ea4ff58555d0.PNG
:alt: The Jellyfin web interface, showing the thumbnails of various YouTube shows.
@ -34,10 +47,52 @@ Visual examples
SoundCloud albums and singles in MusicBee
Why ytdl-sub?
-------------
There is a lack of open-source tools to download media and generate metadata to play it in these players. Most solutions involve using multiple tools or bash scripts to achieve this. ``ytdl-sub`` aims to consolidate all of this logic into a single easy-to-use application that can run automatically once configured.
Motivation
----------
`yt-dlp`_ has grown into a well maintained, central repository of the intricate,
inscrutable, and extensive technical knowledge required to automate downloading media
from online services. When those services change their APIs or otherwise change
behavior, `yt-dlp`_ is the central, low-level tool to update. It does a best-in-class
job at that task, and it does that job more effectively by narrowing focus to just that.
As much knowledge as it encapsulates and as well as it does that, it still requires a
great deal of additional knowledge to make its output accessible to end-users. Mostly
this gap is about extracting and formatting metadata and correctly placing the resulting
output files in a media library.
A number of tools, applications, and other projects have grown up around that central
`yt-dlp`_ pillar to fill in those gaps, and this project was one of the early
entrants. Many are `full-featured services that provide web UIs`_ including some that
`provide media player web UIs`_. Most of those other projects necessarily narrow their
scope to provide a more polished and integrated user experience.
Similarly, ``ytdl-sub`` can run automatically to accomplish the same goals, but aims to
serve users that need lower-level control and/or have use cases not covered by the more
narrow scope of those other projects. To some degree, this makes this project
intrinsically less user friendly and requires more technical experience or learning.
Want something that "Just Works", try one of the other projects; we recommend
`Pinchflat`_ as the next step towards that end. Want to download from more than just
YouTube? Don't like the other restrictions inherent in the goals of those other
projects? Have unique use cases? Then dig in, learn, and we hope ``ytdl-sub`` gives you
enough rope and `a foot-gun`_ to get you there.
.. _`full-featured services that provide web UIs`:
https://github.com/kieraneglin/pinchflat
.. _`provide media player web UIs`:
https://www.tubearchivist.com/
.. _`Pinchflat`: `full-featured services that provide web UIs`_
.. _`a foot-gun`: https://en.wiktionary.org/wiki/footgun
Why download instead of stream?
-------------------------------
We believe it is important to download what you like because there is no guarantee it will stay online forever. We also believe it is important to download it in such a way that it is easy to consume. Most solutions today force you to watch/listen to your downloaded content via file system or web browser. ``ytdl-sub`` aims to format downloaded content for any media player.
Most of the tools in this `yt-dlp`_ ecosystem serve a similar set of larger, more
general use cases, and so does ``ytdl-sub``:
- Don't rely on profit-driven corporate persons to keep more obscure content available.
- Even if they do, don't depend on them to make it possible to use it in different ways.
- Even when you pay, don't count on them not inserting ads later.
- Regardless, don't depend on them to curate content for yourself and/or your family.
- Free yourself and/or your family from what the algorithm would feed them next.

155
docs/source/prebuilt_presets/helpers.rst
View file

@ -1,3 +1,6 @@
==============
Helper Presets
==============
@ -6,11 +9,13 @@ Helper Presets
See how to apply helper presets :doc:`here </prebuilt_presets/index>`
Only Recent
-----------
To only download a recent number of videos, apply the ``Only Recent`` preset. Once a video's
upload date is outside of the range, or you hit max files, older videos will be deleted automatically.
To only download a recent number of videos, apply the ``Only Recent`` preset. Once a
video's upload date is outside of the range, or you hit max files, older videos will be
deleted automatically.
.. code-block:: yaml
@ -31,9 +36,12 @@ To prevent deletion of files, use the preset ``Only Recent Archive`` instead.
Filter Keywords
---------------
``Filter Keywords`` can include or exclude media with any of the listed keywords. Both keywords and title/description are lower-cased before filtering.
``Filter Keywords`` can include or exclude media with any of the listed keywords. Both
keywords and title/description are lower-cased before filtering.
Default behavior for Keyword evaluation is ANY, meaning the filter will succeed if any of the keywords are present. This can be set to ANY or ALL using the respective ``_eval`` variable.
Default behavior for Keyword evaluation is ANY, meaning the filter will succeed if any
of the keywords are present. This can be set to ANY or ALL using the respective
``_eval`` variable.
Supports the following override variables:
@ -71,17 +79,49 @@ Supports the following override variables:
- "maple leafs"
- "highlights"
Filter Duration
---------------
``Filter Duration`` can include or exclude media based on its duration.
Supports the following override variables:
* ``filter_duration_min_s``
* ``filter_duration_max_s``
.. tip::
Use the `~` tilda subscription mode to set a subscription's list override variables.
Tilda mode allows override variables to be set directly underneath it.
.. code-block:: yaml
Plex TV Show by Date | Filter Duration:
= Documentaries:
"~NOVA PBS":
url: "https://www.youtube.com/@novapbs"
filter_duration_min_s: 120 # Only download videos at least 2m long
= Sports:
"~Maple Leafs Highlights":
url: "https://www.youtube.com/@NHL"
filter_duration_max_s: 180 # Only get highlight videos less than 3m long
Chunk Downloads
---------------
If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before
starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the
following preset:
If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's
metadata from newest to oldest before starting any downloads. It is a long process and
not ideal. A better method is to chunk the process by using the following preset:
``Chunk Downloads``
It will download videos starting from the oldest one, and only download 20 at a time by default. You can
change this number by setting the override variable ``chunk_max_downloads``.
It will download videos starting from the oldest one, and only download 20 at a time by
default. You can change this number by setting the override variable
``chunk_max_downloads``.
.. code-block:: yaml
@ -100,5 +140,98 @@ change this number by setting the override variable ``chunk_max_downloads``.
= Documentaries:
"Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
Once the entire channel is downloaded, remove the usage of this preset. It will then pull metadata from newest to
oldest again, and stop once it reaches a video that has already been downloaded.
Once the entire channel is downloaded, remove the usage of this preset. It will then
pull metadata from newest to oldest again, and stop once it reaches a video that has
already been downloaded.
_throttle_protection
--------------------
.. note::
This preset is already a base preset of those higher-level presets that require it,
so users seldom need to use it directly, for example, unless they're writing presets
from scratch.
This preset is primarily a sensible default configuration of :ref:`the
'throttle_protection' plugin <config_reference/plugins:throttle_protection>` along with
an override to disable the plugin:
.. code-block:: yaml
overrides:
# Disable throttle protection:
enable_throttle_protection: false
In addition to throttling by denying download requests, some services also throttle
downloads by only allowing downloads of the lowest resolution quality. At the time of
writing, only YouTube does this by allowing only 360p downloads when throttled. To work
around this kind of throttling, this preset includes :ref:`an assertion
<config_reference/scripting/scripting_functions:error functions>` that will stop
downloading when ``ytdl-sub`` downloads a video at 360p or lower. It supports the
following overrides:
.. code-block:: yaml
overrides:
# Disable resolution quality throttle protection:
enable_resolution_assert: false
# Change the resolution below which to assume downloading is throttled:
resolution_assert_height_gte: 720
.. _resolution assert handling:
Handling Low Quality Videos
~~~~~~~~~~~~~~~~~~~~~~~~~~~
A side effect from throttle protection's resolution assert is, if the only resolution available is 360p or lower, it will
error. You can either disable resolution assert entirely (see above), or ignore specific titles in the subscription
using the ``resolution_assert_ignore_titles`` variable. Add a subset of the title (case-sensitive) as a list entry
to your subscription, like so:
.. code-block:: yaml
# use tilda mode to set override variables to the subscription
"~My Subscription":
url: "https://youtube.com/@channel"
resolution_assert_ignore_titles:
- "This 360p Video Title"
_url
----
All prebuilt presets share the same internal ``_multi_url`` preset which comes equipped with
a few available customizations.
Sibling Metadata
~~~~~~~~~~~~~~~~
*Sibling* refers to any entry within the same *playlist*. For channel downloads, this would
imply **every** video that gets downloaded since yt-dlp treats the channel as the *playlist*.
Setting the variable ``include_sibling_metadata`` will include all sibling metadata within
each individual entry's metadata. This is used specifically for music presets. When downloading
a playlist as an album for example, it will take the max year amongst all the other sibling's metadata
to have a consistent album year that can be used in file or directory naming.
Webpage URL
~~~~~~~~~~~
``ytdl-sub`` performs downloads in two stages.
1. Metadata scrape from the original URL
2. Individual entry downloads
For step 2, ``ytdl-sub`` will use the ``webpage_url`` variable by default for the input URL to yt-dlp.
This can be modified in case it's not working as expected by using the variable ``modified_webpage_url``.
Example:
.. code-block:: yaml
:caption:
Removes yt-dlp smuggle data from the URL
overrides:
modified_webpage_url: >-
{ %regex_sub("#__youtubedl_smuggle=.*", "", webpage_url) }

15
docs/source/prebuilt_presets/index.rst
View file

@ -7,11 +7,13 @@ media in various players.
.. hint::
Apply multiple presets to your subscriptions using pipes. Pipes can define multiple presets and values
on the same line to apply to all subscriptions nested below them.
Apply multiple presets to your subscriptions using pipes. Pipes can define multiple
presets and values on the same line to apply to all subscriptions nested below them.
.. code-block:: yaml
:caption: Applies Max Video Quality preset to all TV shows, and Chunk Downloads preset to some
:caption:
Applies Max Video Quality preset to all TV shows, and Chunk Downloads preset to
some
Plex TV Show by Date | Max Video Quality:
@ -22,9 +24,8 @@ media in various players.
= Documentaries:
"Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
For advanced users, you can review the prebuilt preset
definitions :doc:`here </config_reference/prebuilt_presets/index>`.
For advanced users, you can review the prebuilt preset definitions :doc:`here
</config_reference/prebuilt_presets/index>`.
.. toctree::
:titlesonly:
@ -33,4 +34,4 @@ definitions :doc:`here </config_reference/prebuilt_presets/index>`.
music
music_videos
media_quality
helpers
helpers

8
docs/source/prebuilt_presets/media_quality.rst
View file

@ -6,8 +6,10 @@ Media Quality Presets
See how to apply media quality presets :doc:`here </prebuilt_presets/index>`
Video
-----
The following presets set video quality specifications to yt-dlp.
- ``Max Video Quality``
@ -17,10 +19,12 @@ The following presets set video quality specifications to yt-dlp.
- ``Max 720p``
- ``Max 480p``
Audio
-----
The following presets set audio quality specifications to yt-dlp.
These assume you are only extracting audio (no video).
The following presets set audio quality specifications to yt-dlp. These assume you are
only extracting audio (no video).
- ``Max Audio Quality``, format is determined by the source
- ``Max MP3 Quality``

42
docs/source/prebuilt_presets/music.rst
View file

@ -2,21 +2,16 @@
Music Presets
=============
Music downloadable by yt-dlp comes in many flavors. ``ytdl-sub`` offers a suite
of various presets for handling some of the most popular forms of uploaded music
content.
.. hint::
The subscription *value* (denoted by =) will set the genre tag for all music scraped under its key
for all music presets.
Music downloadable by yt-dlp comes in many flavors. ``ytdl-sub`` offers a suite of
various presets for handling some of the most popular forms of uploaded music content.
YouTube Releases
----------------
Many artists, especially those auto-uploaded as ``Topics`` in YouTube have a section on
their channel named "Releases", or "Albums and Singles". The ``YouTube Releases`` preset aims to
scrape this *playlist of playlists*.
their channel named "Releases", or "Albums and Singles". The ``YouTube Releases`` preset
aims to scrape this *playlist of playlists*.
Playlists are recognized as the album, and videos within it are tracks.
@ -26,8 +21,8 @@ Playlists are recognized as the album, and videos within it are tracks.
= Jazz: # Sets genre tag to "Jazz"
"Thelonious Monk": "https://www.youtube.com/@officialtheloniousmonk/releases"
If you are only interested in a subset of albums, you can provide their playlists as separate values in the form
of an array, like so:
If you are only interested in a subset of albums, you can provide their playlists as
separate values in the form of an array, like so:
.. code-block:: yaml
@ -37,11 +32,13 @@ of an array, like so:
- "https://www.youtube.com/playlist?list=OLAK5uy_lcqINwfzkw73TPnAt6MlpB6V0gM9VzQu8" # Monk on Monk
- "https://www.youtube.com/playlist?list=OLAK5uy_nhuvjuZOO3yLIWCbQzbiWfyzkGapSIuYw" # Late Night Thelonious Monk
YouTube Full Albums
-------------------
In many cases, albums are uploaded to YouTube as a single video, where each track as separated by either
chapters or timestamps in a description. The ``YouTube Full Albums`` preset will take each video and split
it by the chapters to form an album.
In many cases, albums are uploaded to YouTube as a single video, where each track as
separated by either chapters or timestamps in a description. The ``YouTube Full Albums``
preset will take each video and split it by the chapters to form an album.
Videos are recognized as the album, and chapters within it are tracks.
@ -51,8 +48,8 @@ Videos are recognized as the album, and chapters within it are tracks.
= Lofi:
"Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
If you are only interested in a subset of albums, you can provide their video as separate values in the form
of an array, like so:
If you are only interested in a subset of albums, you can provide their video as
separate values in the form of an array, like so:
.. code-block:: yaml
@ -62,11 +59,14 @@ of an array, like so:
- "https://www.youtube.com/watch?v=m7vBrD7LMLI" # Zelda & Sleep Ensemble Collection
- "https://www.youtube.com/watch?v=w0XebCwSpKI" # Study Buddy ~ video game lofi mix
Soundcloud Discography
----------------------
SoundCloud tracks can be uploaded as either a single, part of an album, or a collaboration
with another artist. At this time, ``SoundCloud Discography`` only scrapes singles and albums.
It will attempt to group tracks into albums before falling back to single format.
SoundCloud tracks can be uploaded as either a single, part of an album, or a
collaboration with another artist. At this time, ``SoundCloud Discography`` only scrapes
singles and albums. It will attempt to group tracks into albums before falling back to
single format.
.. code-block:: yaml
@ -77,8 +77,10 @@ It will attempt to group tracks into albums before falling back to single format
"Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
"Earmake": "https://soundcloud.com/earmake"
Bandcamp
--------
Bandcamp albums and singles can be scraped using the ``Bandcamp`` preset.
.. code-block:: yaml

2
docs/source/prebuilt_presets/music_videos.rst
View file

@ -2,4 +2,4 @@
Music Video Presets
===================
WIP
WIP

115
docs/source/prebuilt_presets/tv_shows.rst
View file

@ -2,45 +2,57 @@
TV Show Presets
===============
Player-Specific Presets
=======================
``ytdl-sub`` provides player-specific versions of certain presets, which apply settings to optimize the downloads for that player.
Player-Specific Presets
-----------------------
``ytdl-sub`` provides player-specific versions of certain presets, which apply settings
to optimize the downloads for that player.
The following actions are taken based on the indicated player:
Kodi
--------
~~~~
* Everything that the Jellyfin version does
* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□``
* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with
``□``
Jellyfin
--------
~~~~~~~~
* Places any season-specific poster art in the main show folder
* Generates NFO tags
Emby
----
~~~~
* Places any season-specific poster art in the main show folder
* Generates NFO tags
* For named seasons, creates a ``season.nfo`` file per season
Plex
--------
* :ref:`Special sanitization <config_reference/scripting/entry_variables:title_sanitized_plex>` of numbers so Plex doesn't recognize numbers that are part of the title as the episode number
~~~~~~~~
* :ref:`Special sanitization
<config_reference/scripting/entry_variables:title_sanitized_plex>` of numbers so Plex
doesn't recognize numbers that are part of the title as the episode number
* Converts all downloaded videos to the mp4 format
* Places any season-specific poster art into the season folder
----------------------------------------------
TV Show by Date
===============
TV Show by Date will organize something like a YouTube channel or playlist into a tv show, where seasons and episodes are organized using upload date.
TV Show by Date
---------------
TV Show by Date will organize something like a YouTube channel or playlist into a tv
show, where seasons and episodes are organized using upload date.
Example
-------
~~~~~~~
Must define ``tv_show_directory``. Available presets:
* ``Kodi TV Show by Date``
@ -74,9 +86,10 @@ Must define ``tv_show_directory``. Available presets:
- "https://www.youtube.com/@rickbeato240"
Advanced Usage
--------------
~~~~~~~~~~~~~~
If you prefer a different season/episode organization method, you can set the following override variables.
If you prefer a different season/episode organization method, you can set the following
override variables.
.. code-block:: yaml
@ -95,12 +108,11 @@ Or for a specific preset
tv_show_by_date_season_ordering: "upload-year-month"
tv_show_by_date_episode_ordering: "upload-day"
The following are supported. Be sure the combined season + episode ordering
include the year, month, day, i.e. upload-year + upload-month-day.
The following are supported. Be sure the combined season + episode ordering include the
year, month, day, i.e. upload-year + upload-month-day.
Season Ordering
~~~~~~~~~~~~~~~
"""""""""""""""
``tv_show_by_date_season_ordering`` supports one of the following:
@ -109,23 +121,24 @@ Season Ordering
* ``release-year``
* ``release-year-month``
Episode Ordering
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
""""""""""""""""
``tv_show_by_date_episode_ordering`` supports one of the following:
* ``upload-month-day`` (default)
* ``upload-month-day-reversed``
* Reversed means more recent episodes appear at the top of a season by having a lower value.
* Reversed means more recent episodes appear at the top of a season by having a lower
value.
* ``upload-day``
* ``release-day``
* ``release-month-day``
* ``release-month-day-reversed``
* ``download-index``
* Episodes are numbered by the download order. **NOTE**: this is fetched using the length of the download archive. Do not use if you intend to remove old videos.
* Episodes are numbered by the download order. **NOTE**: this is fetched using the
length of the download archive. Do not use if you intend to remove old videos.
TV Show by Date presets use the following for defaults:
@ -135,21 +148,23 @@ TV Show by Date presets use the following for defaults:
tv_show_by_date_episode_ordering: "upload-month-day"
TV Show Collection
==================
------------------
TV Show Collections set each URL as its own season. If a video belongs to multiple URLs
(i.e. a channel and a channel's playlist), the video will only download once and reside in
the higher-numbered season.
(i.e. a channel and a channel's playlist), the video will only download once and reside
in the higher-numbered season.
Two main use cases of a collection are:
1. Organize a YouTube channel TV show where Season 1 contains any video
not in a 'season playlist', Season 2 for 'Playlist A', Season 3 for
'Playlist B', etc.
2. Organize one or more YouTube channels/playlists, where each season
represents a separate channel/playlist.
1. Organize a YouTube channel TV show where Season 1 contains any video not in a
'season playlist', Season 2 for 'Playlist A', Season 3 for 'Playlist B', etc.
2. Organize one or more YouTube channels/playlists, where each season represents a
separate channel/playlist.
Today, ytdl-supports up to 40 seasons with 11 URLs per season.
Example
-------
~~~~~~~
Must define ``tv_show_directory``. Available presets:
* ``Kodi TV Show Collection``
@ -172,10 +187,34 @@ Must define ``tv_show_directory``. Available presets:
s02_name: "Covers"
s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
Advanced Usage
--------------
Other notable features include:
If you prefer a different episode organization method, you can set the following override variables.
* TV show poster info is pulled from the first URL in s01.
* Duplicate videos in different URLs (channel /videos vs playlist) will not download twice.
* The video will attributed to the season with the highest number.
* Individual seasons support both single and multi URL.
* s00 is supported for specials.
.. code-block:: yaml
"~Beyond the Guitar":
s00_name: "Specials"
s00_url:
- "https://www.youtube.com/watch?v=vXzguOdulAI"
- "https://www.youtube.com/watch?v=IGwYDvaGAz0"
s01_name: "Videos"
s01_url:
- "https://www.youtube.com/c/BeyondTheGuitar"
- "https://www.youtube.com/@BeyondTheGuitarAcademy"
s02_name: "Covers"
s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
Advanced Usage
~~~~~~~~~~~~~~
If you prefer a different episode organization method, you can set the following
override variables.
.. code-block:: yaml
@ -198,9 +237,8 @@ Or for a specific preset
The following are supported.
Episode Ordering
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
""""""""""""""""
``tv_show_collection_episode_ordering`` supports one of the following:
@ -210,7 +248,8 @@ Episode Ordering
* ``release-year-month-day-reversed``
* ``playlist-index``
* Only use ``playlist-index`` episode formatting for playlists that will be fully downloaded once and never again. Otherwise, indices can change.
* Only use ``playlist-index`` episode formatting for playlists that will be fully
downloaded once and never again. Otherwise, indices can change.
* ``playlist-index-reversed``
TV Show Collection presets use upload-year-month-day as the default.

52
docs/source/usage.rst
View file

@ -1,5 +1,5 @@
Usage
=======
=====
.. code-block::
@ -7,10 +7,12 @@ Usage
For Windows users, it would be ``ytdl-sub.exe``
General Options
---------------
General options must be specified before the command (i.e. ``sub``).
CLI options common to all sub-commands. Must be specified before the sub-command, for
example ``$ ytdl-sub --dry-run sub ...``:
.. code-block:: text
@ -20,24 +22,30 @@ General options must be specified before the command (i.e. ``sub``).
path to the config yaml, uses config.yaml if not provided
-d, --dry-run preview what a download would output, does not perform any video downloads or writes to output directories
-l quiet|info|verbose|debug, --log-level quiet|info|verbose|debug
level of logs to print to console, defaults to info
level of logs to print to console, defaults to verbose
-t TRANSACTIONPATH, --transaction-log TRANSACTIONPATH
path to store the transaction log output of all files added, modified, deleted
-st, --suppress-transaction-log
do not output transaction logs to console or file
-nc, --suppress-colors
do not use colors in ytdl-sub output
-m MATCH [MATCH ...], --match MATCH [MATCH ...]
match subscription names to one or more substrings, and only run those subscriptions
Sub Options
-----------
Download all subscriptions specified in each ``SUBPATH``.
Subscriptions Options
---------------------
Download all subscriptions specified in each :doc:`subscriptions file
<./guides/getting_started/subscriptions>`.
.. code-block::
ytdl-sub [GENERAL OPTIONS] sub [SUBPATH ...]
``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided.
It will use the config specified by ``--config``, or ``config.yaml`` if not provided.
``SUBPATH`` is one or more paths to subscription files and defaults to
``./subscriptions.yaml`` if none are given. It will use the config specified by
``--config``, or ``./config.yaml``, if not provided.
.. code-block:: text
:caption: Additional Options
@ -47,16 +55,19 @@ It will use the config specified by ``--config``, or ``config.yaml`` if not prov
-o DL_OVERRIDE, --dl-override DL_OVERRIDE
override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
Download Options
-----------------
Download a single subscription in the form of CLI arguments.
----------------
Download a single subscription in the form of CLI arguments instead of from :doc:`a
subscriptions file <./guides/getting_started/subscriptions>`:
.. code-block::
ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS]
``SUBSCRIPTION ARGUMENTS`` are exactly the same as YAML arguments, but use periods (``.``) instead
of indents for specifying YAML from the CLI. For example, you can represent this subscription:
``SUBSCRIPTION ARGUMENTS`` are the same as YAML arguments, but use periods (``.``)
instead of indents. For example, you can represent this subscription:
.. code-block:: yaml
@ -76,11 +87,15 @@ Using the command:
--overrides.tv_show_name "Rick A" \
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
See how to shorten commands using
`download aliases <https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
See how to shorten commands using `download aliases
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
View Options
-----------------
------------
Preview the source variables for a given URL. Helpful to create new subscriptions:
.. code-block::
ytdl-sub view [-sc] [URL]
@ -91,5 +106,10 @@ View Options
-sc, --split-chapters
View source variables after splitting by chapters
CLI to SUB Options
------------------
Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments.
Preview the source variables for a given URL. Helps when creating new configs.
.. code-block::
ytdl-sub cli-to-sub [YT-DLP ARGS]

6
examples/advanced/tv_show_config.yaml
View file

@ -1,9 +1,9 @@
###############################################################################
# Top-level configurations to apply umask and persist error logs
# Top-level configurations to apply umask and write log files
configuration:
umask: "002"
persist_logs:
logs_directory: './logs'
logs_directory: '/config/logs'
keep_successful_logs: False
presets:
@ -98,4 +98,4 @@ presets:
overrides:
only_recent_date_range: "2months"
only_recent_max_files: 30
only_recent_max_files: 30

2
examples/advanced/tv_show_subscriptions.yaml
View file

@ -24,6 +24,6 @@ TV Show Only Recent:
# to set only for that subscriptions
"~BBC News":
url: "https://www.youtube.com/@BBCNews" # use url2, url3, ... for multi-url in this form
date_range: "2weeks"
only_recent_date_range: "2weeks"
"Frontline PBS": "https://www.youtube.com/@frontline"
"Whitehouse": "https://www.bitchute.com/channel/zWsYVmCOu4JA/" # Supports non-YT sites

10
pyproject.toml
View file

@ -15,7 +15,7 @@ classifiers = [
"Programming Language :: Python :: 3.11",
]
dependencies = [
"yt-dlp[default]==2025.6.30",
"yt-dlp[default]==2026.1.29",
"colorama~=0.4",
"mergedeep~=1.3",
"mediafile~=0.12",
@ -44,15 +44,15 @@ where = ["src"]
test = [
"coverage[toml]>=6.3,<8.0",
"pytest>=7.2,<9.0",
"pytest-rerunfailures>=14,<16",
"pytest-rerunfailures>=14,<17",
]
lint = [
"black==24.10.0",
"isort==6.0.1",
"pylint==3.3.7",
"isort==7.0.0",
"pylint==4.0.1",
]
docs = [
"sphinx>=7,<9",
"sphinx>=7,<10",
"sphinx-rtd-theme>=2,<4",
"sphinx-book-theme~=1.0",
"sphinx-copybutton~=0.5",

7
src/ytdl_sub/cli/entrypoint.py
View file

@ -13,6 +13,7 @@ from yt_dlp.utils import sanitize_filename
from ytdl_sub.cli.output_summary import output_summary
from ytdl_sub.cli.output_transaction_log import _maybe_validate_transaction_log_file
from ytdl_sub.cli.output_transaction_log import output_transaction_log
from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
from ytdl_sub.cli.parsers.dl import DownloadArgsParser
from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME
from ytdl_sub.cli.parsers.main import parser
@ -214,6 +215,10 @@ def main() -> List[Subscription]:
args, extra_args = parser.parse_known_args()
if args.subparser == "cli-to-sub":
print_cli_to_sub(args=extra_args)
return []
# Load the config
if args.config:
config = ConfigFile.from_file_path(args.config)
@ -272,7 +277,7 @@ def main() -> List[Subscription]:
_view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters)
)
else:
raise ValidationException("Must provide one of the commands: sub, dl, view")
raise ValidationException("Must provide one of the commands: sub, dl, view, cli-to-sub")
if not args.suppress_transaction_log:
output_transaction_log(

64
src/ytdl_sub/cli/parsers/cli_to_sub.py Normal file
View file

@ -0,0 +1,64 @@
from typing import List
import yt_dlp
import yt_dlp.options
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.yaml import dump_yaml
logger = Logger.get()
# pylint: disable=missing-function-docstring
##############################################################
# --- BEGIN ----
# Copy of https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
create_parser = yt_dlp.options.create_parser
def parse_patched_options(opts):
patched_parser = create_parser()
patched_parser.defaults.update(
{
"ignoreerrors": False,
"retries": 0,
"fragment_retries": 0,
"extract_flat": False,
"concat_playlist": "never",
"update_self": False,
}
)
yt_dlp.options.create_parser = lambda: patched_parser
try:
return yt_dlp.parse_options(opts)
finally:
yt_dlp.options.create_parser = create_parser
default_opts = parse_patched_options([]).ydl_opts
def cli_to_api(opts, cli_defaults=False):
opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
diff = {k: v for k, v in opts.items() if default_opts[k] != v}
if "postprocessors" in diff:
diff["postprocessors"] = [
pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]
]
return diff
# --- END ----
##############################################################
def print_cli_to_sub(args: List[str]) -> None:
api_args = cli_to_api(args)
if not api_args:
logger.info("Does not resolve to any yt-dlp args")
return
print(dump_yaml({"ytdl_options": api_args}))

4
src/ytdl_sub/cli/parsers/main.py
View file

@ -232,3 +232,7 @@ view_parser.add_argument(
help="View source variables after splitting by chapters",
)
view_parser.add_argument("url", help="URL to view source variables for")
###################################################################################################
# CLI-TO-SUB PARSER
cli_to_sub_parser = subparsers.add_parser("cli-to-sub")

33
src/ytdl_sub/config/config_validator.py
View file

@ -12,6 +12,8 @@ from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH
from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
from ytdl_sub.utils.exceptions import SubscriptionPermissionError
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.validators.file_path_validators import FFmpegFileValidator
from ytdl_sub.validators.file_path_validators import FFprobeFileValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@ -67,7 +69,8 @@ class PersistLogsValidator(StrictDictValidator):
@property
def logs_directory(self) -> str:
"""
Required. The directory to store the logs in.
Write log files to this directory with names like
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. (required)
"""
return self._logs_directory.value
@ -92,7 +95,10 @@ class PersistLogsValidator(StrictDictValidator):
@property
def keep_successful_logs(self) -> bool:
"""
Optional. Whether to store logs when downloading is successful. Defaults to True.
If the ``persist_logs:`` key is in the configuration, then ``ytdl-sub`` *always*
writes log files for the subscription both for successful downloads and when it
encounters an error while downloading. When this key is ``False``, only write
log files for errors. (default ``True``)
"""
return self._keep_successful_logs.value
@ -143,11 +149,17 @@ class ConfigOptions(StrictDictValidator):
key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES
)
if not FileHandler.is_path_writable(self.working_directory):
raise SubscriptionPermissionError(
"ytdl-sub does not have permissions to the working directory: "
f"{self.working_directory}"
)
@property
def working_directory(self) -> str:
"""
The directory to temporarily store downloaded files before moving them into their final
directory. Defaults to .ytdl-sub-working-directory
directory. (default ``./.ytdl-sub-working-directory``)
"""
# Expands tildas to actual paths, use native os sep
return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep))
@ -155,7 +167,7 @@ class ConfigOptions(StrictDictValidator):
@property
def umask(self) -> Optional[str]:
"""
Umask (octal format) to apply to every created file. Defaults to "022".
Umask in octal format to apply to every created file. (default ``022``)
"""
return self._umask.value
@ -214,24 +226,25 @@ class ConfigOptions(StrictDictValidator):
def lock_directory(self) -> str:
"""
The directory to temporarily store file locks, which prevents multiple instances
of ``ytdl-sub`` from running. Note that file locks do not work on network-mounted
directories. Ensure that this directory resides on the host machine. Defaults to ``/tmp``.
of ``ytdl-sub`` from running. Note that file locks do not work on
network-mounted directories. Ensure that this directory resides on the host
machine. (default ``/tmp``)
"""
return self._lock_directory.value
@property
def ffmpeg_path(self) -> str:
"""
Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux, and
``ffmpeg.exe`` for Windows (in the same directory as ytdl-sub).
Path to ffmpeg executable. (default ``/usr/bin/ffmpeg`` for Linux,
``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows)
"""
return self._ffmpeg_path.value
@property
def ffprobe_path(self) -> str:
"""
Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux, and
``ffprobe.exe`` for Windows (in the same directory as ytdl-sub).
Path to ffprobe executable. (default ``/usr/bin/ffprobe`` for Linux,
``./ffprobe.exe`` in the same directory as ytdl-sub for Windows)
"""
return self._ffprobe_path.value

4
src/ytdl_sub/config/defaults.py
View file

@ -2,6 +2,8 @@ import os
from ytdl_sub.utils.system import IS_WINDOWS
# pylint: disable=invalid-name
def _existing_path(*paths: str) -> str:
"""
@ -22,7 +24,7 @@ if IS_WINDOWS:
MAX_FILE_NAME_BYTES = 255
else:
DEFAULT_LOCK_DIRECTORY = "/tmp"
DEFAULT_LOCK_DIRECTORY = ".ytdl-sub-lock"
DEFAULT_FFMPEG_PATH = os.getenv(
"YTDL_SUB_FFMPEG_PATH", _existing_path("/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg")
)

107
src/ytdl_sub/config/overrides.py
View file

@ -1,9 +1,10 @@
from typing import Any
from typing import Dict
from typing import Iterable
from typing import Optional
from typing import Set
import mergedeep
from typing import Type
from typing import TypeVar
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -11,17 +12,21 @@ from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARI
from ytdl_sub.entries.variables.override_variables import OverrideHelpers
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.types.function import BuiltInFunction
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
ExpectedT = TypeVar("ExpectedT")
class Overrides(UnstructuredDictFormatterValidator, Scriptable):
"""
@ -88,6 +93,24 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
return True
def ensure_variable_names_not_a_plugin(self, plugin_names: Iterable[str]) -> None:
"""
Throws an error if an override variable or function has the same name as a
preset key. This is to avoid confusion when accidentally defining things in
overrides that are meant to be in the preset.
"""
for name in self.keys:
if name.startswith("%"):
name = name[1:]
if name in plugin_names:
raise self._validation_exception(
f"Override variable with name {name} cannot be used since it is"
" the name of a plugin. Perhaps you meant to define it as a plugin? If so,"
" indent it left to make it at the same level as overrides.",
exception_class=InvalidVariableNameException,
)
def ensure_variable_name_valid(self, name: str) -> None:
"""
Ensures the variable name does not collide with any entry variables or built-in functions.
@ -115,29 +138,35 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
)
def initial_variables(
self, unresolved_variables: Optional[Dict[str, str]] = None
) -> Dict[str, str]:
self, unresolved_variables: Optional[Dict[str, SyntaxTree]] = None
) -> Dict[str, SyntaxTree]:
"""
Returns
-------
Variables and format strings for all Override variables + additional variables (Optional)
"""
initial_variables: Dict[str, str] = {}
mergedeep.merge(
initial_variables,
self.dict_with_format_strings,
unresolved_variables if unresolved_variables else {},
)
return ScriptUtils.add_sanitized_variables(initial_variables)
initial_variables: Dict[str, SyntaxTree] = self.dict_with_parsed_format_strings
if unresolved_variables:
initial_variables |= unresolved_variables
return ScriptUtils.add_sanitized_parsed_variables(initial_variables)
def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides":
"""
Initialize the override script with any unresolved variables
"""
self.script.add(
self.script.add_parsed(
self.initial_variables(
unresolved_variables={
var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
var_name: SyntaxTree(
ast=[
BuiltInFunction(
name="throw",
args=[
String(f"Plugin variable {var_name} has not been created yet")
],
)
]
)
for var_name in unresolved_variables
}
)
@ -158,10 +187,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
script = entry.script
unresolvable = entry.unresolvable
# Update the script internally so long as we are not supplying overrides
# that could alter the script with one-off state
update = function_overrides is None
try:
return script.resolve_once(
dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
unresolvable=unresolvable,
update=update,
)["tmp_var"]
except ScriptVariableNotResolved as exc:
raise StringFormattingException(
@ -176,7 +210,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Optional[Dict[str, str]] = None,
) -> str:
expected_type: Type[ExpectedT] = str,
) -> ExpectedT:
"""
Parameters
----------
@ -186,6 +221,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
expected_type
The expected type that should return. Defaults to string.
Returns
-------
@ -196,37 +233,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
StringFormattingException
If the formatter that is trying to be resolved cannot
"""
return formatter.post_process(
str(
self._apply_to_resolvable(
formatter=formatter, entry=entry, function_overrides=function_overrides
)
)
out = formatter.post_process(
self._apply_to_resolvable(
formatter=formatter, entry=entry, function_overrides=function_overrides
).native
)
def apply_overrides_formatter_to_native(
self,
formatter: OverridesStringFormatterValidator,
) -> Any:
"""
Parameters
----------
formatter
Overrides formatter to apply
if not isinstance(out, expected_type):
raise StringFormattingException(
f"Expected type {expected_type.__name__}, but received '{out.__class__.__name__}'"
)
Returns
-------
The native python form of the resolved variable
"""
return self._apply_to_resolvable(
formatter=formatter, entry=None, function_overrides=None
).native
def evaluate_boolean(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
) -> bool:
"""
Apply a formatter, and evaluate it to a boolean
"""
output = self.apply_formatter(formatter=formatter, entry=entry)
return ScriptUtils.bool_formatter_output(output)
return out

13
src/ytdl_sub/config/plugin/plugin.py
View file

@ -48,7 +48,7 @@ class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
Returns True if enabled, False if disabled.
"""
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return self.overrides.evaluate_boolean(self.plugin_options.enable)
return self.overrides.apply_formatter(self.plugin_options.enable, expected_type=bool)
return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
@ -119,6 +119,17 @@ class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
"""
return None
def post_completion_entry(self, file_metadata: FileMetadata) -> None:
"""
After the entry file is moved to its final location, run this hook.
Parameters
----------
file_metadata
Metadata about the completed entry's file download
"""
return None
def post_process_subscription(self):
"""
After all downloaded files have been post-processed, apply a subscription-wide post process

8
src/ytdl_sub/config/plugin/plugin_mapping.py
View file

@ -92,6 +92,12 @@ class PluginMapping:
EmbedThumbnailPlugin,
]
_ORDER_POST_COMPLETION: List[Type[Plugin]] = [
# Throttle protection should always be last
# to not sleep over other logic
ThrottleProtectionPlugin
]
@classmethod
def _order_by(
cls, plugin_types: List[Type[Plugin]], operation: PluginOperation
@ -102,6 +108,8 @@ class PluginMapping:
ordering = cls._ORDER_MODIFY_ENTRY
elif operation == PluginOperation.POST_PROCESS:
ordering = cls._ORDER_POST_PROCESS
elif operation == PluginOperation.POST_COMPLETION:
ordering = cls._ORDER_POST_COMPLETION
else:
raise ValueError("PluginOperation does not support ordering")

1
src/ytdl_sub/config/plugin/plugin_operation.py
View file

@ -6,3 +6,4 @@ class PluginOperation(Enum):
MODIFY_ENTRY_METADATA = 0
MODIFY_ENTRY = 1
POST_PROCESS = 2
POST_COMPLETION = 3

33
src/ytdl_sub/config/plugin/preset_plugins.py
View file

@ -1,5 +1,7 @@
from typing import Iterable
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from typing import Type
@ -44,3 +46,34 @@ class PresetPlugins:
if plugin_type in plugin_option_types:
return self.plugin_options[plugin_option_types.index(plugin_type)]
return None
def get_added_and_modified_variables(
self, additional_options: List[OptionsValidator]
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
for plugin_options in self.plugin_options + additional_options:
added_variables: Set[str] = set()
modified_variables: Set[str] = set()
for plugin_added_variables in plugin_options.added_variables(
unresolved_variables=set(),
).values():
added_variables |= set(plugin_added_variables)
for plugin_modified_variables in plugin_options.modified_variables().values():
modified_variables = plugin_modified_variables
yield plugin_options, added_variables, modified_variables
def get_all_variables(self, additional_options: List[OptionsValidator]) -> Set[str]:
"""
Returns set of all added and modified variables' names.
"""
all_variables: Set[str] = set()
for _, added, modified in self.get_added_and_modified_variables(additional_options):
all_variables.update(added)
all_variables.update(modified)
return all_variables

55
src/ytdl_sub/config/preset.py
View file

@ -2,6 +2,7 @@ import copy
from typing import Any
from typing import Dict
from typing import List
from typing import Set
from mergedeep import mergedeep
@ -11,7 +12,6 @@ from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.variable_validation import VariableValidation
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
@ -172,6 +172,37 @@ class Preset(_PresetShell):
mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE)
)
def _initialize_overrides_script(self, overrides: Overrides) -> Overrides:
"""
Do some gymnastics to initialize the Overrides script.
"""
unresolved_variables: Set[str] = set()
for (
plugin_options,
added_variables,
modified_variables,
) in self.plugins.get_added_and_modified_variables(
additional_options=[self.downloader_options, self.output_options]
):
for added_variable in added_variables:
if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable):
# pylint: disable=protected-access
raise plugin_options._validation_exception(
f"Cannot use the variable name {added_variable} because it exists as a"
" built-in ytdl-sub variable name."
)
# pylint: enable=protected-access
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
unresolved_variables |= added_variables | modified_variables
# Initialize overrides with unresolved variables + modified variables to throw an error.
# For modified variables, this is to prevent a resolve(update=True) to setting any
# dependencies until it has been explicitly added
return overrides.initialize_script(unresolved_variables=unresolved_variables)
def __init__(self, config: ConfigValidator, name: str, value: Any):
super().__init__(name=name, value=value)
@ -192,13 +223,10 @@ class Preset(_PresetShell):
)
self.plugins: PresetPlugins = self._validate_and_get_plugins()
self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
VariableValidation(
downloader_options=self.downloader_options,
output_options=self.output_options,
plugins=self.plugins,
).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage()
self.overrides = self._initialize_overrides_script(
overrides=self._validate_key(key="overrides", validator=Overrides, default={})
)
self.overrides.ensure_variable_names_not_a_plugin(plugin_names=PRESET_KEYS)
@property
def name(self) -> str:
@ -227,11 +255,18 @@ class Preset(_PresetShell):
"""
return cls(config=config, name=preset_name, value=preset_dict)
@property
def yaml(self) -> str:
def yaml(self, subscription_only: bool) -> str:
"""
Parameters
----------
subscription_only:
Only include the subscription contents, not the surrounding boiler-plate.
Returns
-------
Preset in YAML format
"""
if subscription_only:
return dump_yaml(self._value)
return dump_yaml({"presets": {self._name: self._value}})

37
src/ytdl_sub/config/preset_options.py
View file

@ -8,6 +8,9 @@ from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.utils.exceptions import SubscriptionPermissionError
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
@ -57,12 +60,24 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
def to_native_dict(self, overrides: Overrides) -> Dict:
"""
Materializes the entire ytdl-options dict from OverrideStringFormatters into
native python
native python.
"""
return {
key: overrides.apply_overrides_formatter_to_native(val)
out = {
key: overrides.apply_formatter(val, expected_type=object)
for key, val in self.dict.items()
}
if "cookiefile" in out:
if not FileHandler.is_file_existent(out["cookiefile"]):
raise ValidationException(
f"Specified cookiefile {out['cookiefile']} but it does not exist as a file."
)
if not FileHandler.is_file_readable(out["cookiefile"]):
raise SubscriptionPermissionError(
f"Cannot read cookiefile {out['cookiefile']} due to permissions issue."
)
return out
# Disable for proper docstring formatting
@ -107,6 +122,7 @@ class OutputOptions(OptionsDictValidator):
"keep_max_files",
"download_archive_standardized_date",
"keep_files_date_eval",
"preserve_mtime",
}
@classmethod
@ -170,6 +186,10 @@ class OutputOptions(OptionsDictValidator):
default=f"{{{v.upload_date_standardized.variable_name}}}",
)
self._preserve_mtime = self._validate_key_if_present(
key="preserve_mtime", validator=BoolValidator, default=False
)
if (
self._keep_files_before or self._keep_files_after or self._keep_max_files
) and not self.maintain_download_archive:
@ -309,6 +329,17 @@ class OutputOptions(OptionsDictValidator):
"""
return self._keep_max_files
@property
def preserve_mtime(self) -> bool:
"""
:expected type: Optional[Boolean]
:description:
Preserve the video's original upload time as the file modification time.
When True, sets the file's mtime to match the video's upload_date from
yt-dlp metadata. Defaults to False.
"""
return self._preserve_mtime.value
def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]:
return {
# PluginOperation.MODIFY_ENTRY_METADATA: {

273
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -1,10 +1,6 @@
import copy
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
@ -13,203 +9,188 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.script import Script
from ytdl_sub.script.script import _is_function
from ytdl_sub.utils.scriptable import BASE_SCRIPT
from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string
from ytdl_sub.script.utils.name_validation import is_function
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import validate_formatters
# Entry variables to mock during validation
_DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
name: to_variable_dependency_format_string(
# pylint: disable=protected-access
script=BASE_SCRIPT,
parsed_format_string=BASE_SCRIPT._variables[name],
# pylint: enable=protected-access
)
for name in BASE_SCRIPT.variable_names
}
class ResolutionLevel:
ORIGINAL = 0
FILL = 1
RESOLVE = 2
INTERNAL = 3
def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
dummy_variables: Dict[str, str] = {}
for var in variables:
dummy_variables[var] = ""
dummy_variables[f"{var}_sanitized"] = ""
@classmethod
def name_of(cls, resolution_level: int) -> str:
"""
Name of the resolution level.
"""
if resolution_level == cls.ORIGINAL:
return "original"
if resolution_level == cls.FILL:
return "fill"
if resolution_level == cls.RESOLVE:
return "resolve"
if resolution_level == cls.INTERNAL:
return "internal"
raise ValueError("Invalid resolution level")
return dummy_variables
def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
# Have the dummy override variable contain all variable deps that it uses in the string
dummy_overrides: Dict[str, str] = {}
for override_name in _override_variables(overrides):
if _is_function(override_name):
continue
# pylint: disable=protected-access
dummy_overrides[override_name] = to_variable_dependency_format_string(
script=overrides.script, parsed_format_string=overrides.script._variables[override_name]
)
# pylint: enable=protected-access
return dummy_overrides
def _get_added_and_modified_variables(
plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
options.append(output_options)
for plugin_options in options:
added_variables: Set[str] = set()
modified_variables: Set[str] = set()
for plugin_added_variables in plugin_options.added_variables(
unresolved_variables=set(),
).values():
added_variables |= set(plugin_added_variables)
for plugin_modified_variables in plugin_options.modified_variables().values():
modified_variables = plugin_modified_variables
yield plugin_options, added_variables, modified_variables
def _override_variables(overrides: Overrides) -> Set[str]:
return set(list(overrides.initial_variables().keys()))
@classmethod
def all(cls) -> List[int]:
"""
All possible resolution levels.
"""
return [cls.ORIGINAL, cls.FILL, cls.RESOLVE, cls.INTERNAL]
class VariableValidation:
def _get_resolve_partial_filter(self) -> Set[str]:
# Exclude sanitized variables from partial validation. This lessens the work
# and prevents double-evaluation, which can lead to bad behavior like double-prints.
return {
name
for name in self.script.variable_names
if name not in self.unresolved_variables and not name.endswith("_sanitized")
}
def _apply_resolution_level(self) -> None:
if self._resolution_level == ResolutionLevel.FILL:
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
# Only partial resolve definitions that are already resolved
self.unresolved_variables |= {
name
for name in self.overrides.keys
if not is_function(name) and not self.script.definition_of(name).maybe_resolvable
}
elif self._resolution_level == ResolutionLevel.RESOLVE:
# Partial resolve everything, but not including internal variables
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
elif self._resolution_level == ResolutionLevel.INTERNAL:
# Partial resolve everything including internal variables
pass
else:
raise ValueError("Invalid resolution level for validation")
self.script = self.script.resolve_partial(
unresolvable=self.unresolved_variables,
output_filter=self._get_resolve_partial_filter(),
)
def __init__(
self,
overrides: Overrides,
downloader_options: MultiUrlValidator,
output_options: OutputOptions,
plugins: PresetPlugins,
resolution_level: int = ResolutionLevel.RESOLVE,
):
self.overrides = overrides
self.downloader_options = downloader_options
self.output_options = output_options
self.plugins = plugins
self.script: Optional[Script] = None
self.resolved_variables: Set[str] = set()
self.unresolved_variables: Set[str] = set()
def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation":
"""
Do some gymnastics to initialize the Overrides script.
"""
override_variables = set(list(overrides.initial_variables().keys()))
# Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable
self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables
plugin_variables: Set[str] = set()
for (
plugin_options,
added_variables,
modified_variables,
) in _get_added_and_modified_variables(
plugins=self.plugins,
downloader_options=self.downloader_options,
output_options=self.output_options,
):
for added_variable in added_variables:
if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable):
# pylint: disable=protected-access
raise plugin_options._validation_exception(
f"Cannot use the variable name {added_variable} because it exists as a"
" built-in ytdl-sub variable name."
)
# pylint: enable=protected-access
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables
plugin_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables
# Initialize overrides with unresolved variables + modified variables to throw an error.
# For modified variables, this is to prevent a resolve(update=True) to setting any
# dependencies until it has been explicitly added
overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables)
# copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script)
self.script.add(
variables=_add_dummy_overrides(overrides=overrides)
| _add_dummy_variables(variables=plugin_variables)
| _DUMMY_ENTRY_VARIABLES
self.script: Script = self.overrides.script
self.unresolved_variables = (
self.plugins.get_all_variables(
additional_options=[self.output_options, self.downloader_options]
)
| UNRESOLVED_VARIABLES
)
self.unresolved_runtime_variables = self.plugins.get_all_variables(
additional_options=[self.output_options, self.downloader_options]
)
self._resolution_level = resolution_level
return self
self._apply_resolution_level()
def _update_script(self) -> None:
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
def _add_subscription_override_variables(self) -> None:
"""
Add dummy subscription variables for script validation
"""
self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
"""
Add dummy variables for script validation
"""
added_variables = options.added_variables(
unresolved_variables=self.unresolved_variables,
unresolved_variables=self.unresolved_runtime_variables,
).get(plugin_op, set())
modified_variables = options.modified_variables().get(plugin_op, set())
resolved_variables = added_variables | modified_variables
self.unresolved_runtime_variables -= added_variables | modified_variables
self.resolved_variables |= resolved_variables
self.unresolved_variables -= resolved_variables
def ensure_proper_usage(self) -> None:
def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict:
"""
Validate variables resolve as plugins are executed, and return
a mock script which contains actualized added variables from the plugins
"""
resolved_subscription: Dict = {}
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
self._add_subscription_override_variables()
self._add_runtime_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
# Always add output options first
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options)
self._add_runtime_variables(
PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options
)
# Metadata variables to be added
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
):
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
self._add_runtime_variables(
PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options
)
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
):
self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
self._add_runtime_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
# Validate that any formatter in the plugin options can resolve
validate_formatters(
resolved_subscription |= validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
unresolved_runtime_variables=self.unresolved_runtime_variables,
validator=plugin_options,
partial_resolve_formatters=partial_resolve_formatters,
)
validate_formatters(
resolved_subscription |= validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
unresolved_runtime_variables=self.unresolved_runtime_variables,
validator=self.output_options,
partial_resolve_formatters=partial_resolve_formatters,
)
assert not self.unresolved_variables
# TODO: make this a function
raw_download_output = validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
unresolved_runtime_variables=self.unresolved_runtime_variables,
validator=self.downloader_options.urls,
partial_resolve_formatters=partial_resolve_formatters,
)
resolved_subscription["download"] = []
for url_output in raw_download_output["download"]:
if isinstance(url_output["url"], list):
url_output["url"] = [url for url in url_output["url"] if bool(url)]
if url_output["url"]:
resolved_subscription["download"].append(url_output)
# TODO: make function
resolved_subscription["overrides"] = {}
for name in self.overrides.keys:
value = self.script.definition_of(name)
if name in self.script.function_names:
# Keep custom functions as-is
resolved_subscription["overrides"][name] = self.overrides.dict_with_format_strings[
name
]
elif resolved := value.maybe_resolvable:
resolved_subscription["overrides"][name] = resolved.native
else:
resolved_subscription["overrides"][name] = ScriptUtils.to_native_script(value)
assert not self.unresolved_runtime_variables
return resolved_subscription

77
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -52,12 +52,12 @@ class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator])
if 0 <= input_url_idx < len(self.plugin_options.urls.list):
validator = self.plugin_options.urls.list[input_url_idx]
if self.overrides.apply_formatter(validator.url) == entry_input_url:
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
return validator
# Match the first validator based on the URL, if one exists
for validator in self.plugin_options.urls.list:
if self.overrides.apply_formatter(validator.url) == entry_input_url:
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
return validator
# Return the first validator if none exist
@ -257,6 +257,16 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
.to_dict()
)
def webpage_url(self, entry: Entry) -> str:
"""
Returns
-------
The webpage_url to use for the actual download
"""
url_idx = entry.get(v.ytdl_sub_input_url_index, int)
webpage_url_formatter = self.plugin_options.urls.list[url_idx].webpage_url
return self.overrides.apply_formatter(webpage_url_formatter, entry=entry)
def metadata_ytdl_options(self, ytdl_option_overrides: Dict) -> Dict:
"""
Returns
@ -358,7 +368,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
else entry.is_thumbnail_downloaded_via_ytdlp
),
url=entry.webpage_url,
url=self.webpage_url(entry=entry),
)
return Entry(
download_entry_dict,
@ -366,13 +376,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
)
def _iterate_child_entries(
self, entries: List[Entry], download_reversed: bool
self, entries: List[Entry], validator: UrlValidator
) -> Iterator[Entry]:
# Iterate a list of entries, and delete the entries after yielding
entries_to_iter: List[Optional[Entry]] = entries
indices = list(range(len(entries_to_iter)))
if download_reversed:
if self.overrides.apply_formatter(validator.download_reverse, expected_type=bool):
indices = reversed(indices)
for idx in indices:
@ -394,17 +404,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
entries_to_iter[idx] = None
def _iterate_parent_entry(
self, parent: EntryParent, download_reversed: bool
self, parent: EntryParent, validator: UrlValidator
) -> Iterator[Entry]:
yield from self._iterate_child_entries(
entries=parent.entry_children(), download_reversed=download_reversed
)
yield from self._iterate_child_entries(entries=parent.entry_children(), validator=validator)
# Recursion the parent's parent entries
for parent_child in reversed(parent.parent_children()):
yield from self._iterate_parent_entry(
parent=parent_child, download_reversed=download_reversed
)
yield from self._iterate_parent_entry(parent=parent_child, validator=validator)
def _download_url_metadata(
self, url: str, include_sibling_metadata: bool, ytdl_options_overrides: Dict
@ -438,7 +444,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
self,
parents: List[EntryParent],
orphans: List[Entry],
download_reversed: bool,
validator: UrlValidator,
) -> Iterator[Entry]:
"""
Downloads the leaf entries from EntryParent trees
@ -446,21 +452,17 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
# Delete info json files afterwards so other collection URLs do not use them
with self._separate_download_archives(clear_info_json_files=True):
for parent in parents:
yield from self._iterate_parent_entry(
parent=parent, download_reversed=download_reversed
)
yield from self._iterate_parent_entry(parent=parent, validator=validator)
yield from self._iterate_child_entries(
entries=orphans, download_reversed=download_reversed
)
yield from self._iterate_child_entries(entries=orphans, validator=validator)
def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]:
metadata_ytdl_options = self.metadata_ytdl_options(
ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides)
)
download_reversed = self.overrides.evaluate_boolean(validator.download_reverse)
include_sibling_metadata = self.overrides.evaluate_boolean(
validator.include_sibling_metadata
include_sibling_metadata = self.overrides.apply_formatter(
validator.include_sibling_metadata, expected_type=bool
)
parents, orphan_entries = self._download_url_metadata(
@ -469,16 +471,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
ytdl_options_overrides=metadata_ytdl_options,
)
# TODO: Encapsulate this logic into its own class
self._url_state = URLDownloadState(
entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries)
entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries),
)
download_logger.info("Beginning downloads for %s", url)
yield from self._iterate_entries(
parents=parents,
orphans=orphan_entries,
download_reversed=download_reversed,
validator=validator,
)
def download_metadata(self) -> Iterable[Entry]:
@ -486,19 +487,25 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
# download the bottom-most urls first since they are top-priority
for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
# URLs can be empty. If they are, then skip
if not (url := self.overrides.apply_formatter(url_validator.url)):
if not (urls := self.overrides.apply_formatter(url_validator.url, expected_type=list)):
continue
for entry in self._download_metadata(url=url, validator=url_validator):
entry.initialize_script(self.overrides).add(
{
v.ytdl_sub_input_url: url,
v.ytdl_sub_input_url_index: idx,
v.ytdl_sub_input_url_count: len(self.collection.urls.list),
}
)
for url in reversed(urls):
assert isinstance(url, str)
yield entry
if not url:
continue
for entry in self._download_metadata(url=url, validator=url_validator):
entry.initialize_script(self.overrides).add(
{
v.ytdl_sub_input_url: url,
v.ytdl_sub_input_url_index: idx,
v.ytdl_sub_input_url_count: len(self.collection.urls.list),
}
)
yield entry
def download(self, entry: Entry) -> Optional[Entry]:
"""

37
src/ytdl_sub/downloaders/url/validators.py
View file

@ -1,5 +1,6 @@
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
@ -21,7 +22,7 @@ class UrlThumbnailValidator(StrictDictValidator):
def __init__(self, name, value):
super().__init__(name, value)
self._name = self._validate_key(key="name", validator=StringFormatterValidator)
self._thumb_name = self._validate_key(key="name", validator=StringFormatterValidator)
self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator)
@property
@ -29,7 +30,7 @@ class UrlThumbnailValidator(StrictDictValidator):
"""
File name for the thumbnail
"""
return self._name
return self._thumb_name
@property
def uid(self) -> OverridesStringFormatterValidator:
@ -43,6 +44,19 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
_inner_list_type = UrlThumbnailValidator
class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator):
def post_process(self, resolved: Any) -> List[str]:
if isinstance(resolved, str):
return [resolved]
if isinstance(resolved, list):
for value in resolved:
if not isinstance(value, str):
raise self._validation_exception("Must be a string or an array of strings.")
return resolved
raise self._validation_exception("Must be a string or an array of strings.")
class UrlValidator(StrictDictValidator):
_required_keys = {"url"}
_optional_keys = {
@ -52,6 +66,7 @@ class UrlValidator(StrictDictValidator):
"download_reverse",
"ytdl_options",
"include_sibling_metadata",
"webpage_url",
}
@classmethod
@ -67,7 +82,7 @@ class UrlValidator(StrictDictValidator):
super().__init__(name, value)
# TODO: url validate using yt-dlp IE
self._url = self._validate_key(key="url", validator=OverridesStringFormatterValidator)
self._url = self._validate_key(key="url", validator=OverridesOneOrManyUrlValidator)
self._variables = self._validate_key_if_present(
key="variables", validator=DictFormatterValidator, default={}
)
@ -89,6 +104,9 @@ class UrlValidator(StrictDictValidator):
validator=OverridesBooleanFormatterValidator,
default="False",
)
self._webpage_url = self._validate_key(
key="webpage_url", validator=StringFormatterValidator, default="{webpage_url}"
)
@property
def url(self) -> OverridesStringFormatterValidator:
@ -180,6 +198,19 @@ class UrlValidator(StrictDictValidator):
"""
return self._include_sibling_metadata
@property
def webpage_url(self) -> StringFormatterValidator:
"""
Optional. After ytdl-sub performs the metadata download, it will inspect each
entry's .info.json file and perform the actual download from yt-dlp using
`webpage_url <config_reference/scripting/entry_variables:webpage_url>`. This
can be overwritten by supplying parameter with a modification to ``webpage_url`` in the
form of an override variable.
Defaults to ``{webpage_url}``.
"""
return self._webpage_url
class UrlStringOrDictValidator(UrlValidator):
"""

2
src/ytdl_sub/downloaders/ytdlp.py
View file

@ -224,7 +224,7 @@ class YTDLP:
except RejectedVideoReached:
cls.logger.debug(
"RejectedVideoReached, stopping additional downloads "
"(Can be disable by setting `date_range.breaking` to False)."
"(Can be disable by setting `date_range.breaks` to False)."
)
except ExistingVideoReached:
cls.logger.debug(

11
src/ytdl_sub/entries/script/variable_definitions.py
View file

@ -1135,6 +1135,16 @@ class VariableDefinitions(
]
}
@cache
def variable_names(self, include_sanitized: bool):
"""
Returns all variable names, and can include sanitized.
"""
var_names: Set[str] = self.scripts().keys()
if include_sanitized:
var_names |= {f"{name}_sanitized" for name in var_names}
return var_names
@cache
def injected_variables(self) -> Set[MetadataVariable]:
"""
@ -1162,6 +1172,7 @@ class VariableDefinitions(
"""
return {
self.uid,
self.extractor,
self.extractor_key,
self.epoch,
self.webpage_url,

10
src/ytdl_sub/entries/script/variable_types.py
View file

@ -22,7 +22,6 @@ VariableT = TypeVar("VariableT", bound="Variable")
def _get(
cast: str,
metadata_variable_name: str,
metadata_key: str,
variable_name: Optional[str],
@ -47,7 +46,7 @@ def _get(
return as_type(
variable_name=variable_name or metadata_key,
metadata_key=metadata_key,
definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}",
definition=f"{{ {out} }}",
)
@ -182,7 +181,6 @@ class MapMetadataVariable(MetadataVariable, MapVariable):
Creates a map variable from entry metadata
"""
return _get(
"map",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
@ -204,7 +202,6 @@ class ArrayMetadataVariable(MetadataVariable, ArrayVariable):
Creates an array variable from entry metadata
"""
return _get(
"array",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
@ -226,7 +223,6 @@ class StringMetadataVariable(MetadataVariable, StringVariable):
Creates a string variable from entry metadata
"""
return _get(
"string",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
@ -245,7 +241,6 @@ class StringMetadataVariable(MetadataVariable, StringVariable):
Creates a string variable from playlist metadata
"""
return _get(
"string",
metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
@ -264,7 +259,6 @@ class StringMetadataVariable(MetadataVariable, StringVariable):
Creates a string variable from source metadata
"""
return _get(
"string",
metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
@ -301,7 +295,6 @@ class IntegerMetadataVariable(MetadataVariable, IntegerVariable):
Creates an int variable from entry metadata
"""
return _get(
"int",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
@ -320,7 +313,6 @@ class IntegerMetadataVariable(MetadataVariable, IntegerVariable):
Creates an int variable from playlist metadata
"""
return _get(
"int",
metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,

20
src/ytdl_sub/entries/variables/override_variables.py
View file

@ -11,9 +11,6 @@ from ytdl_sub.entries.script.variable_types import Variable
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.utils.name_validation import is_valid_name
# TODO: use this
SUBSCRIPTION_ARRAY = "subscription_array"
class SubscriptionVariables:
@staticmethod
@ -55,16 +52,21 @@ class SubscriptionVariables:
@staticmethod
def subscription_indent_i(index: int) -> StringVariable:
"""
For subscriptions in the form of
For subscriptions where the ancestor keys contain the ``= ...`` prefix, the
variables ``subscription_indent_1``, ``subscription_indent_2``, and so on get
set to each subsequent value. For example, given the following subscriptions
file snippet:
.. code-block:: yaml
Preset | = Indent Value 1:
= Indent Value 2:
Preset 1 | = Indent Value 1 | Preset 2:
Preset 3 | = Indent Value 2 | Preset 4:
"Subscription Name": "https://..."
``subscription_indent_1`` and ``subscription_indent_2`` get set to
``Indent Value 1`` and ``Indent Value 2``.
The ``{subscription_indent_1}`` variable will be ``Indent Value 1`` and
``{subscription_indent_2}`` will be ``Indent Value 2``. The most common use of
these variables is to :doc:`set the genre and rating for subscriptions from the
YAML keys <../prebuilt_presets/tv_show>`.
"""
return StringVariable(
variable_name=f"subscription_indent_{index + 1}", definition="{ %string('') }"
@ -158,7 +160,7 @@ class OverrideHelpers:
True if the override name itself is valid. False otherwise.
"""
if name.startswith("%"):
return is_valid_name(name=name[1:])
name = name[1:]
return is_valid_name(name=name)

14
src/ytdl_sub/plugins/date_range.py
View file

@ -25,9 +25,11 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
A string in the format YYYYMMDD or
(now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this.
Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest
granularity possible.
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in
this. Note that yt-dlp will round times to the closest day, meaning that `day` is
the lowest granularity possible. Also note that, considering time zones, it's best
to include a margin of an extra day on either side to be sure it includes the
intended download files.
:Usage:
@ -56,7 +58,7 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
"""
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
Only download videos only before this datetime, not inclusive.
"""
return self._before
@ -65,7 +67,7 @@ class DateRangeOptions(ToggleableOptionsDictValidator):
"""
:expected type: Optional[OverridesFormatter]
:description:
Only download videos after this datetime.
Only download videos after or on this datetime, inclusive.
"""
return self._after
@ -114,7 +116,7 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
date_validator=self.plugin_options.after, overrides=self.overrides
)
after_filter = f"{date_type} >= {after_str}"
if self.overrides.evaluate_boolean(self.plugin_options.breaks):
if self.overrides.apply_formatter(self.plugin_options.breaks, expected_type=bool):
breaking_match_filters.append(after_filter)
else:
match_filters.append(after_filter)

2
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -33,7 +33,7 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
@property
def _embed_thumbnail(self) -> bool:
return self.overrides.evaluate_boolean(self.plugin_options)
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
@classmethod
def _embed_video_thumbnail(cls, entry: Entry) -> None:

11
src/ytdl_sub/plugins/filter_exclude.py
View file

@ -7,13 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("filter-exclude")
class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
"""
Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded.
@ -29,6 +30,8 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
{ %contains( %lower(description), '#short' ) }
"""
_inner_list_type = BooleanFormatterValidator
class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
plugin_options_type = FilterExcludeOptions
@ -52,7 +55,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry
for formatter in self.plugin_options.list:
should_exclude = self.overrides.evaluate_boolean(formatter=formatter, entry=entry)
should_exclude = self.overrides.apply_formatter(
formatter=formatter, entry=entry, expected_type=bool
)
if should_exclude:
logger.info(

12
src/ytdl_sub/plugins/filter_include.py
View file

@ -7,14 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("filter-include")
class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
"""
Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included.
@ -38,6 +38,8 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
}
"""
_inner_list_type = BooleanFormatterValidator
class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
plugin_options_type = FilterIncludeOptions
@ -61,8 +63,8 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry
for formatter in self.plugin_options.list:
should_exclude = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
should_exclude = self.overrides.apply_formatter(
formatter=formatter, entry=entry, expected_type=bool
)
if not should_exclude:
logger.info(

2
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -140,7 +140,7 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC):
if not nfo_tags:
return
if self.overrides.evaluate_boolean(self.plugin_options.kodi_safe):
if self.overrides.apply_formatter(self.plugin_options.kodi_safe, expected_type=bool):
nfo_root = to_max_3_byte_utf8_string(nfo_root)
nfo_tags = {
to_max_3_byte_utf8_string(key): [

2
src/ytdl_sub/plugins/square_thumbnail.py
View file

@ -31,7 +31,7 @@ class SquareThumbnailPlugin(Plugin[SquareThumbnailOptions]):
@property
def _square_thumbnail(self) -> bool:
return self.overrides.evaluate_boolean(self.plugin_options)
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
@classmethod
def _convert_to_square_thumbnail(cls, entry: Entry) -> None:

9
src/ytdl_sub/plugins/static_nfo_tags.py
View file

@ -7,10 +7,11 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
class StaticNfoTagsOptions(SharedNfoTagsOptions):
"""
Adds an NFO file for every entry, but does not link it to an entry in the download archive.
This is intended to produce ``season.nfo``s in each season directory. Each entry within a
season will overwrite this file with its season name. If the entry gets deleted from ytdl-sub,
this file will remain since it's not linked.
Adds an NFO file for every entry, but does not link it to an entry in the download
archive. This is intended to produce ``season.nfo`` files in each season
directory. Each entry within a season will overwrite this file with its season
name. If the entry gets deleted from ytdl-sub, this file will remain since it's not
linked.
Usage:

35
src/ytdl_sub/plugins/subtitles.py
View file

@ -1,8 +1,10 @@
import os
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
@ -11,6 +13,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
@ -57,6 +60,7 @@ class SubtitleOptions(ToggleableOptionsDictValidator):
"subtitles_type",
"embed_subtitles",
"languages",
"languages_required",
"allow_auto_generated_subtitles",
}
@ -76,6 +80,9 @@ class SubtitleOptions(ToggleableOptionsDictValidator):
self._languages = self._validate_key_if_present(
key="languages", validator=StringListValidator, default=["en"]
).list
self._languages_required = self._validate_key_if_present(
key="languages_required", validator=StringListValidator, default=[]
).list
self._allow_auto_generated_subtitles = self._validate_key_if_present(
key="allow_auto_generated_subtitles", validator=BoolValidator, default=False
).value
@ -121,6 +128,16 @@ class SubtitleOptions(ToggleableOptionsDictValidator):
"""
return [lang.value for lang in self._languages]
@property
def languages_required(self) -> Optional[List[str]]:
"""
:expected type: Optional[List[String]]
:description:
Language code(s) that are required to be present for downloads to continue. If missing,
ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles.
"""
return [lang.value for lang in self._languages_required]
@property
def allow_auto_generated_subtitles(self) -> Optional[bool]:
"""
@ -211,7 +228,21 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
if self.plugin_options.embed_subtitles:
file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}")
if self.plugin_options.subtitles_name:
download_subtitle_lang_file_names: List[Tuple[str, str]] = []
for lang in langs:
download_subtitle_file_name = entry.base_filename(
ext=f"{lang}.{self.plugin_options.subtitles_type}"
)
if os.path.isfile(Path(self.working_directory) / download_subtitle_file_name):
download_subtitle_lang_file_names.append((lang, download_subtitle_file_name))
elif lang in self.plugin_options.languages_required:
raise UserThrownRuntimeError(
f"Required the subtitle lang {lang}, but the file could not be found for "
f"the entry {entry.title}."
)
for lang, file_name in download_subtitle_lang_file_names:
output_subtitle_file_name = self.overrides.apply_formatter(
formatter=self.plugin_options.subtitles_name,
entry=entry,
@ -219,9 +250,7 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
)
self.save_file(
file_name=entry.base_filename(
ext=f"{lang}.{self.plugin_options.subtitles_type}"
),
file_name=file_name,
output_file_name=output_subtitle_file_name,
entry=entry,
)

184
src/ytdl_sub/plugins/throttle_protection.py
View file

@ -1,7 +1,10 @@
import random
import time
from abc import ABC
from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
@ -10,68 +13,135 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import FloatValidator
from ytdl_sub.validators.string_formatter_validators import FloatFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesFloatFormatterValidator
from ytdl_sub.validators.validators import ProbabilityValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("throttle-protection")
FloatValidatorT = TypeVar("FloatValidatorT", bound=FloatFormatterValidator)
class RandomizedRangeValidator(StrictDictValidator):
class _RandomizedRangeValidator(StrictDictValidator, ABC):
"""
Validator to specify a float range between [min, max)
Base class for range validation, to support both entry and static overrides.
"""
_float_validator: Type[FloatValidatorT]
_required_keys = {"max"}
_optional_keys = {"min"}
def __init__(self, name, value):
super().__init__(name, value)
self._max = self._validate_key(key="max", validator=FloatValidator).value
self._max = self._validate_key(key="max", validator=self._float_validator)
self._min = self._validate_key_if_present(
key="min", validator=FloatValidator, default=0.0
).value
key="min", validator=self._float_validator, default=0.0
)
if self._min < 0:
raise self._validation_exception("min must be greater than zero")
def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
actualized_min = overrides.apply_formatter(self._min, entry=entry, expected_type=float)
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float)
if self._max < self._min:
if actualized_min < 0:
raise self._validation_exception(
f"max ({self._max}) must be greater than or equal to min ({self._min})"
f"min must be greater than zero, received {actualized_min}"
)
if actualized_max < actualized_min:
raise self._validation_exception(
f"max ({actualized_max}) must be greater than or equal to min ({actualized_min})"
)
def min_value(self) -> float:
"""
Returns
-------
Minimum value
"""
return self._min
return random.uniform(actualized_min, actualized_max)
def max_value(self) -> float:
"""
Returns
-------
Maximum value
"""
return self._max
def randomized_float(self) -> float:
"""
Returns
-------
A random float within the range
"""
return random.uniform(self._min, self._max)
def randomized_int(self) -> int:
def _randomized_int(self, overrides: Overrides, entry: Optional[Entry] = None) -> int:
"""
Returns
-------
A random float within the range, then cast to an integer (floored)
"""
return int(self.randomized_float())
return int(self._randomized_float(overrides, entry=entry))
def _max_value(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
"""
Returns
-------
Max possible value
"""
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float)
if actualized_max < 0:
raise self._validation_exception(
f"max must be greater than zero, received {actualized_max}"
)
return actualized_max
class RandomizedRangeValidator(_RandomizedRangeValidator):
"""
Validator to specify a float range between [min, max) with both
override and entry variable support.
"""
_float_validator = FloatFormatterValidator
def randomized_float(self, overrides: Overrides, entry: Entry) -> float:
"""
Returns
-------
A random float within the range
"""
return self._randomized_float(overrides=overrides, entry=entry)
def randomized_int(self, overrides: Overrides, entry: Entry) -> int:
"""
Returns
-------
A random float within the range, then cast to an integer (floored)
"""
return self._randomized_int(overrides=overrides, entry=entry)
def max_value(self, overrides: Overrides, entry: Entry) -> float:
"""
Returns
-------
Max possible value
"""
return self._max_value(overrides=overrides, entry=entry)
class RandomizedRangeOverridesValidator(_RandomizedRangeValidator):
"""
Validator to specify a float range between [min, max) with
static variable support.
"""
_float_validator = OverridesFloatFormatterValidator
def randomized_float(self, overrides: Overrides) -> float:
"""
Returns
-------
A random float within the range
"""
return self._randomized_float(overrides=overrides)
def randomized_int(self, overrides: Overrides) -> int:
"""
Returns
-------
A random float within the range, then cast to an integer (floored)
"""
return self._randomized_int(overrides=overrides)
def max_value(self, overrides: Overrides) -> float:
"""
Returns
-------
Max possible value
"""
return self._max_value(overrides=overrides)
class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
@ -80,6 +150,9 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
range-based values, a random number will be chosen within the range to avoid sleeps looking
scripted.
Range min and max values support static override variables within their definitions.
``sleep_per_download_s`` supports both static and override variables.
:Usage:
.. code-block:: yaml
@ -115,16 +188,16 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
super().__init__(name, value)
self._sleep_per_request_s = self._validate_key_if_present(
key="sleep_per_request_s", validator=RandomizedRangeValidator
key="sleep_per_request_s", validator=RandomizedRangeOverridesValidator
)
self._sleep_per_download_s = self._validate_key_if_present(
key="sleep_per_download_s", validator=RandomizedRangeValidator
)
self._sleep_per_subscription_s = self._validate_key_if_present(
key="sleep_per_subscription_s", validator=RandomizedRangeValidator
key="sleep_per_subscription_s", validator=RandomizedRangeOverridesValidator
)
self._max_downloads_per_subscription = self._validate_key_if_present(
key="max_downloads_per_subscription", validator=RandomizedRangeValidator
key="max_downloads_per_subscription", validator=RandomizedRangeOverridesValidator
)
self._subscription_download_probability = self._validate_key_if_present(
key="subscription_download_probability", validator=ProbabilityValidator
@ -202,15 +275,25 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
self._subscription_download_counter: int = 0
self._subscription_max_downloads: Optional[int] = None
# Compute this during post-processing using entry metadata.
# Apply the sleep post-completion.
self._entry_sleep_time: Optional[float] = None
# If subscriptions have a max download limit, set it here for the first subscription
if self.plugin_options.max_downloads_per_subscription:
self._subscription_max_downloads = (
self.plugin_options.max_downloads_per_subscription.randomized_int()
self.plugin_options.max_downloads_per_subscription.randomized_int(
overrides=self.overrides
)
)
def ytdl_options(self) -> Optional[Dict]:
if self.plugin_options.sleep_per_request_s is not None:
return {"sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value()}
return {
"sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value(
overrides=self.overrides
)
}
return {}
def initialize_subscription(self) -> bool:
@ -254,12 +337,19 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
self._subscription_download_counter += 1
if self.plugin_options.sleep_per_download_s:
sleep_time = self.plugin_options.sleep_per_download_s.randomized_float()
logger.info("Sleeping between downloads for %0.2f seconds", sleep_time)
self.perform_sleep(sleep_time)
self._entry_sleep_time = self.plugin_options.sleep_per_download_s.randomized_float(
overrides=self.overrides, entry=entry
)
return None
def post_completion_entry(self, file_metadata: FileMetadata) -> None:
if self._entry_sleep_time:
# pylint: disable=logging-fstring-interpolation)
# needed to test logs in unit test
logger.info(f"Sleeping between downloads for {self._entry_sleep_time:.2f} seconds")
self.perform_sleep(self._entry_sleep_time)
def post_process_subscription(self):
# Reset counter to 0 for the next subscription
self._subscription_download_counter = 0
@ -267,10 +357,14 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
# If present, reset max downloads for the next subscription
if self.plugin_options.max_downloads_per_subscription:
self._subscription_max_downloads = (
self.plugin_options.max_downloads_per_subscription.randomized_int
self.plugin_options.max_downloads_per_subscription.randomized_int(
overrides=self.overrides
)
)
if self.plugin_options.sleep_per_subscription_s:
sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float()
sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float(
overrides=self.overrides
)
logger.info("Sleeping between subscriptions for %0.2f seconds", sleep_time)
self.perform_sleep(sleep_time)

2
src/ytdl_sub/plugins/video_tags.py
View file

@ -34,7 +34,7 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]):
Tags the entry's audio file using values defined in the metadata options
"""
tags_to_write: Dict[str, str] = {}
for tag_name, tag_formatter in self.plugin_options.dict.items():
for tag_name, tag_formatter in sorted(self.plugin_options.dict.items()):
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
tags_to_write[tag_name] = tag_value

3
src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml
View file

@ -11,8 +11,7 @@ presets:
# Set the default date_range to 2 months
overrides:
date_range: "2months" # keep for legacy-reasons
only_recent_date_range: "{date_range}"
only_recent_date_range: "2months"
#############################################################################
# Only Recent

34
src/ytdl_sub/prebuilt_presets/helpers/filter_duration.yaml Normal file
View file

@ -0,0 +1,34 @@
presets:
#############################################################################
# Filter Duration
# Include or exclude media based on its play time duration
Filter Duration:
overrides:
filter_duration_min_s: 0
filter_duration_max_s: 4294967296
"%filter_duration_ensure_numeric": >-
{
%assert_then(
%is_numeric($0),
$0,
"filter_duration args must be numeric"
)
}
filter_duration_zero_msg: "Duration metadata for {title} is missing, cannot perform filter."
"%filter_duration_eval": >-
{
%if(
%eq(duration, 0),
%print(filter_duration_zero_msg, False)
$0
)
}
filter_exclude:
- "{ %filter_duration_eval( %lt(duration, %filter_duration_ensure_numeric(filter_duration_min_s)) ) }"
- "{ %filter_duration_eval( %gt(duration, %filter_duration_ensure_numeric(filter_duration_max_s)) ) }"

0
src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml → src/ytdl_sub/prebuilt_presets/helpers/filter_keywords.yaml
View file

64
src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml
View file

@ -14,8 +14,15 @@ presets:
)
}
sleep_per_request_s:
min: 3.5
max: 3.5
# yt-dlp processes eath request synchronously, so it's intrinsic delay between
# requests already represent a maximum close to real app/client behavior in the
# field. Add a small additional margin matching the example values from yt-dlp
# to be conservative:
max: 0.75
# Not used at time of writing, but choose a value that would mimic real
# app/client behavior in the field. The next request is usually sent right away
# if processing the previous request is done asynchronously:
min: 0.0
sleep_per_download_s:
min: 13.8
max: 28.4
@ -23,4 +30,55 @@ presets:
min: 16.3
max: 26.1
overrides:
enable_throttle_protection: True
enable_throttle_protection: True
##### Resolution Assert
enable_resolution_assert: True
resolution_assert_height_gte: 361
resolution_assert_print: >-
{
%print(
%if(
enable_resolution_assert,
"Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`",
"Resolution assert is disabled. Use at your own risk!"
),
enable_resolution_assert,
-1
)
}
# list variable that contains partial titles to ignore
resolution_assert_ignore_titles: "{ [] }"
resolution_assert_is_ignored: >-
{
%print_if_true(
%concat(title, " has a match in resolution_assert_ignore_titles, skipping resolution assert."),
%contains_any(title, resolution_assert_ignore_titles)
)
}
resolution_readable: "{width}x{height}"
# height is not a guaranteed populated variable, do not assert if it's missing (which defaults to 0)
resolution_assert: >-
{
%if(
%and(
enable_resolution_assert,
%ne( height, 0 ),
%not(resolution_assert_is_ignored)
),
%assert(
%gte( height, resolution_assert_height_gte ),
%concat(
"Entry ",
title,
" downloaded at a low resolution (",
resolution_readable,
"), you've probably been throttled. ",
"Stopping further downloads, wait a few hours and try again. ",
"Disable using the override variable `enable_resolution_assert: False`."
)
),
"false is no-op"
)
}

919
src/ytdl_sub/prebuilt_presets/helpers/url.yaml
View file

@ -14,7 +14,7 @@ presets:
#
_multi_url:
download:
- url: "{url}"
- url: "{ %array_at(urls, 0) }"
playlist_thumbnails:
- name: "{avatar_uncropped_thumbnail_file_name}"
uid: "avatar_uncropped"
@ -26,411 +26,135 @@ presets:
- name: "{banner_uncropped_thumbnail_file_name}"
uid: "banner_uncropped"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url2}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url3}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url4}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url5}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url6}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url7}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url8}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url9}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url10}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url11}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url12}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url13}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url14}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url15}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url16}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url17}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url18}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url19}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url20}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url21}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url22}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url23}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url24}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url25}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url26}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url27}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url28}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url29}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url30}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url31}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url32}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url33}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url34}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url35}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url36}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url37}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url38}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url39}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url40}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url41}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url42}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url43}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url44}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url45}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url46}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url47}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url48}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url49}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url50}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url51}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url52}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url53}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url54}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url55}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url56}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url57}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url58}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url59}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url60}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url61}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url62}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url63}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url64}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url65}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url66}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url67}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url68}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url69}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url70}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url71}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url72}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url73}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url74}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url75}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url76}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url77}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url78}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url79}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url80}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url81}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url82}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url83}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url84}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url85}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url86}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url87}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url88}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url89}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url90}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url91}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url92}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url93}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url94}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url95}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url96}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url97}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url98}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url99}"
include_sibling_metadata: "{include_sibling_metadata}"
- url: "{url100}"
webpage_url: "{modified_webpage_url}"
- url: "{ %array_slice(urls, 1) }"
include_sibling_metadata: "{include_sibling_metadata}"
webpage_url: "{modified_webpage_url}"
overrides:
avatar_uncropped_thumbnail_file_name: ""
banner_uncropped_thumbnail_file_name: ""
include_sibling_metadata: False
modified_webpage_url: "{webpage_url}"
subscription_array: >-
{
[
url, url2, url3, url4, url5, url6, url7, url8, url9, url10,
url11, url12, url13, url14, url15, url16, url17, url18, url19, url20,
url21, url22, url23, url24, url25, url26, url27, url28, url29, url30,
url31, url32, url33, url34, url35, url36, url37, url38, url39, url40,
url41, url42, url43, url44, url45, url46, url47, url48, url49, url50,
url51, url52, url53, url54, url55, url56, url57, url58, url59, url60,
url61, url62, url63, url64, url65, url66, url67, url68, url69, url70,
url71, url72, url73, url74, url75, url76, url77, url78, url79, url80,
url81, url82, url83, url84, url85, url86, url87, url88, url89, url90,
url91, url92, url93, url94, url95, url96, url97, url98, url99, url100
]
}
urls: "{subscription_array}"
subscription_array: "{ [] }"
subscription_value: ""
subscription_value_2: ""
subscription_value_3: ""
subscription_value_4: ""
subscription_value_5: ""
subscription_value_6: ""
subscription_value_7: ""
subscription_value_8: ""
subscription_value_9: ""
subscription_value_10: ""
subscription_value_11: ""
subscription_value_12: ""
subscription_value_13: ""
subscription_value_14: ""
subscription_value_15: ""
subscription_value_16: ""
subscription_value_17: ""
subscription_value_18: ""
subscription_value_19: ""
subscription_value_20: ""
subscription_value_21: ""
subscription_value_22: ""
subscription_value_23: ""
subscription_value_24: ""
subscription_value_25: ""
subscription_value_26: ""
subscription_value_27: ""
subscription_value_28: ""
subscription_value_29: ""
subscription_value_30: ""
subscription_value_31: ""
subscription_value_32: ""
subscription_value_33: ""
subscription_value_34: ""
subscription_value_35: ""
subscription_value_36: ""
subscription_value_37: ""
subscription_value_38: ""
subscription_value_39: ""
subscription_value_40: ""
subscription_value_41: ""
subscription_value_42: ""
subscription_value_43: ""
subscription_value_44: ""
subscription_value_45: ""
subscription_value_46: ""
subscription_value_47: ""
subscription_value_48: ""
subscription_value_49: ""
subscription_value_50: ""
subscription_value_51: ""
subscription_value_52: ""
subscription_value_53: ""
subscription_value_54: ""
subscription_value_55: ""
subscription_value_56: ""
subscription_value_57: ""
subscription_value_58: ""
subscription_value_59: ""
subscription_value_60: ""
subscription_value_61: ""
subscription_value_62: ""
subscription_value_63: ""
subscription_value_64: ""
subscription_value_65: ""
subscription_value_66: ""
subscription_value_67: ""
subscription_value_68: ""
subscription_value_69: ""
subscription_value_70: ""
subscription_value_71: ""
subscription_value_72: ""
subscription_value_73: ""
subscription_value_74: ""
subscription_value_75: ""
subscription_value_76: ""
subscription_value_77: ""
subscription_value_78: ""
subscription_value_79: ""
subscription_value_80: ""
subscription_value_81: ""
subscription_value_82: ""
subscription_value_83: ""
subscription_value_84: ""
subscription_value_85: ""
subscription_value_86: ""
subscription_value_87: ""
subscription_value_88: ""
subscription_value_89: ""
subscription_value_90: ""
subscription_value_91: ""
subscription_value_92: ""
subscription_value_93: ""
subscription_value_94: ""
subscription_value_95: ""
subscription_value_96: ""
subscription_value_97: ""
subscription_value_98: ""
subscription_value_99: ""
subscription_value_100: ""
url: "{subscription_value}"
url2: "{subscription_value_2}"
url3: "{subscription_value_3}"
url4: "{subscription_value_4}"
url5: "{subscription_value_5}"
url6: "{subscription_value_6}"
url7: "{subscription_value_7}"
url8: "{subscription_value_8}"
url9: "{subscription_value_9}"
url10: "{subscription_value_10}"
url11: "{subscription_value_11}"
url12: "{subscription_value_12}"
url13: "{subscription_value_13}"
url14: "{subscription_value_14}"
url15: "{subscription_value_15}"
url16: "{subscription_value_16}"
url17: "{subscription_value_17}"
url18: "{subscription_value_18}"
url19: "{subscription_value_19}"
url20: "{subscription_value_20}"
url21: "{subscription_value_21}"
url22: "{subscription_value_22}"
url23: "{subscription_value_23}"
url24: "{subscription_value_24}"
url25: "{subscription_value_25}"
url26: "{subscription_value_26}"
url27: "{subscription_value_27}"
url28: "{subscription_value_28}"
url29: "{subscription_value_29}"
url30: "{subscription_value_30}"
url31: "{subscription_value_31}"
url32: "{subscription_value_32}"
url33: "{subscription_value_33}"
url34: "{subscription_value_34}"
url35: "{subscription_value_35}"
url36: "{subscription_value_36}"
url37: "{subscription_value_37}"
url38: "{subscription_value_38}"
url39: "{subscription_value_39}"
url40: "{subscription_value_40}"
url41: "{subscription_value_41}"
url42: "{subscription_value_42}"
url43: "{subscription_value_43}"
url44: "{subscription_value_44}"
url45: "{subscription_value_45}"
url46: "{subscription_value_46}"
url47: "{subscription_value_47}"
url48: "{subscription_value_48}"
url49: "{subscription_value_49}"
url50: "{subscription_value_50}"
url51: "{subscription_value_51}"
url52: "{subscription_value_52}"
url53: "{subscription_value_53}"
url54: "{subscription_value_54}"
url55: "{subscription_value_55}"
url56: "{subscription_value_56}"
url57: "{subscription_value_57}"
url58: "{subscription_value_58}"
url59: "{subscription_value_59}"
url60: "{subscription_value_60}"
url61: "{subscription_value_61}"
url62: "{subscription_value_62}"
url63: "{subscription_value_63}"
url64: "{subscription_value_64}"
url65: "{subscription_value_65}"
url66: "{subscription_value_66}"
url67: "{subscription_value_67}"
url68: "{subscription_value_68}"
url69: "{subscription_value_69}"
url70: "{subscription_value_70}"
url71: "{subscription_value_71}"
url72: "{subscription_value_72}"
url73: "{subscription_value_73}"
url74: "{subscription_value_74}"
url75: "{subscription_value_75}"
url76: "{subscription_value_76}"
url77: "{subscription_value_77}"
url78: "{subscription_value_78}"
url79: "{subscription_value_79}"
url80: "{subscription_value_80}"
url81: "{subscription_value_81}"
url82: "{subscription_value_82}"
url83: "{subscription_value_83}"
url84: "{subscription_value_84}"
url85: "{subscription_value_85}"
url86: "{subscription_value_86}"
url87: "{subscription_value_87}"
url88: "{subscription_value_88}"
url89: "{subscription_value_89}"
url90: "{subscription_value_90}"
url91: "{subscription_value_91}"
url92: "{subscription_value_92}"
url93: "{subscription_value_93}"
url94: "{subscription_value_94}"
url95: "{subscription_value_95}"
url96: "{subscription_value_96}"
url97: "{subscription_value_97}"
url98: "{subscription_value_98}"
url99: "{subscription_value_99}"
url100: "{subscription_value_100}"
url2: ""
url3: ""
url4: ""
url5: ""
url6: ""
url7: ""
url8: ""
url9: ""
url10: ""
url11: ""
url12: ""
url13: ""
url14: ""
url15: ""
url16: ""
url17: ""
url18: ""
url19: ""
url20: ""
url21: ""
url22: ""
url23: ""
url24: ""
url25: ""
url26: ""
url27: ""
url28: ""
url29: ""
url30: ""
url31: ""
url32: ""
url33: ""
url34: ""
url35: ""
url36: ""
url37: ""
url38: ""
url39: ""
url40: ""
url41: ""
url42: ""
url43: ""
url44: ""
url45: ""
url46: ""
url47: ""
url48: ""
url49: ""
url50: ""
url51: ""
url52: ""
url53: ""
url54: ""
url55: ""
url56: ""
url57: ""
url58: ""
url59: ""
url60: ""
url61: ""
url62: ""
url63: ""
url64: ""
url65: ""
url66: ""
url67: ""
url68: ""
url69: ""
url70: ""
url71: ""
url72: ""
url73: ""
url74: ""
url75: ""
url76: ""
url77: ""
url78: ""
url79: ""
url80: ""
url81: ""
url82: ""
url83: ""
url84: ""
url85: ""
url86: ""
url87: ""
url88: ""
url89: ""
url90: ""
url91: ""
url92: ""
url93: ""
url94: ""
url95: ""
url96: ""
url97: ""
url98: ""
url99: ""
url100: ""
# multi-url with bilateral scraping built into it via
@ -441,406 +165,11 @@ presets:
- "_url_bilateral_overrides"
download:
- url: "{%bilateral_url(url) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url2) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url3) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url4) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url5) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url6) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url7) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url8) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url9) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url10) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url11) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url12) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url13) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url14) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url15) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url16) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url17) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url18) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url19) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url20) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url21) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url22) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url23) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url24) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url25) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url26) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url27) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url28) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url29) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url30) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url31) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url32) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url33) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url34) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url35) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url36) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url37) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url38) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url39) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url40) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url41) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url42) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url43) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url44) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url45) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url46) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url47) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url48) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url49) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url50) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url51) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url52) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url53) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url54) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url55) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url56) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url57) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url58) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url59) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url60) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url61) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url62) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url63) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url64) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url65) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url66) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url67) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url68) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url69) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url70) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url71) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url72) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url73) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url74) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url75) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url76) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url77) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url78) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url79) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url80) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url81) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url82) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url83) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url84) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url85) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url86) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url87) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url88) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url89) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url90) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url91) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url92) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url93) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url94) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url95) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url96) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url97) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url98) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(url99) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{%bilateral_url(url100) }"
- url: "{ %array_apply(urls, %bilateral_url) }"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
webpage_url: "{modified_webpage_url}"
_multi_url_bilateral:
preset:

101
src/ytdl_sub/prebuilt_presets/helpers/url_categorized.yaml
View file

@ -108,103 +108,4 @@ presets:
}
subscription_map: "{ {} }"
url: "{ %get_url_i(1) }"
url2: "{ %get_url_i(2) }"
url3: "{ %get_url_i(3) }"
url4: "{ %get_url_i(4) }"
url5: "{ %get_url_i(5) }"
url6: "{ %get_url_i(6) }"
url7: "{ %get_url_i(7) }"
url8: "{ %get_url_i(8) }"
url9: "{ %get_url_i(9) }"
url10: "{ %get_url_i(10) }"
url11: "{ %get_url_i(11) }"
url12: "{ %get_url_i(12) }"
url13: "{ %get_url_i(13) }"
url14: "{ %get_url_i(14) }"
url15: "{ %get_url_i(15) }"
url16: "{ %get_url_i(16) }"
url17: "{ %get_url_i(17) }"
url18: "{ %get_url_i(18) }"
url19: "{ %get_url_i(19) }"
url20: "{ %get_url_i(20) }"
url21: "{ %get_url_i(21) }"
url22: "{ %get_url_i(22) }"
url23: "{ %get_url_i(23) }"
url24: "{ %get_url_i(24) }"
url25: "{ %get_url_i(25) }"
url26: "{ %get_url_i(26) }"
url27: "{ %get_url_i(27) }"
url28: "{ %get_url_i(28) }"
url29: "{ %get_url_i(29) }"
url30: "{ %get_url_i(30) }"
url31: "{ %get_url_i(31) }"
url32: "{ %get_url_i(32) }"
url33: "{ %get_url_i(33) }"
url34: "{ %get_url_i(34) }"
url35: "{ %get_url_i(35) }"
url36: "{ %get_url_i(36) }"
url37: "{ %get_url_i(37) }"
url38: "{ %get_url_i(38) }"
url39: "{ %get_url_i(39) }"
url40: "{ %get_url_i(40) }"
url41: "{ %get_url_i(41) }"
url42: "{ %get_url_i(42) }"
url43: "{ %get_url_i(43) }"
url44: "{ %get_url_i(44) }"
url45: "{ %get_url_i(45) }"
url46: "{ %get_url_i(46) }"
url47: "{ %get_url_i(47) }"
url48: "{ %get_url_i(48) }"
url49: "{ %get_url_i(49) }"
url50: "{ %get_url_i(50) }"
url51: "{ %get_url_i(51) }"
url52: "{ %get_url_i(52) }"
url53: "{ %get_url_i(53) }"
url54: "{ %get_url_i(54) }"
url55: "{ %get_url_i(55) }"
url56: "{ %get_url_i(56) }"
url57: "{ %get_url_i(57) }"
url58: "{ %get_url_i(58) }"
url59: "{ %get_url_i(59) }"
url60: "{ %get_url_i(60) }"
url61: "{ %get_url_i(61) }"
url62: "{ %get_url_i(62) }"
url63: "{ %get_url_i(63) }"
url64: "{ %get_url_i(64) }"
url65: "{ %get_url_i(65) }"
url66: "{ %get_url_i(66) }"
url67: "{ %get_url_i(67) }"
url68: "{ %get_url_i(68) }"
url69: "{ %get_url_i(69) }"
url70: "{ %get_url_i(70) }"
url71: "{ %get_url_i(71) }"
url72: "{ %get_url_i(72) }"
url73: "{ %get_url_i(73) }"
url74: "{ %get_url_i(74) }"
url75: "{ %get_url_i(75) }"
url76: "{ %get_url_i(76) }"
url77: "{ %get_url_i(77) }"
url78: "{ %get_url_i(78) }"
url79: "{ %get_url_i(79) }"
url80: "{ %get_url_i(80) }"
url81: "{ %get_url_i(81) }"
url82: "{ %get_url_i(82) }"
url83: "{ %get_url_i(83) }"
url84: "{ %get_url_i(84) }"
url85: "{ %get_url_i(85) }"
url86: "{ %get_url_i(86) }"
url87: "{ %get_url_i(87) }"
url88: "{ %get_url_i(88) }"
url89: "{ %get_url_i(89) }"
url90: "{ %get_url_i(90) }"
url91: "{ %get_url_i(91) }"
url92: "{ %get_url_i(92) }"
url93: "{ %get_url_i(93) }"
url94: "{ %get_url_i(94) }"
url95: "{ %get_url_i(95) }"
url96: "{ %get_url_i(96) }"
url97: "{ %get_url_i(97) }"
url98: "{ %get_url_i(98) }"
url99: "{ %get_url_i(99) }"
url100: "{ %get_url_i(100) }"
urls: "{ %array_apply(%range(100, 1), %get_url_i) }"

2
src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml
View file

@ -8,7 +8,7 @@ presets:
urls:
# The first URL will be all the artist's tracks.
# Treat these as singles - an album with a single track
- url: "{url}/tracks"
- url: "{url}"
include_sibling_metadata: False
variables:
sc_track_album: "{title}"

356
src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml
View file

@ -124,7 +124,7 @@ presets:
- "_tv_show_collection_asserts"
download:
- url: "{collection_season_1_url}"
- url: "{ %array_at( %get_season_urls(collection_season_1_url), 0) }"
variables:
collection_season_number: "1"
collection_season_name: "{collection_season_1_name}"
@ -142,318 +142,493 @@ presets:
uid: "avatar_uncropped"
- name: "{tv_show_fanart_file_name}"
uid: "banner_uncropped"
- url: "{ %array_slice( %get_season_urls(collection_season_1_url), 1) }"
variables:
collection_season_number: "1"
collection_season_name: "{collection_season_1_name}"
- url: "{collection_season_2_url}"
- url: "{ %array_at( %get_season_urls(collection_season_2_url), 0) }"
variables:
collection_season_number: "2"
collection_season_name: "{collection_season_2_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_2_url), 1) }"
variables:
collection_season_number: "2"
collection_season_name: "{collection_season_2_name}"
- url: "{collection_season_3_url}"
- url: "{ %array_at( %get_season_urls(collection_season_3_url), 0) }"
variables:
collection_season_number: "3"
collection_season_name: "{collection_season_3_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_3_url), 1) }"
variables:
collection_season_number: "3"
collection_season_name: "{collection_season_3_name}"
- url: "{collection_season_4_url}"
- url: "{ %array_at( %get_season_urls(collection_season_4_url), 0) }"
variables:
collection_season_number: "4"
collection_season_name: "{collection_season_4_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_4_url), 1) }"
variables:
collection_season_number: "4"
collection_season_name: "{collection_season_4_name}"
- url: "{collection_season_5_url}"
- url: "{ %array_at( %get_season_urls(collection_season_5_url), 0) }"
variables:
collection_season_number: "5"
collection_season_name: "{collection_season_5_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_5_url), 1) }"
variables:
collection_season_number: "5"
collection_season_name: "{collection_season_5_name}"
- url: "{collection_season_6_url}"
- url: "{ %array_at( %get_season_urls(collection_season_6_url), 0) }"
variables:
collection_season_number: "6"
collection_season_name: "{collection_season_6_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_6_url), 1) }"
variables:
collection_season_number: "6"
collection_season_name: "{collection_season_6_name}"
- url: "{collection_season_7_url}"
- url: "{ %array_at( %get_season_urls(collection_season_7_url), 0) }"
variables:
collection_season_number: "7"
collection_season_name: "{collection_season_7_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_7_url), 1) }"
variables:
collection_season_number: "7"
collection_season_name: "{collection_season_7_name}"
- url: "{collection_season_8_url}"
- url: "{ %array_at( %get_season_urls(collection_season_8_url), 0) }"
variables:
collection_season_number: "8"
collection_season_name: "{collection_season_8_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_8_url), 1) }"
variables:
collection_season_number: "8"
collection_season_name: "{collection_season_8_name}"
- url: "{collection_season_9_url}"
- url: "{ %array_at( %get_season_urls(collection_season_9_url), 0) }"
variables:
collection_season_number: "9"
collection_season_name: "{collection_season_9_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_9_url), 1) }"
variables:
collection_season_number: "9"
collection_season_name: "{collection_season_9_name}"
- url: "{collection_season_10_url}"
- url: "{ %array_at( %get_season_urls(collection_season_10_url), 0) }"
variables:
collection_season_number: "10"
collection_season_name: "{collection_season_10_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_10_url), 1) }"
variables:
collection_season_number: "10"
collection_season_name: "{collection_season_10_name}"
- url: "{collection_season_11_url}"
- url: "{ %array_at( %get_season_urls(collection_season_11_url), 0) }"
variables:
collection_season_number: "11"
collection_season_name: "{collection_season_11_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_11_url), 1) }"
variables:
collection_season_number: "11"
collection_season_name: "{collection_season_11_name}"
- url: "{collection_season_12_url}"
- url: "{ %array_at( %get_season_urls(collection_season_12_url), 0) }"
variables:
collection_season_number: "12"
collection_season_name: "{collection_season_12_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_12_url), 1) }"
variables:
collection_season_number: "12"
collection_season_name: "{collection_season_12_name}"
- url: "{collection_season_13_url}"
- url: "{ %array_at( %get_season_urls(collection_season_13_url), 0) }"
variables:
collection_season_number: "13"
collection_season_name: "{collection_season_13_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_13_url), 1) }"
variables:
collection_season_number: "13"
collection_season_name: "{collection_season_13_name}"
- url: "{collection_season_14_url}"
- url: "{ %array_at( %get_season_urls(collection_season_14_url), 0) }"
variables:
collection_season_number: "14"
collection_season_name: "{collection_season_14_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_14_url), 1) }"
variables:
collection_season_number: "14"
collection_season_name: "{collection_season_14_name}"
- url: "{collection_season_15_url}"
- url: "{ %array_at( %get_season_urls(collection_season_15_url), 0) }"
variables:
collection_season_number: "15"
collection_season_name: "{collection_season_15_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_15_url), 1) }"
variables:
collection_season_number: "15"
collection_season_name: "{collection_season_15_name}"
- url: "{collection_season_16_url}"
- url: "{ %array_at( %get_season_urls(collection_season_16_url), 0) }"
variables:
collection_season_number: "16"
collection_season_name: "{collection_season_16_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_16_url), 1) }"
variables:
collection_season_number: "16"
collection_season_name: "{collection_season_16_name}"
- url: "{collection_season_17_url}"
- url: "{ %array_at( %get_season_urls(collection_season_17_url), 0) }"
variables:
collection_season_number: "17"
collection_season_name: "{collection_season_17_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_17_url), 1) }"
variables:
collection_season_number: "17"
collection_season_name: "{collection_season_17_name}"
- url: "{collection_season_18_url}"
- url: "{ %array_at( %get_season_urls(collection_season_18_url), 0) }"
variables:
collection_season_number: "18"
collection_season_name: "{collection_season_18_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_18_url), 1) }"
variables:
collection_season_number: "18"
collection_season_name: "{collection_season_18_name}"
- url: "{collection_season_19_url}"
- url: "{ %array_at( %get_season_urls(collection_season_19_url), 0) }"
variables:
collection_season_number: "19"
collection_season_name: "{collection_season_19_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_19_url), 1) }"
variables:
collection_season_number: "19"
collection_season_name: "{collection_season_19_name}"
- url: "{collection_season_20_url}"
- url: "{ %array_at( %get_season_urls(collection_season_20_url), 0) }"
variables:
collection_season_number: "20"
collection_season_name: "{collection_season_20_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_20_url), 1) }"
variables:
collection_season_number: "20"
collection_season_name: "{collection_season_20_name}"
- url: "{collection_season_21_url}"
- url: "{ %array_at( %get_season_urls(collection_season_21_url), 0) }"
variables:
collection_season_number: "21"
collection_season_name: "{collection_season_21_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_21_url), 1) }"
variables:
collection_season_number: "21"
collection_season_name: "{collection_season_21_name}"
- url: "{collection_season_22_url}"
- url: "{ %array_at( %get_season_urls(collection_season_22_url), 0) }"
variables:
collection_season_number: "22"
collection_season_name: "{collection_season_22_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_22_url), 1) }"
variables:
collection_season_number: "22"
collection_season_name: "{collection_season_22_name}"
- url: "{collection_season_23_url}"
- url: "{ %array_at( %get_season_urls(collection_season_23_url), 0) }"
variables:
collection_season_number: "23"
collection_season_name: "{collection_season_23_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_23_url), 1) }"
variables:
collection_season_number: "23"
collection_season_name: "{collection_season_23_name}"
- url: "{collection_season_24_url}"
- url: "{ %array_at( %get_season_urls(collection_season_24_url), 0) }"
variables:
collection_season_number: "24"
collection_season_name: "{collection_season_24_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_24_url), 1) }"
variables:
collection_season_number: "24"
collection_season_name: "{collection_season_24_name}"
- url: "{collection_season_25_url}"
- url: "{ %array_at( %get_season_urls(collection_season_25_url), 0) }"
variables:
collection_season_number: "25"
collection_season_name: "{collection_season_25_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_25_url), 1) }"
variables:
collection_season_number: "25"
collection_season_name: "{collection_season_25_name}"
- url: "{collection_season_26_url}"
- url: "{ %array_at( %get_season_urls(collection_season_26_url), 0) }"
variables:
collection_season_number: "26"
collection_season_name: "{collection_season_26_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_26_url), 1) }"
variables:
collection_season_number: "26"
collection_season_name: "{collection_season_26_name}"
- url: "{collection_season_27_url}"
- url: "{ %array_at( %get_season_urls(collection_season_27_url), 0) }"
variables:
collection_season_number: "27"
collection_season_name: "{collection_season_27_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_27_url), 1) }"
variables:
collection_season_number: "27"
collection_season_name: "{collection_season_27_name}"
- url: "{collection_season_28_url}"
- url: "{ %array_at( %get_season_urls(collection_season_28_url), 0) }"
variables:
collection_season_number: "28"
collection_season_name: "{collection_season_28_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_28_url), 1) }"
variables:
collection_season_number: "28"
collection_season_name: "{collection_season_28_name}"
- url: "{collection_season_29_url}"
- url: "{ %array_at( %get_season_urls(collection_season_29_url), 0) }"
variables:
collection_season_number: "29"
collection_season_name: "{collection_season_29_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_29_url), 1) }"
variables:
collection_season_number: "29"
collection_season_name: "{collection_season_29_name}"
- url: "{collection_season_30_url}"
- url: "{ %array_at( %get_season_urls(collection_season_30_url), 0) }"
variables:
collection_season_number: "30"
collection_season_name: "{collection_season_30_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_30_url), 1) }"
variables:
collection_season_number: "30"
collection_season_name: "{collection_season_30_name}"
- url: "{collection_season_31_url}"
- url: "{ %array_at( %get_season_urls(collection_season_31_url), 0) }"
variables:
collection_season_number: "31"
collection_season_name: "{collection_season_31_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_31_url), 1) }"
variables:
collection_season_number: "31"
collection_season_name: "{collection_season_31_name}"
- url: "{collection_season_32_url}"
- url: "{ %array_at( %get_season_urls(collection_season_32_url), 0) }"
variables:
collection_season_number: "32"
collection_season_name: "{collection_season_32_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_32_url), 1) }"
variables:
collection_season_number: "32"
collection_season_name: "{collection_season_32_name}"
- url: "{collection_season_33_url}"
- url: "{ %array_at( %get_season_urls(collection_season_33_url), 0) }"
variables:
collection_season_number: "33"
collection_season_name: "{collection_season_33_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_33_url), 1) }"
variables:
collection_season_number: "33"
collection_season_name: "{collection_season_33_name}"
- url: "{collection_season_34_url}"
- url: "{ %array_at( %get_season_urls(collection_season_34_url), 0) }"
variables:
collection_season_number: "34"
collection_season_name: "{collection_season_34_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_34_url), 1) }"
variables:
collection_season_number: "34"
collection_season_name: "{collection_season_34_name}"
- url: "{collection_season_35_url}"
- url: "{ %array_at( %get_season_urls(collection_season_35_url), 0) }"
variables:
collection_season_number: "35"
collection_season_name: "{collection_season_35_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_35_url), 1) }"
variables:
collection_season_number: "35"
collection_season_name: "{collection_season_35_name}"
- url: "{collection_season_36_url}"
- url: "{ %array_at( %get_season_urls(collection_season_36_url), 0) }"
variables:
collection_season_number: "36"
collection_season_name: "{collection_season_36_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_36_url), 1) }"
variables:
collection_season_number: "36"
collection_season_name: "{collection_season_36_name}"
- url: "{collection_season_37_url}"
- url: "{ %array_at( %get_season_urls(collection_season_37_url), 0) }"
variables:
collection_season_number: "37"
collection_season_name: "{collection_season_37_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_37_url), 1) }"
variables:
collection_season_number: "37"
collection_season_name: "{collection_season_37_name}"
- url: "{collection_season_38_url}"
- url: "{ %array_at( %get_season_urls(collection_season_38_url), 0) }"
variables:
collection_season_number: "38"
collection_season_name: "{collection_season_38_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_38_url), 1) }"
variables:
collection_season_number: "38"
collection_season_name: "{collection_season_38_name}"
- url: "{collection_season_39_url}"
- url: "{ %array_at( %get_season_urls(collection_season_39_url), 0) }"
variables:
collection_season_number: "39"
collection_season_name: "{collection_season_39_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_39_url), 1) }"
variables:
collection_season_number: "39"
collection_season_name: "{collection_season_39_name}"
- url: "{collection_season_40_url}"
- url: "{ %array_at( %get_season_urls(collection_season_40_url), 0) }"
variables:
collection_season_number: "40"
collection_season_name: "{collection_season_40_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_40_url), 1) }"
variables:
collection_season_number: "40"
collection_season_name: "{collection_season_40_name}"
# Place season 0 at end
- url: "{ %array_at( %get_season_urls(collection_season_0_url), 0) }"
variables:
collection_season_number: "0"
collection_season_name: "{collection_season_0_name}"
playlist_thumbnails:
- name: "{season_poster_file_name}"
uid: "latest_entry"
- url: "{ %array_slice( %get_season_urls(collection_season_0_url), 1) }"
variables:
collection_season_number: "0"
collection_season_name: "{collection_season_0_name}"
output_directory_nfo_tags:
tags:
@ -625,6 +800,7 @@ presets:
collection_season_38_name: "{s38_name}"
collection_season_39_name: "{s39_name}"
collection_season_40_name: "{s40_name}"
collection_season_0_name: "{s00_name}"
# Legacy url variable
collection_season_1_url: "{s01_url}"
@ -667,6 +843,7 @@ presets:
collection_season_38_url: "{s38_url}"
collection_season_39_url: "{s39_url}"
collection_season_40_url: "{s40_url}"
collection_season_0_url: "{s00_url}"
s01_name: ""
s02_name: ""
@ -708,6 +885,7 @@ presets:
s38_name: ""
s39_name: ""
s40_name: ""
s00_name: ""
s01_url: ""
s02_url: ""
@ -749,13 +927,22 @@ presets:
s38_url: ""
s39_url: ""
s40_url: ""
s00_url: ""
"%get_season_urls": >-
{ %if( %is_array( $0 ), $0, [ $0 ] ) }
# $0 - season url variable
# $1 - get the i'th url from the array
"%get_season_url": >-
{ %array_at( %get_season_urls($0), $1, "" ) }
_tv_show_collection_bilateral:
preset:
- "_url_bilateral_overrides"
download:
- url: "{ %bilateral_url(collection_season_1_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_1_url), %bilateral_url) }"
variables:
collection_season_number: "1"
collection_season_name: "{collection_season_1_name}"
@ -763,7 +950,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_2_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_2_url), %bilateral_url) }"
variables:
collection_season_number: "2"
collection_season_name: "{collection_season_2_name}"
@ -771,7 +958,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_3_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_3_url), %bilateral_url) }"
variables:
collection_season_number: "3"
collection_season_name: "{collection_season_3_name}"
@ -779,7 +966,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_4_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_4_url), %bilateral_url) }"
variables:
collection_season_number: "4"
collection_season_name: "{collection_season_4_name}"
@ -787,7 +974,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_5_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_5_url), %bilateral_url) }"
variables:
collection_season_number: "5"
collection_season_name: "{collection_season_5_name}"
@ -795,7 +982,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_6_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_6_url), %bilateral_url) }"
variables:
collection_season_number: "6"
collection_season_name: "{collection_season_6_name}"
@ -803,7 +990,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_7_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_7_url), %bilateral_url) }"
variables:
collection_season_number: "7"
collection_season_name: "{collection_season_7_name}"
@ -811,7 +998,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_8_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_8_url), %bilateral_url) }"
variables:
collection_season_number: "8"
collection_season_name: "{collection_season_8_name}"
@ -819,7 +1006,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_9_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_9_url), %bilateral_url) }"
variables:
collection_season_number: "9"
collection_season_name: "{collection_season_9_name}"
@ -827,7 +1014,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_10_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_10_url), %bilateral_url) }"
variables:
collection_season_number: "10"
collection_season_name: "{collection_season_10_name}"
@ -835,7 +1022,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_11_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_11_url), %bilateral_url) }"
variables:
collection_season_number: "11"
collection_season_name: "{collection_season_11_name}"
@ -843,7 +1030,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_12_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_12_url), %bilateral_url) }"
variables:
collection_season_number: "12"
collection_season_name: "{collection_season_12_name}"
@ -851,7 +1038,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_13_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_13_url), %bilateral_url) }"
variables:
collection_season_number: "13"
collection_season_name: "{collection_season_13_name}"
@ -859,7 +1046,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_14_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_14_url), %bilateral_url) }"
variables:
collection_season_number: "14"
collection_season_name: "{collection_season_14_name}"
@ -867,7 +1054,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_15_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_15_url), %bilateral_url) }"
variables:
collection_season_number: "15"
collection_season_name: "{collection_season_15_name}"
@ -875,7 +1062,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_16_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_16_url), %bilateral_url) }"
variables:
collection_season_number: "16"
collection_season_name: "{collection_season_16_name}"
@ -883,7 +1070,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_17_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_17_url), %bilateral_url) }"
variables:
collection_season_number: "17"
collection_season_name: "{collection_season_17_name}"
@ -891,7 +1078,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_18_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_18_url), %bilateral_url) }"
variables:
collection_season_number: "18"
collection_season_name: "{collection_season_18_name}"
@ -899,7 +1086,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_19_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_19_url), %bilateral_url) }"
variables:
collection_season_number: "19"
collection_season_name: "{collection_season_19_name}"
@ -907,7 +1094,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_20_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_20_url), %bilateral_url) }"
variables:
collection_season_number: "20"
collection_season_name: "{collection_season_20_name}"
@ -915,7 +1102,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_21_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_21_url), %bilateral_url) }"
variables:
collection_season_number: "21"
collection_season_name: "{collection_season_21_name}"
@ -923,7 +1110,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_22_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_22_url), %bilateral_url) }"
variables:
collection_season_number: "22"
collection_season_name: "{collection_season_22_name}"
@ -931,7 +1118,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_23_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_23_url), %bilateral_url) }"
variables:
collection_season_number: "23"
collection_season_name: "{collection_season_23_name}"
@ -939,7 +1126,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_24_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_24_url), %bilateral_url) }"
variables:
collection_season_number: "24"
collection_season_name: "{collection_season_24_name}"
@ -947,7 +1134,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_25_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_25_url), %bilateral_url) }"
variables:
collection_season_number: "25"
collection_season_name: "{collection_season_25_name}"
@ -955,7 +1142,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_26_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_26_url), %bilateral_url) }"
variables:
collection_season_number: "26"
collection_season_name: "{collection_season_26_name}"
@ -963,7 +1150,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_27_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_27_url), %bilateral_url) }"
variables:
collection_season_number: "27"
collection_season_name: "{collection_season_27_name}"
@ -971,7 +1158,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_28_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_28_url), %bilateral_url) }"
variables:
collection_season_number: "28"
collection_season_name: "{collection_season_28_name}"
@ -979,7 +1166,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_29_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_29_url), %bilateral_url) }"
variables:
collection_season_number: "29"
collection_season_name: "{collection_season_29_name}"
@ -987,7 +1174,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_30_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_30_url), %bilateral_url) }"
variables:
collection_season_number: "30"
collection_season_name: "{collection_season_30_name}"
@ -995,7 +1182,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_31_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_31_url), %bilateral_url) }"
variables:
collection_season_number: "31"
collection_season_name: "{collection_season_31_name}"
@ -1003,7 +1190,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_32_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_32_url), %bilateral_url) }"
variables:
collection_season_number: "32"
collection_season_name: "{collection_season_32_name}"
@ -1011,7 +1198,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_33_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_33_url), %bilateral_url) }"
variables:
collection_season_number: "33"
collection_season_name: "{collection_season_33_name}"
@ -1019,7 +1206,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_34_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_34_url), %bilateral_url) }"
variables:
collection_season_number: "34"
collection_season_name: "{collection_season_34_name}"
@ -1027,7 +1214,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_35_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_35_url), %bilateral_url) }"
variables:
collection_season_number: "35"
collection_season_name: "{collection_season_35_name}"
@ -1035,7 +1222,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_36_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_36_url), %bilateral_url) }"
variables:
collection_season_number: "36"
collection_season_name: "{collection_season_36_name}"
@ -1043,7 +1230,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_37_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_37_url), %bilateral_url) }"
variables:
collection_season_number: "37"
collection_season_name: "{collection_season_37_name}"
@ -1051,7 +1238,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_38_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_38_url), %bilateral_url) }"
variables:
collection_season_number: "38"
collection_season_name: "{collection_season_38_name}"
@ -1059,7 +1246,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_39_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_39_url), %bilateral_url) }"
variables:
collection_season_number: "39"
collection_season_name: "{collection_season_39_name}"
@ -1067,7 +1254,7 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
- url: "{ %bilateral_url(collection_season_40_url) }"
- url: "{ %array_apply( %get_season_urls(collection_season_40_url), %bilateral_url) }"
variables:
collection_season_number: "40"
collection_season_name: "{collection_season_40_name}"
@ -1075,6 +1262,15 @@ presets:
ytdl_options:
playlist_items: "-1:0:-1"
# Season 0 at end (to download first)
- url: "{ %array_apply( %get_season_urls(collection_season_0_url), %bilateral_url) }"
variables:
collection_season_number: "0"
collection_season_name: "{collection_season_0_name}"
download_reverse: False
ytdl_options:
playlist_items: "-1:0:-1"
_tv_show_collection_asserts:
overrides:
url: ""

17
src/ytdl_sub/script/functions/numeric_functions.py
View file

@ -1,5 +1,7 @@
import math
from typing import Optional
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Float
from ytdl_sub.script.types.resolvable import Integer
@ -96,3 +98,18 @@ class NumericFunctions:
Returns min of all values.
"""
return _to_numeric(min(val.value for val in values))
@staticmethod
def range(
end: Integer, start: Optional[Integer] = None, step: Optional[Integer] = None
) -> Array:
"""
:description:
Returns the desired range of Integers in the form of an Array.
"""
if start is None:
start = Integer(0)
if step is None:
step = Integer(1)
return Array(value=[Integer(idx) for idx in range(start.value, end.value, step.value)])

17
src/ytdl_sub/script/functions/print_functions.py
View file

@ -34,9 +34,8 @@ class PrintFunctions:
) -> ReturnableArgument:
"""
:description:
Print the ``message`` and return ``passthrough``.
Optionally can pass level, where < 0 is debug, 0 is info, 1 is warning, > 1 is error.
Defaults to info.
Log the ``message`` and return ``passthrough``. Optionally can pass level,
where < 0 is debug, 0 is info, 1 is warning, > 1 is error. (default ``0``)
"""
_log(message=message, level=level)
return passthrough
@ -47,9 +46,9 @@ class PrintFunctions:
) -> ReturnableArgument:
"""
:description:
Print the ``message`` if ``passthrough`` evaluates to ``true``. Return ``passthrough``.
Optionally can pass level, where < 0 is debug, 0 is info, 1 is warning, > 1 is error.
Defaults to info.
Log the ``message`` if ``passthrough`` evaluates to ``true``. Return
``passthrough``. Optionally can pass level, where < 0 is debug, 0 is info, 1
is warning, > 1 is error. (default ``0``)
"""
if passthrough.value:
_log(message=message, level=level)
@ -61,9 +60,9 @@ class PrintFunctions:
) -> ReturnableArgument:
"""
:description:
Print the ``message`` if ``passthrough`` evaluates to ``false``. Return ``passthrough``.
Optionally can pass level, where < 0 is debug, 0 is info, 1 is warning, > 1 is error.
Defaults to info.
Log the ``message`` if ``passthrough`` evaluates to ``false``. Return
``passthrough``. Optionally can pass level, where < 0 is debug, 0 is info, 1
is warning, > 1 is error. (default ``0``)
"""
if not passthrough.value:
_log(message=message, level=level)

7
src/ytdl_sub/script/parser.py
View file

@ -31,6 +31,8 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
from ytdl_sub.script.utils.exceptions import InvalidVariableName
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
from ytdl_sub.script.utils.name_validation import is_function
from ytdl_sub.script.utils.name_validation import to_function_name
from ytdl_sub.script.utils.name_validation import validate_variable_name
# pylint: disable=invalid-name
@ -144,6 +146,9 @@ class _Parser:
):
self._text = text
self._name = name
if name and is_function(name):
self._name = to_function_name(name)
self._custom_function_names = custom_function_names
self._variable_names = variable_names
self._pos = 0
@ -605,7 +610,7 @@ def parse(
name=name,
custom_function_names=custom_function_names,
variable_names=variable_names,
).ast
).ast.maybe_resolvable_casted()
# pylint: enable=invalid-name

366
src/ytdl_sub/script/script.py
View file

@ -1,4 +1,5 @@
# pylint: disable=missing-raises-doc
from collections import defaultdict
from typing import Dict
from typing import List
from typing import Optional
@ -7,40 +8,28 @@ from typing import Set
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script_output import ScriptOutput
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.types.variable import FunctionArgument
from ytdl_sub.script.types.variable import Variable
from ytdl_sub.script.types.variable_dependency import VariableDependency
from ytdl_sub.script.utils.exceptions import UNREACHABLE
from ytdl_sub.script.utils.exceptions import CycleDetected
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.script.utils.name_validation import is_function
from ytdl_sub.script.utils.name_validation import to_function_definition_name
from ytdl_sub.script.utils.name_validation import to_function_name
from ytdl_sub.script.utils.name_validation import validate_variable_name
from ytdl_sub.script.utils.type_checking import FunctionSpec
def _is_function(override_name: str):
return override_name.startswith("%")
def _function_name(function_key: str) -> str:
"""
Drop the % in %custom_function
"""
return function_key[1:]
def _to_function_definition_name(function_key: str) -> str:
"""
Add % in %custom_function
"""
return f"%{function_key}"
class Script:
"""
Takes a dictionary of both
@ -49,43 +38,71 @@ class Script:
``{ %custom_function: syntax }``
"""
def _ensure_no_cycle(
def _throw_cycle_error(
self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree]
):
if dep not in definitions:
return # does not exist, will throw downstream in parser
type_name, pre = (
("custom functions", "%") if definitions is self._functions else ("variables", "")
)
cycle_deps = [name] + deps + [dep]
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
if name in deps + [dep]:
type_name, pre = (
("custom functions", "%") if definitions is self._functions else ("variables", "")
)
cycle_deps = [name] + deps + [dep]
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}")
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}")
def _traverse_variable_dependencies(
self,
variable_name: str,
variable_dependency: SyntaxTree,
deps: List[str],
ensured: Dict[str, Set[str]],
) -> None:
for dep in variable_dependency.variables:
self._ensure_no_cycle(
name=variable_name, dep=dep.name, deps=deps, definitions=self._variables
)
if variable_name == dep.name:
self._throw_cycle_error(
name=variable_name, dep=dep.name, deps=deps, definitions=self._variables
)
if dep.name in ensured[variable_name]:
continue
self._traverse_variable_dependencies(
variable_name=variable_name,
variable_dependency=self._variables[dep.name],
deps=deps + [dep.name],
ensured=ensured,
)
ensured[variable_name].add(dep.name)
for custom_func in variable_dependency.custom_function_dependencies(
custom_function_definitions=self._functions
):
for dep in self._functions[custom_func.name].variables:
if variable_name == dep.name:
self._throw_cycle_error(
name=variable_name,
dep=dep.name,
deps=deps + [custom_func.definition_name()],
definitions=self._variables,
)
if dep.name in ensured[variable_name]:
continue
self._traverse_variable_dependencies(
variable_name=variable_name,
variable_dependency=self._variables[dep.name],
deps=deps + [custom_func.definition_name(), dep.name],
ensured=ensured,
)
ensured[variable_name].add(dep.name)
def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]):
ensured: Dict[str, Set[str]] = defaultdict(set)
for variable_name, variable_definition in variables.items():
self._traverse_variable_dependencies(
variable_name=variable_name,
variable_dependency=variable_definition,
deps=[],
ensured=ensured,
)
def _traverse_custom_function_dependencies(
@ -95,9 +112,10 @@ class Script:
deps: List[str],
) -> None:
for dep in custom_function_dependency.custom_functions:
self._ensure_no_cycle(
name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions
)
if custom_function_name == dep.name:
self._throw_cycle_error(
name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions
)
self._traverse_custom_function_dependencies(
custom_function_name=custom_function_name,
custom_function_dependency=self._functions[dep.name],
@ -240,23 +258,23 @@ class Script:
def __init__(self, script: Dict[str, str]):
function_names: Set[str] = {
_function_name(name) for name in script.keys() if _is_function(name)
to_function_name(name) for name in script.keys() if is_function(name)
}
variable_names: Set[str] = {
validate_variable_name(name) for name in script.keys() if not _is_function(name)
validate_variable_name(name) for name in script.keys() if not is_function(name)
}
self._functions: Dict[str, SyntaxTree] = {
# custom_function_name must be passed to properly type custom function
# arguments uniquely if they're nested (i.e. $0 to $custom_func___0)
_function_name(function_key): parse(
to_function_name(function_key): parse(
text=function_value,
name=_function_name(function_key),
name=to_function_name(function_key),
custom_function_names=function_names,
variable_names=variable_names,
)
for function_key, function_value in script.items()
if _is_function(function_key)
if is_function(function_key)
}
self._variables: Dict[str, SyntaxTree] = {
@ -267,13 +285,13 @@ class Script:
variable_names=variable_names,
)
for variable_key, variable_value in script.items()
if not _is_function(variable_key)
if not is_function(variable_key)
}
self._validate()
def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None:
for variable_name, resolved in resolved_variables.items():
self._variables[variable_name] = SyntaxTree(ast=[resolved])
self._variables[variable_name] = ResolvedSyntaxTree(ast=[resolved])
def _recursive_get_unresolved_output_filter_variables(
self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable]
@ -302,14 +320,21 @@ class Script:
unresolvable=unresolvable,
)
for lambda_func in current_var.lambdas:
if lambda_func.value in self._functions:
subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables(
current_var=self._functions[lambda_func.value],
subset_to_resolve=subset_to_resolve,
unresolvable=unresolvable,
)
return subset_to_resolve
def _get_unresolved_output_filter(
self,
unresolved: Dict[Variable, SyntaxTree],
output_filter: Set[str],
unresolvable: Set[Variable],
) -> Dict[Variable, SyntaxTree]:
) -> Set[str]:
"""
When an output filter is applied, only a subset of variables that the filter
depends on need to be resolved.
@ -330,7 +355,7 @@ class Script:
unresolvable=unresolvable,
)
return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve}
return subset_to_resolve
def _resolve(
self,
@ -366,18 +391,21 @@ class Script:
unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})}
unresolved_filter = set(resolved.keys()).union(unresolvable)
unresolved: Dict[Variable, SyntaxTree] = {
Variable(name): ast
for name, ast in self._variables.items()
if Variable(name) not in unresolved_filter
}
if output_filter:
unresolved = self._get_unresolved_output_filter(
unresolved=unresolved,
output_filter=output_filter,
unresolvable=unresolvable,
)
unresolved = {
Variable(name): self._variables[name]
for name in self._get_unresolved_output_filter(
output_filter=output_filter,
unresolvable=unresolvable,
)
}
else:
unresolved = {
Variable(name): ast
for name, ast in self._variables.items()
if Variable(name) not in unresolved_filter
}
while unresolved:
unresolved_count: int = len(unresolved)
@ -398,8 +426,8 @@ class Script:
# Otherwise, if it has dependencies that are all resolved, then
# resolve the definition
elif not definition.is_subset_of(
variables=resolved.keys(), custom_function_definitions=self._functions
elif definition.is_subset_of(
variables=resolved, custom_function_definitions=self._functions
):
resolved[variable] = unresolved[variable].resolve(
resolved_variables=resolved,
@ -482,12 +510,12 @@ class Script:
added_variables_to_validate: Set[str] = set()
functions_to_add = {
_function_name(name): definition
to_function_name(name): definition
for name, definition in variables.items()
if _is_function(name)
if is_function(name)
}
variables_to_add = {
name: definition for name, definition in variables.items() if not _is_function(name)
name: definition for name, definition in variables.items() if not is_function(name)
}
custom_function_names = set(self._functions.keys()) | functions_to_add.keys()
@ -517,11 +545,52 @@ class Script:
return self
def add_parsed(self, variables: Dict[str, SyntaxTree]) -> "Script":
"""
Adds already parsed, new variables to the script.
Parameters
----------
variables
Mapping containing variable name to definition.
Returns
-------
Script
self
"""
added_variables_to_validate: Set[str] = set()
functions_to_add = {
to_function_name(name): definition
for name, definition in variables.items()
if is_function(name)
}
variables_to_add = {
name: definition for name, definition in variables.items() if not is_function(name)
}
for definitions in [functions_to_add, variables_to_add]:
for name, parsed in definitions.items():
if parsed.maybe_resolvable is None:
added_variables_to_validate.add(name)
if name in functions_to_add:
self._functions[name] = parsed
else:
self._variables[name] = parsed
if added_variables_to_validate:
self._validate(added_variables=added_variables_to_validate)
return self
def resolve_once(
self,
variable_definitions: Dict[str, str],
resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None,
update: bool = False,
) -> Dict[str, Resolvable]:
"""
Given a new set of variable definitions, resolve them using the Script, but do not
@ -536,6 +605,8 @@ class Script:
unresolvable
Optional. Unresolvable variables that will be ignored in resolution, including all
variables with a dependency to them.
update
Whether to update the script's state with resolved variables. Defaults to False.
Returns
-------
@ -548,11 +619,51 @@ class Script:
pre_resolved=resolved,
unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())),
update=update,
).output
finally:
for name in variable_definitions.keys():
if name in self._variables:
del self._variables[name]
self._variables.pop(name, None)
def resolve_once_parsed(
self,
variable_definitions: Dict[str, SyntaxTree],
resolved: Optional[Dict[str, Resolvable]] = None,
unresolvable: Optional[Set[str]] = None,
update: bool = False,
) -> Dict[str, Resolvable]:
"""
Given a new set of variable definitions, resolve them using the Script, but do not
add them to the Script itself.
Parameters
----------
variable_definitions
Variables to resolve, but not store in the Script
resolved
Optional. Pre-resolved variables that should be used instead of what is in the script.
unresolvable
Optional. Unresolvable variables that will be ignored in resolution, including all
variables with a dependency to them.
update
Whether to update the script's state with resolved variables. Defaults to False.
Returns
-------
Dict[str, Resolvable]
Dict containing the variable names to their resolved values.
"""
try:
self.add_parsed(variable_definitions)
return self._resolve(
pre_resolved=resolved,
unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())),
update=update,
).output
finally:
for name in variable_definitions.keys():
self._variables.pop(name, None)
def get(self, variable_name: str) -> Resolvable:
"""
@ -581,6 +692,18 @@ class Script:
raise RuntimeException(f"Tried to get unresolved variable {variable_name}")
def definition_of(self, name: str) -> SyntaxTree:
"""
Returns
-------
The definition of the variable or function.
"""
if name.startswith("%") and name[1:] in self._functions:
return self._functions[name[1:]]
if name in self._variables:
return self._variables[name]
raise RuntimeException(f"Tried to get non-existent definition with name {name}")
@property
def variable_names(self) -> Set[str]:
"""
@ -599,4 +722,121 @@ class Script:
Set[str]
Names of all functions within the Script.
"""
return set(_to_function_definition_name(name) for name in self._functions.keys())
return set(to_function_definition_name(name) for name in self._functions.keys())
def _resolve_partial_loop(
self,
output_filter: Optional[Set[str]],
resolved: Dict[Variable, Resolvable],
unresolved: Dict[Variable, Argument],
unresolvable: Optional[Set[str]],
):
to_partially_resolve: Set[Variable] = (
{Variable(name) for name in output_filter} if output_filter else set(unresolved.keys())
)
partially_resolved = True
while partially_resolved:
partially_resolved = False
for variable in list(to_partially_resolve):
definition = unresolved[variable]
maybe_resolved = definition
if isinstance(definition, Variable) and definition.name not in unresolvable:
if definition in resolved:
maybe_resolved = resolved[definition]
elif definition in unresolved:
maybe_resolved = unresolved[definition]
else:
raise UNREACHABLE
elif isinstance(definition, VariableDependency):
maybe_resolved = definition.partial_resolve(
resolved_variables=resolved,
unresolved_variables=unresolved,
custom_functions=self._functions,
)
if isinstance(maybe_resolved, Resolvable):
resolved[variable] = maybe_resolved
del unresolved[variable]
to_partially_resolve.remove(variable)
partially_resolved = True
else:
unresolved[variable] = maybe_resolved
# If the definition changed, then the script changed
# which means we can iterate again
partially_resolved |= definition != maybe_resolved
def _resolve_partial(
self,
unresolvable: Optional[Set[str]] = None,
output_filter: Optional[Set[str]] = None,
) -> Dict[str, SyntaxTree]:
"""
Returns
-------
New (deep-copied) script that resolves inner variables as much
as possible.
"""
unresolvable: Set[str] = unresolvable or {}
resolved: Dict[Variable, Resolvable] = {}
unresolved: Dict[Variable, Argument] = {
Variable(name): definition
for name, definition in self._variables.items()
if name not in unresolvable
}
self._resolve_partial_loop(
output_filter=output_filter,
resolved=resolved,
unresolved=unresolved,
unresolvable=unresolvable,
)
if output_filter:
out: Dict[str, SyntaxTree] = {}
for name in output_filter:
variable_name = Variable(name)
if variable_name in resolved:
out[name] = ResolvedSyntaxTree(ast=[resolved[variable_name]])
else:
out[name] = SyntaxTree(ast=[unresolved[variable_name]])
return out
return {
var.name: ResolvedSyntaxTree(ast=[definition]) for var, definition in resolved.items()
} | {var.name: SyntaxTree(ast=[definition]) for var, definition in unresolved.items()}
def resolve_partial(
self,
unresolvable: Optional[Set[str]] = None,
output_filter: Optional[Set[str]] = None,
) -> "Script":
"""
Updates the internal script to resolve as much as possible.
"""
out = self._resolve_partial(unresolvable=unresolvable, output_filter=output_filter)
for var_name, definition in out.items():
self._variables[var_name] = definition
return self
def resolve_partial_once(
self, variable_definitions: Dict[str, SyntaxTree], unresolvable: Optional[Set[str]] = None
) -> Dict[str, SyntaxTree]:
"""
Partially resolves the input variable definitions as much as possible.
"""
try:
self.add_parsed(variable_definitions)
return self._resolve_partial(
unresolvable=unresolvable,
output_filter=set(list(variable_definitions.keys())),
)
finally:
for name in variable_definitions.keys():
self._variables.pop(name, None)

23
src/ytdl_sub/script/types/array.py
View file

@ -28,7 +28,7 @@ class UnresolvedArray(_Array, VariableDependency, FutureResolvable):
value: List[Argument]
@property
def _iterable_arguments(self) -> List[Argument]:
def iterable_arguments(self) -> List[Argument]:
return self.value
def resolve(
@ -47,6 +47,27 @@ class UnresolvedArray(_Array, VariableDependency, FutureResolvable):
]
)
def partial_resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, VariableDependency],
) -> Argument | Resolvable:
maybe_resolvable_values, is_resolvable = VariableDependency.try_partial_resolve(
args=self.value,
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
out = UnresolvedArray(value=maybe_resolvable_values)
if is_resolvable:
return out.resolve(
resolved_variables=resolved_variables, custom_functions=custom_functions
)
return out
def future_resolvable_type(self) -> Type[Resolvable]:
return Array

189
src/ytdl_sub/script/types/function.py
View file

@ -1,10 +1,10 @@
import copy
import functools
from abc import ABC
from dataclasses import dataclass
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
from typing import Union
@ -12,9 +12,11 @@ from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.array import UnresolvedArray
from ytdl_sub.script.types.resolvable import Argument
from ytdl_sub.script.types.resolvable import Boolean
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
from ytdl_sub.script.types.resolvable import FunctionType
from ytdl_sub.script.types.resolvable import FutureResolvable
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import Lambda
from ytdl_sub.script.types.resolvable import NamedCustomFunction
from ytdl_sub.script.types.resolvable import Resolvable
@ -36,7 +38,7 @@ from ytdl_sub.script.utils.type_checking import is_union
@dataclass(frozen=True)
class Function(FunctionType, VariableDependency, ABC):
@property
def _iterable_arguments(self) -> List[Argument]:
def iterable_arguments(self) -> List[Argument]:
return self.args
@ -58,27 +60,79 @@ class CustomFunction(Function, NamedCustomFunction):
# Should be validated in the Script
raise UNREACHABLE
resolved_variables_with_args = copy.deepcopy(resolved_variables)
function_args: List[FunctionArgument] = []
for i, arg in enumerate(resolved_args):
function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name)
if function_arg in resolved_variables_with_args:
if function_arg in resolved_variables:
# function args should always be unique since they are only defined once
# in the custom function as %custom_function_name___idx
# and returned as a set from each custom function.
raise UNREACHABLE
resolved_variables_with_args[function_arg] = arg
resolved_variables[function_arg] = arg
function_args.append(function_arg)
return custom_functions[self.name].resolve(
resolved_variables=resolved_variables_with_args,
out = custom_functions[self.name].resolve(
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
for function_arg in function_args:
del resolved_variables[function_arg]
return out
# Implies the custom function does not exist. This should have
# been checked in the parser with
raise UNREACHABLE
def partial_resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, VariableDependency],
) -> Argument | Resolvable:
maybe_resolvable_args, _ = VariableDependency.try_partial_resolve(
args=self.args,
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
for i in range(len(self.args)):
function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name)
function_value = maybe_resolvable_args[i]
if isinstance(function_value, Resolvable):
resolved_variables[function_arg] = function_value
else:
unresolved_variables[function_arg] = function_value
assert len(custom_functions[self.name].iterable_arguments) == 1
custom_function_definition = custom_functions[self.name].iterable_arguments[0]
maybe_resolvable_custom_function, is_resolvable = VariableDependency.try_partial_resolve(
args=[custom_function_definition],
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
for i in range(len(self.args)):
function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name)
if isinstance(maybe_resolvable_args[i], Resolvable):
del resolved_variables[function_arg]
else:
del unresolved_variables[function_arg]
if is_resolvable:
return maybe_resolvable_custom_function[0]
# Did not resolve custom function arguments, do not proceed
return CustomFunction(name=self.name, args=maybe_resolvable_args)
class BuiltInFunction(Function, BuiltInFunctionType):
def validate_args(self) -> "BuiltInFunction":
@ -307,5 +361,126 @@ class BuiltInFunction(Function, BuiltInFunctionType):
f"Runtime error occurred when executing the function %{self.name}: {str(exc)}"
) from exc
def _partial_resolve_conditional(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, "VariableDependency"],
):
"""
If the conditional partially resolvable enough to warrant evaluation,
perform it here.
"""
if self.name == "if":
maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve(
args=[self.args[0]],
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
if is_resolvable:
boolean_output = maybe_resolvable_arg[0]
assert isinstance(boolean_output, Boolean)
return self.args[1] if boolean_output.native else self.args[2]
if self.name == "elif":
for idx in range(0, len(self.args), 2):
maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve(
args=[self.args[idx]],
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
if is_resolvable:
boolean_output = maybe_resolvable_arg[0]
assert isinstance(boolean_output, Boolean)
if boolean_output.native:
return self.args[idx + 1]
else:
break
if self.name == "assert_then":
maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve(
args=[self.args[0]],
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
if is_resolvable:
boolean_output = maybe_resolvable_arg[0]
assert isinstance(boolean_output, Boolean)
if boolean_output.native:
return self.args[1]
return self
def _try_optimized_partial_resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, "VariableDependency"],
) -> Optional[Argument]:
"""
If a function has enough (but not all) resolved parameters to warrant a return,
perform it here.
"""
if self.name == "array_at":
if (
isinstance(self.args[0], UnresolvedArray)
and isinstance(self.args[1], Integer)
and len(self.args[0].value) >= self.args[1].value
):
maybe_resolvable_values, _ = VariableDependency.try_partial_resolve(
args=[self.args[0].value[self.args[1].value]],
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
return maybe_resolvable_values[0]
return None
def partial_resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, VariableDependency],
) -> Argument | Resolvable:
conditional_return_args = self.function_spec.conditional_arg_indices(
num_input_args=len(self.args)
)
if conditional_return_args:
return self._partial_resolve_conditional(
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
if partial_resolved := self._try_optimized_partial_resolve(
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
):
return partial_resolved
maybe_resolvable_values, is_resolvable = VariableDependency.try_partial_resolve(
args=self.args,
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
out = BuiltInFunction(name=self.name, args=maybe_resolvable_values)
if is_resolvable:
return out.resolve(
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
return out
def __hash__(self):
return hash((self.name, *self.args))

31
src/ytdl_sub/script/types/map.py
View file

@ -31,7 +31,7 @@ class UnresolvedMap(_Map, VariableDependency, FutureResolvable):
value: Dict[Argument, Argument]
@property
def _iterable_arguments(self) -> List[Argument]:
def iterable_arguments(self) -> List[Argument]:
return list(itertools.chain(*self.value.items()))
def resolve(
@ -55,6 +55,35 @@ class UnresolvedMap(_Map, VariableDependency, FutureResolvable):
return Map(output)
def partial_resolve(
self,
resolved_variables: Dict[Variable, Resolvable],
unresolved_variables: Dict[Variable, Argument],
custom_functions: Dict[str, VariableDependency],
) -> Argument | Resolvable:
maybe_resolvable_keys, is_keys_resolvable = VariableDependency.try_partial_resolve(
args=self.value.keys(),
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
maybe_resolvable_values, is_values_resolvable = VariableDependency.try_partial_resolve(
args=self.value.values(),
resolved_variables=resolved_variables,
unresolved_variables=unresolved_variables,
custom_functions=custom_functions,
)
out = UnresolvedMap(value=dict(zip(maybe_resolvable_keys, maybe_resolvable_values)))
if is_keys_resolvable and is_values_resolvable:
return out.resolve(
resolved_variables=resolved_variables,
custom_functions=custom_functions,
)
return out
def future_resolvable_type(self) -> Type[Resolvable]:
return Map

8
src/ytdl_sub/script/types/resolvable.py
View file

@ -193,6 +193,14 @@ class NamedCustomFunction(NamedArgument, ABC):
class ParsedCustomFunction(NamedCustomFunction):
num_input_args: int
def definition_name(self) -> str:
"""
Returns
-------
The function definition name, including the %
"""
return f"%{self.name}"
@dataclass(frozen=True)
class FunctionType(NamedArgument, ABC):

Some files were not shown because too many files have changed in this diff Show more