danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into j/music-extras

Browse source
This commit is contained in:
Jesse Bannon 2023-12-29 22:38:47 -08:00
commit d14506dbef
237 changed files with 16190 additions and 4643 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -149,3 +149,5 @@ docker/testing/volumes
ffmpeg.exe
ffprobe.exe
tools/docgen/out

7
.readthedocs.yaml
View file

@ -1,16 +1,17 @@
version: 2
build:
os: ubuntu-20.04
os: "ubuntu-22.04"
tools:
python: "3.10"
sphinx:
configuration: docs/conf.py
configuration: docs/source/conf.py
python:
install:
- requirements: docs/source/requirements.txt
- method: pip
path: .
extra_requirements:
- docs
- docs

8
Makefile
View file

@ -17,12 +17,10 @@ lint:
@-isort .
@-black .
@-pylint src/
@-pydocstyle src/*
check_lint:
isort . --check-only --diff \
&& black . --check \
&& pylint src/ \
&& pydocstyle src/*
&& pylint src/
wheel: clean
$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py)
cat src/ytdl_sub/__init__.py
@ -41,14 +39,14 @@ executable: clean
pyinstaller ytdl-sub.spec
mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX}
docs:
sphinx-build -a -b html docs docs/_html
sphinx-build -M html docs/source/ docs/build/
clean:
rm -rf \
.pytest_cache/ \
build/ \
dist/ \
src/ytdl_sub.egg-info/ \
docs/_html/ \
docs/build/ \
.coverage \
docker/root/*.whl \
docker/root/defaults/examples \

35
README.md
View file

@ -62,9 +62,10 @@ __preset__:
# Root folder of all ytdl-sub Music Videos
music_video_directory: "/music_videos"
# For 'Only Recent' preset, only keep vids uploaded in this range
date_range: "2months"
# For 'Only Recent' preset, only keep vids within this range and limit
only_recent_date_range: "2months"
only_recent_max_files: 30
# Pass any arg directly to yt-dlp's Python API
ytdl_options:
cookiefile: "/config/cookie.txt"
@ -185,24 +186,24 @@ Any part of this process is modifiable by using custom configs. See our
on how to build your first config from scratch. Ready-to-use
[example configurations](https://github.com/jmbannon/ytdl-sub/tree/master/examples)
can be found here alongside our
[readthedocs](https://ytdl-sub.readthedocs.io/en/latest/config.html#)
[readthedocs](https://ytdl-sub.readthedocs.io/en/latest/index.html)
for detailed information on all config fields.
## Installation
`ytdl-sub` can be installed on the following platforms.
- [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker-compose)
- [Web-GUI](https://ytdl-sub.readthedocs.io/en/latest/install.html#gui)
- [Headless](https://ytdl-sub.readthedocs.io/en/latest/install.html#headless)
- [CPU / GPU Passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#passthrough)
- [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker)
- [Windows](https://ytdl-sub.readthedocs.io/en/latest/install.html#windows)
- [Unraid](https://ytdl-sub.readthedocs.io/en/latest/install.html#unraid)
- [Linux](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux)
- [Linux ARM](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux-arm)
- [PIP](https://ytdl-sub.readthedocs.io/en/latest/install.html#pip)
- [Local Install](https://ytdl-sub.readthedocs.io/en/latest/install.html#local-install)
- [Local Docker Build](https://ytdl-sub.readthedocs.io/en/latest/install.html#local-docker-build)
- [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose)
- [Web-GUI](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose)
- [Headless](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose)
- [CPU / GPU Passthrough](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#device-passthrough)
- [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#docker-cli)
- [Windows](https://ytdl-sub.readthedocs.io/en/latest/guides/install/windows.html)
- [Unraid](https://ytdl-sub.readthedocs.io/en/latest/guides/install/unraid.html)
- [Linux](https://ytdl-sub.readthedocs.io/en/latest/guides/install/linux.html)
- [Linux ARM](https://ytdl-sub.readthedocs.io/en/latest/guides/install/linux.html)
- [PIP](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#pip-install)
- [Local Install](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#local-install)
- [Local Docker Build](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#local-docker-build)
### Docker Installation
Docker installs can be either headless or use the Web-GUI image, which comprises
@ -222,5 +223,5 @@ pick up a bug.
We are pretty active in our
[Discord channel](https://discord.gg/v8j9RAHb4k)
if you have any questions. Also see our
[FAQ](https://github.com/jmbannon/ytdl-sub/wiki/FAQ)
[FAQ](https://ytdl-sub.readthedocs.io/en/latest/faq/index.html)
for commonly asked questions.

2
docker/Dockerfile
View file

@ -15,6 +15,7 @@ RUN mkdir -p /config && \
g++ \
nano \
make \
libffi-dev \
"python3>=3.10" \
py3-pip \
fontconfig \
@ -46,6 +47,7 @@ RUN mkdir -p /config && \
apk del \
g++ \
make \
libffi-dev \
py3-pip \
py3-setuptools

5
docker/root/defaults/subscriptions.yaml
View file

@ -14,8 +14,9 @@ __preset__:
# Root folder of all ytdl-sub Music Videos
music_video_directory: "/music_videos"
# For 'Only Recent' preset, only keep vids uploaded in this range
date_range: "2months"
# For 'Only Recent' preset, only keep vids within this range and limit
only_recent_date_range: "2months"
only_recent_max_files: 30
# Pass any arg directly to yt-dlp's Python API
ytdl_options:

4
docs/Makefile
View file

@ -5,8 +5,8 @@
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = .
BUILDDIR = _build
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:

67
docs/conf.py
View file

@ -1,67 +0,0 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath("../src"))
# -- Project information -----------------------------------------------------
project = "ytdl-sub"
copyright = "2022, Jesse Bannon"
author = "Jesse Bannon"
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
"sphinx.ext.autodoc",
"sphinx.ext.napoleon",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"
html_theme_options = {"navigation_depth": 10}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ["_static"]
# Do not show full module path in api docs
add_module_names = False
python_use_unqualified_type_names = False
napoleon_numpy_docstrings = True
napoleon_use_rtype = False
# -- Options for autodocs -------------------------------------------------
autodoc_default_options = {"autodoc_typehints_format": "short"}

454
docs/config.rst
View file

@ -1,454 +0,0 @@
Config
======
ytdl-sub is configured using a ``config.yaml`` file.
.. _config:
config.yaml
-----------
The ``config.yaml`` is made up of two sections:
.. code-block:: yaml
configuration:
presets:
You can jump to any section and subsection of the config using the navigation
section to the left.
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
If you wish to represent paths like Windows, you will need to ``C:\\double\\bashslash\\paths``
in order to escape the backslash character.
configuration
^^^^^^^^^^^^^
The ``configuration`` section contains app-wide configs applied to all presets
and subscriptions.
.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
:members:
:member-order: bysource
:exclude-members: subscription_value, persist_logs, experimental
persist_logs
""""""""""""
Within ``configuration``, define whether logs from subscription downloads
should be persisted.
.. code-block:: yaml
configuration:
persist_logs:
logs_directory: "/path/to/log/directory"
Log files are stored as
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
:members:
:member-order: bysource
presets
^^^^^^^
``presets`` define a `formula` for how to format downloaded media and metadata.
download_strategy
"""""""""""""""""
Download strategies dictate what is getting downloaded from a source. Each
download strategy has its own set of parameters.
.. _url:
url
'''
.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions()
:members: url, playlist_thumbnails, source_thumbnails, download_reverse
:member-order: bysource
multi_url
'''''''''
.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions()
:members: urls, variables
-------------------------------------------------------------------------------
output_options
""""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
:members:
:member-order: bysource
:exclude-members: get_upload_date_range_to_keep, partial_validate
-------------------------------------------------------------------------------
.. _ytdl_options:
ytdl_options
""""""""""""
.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions()
-------------------------------------------------------------------------------
.. _overrides:
overrides
"""""""""
.. autoclass:: ytdl_sub.config.preset_options.Overrides()
.. _parent preset:
preset
""""""
Presets support inheritance by defining a parent preset:
.. code-block:: yaml
presets:
custom_preset:
...
parent_preset:
...
child_preset:
preset: "parent_preset"
In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
It is advantageous to use parent presets where possible to reduce duplicate yaml
definitions.
Presets also support inheritance from multiple presets:
.. code-block:: yaml
child_preset:
preset:
- "custom_preset"
- "parent_preset"
In this example, ``child_preset`` will inherit all fields from ``custom_preset``
and ``parent_preset`` in that order. The bottom-most preset has the highest
priority.
If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is
valid YAML. Inheriting from multiple presets require use of a list.
-------------------------------------------------------------------------------
Plugins
"""""""
Plugins are used to perform any type of post-processing to the already downloaded files.
audio_extract
'''''''''''''
.. autoclass:: ytdl_sub.plugins.audio_extract.AudioExtractOptions()
:members:
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
chapters
''''''''
.. autoclass:: ytdl_sub.plugins.chapters.ChaptersOptions()
:members:
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
date_range
''''''''''
.. autoclass:: ytdl_sub.plugins.date_range.DateRangeOptions()
:members:
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
embed_thumbnail
''''''''''''''''
.. autoclass:: ytdl_sub.plugins.embed_thumbnail.EmbedThumbnailOptions()
-------------------------------------------------------------------------------
file_convert
''''''''''''
.. autoclass:: ytdl_sub.plugins.file_convert.FileConvertOptions()
:members:
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
format
''''''
.. autoclass:: ytdl_sub.plugins.format.FormatOptions()
-------------------------------------------------------------------------------
match_filters
'''''''''''''
.. autoclass:: ytdl_sub.plugins.match_filters.MatchFiltersOptions()
:members:
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
music_tags
''''''''''
.. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions()
-------------------------------------------------------------------------------
nfo_tags
''''''''
.. autoclass:: ytdl_sub.plugins.nfo_tags.NfoTagsOptions()
:members: nfo_name, nfo_root, tags, kodi_safe
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
output_directory_nfo_tags
'''''''''''''''''''''''''
.. autoclass:: ytdl_sub.plugins.output_directory_nfo_tags.OutputDirectoryNfoTagsOptions()
:members: nfo_name, nfo_root, tags, kodi_safe
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
regex
'''''
.. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
:members: skip_if_match_fails
.. autoclass:: ytdl_sub.plugins.regex.VariableRegex()
:members: match, capture_group_names, capture_group_defaults, exclude
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
split_by_chapters
'''''''''''''''''
.. autoclass:: ytdl_sub.plugins.split_by_chapters.SplitByChaptersOptions()
:members: when_no_chapters
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
subtitles
'''''''''
.. autoclass:: ytdl_sub.plugins.subtitles.SubtitleOptions()
:members: subtitles_name, subtitles_type, embed_subtitles, languages, allow_auto_generated_subtitles
:member-order: bysource
:exclude-members: partial_validate
-------------------------------------------------------------------------------
throttle_protection
'''''''''''''''''''
.. autoclass:: ytdl_sub.plugins.throttle_protection.ThrottleProtectionOptions()
:members:
:member-order: bysource
-------------------------------------------------------------------------------
video_tags
''''''''''
.. autoclass:: ytdl_sub.plugins.video_tags.VideoTagsOptions()
-------------------------------------------------------------------------------
.. _subscription_yaml:
subscription.yaml
-----------------
The ``subscription.yaml`` file is where we use our `presets`_ in the `config.yaml`_
to define a `subscription`: something we want to recurrently download such as a specific
channel or playlist.
The only difference between a ``subscription`` and ``preset`` is that the subscription
must have all required fields and ``{variables}`` defined so it can perform a download.
Below is an example that downloads a YouTube playlist:
.. code-block:: yaml
:caption: config.yaml
presets:
playlist_preset_ex:
download:
download_strategy: "url"
url: "{url}"
output_options:
output_directory: "{output_directory}/{playlist_name}"
file_name: "{playlist_name}.{title}.{ext}"
overrides:
output_directory: "/path/to/ytdl-sub-videos"
.. code-block:: yaml
:caption: subscription.yaml
my_subscription_name:
preset: "playlist_preset_ex"
overrides:
playlist_name: "diy-playlist"
url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
Our preset ``playlist_preset_ex`` defines three
custom variables: ``{output_directory}``, ``{playlist_name}``, and ``{url}``. The subscription sets
the `parent preset`_ to ``playlist_preset_ex``, and must define the variables ``{playlist_name}``
and ``{url}`` since the preset did not.
.. _beautifying subscriptions:
Beautifying Subscriptions
^^^^^^^^^^^^^^^^^^^^^^^^^
Subscriptions support using presets as keys, and using keys to set override variables as values.
For example:
.. code-block:: yaml
:caption: subscription.yaml
TV Show Full Archive:
= News:
"Breaking News": "https://www.youtube.com/@SomeBreakingNews"
TV Show Only Recent:
= Tech | TV-Y:
"Two Minute Papers": "https://www.youtube.com/@TwoMinutePapers"
Will create two subscriptions named "Breaking News" and "Two Minute Papers", equivalent to:
.. code-block:: yaml
"Breaking News":
preset:
- "TV Show Full Archive"
overrides:
subscription_indent_1: "News"
subscription_name: "Breaking News"
subscription_value: "https://www.youtube.com/@SomeBreakingNews"
"Two Minute Papers":
preset:
- "TV Show Only Recent"
overrides:
subscription_indent_1: "Tech"
subscription_indent_2: "TV-Y"
subscription_name: "Two Minute Papers"
subscription_value: "https://www.youtube.com/@TwoMinutePapers"
You can provide as many parent presets in the form of keys, and subscription indents as ``=keys``.
This can drastically simplify subscription definitions by setting things like so in your
parent preset:
.. code-block:: yaml
presets:
"TV Show Preset":
overrides:
subscription_indent_1: "default-genre"
subscription_indent_2: "default-content-rating"
tv_show_name: "{subscription_name}"
url: "{subscription_value}"
genre: "{subscription_indent_1}"
content_rating: "{subscription_indent_2}"
.. _subscription value:
File Preset
^^^^^^^^^^^
NOTE: This is deprecated in favor of using the method in :ref:`beautifying subscriptions`.
You can apply a preset to all subscriptions in the ``subscription.yaml`` file
by using the file-wide ``__preset__``:
.. code-block:: yaml
:caption: subscription.yaml
__preset__:
preset: "playlist_preset_ex"
my_subscription_name:
overrides:
url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
playlist_name: "diy-playlist"
This ``subscription.yaml`` is equivalent to the one above it because all
subscriptions automatically set ``__preset__`` as a `parent preset`_.
Subscription Value
^^^^^^^^^^^^^^^^^^^
NOTE: This is deprecated in favor of using the method in :ref:`beautifying subscriptions`.
With a clever config and use of ``__preset__``, your subscriptions can typically boil
down to a name and url. You can set ``__value__`` to the name of an override variable,
and use the override variable ``subscription_name`` to achieve one-liner subscriptions.
Using the example above, we can do:
.. code-block:: yaml
:caption: subscription.yaml
__preset__:
preset:
- "tv_show"
overrides:
tv_show_name: "{subscription_name}"
__value__: "url"
# single-line subscription, sets "Brandon Acker" and the subscription value
# to the override variables tv_show_name and url
"Brandon Acker": "https://www.youtube.com/@brandonacker"
Traditional subscriptions that can override presets will still work when using ``__value__``.
``__value__`` can also be set within a :ref:`config`.
-------------------------------------------------------------------------------
.. _source-variables:
Source Variables
----------------
.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables
:members:
:inherited-members:
:undoc-members:
Override Variables
------------------
.. autoclass:: ytdl_sub.entries.variables.override_variables.OverrideVariables()
:members:
:member-order: bysource
-------------------------------------------------------------------------------
Config Types
------------
The `config.yaml`_ uses various types for its configurable fields. Below is a definition for each type.
.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator()
.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator()
.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator()
.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator()

29
docs/getting_started.rst
View file

@ -1,29 +0,0 @@
Getting Started
===============
Walk-through Guide
-------------------
If you haven't read it yet, it's highly recommended to go through our
`walk-through guide <https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction>`_
to get familiar with how ``ytdl-sub`` works.
Example Configs
---------------
If you are ready to start downloading, see our
`examples directory <https://github.com/jmbannon/ytdl-sub/tree/master/examples>`_
for ready-to-use configs and subscriptions. Read through them carefully before use.
Using Example Configs
^^^^^^^^^^^^^^^^^^^^^^
Copy and paste the examples into local yaml files, modify the
``working_directory`` and ``output_directory`` with your desired paths,
and perform a dry-run using
.. code-block:: bash
ytdl-sub \
--dry-run \
--config path/to/config.yaml \
sub path/to/subscriptions.yaml
This will simulate what a download will look like.

29
docs/index.rst
View file

@ -1,29 +0,0 @@
ytdl-sub readthedocs
====================
Our readthedocs page is dedicated towards ytdl-sub config documentation.
If you are new to ytdl-sub, head over to the
`GitHub Wiki <https://github.com/jmbannon/ytdl-sub/wiki>`_
to see our
`walkthrough <https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction>`_ and
`FAQ <https://github.com/jmbannon/ytdl-sub/wiki/FAQ>`_. For full examples of
ytdl-sub configs, check out the
`examples directory <https://github.com/jmbannon/ytdl-sub/tree/master/examples>`_.
For navigating config docs, use the left-side bar on the
:ref:`config` page to find every available ytdl-sub field.
Contents
========
.. toctree::
:maxdepth: 2
install
usage
getting_started
presets
config
deprecation_notices

256
docs/install.rst
View file

@ -1,256 +0,0 @@
Install
=======
``ytdl-sub`` can be installed on the following platforms.
.. contents::
:depth: 3
All installations require a 64-bit CPU. 32-bit is not supported.
Docker Compose
--------------
The ytdl-sub Docker images use
`LSIO-based images <https://www.linuxserver.io/>`_
and installs ytdl-sub on top. There are a few flavors to choose from.
For automating ``subscriptions.yaml`` downloads to pull new media, see
`this guide <https://github.com/jmbannon/ytdl-sub/wiki/7.-Automate-Downloading-New-Content-Using-Your-Configs/>`_
on how set up a cron job in any of the docker containers.
GUI
^^^^
The GUI image uses LSIO's
`code-server <https://hub.docker.com/r/linuxserver/code-server>`_
for its base image. More info on other code-server environment variables
can be found within its documentation. This is the recommended way to use ``ytdl-sub``.
After starting, code-server will be running at http://localhost:8443/
.. code-block:: yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
ports:
- 8443:8443
restart: unless-stopped
Headless
^^^^^^^^^^
The headless image uses LSIO's
`baseimage-alpine <https://github.com/linuxserver/docker-baseimage-alpine>`_
for its base image. With this image, ``ytdl-sub`` is meant to be ran from console
via exec'ing into the image using the command:
.. code-block:: bash
docker exec -u abc -it ytdl-sub /bin/bash
.. code-block:: yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- DOCKER_MODS=linuxserver/mods:universal-cron
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
restart: unless-stopped
Passthrough
^^^^^^^^^^^
For CPU or GPU passthrough, you must use either the GUI image or the headless Ubuntu image
``ghcr.io/jmbannon/ytdl-sub:ubuntu-latest``.
The docker-compose examples use the GUI image.
CPU
____
.. code-block:: yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
ports:
- 8443:8443
devices:
- /dev/dri:/dev/dri # CPU passthrough
restart: unless-stopped
GPU
____
.. code-block:: yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args
- NVIDIA_VISIBLE_DEVICES=all
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
ports:
- 8443:8443
deploy:
resources:
reservations:
devices:
- capabilities: [gpu] # GPU passthrough
restart: unless-stopped
Docker
--------------
.. code-block:: bash
docker run -d \
--name=ytdl-sub \
-e PUID=1000 \
-e PGID=1000 \
-e TZ=America/Los_Angeles \
-p 8443:8443 \
-v <path/to/ytdl-sub/config>:/config \
-v <OPTIONAL/path/to/tv_shows>:/tv_shows \
-v <OPTIONAL/path/to/movies>:/movies \
-v <OPTIONAL/path/to/music_videos>:/music_videos \
-v <OPTIONAL/path/to/music>:/music \
--restart unless-stopped \
ghcr.io/jmbannon/ytdl-sub-gui:latest
Windows
--------------
From powershell, run:
.. code-block:: powershell
# Download ffmpeg/ffprobe dependencies from yt-dlp
curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
tar -xf ffmpeg.zip
move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe"
move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
# Download ytdl-sub
curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe
ytdl-sub.exe -h
Unraid
--------------
See the
`community app <https://unraid.net/community/apps?q=ytdl-sub#r>`_
``ytdl-sub``. Uses Docker under the hood.
Linux
--------------
Requires ffmpeg as a dependency. Can typically be installed with any Linux package manager.
.. code-block:: bash
curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub
chmod +x ytdl-sub
ytdl-sub -h
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
.. code-block:: bash
curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xf ffmpeg.tar.gz
chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg
chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe
# May need sudo / root permissions to perform
mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg
mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe
Linux ARM
--------------
Requires ffmpeg as a dependency. Can typically be installed with any Linux package manager.
.. code-block:: bash
curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub_aarch64
chmod +x ytdl-sub
ytdl-sub -h
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
.. code-block:: bash
curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz
tar -xf ffmpeg.tar.gz
chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg
chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe
# May need sudo / root permissions to perform
mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg
mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe
PIP
--------------
You can install our
`PyPI package <https://pypi.org/project/ytdl-sub/>`_.
Both ffmpeg and Python 3.10 or greater are required.
.. code-block:: bash
python3 -m pip install -U ytdl-sub
Local Install
--------------
With a Python 3.10 virtual environment, you can clone and install the repo using
.. code-block:: bash
git clone https://github.com/jmbannon/ytdl-sub.git
cd ytdl-sub
pip install -e .
Local Docker Build
-------------------
Run ``make docker`` in the root directory of this repo to build the image. This
will build the python wheel and install it in the Dockerfile.

4
docs/make.bat
View file

@ -7,8 +7,8 @@ REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
set SOURCEDIR=source
set BUILDDIR=build
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (

183
docs/presets.rst
View file

@ -1,183 +0,0 @@
Presets
=======
``ytdl-sub`` offers a number of built-in presets using best practices for formatting
media in various players. For advanced users, you can find the prebuilt preset
definitions
`here <https://github.com/jmbannon/ytdl-sub/tree/master/src/ytdl_sub/prebuilt_presets>`_.
TV Shows
--------
There are two main methods for downloading and formatting videos as a TV show.
TV Show by Date
^^^^^^^^^^^^^^^
TV Show by Date will organize something like a YouTube channel or playlist
into a tv show, where seasons and episodes are organized using upload date.
Player Presets
""""""""""""""
* ``kodi_tv_show_by_date``
* ``jellyfin_tv_show_by_date``
* ``plex_tv_show_by_date``
Episode Formatting Presets
""""""""""""""""""""""""""
* ``season_by_year__episode_by_month_day``
* ``season_by_year_month__episode_by_day``
* ``season_by_year__episode_by_month_day_reversed``
* Episode numbers are reversed, meaning more recent episodes appear at the
top of a season by having a lower value.
* ``season_by_year__episode_by_download_index``
* Episodes are numbered by the download order. NOTE that this fetched using
the length of the download archive. Do not use if you intend to remove
old videos.
Usage
"""""
A preset/subscription requires specifying a player and episode formatting preset
and overriding the following variables:
.. code-block:: yaml
rick_a_tv_show_by_date:
preset:
- "jellyfin_tv_show_by_date"
- "season_by_year__episode_by_month_day"
overrides:
# required
tv_show_name: "Rick A"
tv_show_directory: "/path/to/youtube_shows"
url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
# can be modified from their default value
# tv_show_genre: "ytdl-sub"
# tv_show_content_rating: "TV-14"
# episode_title: "{upload_date_standardized} - {title}"
# episode_description: "{webpage_url}"
In addition, you can add additional URLs to create a single TV by using the override variables
``url2``, ``url3``, ..., ``url20``:
.. code-block:: yaml
overrides:
tv_show_name: "Rick A"
tv_show_directory: "/path/to/youtube_shows"
url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
url2: "https://www.youtube.com/@just.rick_6"
TV Show Collection
^^^^^^^^^^^^^^^^^^
TV Show Collections are made up from multiple URLs, where each URL is a season.
If a video belongs to multiple URLs (i.e. a channel and a channel's playlist),
it will resolve to the bottom-most season.
Two main use cases of a collection are:
1. Organize a YouTube channel TV show where Season 1 contains any video
not in a 'season playlist', Season 2 for 'Playlist A', Season 3 for
'Playlist B', etc.
2. Organize one or more YouTube channels/playlists, where each season
represents a separate channel/playlist.
Player Presets
""""""""""""""
* ``kodi_tv_show_collection``
* ``jellyfin_tv_show_collection``
* ``plex_tv_show_collection``
Episode Formatting Presets
""""""""""""""""""""""""""
* ``season_by_collection__episode_by_year_month_day``
* ``season_by_collection__episode_by_year_month_day_reversed``
* ``season_by_collection__episode_by_playlist_index``
* Only use playlist_index episode formatting for playlists that
will be fully downloaded once and never again. Otherwise,
indices can change.
* ``season_by_collection__episode_by_playlist_index_reversed``
Season Presets
""""""""""""""
* ``collection_season_1``
* ``collection_season_2``
* ``collection_season_3``
* ``collection_season_4``
* ``...``
* ``collection_season_40``
Example
"""""""
A preset/subscription requires specifying a player, episode formatting, and
one or more season presets, with the following override variables:
.. code-block:: yaml
rick_a_tv_show_collection:
preset:
- "jellyfin_tv_show_collection"
- "season_by_collection__episode_by_year_month_day_reversed"
- "collection_season_1"
- "collection_season_2"
overrides:
# required
tv_show_name: "Rick A"
tv_show_directory: "/path/to/youtube_shows"
collection_season_1_url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
collection_season_1_name: "All Videos"
collection_season_2_url: "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
collection_season_2_name: "Official Music Videos"
# can be modified from their default value
# tv_show_genre: "ytdl-sub"
# episode_title: "{upload_date_standardized} - {title}"
# episode_description: "{webpage_url}"
Common
------
Common presets are applicable to any config.
Best Video Quality
^^^^^^^^^^^^^^^^^^
Add the following preset to download the best available video and audio quality, and remux
it into an MP4 container:
* ``best_video_quality``
Max 1080p
^^^^^^^^^^^^^^^^^^
Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container:
* ``max_1080p``
Chunk Initial Download
^^^^^^^^^^^^^^^^^^^^^^
If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from
newest to oldest before starting any downloads. It is a long process and not ideal. A better method
is to chunk the process by using the following preset:
* ``chunk_initial_download``
It will download videos starting from the oldest one, and only download 20 at a time. You can
change this number by setting:
.. code-block:: yaml
ytdl_options:
max_downloads: 30 # Desired number to download per invocation
Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to
oldest again, and stop pulling additional metadata once it reaches a video that has already been
downloaded.

0
docs/source/_static/custom.css Normal file
View file

88
docs/source/conf.py Normal file
View file

@ -0,0 +1,88 @@
# Configuration file for the Sphinx documentation builder.
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
project = "ytdl-sub"
copyright = "2023, Jesse Bannon"
author = "Jesse Bannon"
release = "2023.12.15"
# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [
"sphinx.ext.autodoc",
"sphinx.ext.autosectionlabel",
"sphinx.ext.extlinks",
"sphinx.ext.napoleon",
"sphinx_copybutton",
"sphinx_design",
]
templates_path = ["_templates"]
exclude_patterns = []
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
html_theme = "sphinx_book_theme"
html_theme_options = {
"icon_links": [
{
"name": "GitHub",
"url": "https://github.com/jmbannon/ytdl-sub",
"icon": "fa-brands fa-square-github",
"type": "fontawesome",
},
{
"name": "Discord",
"url": "https://discord.gg/v8j9RAHb4k",
"icon": "https://img.shields.io/discord/994270357957648404?logo=Discord",
"type": "url",
},
],
"announcement": ("Please excuse our mess as we update these documents"),
"navigation_depth": 10,
"show_toc_level": 10,
}
html_static_path = ["_static"]
html_css_files = ["custom.css"]
# Make sure the all autosectionlabel targets are unique
autosectionlabel_prefix_document = True
suppress_warnings = [
"autosectionlabel.*",
]
extlinks = {
"yt-dlp": ("https://github.com/yt-dlp/yt-dlp/%s", "yt-dlp%s"),
"unraid": ("https://unraid.net/%s", "unraid%s"),
"lsio": ("https://www.linuxserver.io/%s", "lsio%s"),
"lsio-gh": ("https://github.com/linuxserver/%s", "%s image"),
"ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"),
}
# -- Options for autodoc ----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
# Automatically extract typehints when specified and place them in
# descriptions of the relevant function/method.
autodoc_default_options = {
"autodoc_typehints_format": "short",
"autodoc_class_signature": "separated",
"add_module_names": False,
# "add_class_names": False,
}
python_use_unqualified_type_names = True
napoleon_numpy_docstring = True
napoleon_use_rtype = False

90
docs/source/config_reference/config_yaml.rst Normal file
View file

@ -0,0 +1,90 @@
==================
Configuration File
==================
-----------
config.yaml
-----------
ytdl-sub is configured using a ``config.yaml`` file.
The ``config.yaml`` is made up of two sections:
.. code-block:: yaml
configuration:
presets:
You can jump to any section and subsection of the config using the navigation
section to the left.
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
If you wish to represent paths like Windows, you will need to ``C:\\double\\bashslash\\paths``
in order to escape the backslash character.
configuration
~~~~~~~~~~~~~
The ``configuration`` section contains app-wide configs applied to all presets
and subscriptions.
.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
:members:
:member-order: bysource
:exclude-members: subscription_value, persist_logs, experimental
persist_logs
""""""""""""
Within ``configuration``, define whether logs from subscription downloads
should be persisted.
.. code-block:: yaml
configuration:
persist_logs:
logs_directory: "/path/to/log/directory"
Log files are stored as
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
:members:
:member-order: bysource
presets
~~~~~~~
``presets`` define a `formula` for how to format downloaded media and metadata.
This section is work-in-progress!
preset
""""""
Presets support inheritance by defining a parent preset:
.. code-block:: yaml
presets:
custom_preset:
...
parent_preset:
...
child_preset:
preset: "parent_preset"
In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
It is advantageous to use parent presets where possible to reduce duplicate yaml
definitions.
Presets also support inheritance from multiple presets:
.. code-block:: yaml
child_preset:
preset:
- "custom_preset"
- "parent_preset"
In this example, ``child_preset`` will inherit all fields from ``custom_preset``
and ``parent_preset`` in that order. The bottom-most preset has the highest
priority.
If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is
valid YAML. Inheriting from multiple presets require use of a list.

12
docs/source/config_reference/index.rst Normal file
View file

@ -0,0 +1,12 @@
=========
Reference
=========
This section contains direct references to the code of ``ytdl-sub`` and information on how it functions.
.. toctree::
config_yaml
subscriptions_yaml
plugins
scripting/index
prebuilt_presets/index

1015
docs/source/config_reference/plugins.rst Normal file
View file

File diff suppressed because it is too large Load diff

7
docs/source/config_reference/prebuilt_presets/helpers_common.rst Normal file
View file

@ -0,0 +1,7 @@
=======================
Common Preset Reference
=======================
.. highlight:: yaml
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/common.yaml

7
docs/source/config_reference/prebuilt_presets/helpers_players.rst Normal file
View file

@ -0,0 +1,7 @@
========================
Players Preset Reference
========================
.. highlight:: yaml
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/players.yaml

7
docs/source/config_reference/prebuilt_presets/helpers_url.rst Normal file
View file

@ -0,0 +1,7 @@
====================
URL Preset Reference
====================
.. highlight:: yaml
.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/url.yaml

12
docs/source/config_reference/prebuilt_presets/index.rst Normal file
View file

@ -0,0 +1,12 @@
=========================
Prebuilt Preset Reference
=========================
This section contains the code for the prebuilt presets. If you just want to understand how to use the presets, check :doc:`this section instead</prebuilt_presets/index>`.
.. toctree::
helpers_common
helpers_players
helpers_url

649
docs/source/config_reference/scripting/entry_variables.rst Normal file
View file

@ -0,0 +1,649 @@
Entry Variables
===============
Entry Variables
---------------
channel
~~~~~~~
:type: ``String``
:description:
The channel name if it exists, otherwise returns the uploader.
channel_id
~~~~~~~~~~
:type: ``String``
:description:
The channel id if it exists, otherwise returns the entry uploader ID.
chapters
~~~~~~~~
:type: ``Array``
:description:
Chapters if they exist
comments
~~~~~~~~
:type: ``Array``
:description:
Comments if they are requested
creator
~~~~~~~
:type: ``String``
:description:
The creator name if it exists, otherwise returns the channel.
description
~~~~~~~~~~~
:type: ``String``
:description:
The description if it exists. Otherwise, returns an emtpy string.
duration
~~~~~~~~
:type: ``Integer``
:description:
The duration of the entry in seconds if it exists. Defaults to zero otherwise.
epoch
~~~~~
:type: ``Integer``
:description:
The unix epoch of when the metadata was scraped by yt-dlp.
epoch_date
~~~~~~~~~~
:type: ``String``
:description:
The epoch's date, in YYYYMMDD format.
epoch_hour
~~~~~~~~~~
:type: ``String``
:description:
The epoch's hour
ext
~~~
:type: ``String``
:description:
The downloaded entry's file extension
extractor
~~~~~~~~~
:type: ``String``
:description:
The yt-dlp extractor name
extractor_key
~~~~~~~~~~~~~
:type: ``String``
:description:
The yt-dlp extractor key
ie_key
~~~~~~
:type: ``String``
:description:
The ie_key, used in legacy yt-dlp things as the 'info-extractor key'.
If it does not exist, return ``extractor_key``
info_json_ext
~~~~~~~~~~~~~
:type: ``String``
:description:
The "info.json" extension
requested_subtitles
~~~~~~~~~~~~~~~~~~~
:type: ``Map``
:description:
Subtitles if they are requested and exist
sponsorblock_chapters
~~~~~~~~~~~~~~~~~~~~~
:type: ``Array``
:description:
Sponsorblock Chapters if they are requested and exist
thumbnail_ext
~~~~~~~~~~~~~
:type: ``String``
:description:
The download entry's thumbnail extension. Will always return 'jpg'. Until there is a
need to support other image types, we always convert to jpg.
title
~~~~~
:type: ``String``
:description:
The title of the entry. If a title does not exist, returns its unique ID.
title_sanitized_plex
~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The sanitized title with additional sanitizing for Plex. It replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
uid
~~~
:type: ``String``
:description:
The entry's unique ID
uid_sanitized_plex
~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The sanitized uid with additional sanitizing for Plex. Replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
uploader
~~~~~~~~
:type: ``String``
:description:
The uploader if it exists, otherwise return the uploader ID.
uploader_id
~~~~~~~~~~~
:type: ``String``
:description:
The uploader id if it exists, otherwise return the unique ID.
uploader_url
~~~~~~~~~~~~
:type: ``String``
:description:
The uploader url if it exists, otherwise returns the webpage_url.
webpage_url
~~~~~~~~~~~
:type: ``String``
:description:
The url to the webpage.
----------------------------------------------------------------------------------------------------
Metadata Variables
------------------
entry_metadata
~~~~~~~~~~~~~~
:type: ``Map``
:description:
The entry's info.json
playlist_metadata
~~~~~~~~~~~~~~~~~
:type: ``Map``
:description:
Metadata from the playlist (i.e. the parent metadata, like playlist -> entry)
sibling_metadata
~~~~~~~~~~~~~~~~
:type: ``Array``
:description:
Metadata from any sibling entries that reside in the same playlist as this entry.
source_metadata
~~~~~~~~~~~~~~~
:type: ``Map``
:description:
Metadata from the source
(i.e. the grandparent metadata, like channel -> playlist -> entry)
----------------------------------------------------------------------------------------------------
Playlist Variables
------------------
playlist_count
~~~~~~~~~~~~~~
:type: ``Integer``
:description:
Playlist count if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
playlist_description
~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The playlist description if it exists, otherwise returns the entry's description.
playlist_index
~~~~~~~~~~~~~~
:type: ``Integer``
:description:
Playlist index if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
playlist_index_padded
~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
playlist_index padded two digits
playlist_index_padded6
~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
playlist_index padded six digits.
playlist_index_reversed
~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
Playlist index reversed via ``playlist_count - playlist_index + 1``
playlist_index_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
playlist_index_reversed padded two digits
playlist_index_reversed_padded6
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
playlist_index_reversed padded six digits.
playlist_max_upload_date
~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
Max upload_date for all entries in this entry's playlist if it exists, otherwise returns
``upload_date``
playlist_max_upload_year
~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
Max upload_year for all entries in this entry's playlist if it exists, otherwise returns
``upload_year``
playlist_max_upload_year_truncated
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The max playlist truncated upload year for all entries in this entry's playlist if it
exists, otherwise returns ``upload_year_truncated``.
playlist_title
~~~~~~~~~~~~~~
:type: ``String``
:description:
Name of its parent playlist/channel if it exists, otherwise returns its title.
playlist_uid
~~~~~~~~~~~~
:type: ``String``
:description:
The playlist unique ID if it exists, otherwise return the entry unique ID.
playlist_uploader
~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The playlist uploader if it exists, otherwise return the entry uploader.
playlist_uploader_id
~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The playlist uploader id if it exists, otherwise returns the entry uploader ID.
playlist_uploader_url
~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The playlist uploader url if it exists, otherwise returns the playlist webpage_url.
playlist_webpage_url
~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
----------------------------------------------------------------------------------------------------
Release Date Variables
----------------------
release_date
~~~~~~~~~~~~
:type: ``String``
:description:
The entry’s release date, in YYYYMMDD format. If not present, return the upload date.
release_date_standardized
~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The uploaded date formatted as YYYY-MM-DD
release_day
~~~~~~~~~~~
:type: ``Integer``
:description:
The upload day as an integer (no padding).
release_day_of_year
~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The day of the year, i.e. February 1st returns ``32``
release_day_of_year_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The upload day of year, but padded i.e. February 1st returns "032"
release_day_of_year_reversed
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``,
i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
release_day_of_year_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The reversed upload day of year, but padded i.e. December 31st returns "001"
release_day_padded
~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The entry's upload day padded to two digits, i.e. the fifth returns "05"
release_day_reversed
~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``,
i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24``
release_day_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The reversed upload day, but padded. i.e. August 30th returns "02".
release_month
~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload month as an integer (no padding).
release_month_padded
~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The entry's upload month padded to two digits, i.e. March returns "03"
release_month_reversed
~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10``
release_month_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The reversed upload month, but padded. i.e. November returns "02"
release_year
~~~~~~~~~~~~
:type: ``Integer``
:description:
The entry's upload year
release_year_truncated
~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The last two digits of the upload year, i.e. 22 in 2022
release_year_truncated_reversed
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
----------------------------------------------------------------------------------------------------
Source Variables
----------------
source_count
~~~~~~~~~~~~
:type: ``Integer``
:description:
The source count if it exists, otherwise returns ``1``.
source_description
~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The source description if it exists, otherwise returns the playlist description.
source_index
~~~~~~~~~~~~
:type: ``Integer``
:description:
Source index if it exists, otherwise returns ``1``.
It is recommended to not use this unless you know the source will never add new content
(it is easy for this value to change).
source_index_padded
~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The source index, padded two digits.
source_title
~~~~~~~~~~~~
:type: ``String``
:description:
Name of the source (i.e. channel with multiple playlists) if it exists, otherwise
returns its playlist_title.
source_uid
~~~~~~~~~~
:type: ``String``
:description:
The source unique id if it exists, otherwise returns the playlist unique ID.
source_uploader
~~~~~~~~~~~~~~~
:type: ``String``
:description:
The source uploader if it exists, otherwise return the playlist_uploader
source_uploader_id
~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The source uploader id if it exists, otherwise returns the playlist_uploader_id
source_uploader_url
~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The source uploader url if it exists, otherwise returns the source webpage_url.
source_webpage_url
~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The source webpage url if it exists, otherwise returns the playlist webpage url.
----------------------------------------------------------------------------------------------------
Upload Date Variables
---------------------
upload_date
~~~~~~~~~~~
:type: ``String``
:description:
The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date.
upload_date_standardized
~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The uploaded date formatted as YYYY-MM-DD
upload_day
~~~~~~~~~~
:type: ``Integer``
:description:
The upload day as an integer (no padding).
upload_day_of_year
~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The day of the year, i.e. February 1st returns ``32``
upload_day_of_year_padded
~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The upload day of year, but padded i.e. February 1st returns "032"
upload_day_of_year_reversed
~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``,
i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
upload_day_of_year_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The reversed upload day of year, but padded i.e. December 31st returns "001"
upload_day_padded
~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The entry's upload day padded to two digits, i.e. the fifth returns "05"
upload_day_reversed
~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``,
i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24``
upload_day_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The reversed upload day, but padded. i.e. August 30th returns "02".
upload_month
~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload month as an integer (no padding).
upload_month_padded
~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The entry's upload month padded to two digits, i.e. March returns "03"
upload_month_reversed
~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10``
upload_month_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The reversed upload month, but padded. i.e. November returns "02"
upload_year
~~~~~~~~~~~
:type: ``Integer``
:description:
The entry's upload year
upload_year_truncated
~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The last two digits of the upload year, i.e. 22 in 2022
upload_year_truncated_reversed
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
----------------------------------------------------------------------------------------------------
Ytdl-Sub Variables
------------------
download_index
~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The i'th entry downloaded. NOTE that this is fetched dynamically from the download
archive.
download_index_padded6
~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The download_index padded six digits
upload_date_index
~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
The i'th entry downloaded with this upload date.
upload_date_index_padded
~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The upload_date_index padded two digits
upload_date_index_reversed
~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``Integer``
:description:
100 - upload_date_index
upload_date_index_reversed_padded
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The upload_date_index padded two digits
ytdl_sub_input_url
~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The input URL used in ytdl-sub to create this entry.

172
docs/source/config_reference/scripting/index.rst Normal file
View file

@ -0,0 +1,172 @@
=========
Scripting
=========
``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The links below
contain reference documentation for each built-in variable and scripting function.
.. toctree::
:maxdepth: 1
entry_variables
override_variables
scripting_functions
scripting_types
How it Works
------------
Fields in the config that support ``formatters`` mean they support scripting, and will
*format* the field using its defined script.
In its most basic form, a script is a string comprised of variables and/or functions.
Static String
~~~~~~~~~~~~~
The following example sets ``ytdl-sub``'s output directory. It is
considered *static* because it does not depend on anything from an entry.
.. code-block:: yaml
output_options:
output_directory: "Custom YTDL-SUB TV Show"
Static Variables
~~~~~~~~~~~~~~~~
``ytdl-sub`` offers a few built-in static variables, including ``subscription_name``.
We can use this instead of hard-coding it above:
.. code-block:: yaml
output_options:
output_directory: "{subscription_name}"
The syntax for variable usage is curly-braces with the variable name within it. Assuming
our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub``
will actually write to that directory.
Entry Variables
~~~~~~~~~~~~~~~
For context, an *entry* is a video or audio file downloaded from ``yt-dlp``.
*Entry variables* are variables that are derived from an entry's ``info.json`` file. This file
comes from ``yt-dlp`` and contains every piece of metadata that it scraped.
These variables are not considered static since they change per entry download. There are a
few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For others,
we are free to use values that derive from an entry.
Suppose we want to customize the name of an entry's output file and thumbnail to include its
title in its name. We can do that using entry variables:
.. code-block:: yaml
output_options:
output_directory: "{subscription_name}"
file_name: "{title}.{ext}"
thumbnail_name: "{title}.{thumbnail_ext}"
Creating Custom Variables
~~~~~~~~~~~~~~~~~~~~~~~~~
Suppose we want to include the date in our file names. This means we'd need to update
both the ``file_name`` and ``thumbnail_name`` fields to include it.
Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method
for creating and overriding custom variables.
These are created in the ``overrides`` section. Let's take our above example and create
a ``custom_file_name`` variable to use for the entry file and thumbnail fields:
.. code-block:: yaml
output_options:
output_directory: "{subscription_name}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
overrides:
custom_file_name: "{upload_date_standardized} {title}"
Sanitizing Variables
~~~~~~~~~~~~~~~~~~~~
For experienced ``yt-dlp`` scrapers, you may be thinking:
- What if the title has characters that do not play nice with my operating system?
``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic characters
with safe alternatives that can be used in file names. We can ensure our file names and directories
are safe by using:
.. code-block:: yaml
output_options:
output_directory: "{subscription_name_sanitized}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
overrides:
custom_file_name: "{upload_date_standardized} {title_sanitized}"
Simply add a ``_sanitized`` suffix to any variable name to make it sanitized.
.. note::
Make sure you do not sanitize custom variables that intentionally create directories, otherwise
they will... be sanitized and not resolve to directories!
Using Scripting Functions
~~~~~~~~~~~~~~~~~~~~~~~~~
Let's suppose you are an avid command-line user, and like all of your file names to be
``snake_cased_with_no_spaces``. We can use the
`replace <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_
*scripting function* to create and use a snake-cased title.
.. code-block:: yaml
output_options:
output_directory: "{subscription_name_sanitized}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
overrides:
snake_cased_title: >-
{
%replace( title, ' ', '_' )
}
custom_file_name: "{upload_date_standardized}_{snake_cased_title_sanitized}"
Scripting functions are similar to variables - they must be used within curly-braces.
It is good practice to use ``>-`` when defining variables that use functions. It is YAML's way of
saying:
- Allow a string to be multi-lined, and do not include newlines before or after it.
See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=canonical_yaml>`_.
Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make
scripting function usage readable.
.. important::
It is important to use ``>-`` over other YAML new-line directives like ``>`` because they
add newlines before or after curly-braces, and will be included in your variable's output string.
Advanced Scripting
------------------
Accessing ``info.json`` Fields
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
WIP
Creating Custom Functions
~~~~~~~~~~~~~~~~~~~~~~~~~
WIP
Parsing Maps and Arrays
~~~~~~~~~~~~~~~~~~~~~~~
WIP

70
docs/source/config_reference/scripting/override_variables.rst Normal file
View file

@ -0,0 +1,70 @@
Override Variables
==================
subscription_indent_i
---------------------
For subscriptions in the form of
.. code-block:: yaml
Preset | = Indent Value 1:
= Indent Value 2:
"Subscription Name": "https://..."
``subscription_indent_1`` and ``subscription_indent_2`` get set to
``Indent Value 1`` and ``Indent Value 2``.
subscription_map
----------------
For subscriptions in the form of
.. code-block:: yaml
+ Subscription Name:
Music Videos:
- "https://url1.com/..."
Concerts:
- "https://url2.com/..."
Stores all the contents under the subscription name into the override variable
``subscription_map`` as a Map value. The above example is stored as:
.. code-block:: python
{
"Music Videos": [
"https://url1.com/..."
],
"Concerts: [
"https://url2.com/..."
]
}
subscription_name
-----------------
Name of the subscription
subscription_value
------------------
For subscriptions in the form of
.. code-block:: yaml
"Subscription Name": "https://..."
``subscription_value`` gets set to ``https://...``.
subscription_value_i
--------------------
For subscriptions in the form of
.. code-block:: yaml
"Subscription Name":
- "https://url1.com/..."
- "https://url2.com/..."
``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...``
and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to
``subscription_value``.

631
docs/source/config_reference/scripting/scripting_functions.rst Normal file
View file

@ -0,0 +1,631 @@
Scripting Functions
===================
Array Functions
---------------
array
~~~~~
:spec: ``array(maybe_array: AnyArgument) -> Array``
:description:
Tries to cast an unknown variable type to an Array.
array_apply
~~~~~~~~~~~
:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array``
:description:
Apply a lambda function on every element in the Array.
:usage:
.. code-block:: python
{
%array_apply( [1, 2, 3] , %string )
}
# ["1", "2", "3"]
array_apply_fixed
~~~~~~~~~~~~~~~~~
:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array``
:description:
Apply a lambda function on every element in the Array, with ``fixed_argument``
passed as a second argument to every invocation.
array_at
~~~~~~~~
:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument``
:description:
Return the element in the Array at index ``idx``.
array_contains
~~~~~~~~~~~~~~
:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean``
:description:
Return True if the value exists in the Array. False otherwise.
array_enumerate
~~~~~~~~~~~~~~~
:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array``
:description:
Apply a lambda function on every element in the Array, where each arg
passed to the lambda function is ``idx, element`` as two separate args.
array_extend
~~~~~~~~~~~~
:spec: ``array_extend(arrays: Array, ...) -> Array``
:description:
Combine multiple Arrays into a single Array.
array_first
~~~~~~~~~~~
:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument``
:description:
Returns the first element whose boolean conversion is True. Returns fallback
if all elements evaluate to False.
array_flatten
~~~~~~~~~~~~~
:spec: ``array_flatten(array: Array) -> Array``
:description:
Flatten any nested Arrays into a single-dimensional Array.
array_index
~~~~~~~~~~~
:spec: ``array_index(array: Array, value: AnyArgument) -> Integer``
:description:
Return the index of the value within the Array if it exists. If it does not, it will
throw an error.
array_overlay
~~~~~~~~~~~~~
:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array``
:description:
Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
array_product
~~~~~~~~~~~~~
:spec: ``array_product(arrays: Array, ...) -> Array``
:description:
Returns the Cartesian product of elements from different arrays
array_reduce
~~~~~~~~~~~~
:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument``
:description:
Apply a reduce function on pairs of elements in the Array, until one element remains.
Executes using the left-most and reduces in the right direction.
array_reverse
~~~~~~~~~~~~~
:spec: ``array_reverse(array: Array) -> Array``
:description:
Reverse an Array.
array_size
~~~~~~~~~~
:spec: ``array_size(array: Array) -> Integer``
:description:
Returns the size of an Array.
array_slice
~~~~~~~~~~~
:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array``
:description:
Returns the slice of the Array.
----------------------------------------------------------------------------------------------------
Boolean Functions
-----------------
and
~~~
:spec: ``and(values: AnyArgument, ...) -> Boolean``
:description:
``and`` operator. Returns True if all values evaluate to True. False otherwise.
bool
~~~~
:spec: ``bool(value: AnyArgument) -> Boolean``
:description:
Cast any type to a Boolean.
eq
~~
:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``==`` operator. Returns True if left == right. False otherwise.
gt
~~
:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``>`` operator. Returns True if left > right. False otherwise.
gte
~~~
:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``>=`` operator. Returns True if left >= right. False otherwise.
is_null
~~~~~~~
:spec: ``is_null(value: AnyArgument) -> Boolean``
:description:
Returns True if a value is null (i.e. an empty string). False otherwise.
lt
~~
:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``<`` operator. Returns True if left < right. False otherwise.
lte
~~~
:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``<=`` operator. Returns True if left <= right. False otherwise.
ne
~~
:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean``
:description:
``!=`` operator. Returns True if left != right. False otherwise.
not
~~~
:spec: ``not(value: Boolean) -> Boolean``
:description:
``not`` operator. Returns the opposite of value.
or
~~
:spec: ``or(values: AnyArgument, ...) -> Boolean``
:description:
``or`` operator. Returns True if any value evaluates to True. False otherwise.
xor
~~~
:spec: ``xor(values: AnyArgument, ...) -> Boolean``
:description:
``^`` operator. Returns True if exactly one value is set to True. False otherwise.
----------------------------------------------------------------------------------------------------
Conditional Functions
---------------------
if
~~
:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:description:
Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
depending on the ``condition`` value.
if_passthrough
~~~~~~~~~~~~~~
:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
:description:
Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
otherwise returns ``else_arg``.
----------------------------------------------------------------------------------------------------
Date Functions
--------------
datetime_strftime
~~~~~~~~~~~~~~~~~
:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String``
:description:
Converts a posix timestamp to a date using strftime formatting.
----------------------------------------------------------------------------------------------------
Error Functions
---------------
assert
~~~~~~
:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``value``.
assert_eq
~~~~~~~~~
:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` does not equal
``equals``. If they do equal, then return ``value``.
assert_ne
~~~~~~~~~
:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` equals
``equals``. If they do equal, then return ``value``.
assert_then
~~~~~~~~~~~
:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument``
:description:
Explicitly throw an error with the provided assert message if ``value`` evaluates to
False. If it evaluates to True, it will return ``ret``.
throw
~~~~~
:spec: ``throw(error_message: String) -> AnyArgument``
:description:
Explicitly throw an error with the provided error message.
----------------------------------------------------------------------------------------------------
Json Functions
--------------
from_json
~~~~~~~~~
:spec: ``from_json(argument: String) -> AnyArgument``
:description:
Converts a JSON string into an actual type.
----------------------------------------------------------------------------------------------------
Map Functions
-------------
map
~~~
:spec: ``map(maybe_mapping: AnyArgument) -> Map``
:description:
Tries to cast an unknown variable type to a Map.
map_apply
~~~~~~~~~
:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array``
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``key, value`` as two separate args.
map_contains
~~~~~~~~~~~~
:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean``
:description:
Returns True if the key is in the Map. False otherwise.
map_enumerate
~~~~~~~~~~~~~
:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array``
:description:
Apply a lambda function on the Map, where each arg
passed to the lambda function is ``idx, key, value`` as three separate args.
map_get
~~~~~~~
:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
:description:
Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
provided, it will return ``default``. Otherwise, will error.
map_get_non_empty
~~~~~~~~~~~~~~~~~
:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument``
:description:
Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
return ``default``. Otherwise, will error.
map_size
~~~~~~~~
:spec: ``map_size(mapping: Map) -> Integer``
:description:
Returns the size of a Map.
----------------------------------------------------------------------------------------------------
Numeric Functions
-----------------
add
~~~
:spec: ``add(values: Numeric, ...) -> Numeric``
:description:
``+`` operator. Returns the sum of all values.
div
~~~
:spec: ``div(left: Numeric, right: Numeric) -> Numeric``
:description:
``/`` operator. Returns ``left / right``.
float
~~~~~
:spec: ``float(value: AnyArgument) -> Float``
:description:
Cast to Float.
int
~~~
:spec: ``int(value: AnyArgument) -> Integer``
:description:
Cast to Integer.
max
~~~
:spec: ``max(values: Numeric, ...) -> Numeric``
:description:
Returns max of all values.
min
~~~
:spec: ``min(values: Numeric, ...) -> Numeric``
:description:
Returns min of all values.
mod
~~~
:spec: ``mod(left: Numeric, right: Numeric) -> Numeric``
:description:
``%`` operator. Returns ``left % right``.
mul
~~~
:spec: ``mul(values: Numeric, ...) -> Numeric``
:description:
``*`` operator. Returns the product of all values.
pow
~~~
:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric``
:description:
``**`` operator. Returns the exponential of the base and exponent value.
sub
~~~
:spec: ``sub(values: Numeric, ...) -> Numeric``
:description:
``-`` operator. Subtracts all values from left to right.
----------------------------------------------------------------------------------------------------
Regex Functions
---------------
regex_capture_groups
~~~~~~~~~~~~~~~~~~~~
:spec: ``regex_capture_groups(regex: String) -> Integer``
:description:
Returns number of capture groups in regex
regex_fullmatch
~~~~~~~~~~~~~~~
:spec: ``regex_fullmatch(regex: String, string: String) -> Array``
:description:
Checks for entire string to be a match. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_match
~~~~~~~~~~~
:spec: ``regex_match(regex: String, string: String) -> Array``
:description:
Checks for a match only at the beginning of the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
regex_search
~~~~~~~~~~~~
:spec: ``regex_search(regex: String, string: String) -> Array``
:description:
Checks for a match anywhere in the string. If a match exists, returns
the string as the first element of the Array. If there are capture groups, returns each
group as a subsequent element in the Array.
----------------------------------------------------------------------------------------------------
String Functions
----------------
capitalize
~~~~~~~~~~
:spec: ``capitalize(string: String) -> String``
:description:
Capitalize the first character in the string.
concat
~~~~~~
:spec: ``concat(values: String, ...) -> String``
:description:
Concatenate multiple Strings into a single String.
contains
~~~~~~~~
:spec: ``contains(string: String, contains: String) -> Boolean``
:description:
Returns True if ``contains`` is in ``string``. False otherwise.
lower
~~~~~
:spec: ``lower(string: String) -> String``
:description:
Lower-case the entire String.
pad
~~~
:spec: ``pad(string: String, length: Integer, char: String) -> String``
:description:
Pads the string to the given length
pad_zero
~~~~~~~~
:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String``
:description:
Pads a numeric with zeros to the given length
replace
~~~~~~~
:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String``
:description:
Replace the ``old`` part of the String with the ``new``. Optionally only replace it
``count`` number of times.
slice
~~~~~
:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String``
:description:
Returns the slice of the Array.
string
~~~~~~
:spec: ``string(value: AnyArgument) -> String``
:description:
Cast to String.
titlecase
~~~~~~~~~
:spec: ``titlecase(string: String) -> String``
:description:
Capitalize each word in the string.
upper
~~~~~
:spec: ``upper(string: String) -> String``
:description:
Upper-case the entire String.
----------------------------------------------------------------------------------------------------
Ytdl-Sub Functions
------------------
legacy_bracket_safety
~~~~~~~~~~~~~~~~~~~~~
:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument``
ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not
interfere with its legacy variable scripting system. This function replicates that
behavior.
sanitize
~~~~~~~~
:spec: ``sanitize(value: AnyArgument) -> String``
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS.
sanitize_plex_episode
~~~~~~~~~~~~~~~~~~~~~
:spec: ``sanitize_plex_episode(string: String) -> String``
Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width
numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the
season and/or episode.
to_date_metadata
~~~~~~~~~~~~~~~~
:spec: ``to_date_metadata(yyyymmdd: String) -> Map``
Takes a date in the form of YYYYMMDD and returns a Map containing:
- date (String, YYYYMMDD)
- date_standardized (String, YYYY-MM-DD)
- year (Integer)
- month (Integer)
- day (Integer)
- year_truncated (Integer, YY from YY[YY])
- month_padded (String)
- day_padded (String)
- year_truncated_reversed (Integer, 100 - year_truncated)
- month_reversed (Integer, 13 - month)
- month_reversed_padded (String)
- day_reversed (Integer, total_days_in_month + 1 - day)
- day_reversed_padded (String)
- day_of_year (Integer)
- day_of_year_padded (String, padded 3)
- day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year)
- day_of_year_reversed_padded (String, padded 3)
to_native_filepath
~~~~~~~~~~~~~~~~~~
:spec: ``to_native_filepath(filepath: String) -> String``
Convert any unix-based path separators ('/') with the OS's native
separator.
truncate_filepath_if_too_long
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:spec: ``truncate_filepath_if_too_long(filepath: String) -> String``
If a file-path is too long for the OS, this function will truncate it while preserving
the extension.

319
docs/source/config_reference/scripting/scripting_types.rst Normal file
View file

@ -0,0 +1,319 @@
Scripting Types
===============
Types
-----
String
~~~~~~
Strings are a series of characters surrounded by quotes and can be defined in a few ways, including:
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
string_variable: "This is a String variable"
.. tab-item:: In-Line
.. code-block:: yaml
string_variable: "{ %string('This is a String variable') }"
.. tab-item:: Single Quote
.. code-block:: yaml
string_variable: >-
{
%string('This is a String variable')
}
.. tab-item:: Double Quote
.. code-block:: yaml
string_variable: >-
{
%string("This is a String variable")
}
.. tab-item:: Triple Quote
.. code-block:: yaml
string_variable: >-
{
%string('''This is a String variable''')
}
.. tab-item:: Triple-Double Quote
.. code-block:: yaml
string_variable: >-
{
%string("""This is a String variable""")
}
.. note::
For non-String types, they must be defined as parameters to scripting functions. This is because
anything in a variable definition that is not within curly-braces gets evaluated as a String.
Integer
~~~~~~~
Integers are whole numbers with no decimal.
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
int_variable: >-
{
%int(2022)
}
.. tab-item:: In-Line
.. code-block:: yaml
int_variable: "{ %int(2022) }"
Float
~~~~~
Floats are floating-point decimals numbers.
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
float_variable: >-
{
%float(3.14)
}
.. tab-item:: In-Line
.. code-block:: yaml
float_variable: "{ %float(3.14) }"
Boolean
~~~~~~~
A type is considered boolean if it spells out ``True`` or ``False``, case-insensitive.
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
bool_variable: >-
{
%bool(True)
}
.. tab-item:: In-Line
.. code-block:: yaml
bool_variable: "{ %bool(FALSE) }"
Array
~~~~~
An Array contains multiple types of any kind, including nested Arrays and Maps.
Arrays are defined using brackets (``[ ]``), and are accessed using zero-based indexing.
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
array_variable: >-
{
[
"element with index 0",
1,
2.0,
[ "Nested Array 3" ]
]
}
element_0: >-
{
%array_at(array_variable, 0)
}
.. tab-item:: In-Line
.. code-block:: yaml
array_variable: "{ ['element with index 0', 1, 2.0, ['Nested Array 3' ]] }"
element_0: "{ %array_at(array_variable, 0) }"
Map
~~~
A Map is a key-value store, containing mappings between keys and values.
Maps are defined using curley-braces (``{ }``), and are accessed using their keys.
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
map_variable: >-
{
{
"string_key": "string_value",
1: "int_key",
"list_value": [ "elem0", 1, 2.0 ]
}
}
string_value: >-
{
%map_get(map_variable, "string_key")
}
.. tab-item:: In-Line
.. code-block:: yaml
map_variable: "{ {'string_key': 'string_value', 1: 'int_key', 'list_value': [ 'elem0', 1, 2.0 ]} }"
string_value: "{ %map_get(map_variable, 'string_key') }"
Null
~~~~
Null is represented by an empty String, and can be conveyed by spelling out ``null``,
case-insensitive.
.. tab-set::
.. tab-item:: Literal
.. code-block:: yaml
null_variable: ""
.. tab-item:: In-Line
.. code-block:: yaml
null_variable: "{ %string(null) }"
Function Type-Hints
-------------------
AnyArgument
~~~~~~~~~~~
AnyArgument means any of the above Types are valid as input or output to a scripting function.
.. note::
Strict typing is enforced. For functions that return ``AnyArgument`` need to be casted before
passing into functions that expect a particular type.
Numeric
~~~~~~~
Numeric refers to either an Integer or Float.
Optional
~~~~~~~~
Optional means a particular scripting function argument can be either provided or not included.
For example, the function
`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
has an optional default value. Both of these usages are valid:
.. tab-set::
.. tab-item:: Map Get
.. code-block:: yaml
will_throw_key_does_not_exist_error: "{ %map_get( {}, 'key' ) }"
.. tab-item:: Map Get with Optional Default Value
.. code-block:: yaml
will_return_default: "{ %map_get( {}, 'key', 'default value' ) }"
Lambda
~~~~~~
Lambda parameters are a reference to a function, and will call that lambda function
on the input. In this example,
.. code-block:: yaml
lambda_array_numeric_to_string: >-
{
%array_apply( [ 1, 2, 3, 4], %string )
}
We apply ``%string`` as a lambda function to
`array_apply <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_,
which is called on every element in the input array. The output becomes ``["1", "2", "3", "4"]``.
This example has one input-argument being passed into the lambda. For other lambda-based functions
like `array_enumerate <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_,
it expects the lambda function to have two input arguments. These are denoted using
``LambdaTwo``, ``LambdaThree``, etc within the function spec.
LambdaReduce
~~~~~~~~~~~~
LambdaReduce is special type of lambda that reduces an Array to a single value by calling the
LabmdaReduce function repeatedly on two elements in the Array until it is reduced to a single value.
In this example,
.. code-block:: yaml
lambda_reduce_sum: >-
{
%array_reduce( [ 1, 2, 3, 4], %add )
}
We call
`array_reduce <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_
on the input array, using
`add <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_
as the LambdaReduce function. This will reduce the Array to a single value by internally calling
.. code-block::
- %add(1, 2) = 3
- %add(3, 3) = 6
- %add(6, 4) = 10
And evaluate to ``10``.
ReturnableArguments
~~~~~~~~~~~~~~~~~~~
Returnable arguments are used in conditional functions like
`if <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_,
which implies the argument passed into the function is the function's output. For example,
.. code-block:: yaml
conditional_function: >-
{
%if( True, "Return this if True", "Return this if False" )
}
is going to return ``"Return this if True"`` since the condition parameter is ``True``.

149
docs/source/config_reference/subscriptions_yaml.rst Normal file
View file

@ -0,0 +1,149 @@
==================
Subscriptions File
==================
------------------
subscriptions.yaml
------------------
The ``subscriptions.yaml`` file is where we use our :ref:`config_reference/config_yaml:presets` in the :ref:`config_reference/config_yaml:config.yaml`
to define a ``subscription``: something we want to recurrently download such as a specific
channel or playlist.
The only difference between a ``subscription`` and ``preset`` is that the subscription
must have all required fields and ``{override_variables}`` defined so it can perform a download.
Below is an example that downloads a YouTube playlist:
.. code-block:: yaml
:caption: config.yaml
presets:
playlist_preset_ex:
download:
download_strategy: "url"
url: "{url}"
output_options:
output_directory: "{output_directory}/{playlist_name}"
file_name: "{playlist_name}.{title}.{ext}"
overrides:
output_directory: "/path/to/ytdl-sub-videos"
.. code-block:: yaml
:caption: subscription.yaml
my_subscription_name:
preset: "playlist_preset_ex"
overrides:
playlist_name: "diy-playlist"
url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
Our preset ``playlist_preset_ex`` defines three
custom variables: ``{output_directory}``, ``{playlist_name}``, and ``{url}``. The subscription sets
the ``parent preset`` to ``playlist_preset_ex``, and must define the variables ``{playlist_name}``
and ``{url}`` since the preset did not.
Beautifying Subscriptions
~~~~~~~~~~~~~~~~~~~~~~~~~
Subscriptions support using presets as keys, and using keys to set override variables as values.
For example:
.. code-block:: yaml
:caption: subscription.yaml
TV Show Full Archive:
= News:
"Breaking News": "https://www.youtube.com/@SomeBreakingNews"
TV Show Only Recent:
= Tech | TV-Y:
"Two Minute Papers": "https://www.youtube.com/@TwoMinutePapers"
Will create two subscriptions named "Breaking News" and "Two Minute Papers", equivalent to:
.. code-block:: yaml
"Breaking News":
preset:
- "TV Show Full Archive"
overrides:
subscription_indent_1: "News"
subscription_name: "Breaking News"
subscription_value: "https://www.youtube.com/@SomeBreakingNews"
"Two Minute Papers":
preset:
- "TV Show Only Recent"
overrides:
subscription_indent_1: "Tech"
subscription_indent_2: "TV-Y"
subscription_name: "Two Minute Papers"
subscription_value: "https://www.youtube.com/@TwoMinutePapers"
You can provide as many parent presets in the form of ``keys``, and subscription indents as ``= keys``.
This can drastically simplify subscription definitions by setting things like so in your
parent preset:
.. code-block:: yaml
presets:
"TV Show Preset":
overrides:
subscription_indent_1: "default-genre"
subscription_indent_2: "default-content-rating"
tv_show_name: "{subscription_name}"
url: "{subscription_value}"
genre: "{subscription_indent_1}"
content_rating: "{subscription_indent_2}"
.. _subscription value:
File Preset
~~~~~~~~~~~
You can apply a preset to all subscriptions in the ``subscription.yaml`` file
by using the file-wide ``__preset__``:
.. code-block:: yaml
:caption: subscription.yaml
__preset__:
preset: "playlist_preset_ex"
my_subscription_name:
overrides:
url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
playlist_name: "diy-playlist"
This ``subscription.yaml`` is equivalent to the one above it because all
subscriptions automatically set ``__preset__`` as a ``parent preset``.
Subscription Value
~~~~~~~~~~~~~~~~~~~
NOTE: This is deprecated in favor of using the method in :ref:`config_reference/subscriptions_yaml:beautifying subscriptions`.
With a clever config and use of ``__preset__``, your subscriptions can typically boil
down to a name and url. You can set ``__value__`` to the name of an override variable,
and use the override variable ``subscription_name`` to achieve one-liner subscriptions.
Using the example above, we can do:
.. code-block:: yaml
:caption: subscription.yaml
__preset__:
preset:
- "tv_show"
overrides:
tv_show_name: "{subscription_name}"
__value__: "url"
# single-line subscription, sets "Brandon Acker" and the subscription value
# to the override variables tv_show_name and url
"Brandon Acker": "https://www.youtube.com/@brandonacker"
Traditional subscriptions that can override presets will still work when using ``__value__``.
``__value__`` can also be set within a :ref:`config_reference/config_yaml:config.yaml`.

40
docs/deprecation_notices.rst → docs/source/deprecation_notices.rst
View file

@ -5,55 +5,55 @@ Oct 2023
--------
subscription preset and value
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The use of ``__value__`` will go away in Dec 2023 in favor of the method found in
:ref:`beautifying subscriptions`. ``__preset__`` will still be supported for the time being.
:ref:`config_reference/subscriptions_yaml:beautifying subscriptions`. ``__preset__`` will still be supported for the time being.
July 2023
---------
music_tags
^^^^^^^^^^
~~~~~~~~~~
Music tags are getting simplified. ``tags`` will now reside directly under music_tags, and
``embed_thumbnail`` is getting moved to its own plugin (supports video files as well). Convert from:
.. code-block:: yaml
my_example_preset:
music_tags:
embed_thumbnail: True
tags:
artist: "Elvis Presley"
my_example_preset:
music_tags:
embed_thumbnail: True
tags:
artist: "Elvis Presley"
To the following:
.. code-block:: yaml
my_example_preset:
embed_thumbnail: True
music_tags:
artist: "Elvis Presley"
my_example_preset:
embed_thumbnail: True
music_tags:
artist: "Elvis Presley"
The old format will be removed in October 2023.
video_tags
^^^^^^^^^^
~~~~~~~~~~
Video tags are getting simplified as well. ``tags`` will now reside directly under video_tags.
Convert from:
.. code-block:: yaml
my_example_preset:
video_tags:
tags:
title: "Elvis Presley Documentary"
my_example_preset:
video_tags:
tags:
title: "Elvis Presley Documentary"
To the following:
.. code-block:: yaml
my_example_preset:
video_tags:
title: "Elvis Presley Documentary"
my_example_preset:
video_tags:
title: "Elvis Presley Documentary"

59
docs/source/faq/index.rst Normal file
View file

@ -0,0 +1,59 @@
FAQ
===
Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as
more questions get asked.
.. contents:: Frequently Asked Questions
:depth: 3
How do I...
-----------
...download age-restricted YouTube videos?
''''''''''''''''''''''''''''''''''''''''''
See
`ytdls recommended way <https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_
to download your YouTube cookie, then add it to your
`ytdl options <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl-options>`_ section of your config:
.. code-block:: yaml
ytdl_options:
cookiefile: "/path/to/cookies/file.txt"
...automate my downloads?
'''''''''''''''''''''''''
`This part of the wiki <https://github.com/jmbannon/ytdl-sub/wiki/7.-Automate-Downloading-New-Content-Using-Your-Configs>`_ shows how to set up ``ytdl-sub`` to run in a cron job within Docker.
There is a bug where...
-----------------------
...date_range is not downloading older videos after I changed the range
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether.
.. code-block:: yaml
ytdl_options:
break_on_existing: False
After your download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads.
...it is downloading non-English title and description metadata
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using
.. code-block:: yaml
ytdl_options:
extractor_args:
youtube:
lang:
- "en"
...Plex is not showing my TV shows correctly
''''''''''''''''''''''''''''''''''''''''''''
Set the following
`Scanner and Agent <https://i.imgur.com/zdZhCLZ.png>`_
for your library.

5
docs/source/guides/development/index.rst Normal file
View file

@ -0,0 +1,5 @@
Development and Contributing
============================
.. toctree::

21
docs/source/guides/getting_started/advanced_configuration.rst Normal file
View file

@ -0,0 +1,21 @@
Advanced Configuration
======================
If the :doc:`prebuilt presets </prebuilt_presets/index>` aren't suitable for your needs, you may want to set up an advanced configuration.
Layout of a Config file
-----------------------
The layout of the ``config.yaml`` file is relatively straightforward:
.. code-block:: yaml
presets:
preset_name:
plugin1:
plugin1_option1: value1
Preset Inheritance
------------------

140
docs/source/guides/getting_started/automating_downloads.rst Normal file
View file

@ -0,0 +1,140 @@
Automating Downloads
====================
One of the key capabilities of ``ytdl-sub`` is how well it runs without user input, but to take advantage of this you must set up scheduling to execute the commands at some interval. How you set up this scheduling depends on which version of ``ytdl-sub`` you downloaded.
:ref:`Guide for Docker and Unraid Containers <guides/getting_started/automating_downloads:docker and unraid>`
:ref:`Guide for Linux <guides/getting_started/automating_downloads:linux>`
:ref:`Guide for Windows <guides/getting_started/automating_downloads:windows>`
.. _cron tab manpage: https://man7.org/linux/man-pages/man5/crontab.5.html#EXAMPLE_CRON_FILE
.. _docker-unraid-setup:
Docker and Unraid
-----------------
.. tab-set::
.. tab-item:: GUI Image
The script that will execute automatically is located at ``/config/ytdl-sub-configs/run-cron``.
Access your container at http://localhost:8443/, then in the GUI terminal run these commands:
.. code-block:: shell
echo '#!/bin/bash' > /config/ytdl-sub-configs/run_cron
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/ytdl-sub-configs/run_cron
echo "echo 'Cron started, running ytdl-sub...'" >> /config/ytdl-sub-configs/run_cron
echo "cd /config/ytdl-sub-configs" >> /config/ytdl-sub-configs/run_cron
echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/ytdl-sub-configs/run_cron
chmod +x /config/ytdl-sub-configs/run_cron
chown abc:abc /config/ytdl-sub-configs/run_cron
You can test the newly created script by running:
.. code-block:: shell
/config/ytdl-sub-configs/run_cron
To create the cron definition, run the following command:
.. code-block:: shell
echo "# min hour day month weekday command" > /config/crontabs/abc
echo " 0 */6 * * * /config/ytdl-sub-configs/run_cron" >> /config/crontabs/abc
This will run the script every 6 hours. To run every hour, change ``*/6`` to ``*/1``, or to run once a day, change the same value to the hour (in 24hr format) that you want it to run at. See the `cron tab manpage`_ for more options.
.. tab-item:: Headless Image
.. _LinuxServer's Universal Cron mod: https://github.com/linuxserver/docker-mods/tree/universal-cron
The first step is to ensure you have `LinuxServer's Universal Cron mod`_ enabled via the environment variable. For the GUI image, this is already included (no need to add it).
.. code-block:: yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- DOCKER_MODS=linuxserver/mods:universal-cron # <-- Make sure you have this!
volumes:
# ensure directories have user permissions
- </path/to/ytdl-sub/config>:/config
- </path/to/ytdl-sub/tv_shows>:/tv_shows
restart: unless-stopped
This line will tell your container to install and enable cron on start.
If you had to add this line, you will need to restart your container.
.. code-block:: shell
docker compose restart
The script that will execute automatically is located at ``/config/run-cron``.
Access your container from the terminal by running:
.. code-block:: shell
docker exec -itu abc ytdl-sub /bin/bash
then in the terminal run these commands:
.. code-block:: shell
echo '#!/bin/bash' > /config/ytdl-sub-configs/run_cron
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/ytdl-sub-configs/run_cron
echo "echo 'Cron started, running ytdl-sub...'" >> /config/ytdl-sub-configs/run_cron
echo "cd /config/ytdl-sub-configs" >> /config/ytdl-sub-configs/run_cron
echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/ytdl-sub-configs/run_cron
chmod +x /config/ytdl-sub-configs/run_cron
chown abc:abc /config/ytdl-sub-configs/run_cron
You can test the newly created script by running:
.. code-block::
/config/run_cron
To create the cron definition, run the following command:
.. code-block:: shell
echo "# min hour day month weekday command" > /config/crontabs/abc
echo " 0 */6 * * * /config/run_cron" >> /config/crontabs/abc
This will run the script every 6 hours. To run every hour, change ``*/6`` to ``*/1``, or to run once a day, change the same value to the hour (in 24hr format) that you want it to run at. See the `cron tab manpage`_ for more options.
.. _linux-setup:
Linux
-----
.. code-block:: shell
crontab -e
0 */6 * * * /config/run_cron
.. _windows-setup:
Windows
-------
To be tested (please contact code owner or join the discord server if you can test this out for us)
.. code-block:: powershell
ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml

15
docs/source/guides/getting_started/examples.rst Normal file
View file

@ -0,0 +1,15 @@
=====================
Using Example Configs
=====================
Copy and paste the examples into local yaml files, modify the
``working_directory`` and ``output_directory`` with your desired paths,
and perform a dry-run using
.. code-block:: bash
ytdl-sub \
--dry-run \
--config path/to/config.yaml \
sub path/to/subscriptions.yaml
This will simulate what a download will look like.

25
docs/source/guides/getting_started/first_config.rst Normal file
View file

@ -0,0 +1,25 @@
Basic Configuration
===================
Your first configuration will look pretty simple:
.. code-block:: yaml
:linenos:
configuration:
working_directory: '.ytdl-sub-downloads'
The first two lines in this ``config.yaml`` file are the ``configuration``, and define the ``working_directory``, which is described near the bottom of :ref:`this section <guides/getting_started/index:quick overview of \`\`ytdl-sub\`\`>`
Line 4 begins the definition of your custom ``presets``, with line 5 being the name of your first custom ``preset``.
Lines 7 and 8 tell ``ytdl-sub`` which :doc:`/prebuilt_presets/index` to expand on; these ``presets`` already indicate that the downloaded files should be:
- in a format usable by, and with metadata accessible to, Jellyfin
- sorted by upload date, and
- only uploaded in the last 2 months (and will also delete any files in the media library which were uploaded over 2 months ago)
Line 11 is an override variable, ``tv_show_directory``, that tells ``ytdl-sub`` where to save your downloaded files once they've been processed, also known as the ``output_directory``. In this case, the downloaded files will be saved to the ``youtube`` folder in the root ``tv_shows`` directory.

30
docs/source/guides/getting_started/first_download.rst Normal file
View file

@ -0,0 +1,30 @@
Initial Download
================
Once you have the ``config.yaml`` and ``subscriptions.yaml`` files created and filled out, you can perform your first download. Access ``ytdl-sub``, navigate to the directory containing your ``config.yaml`` and ``subscriptions.yaml`` files, then run the below command:
.. tab-set::
.. tab-item:: Dry run
A dry run lets you check that your configuration doesn't throw any errors and what the expected output files of actually doing the download are, without actually downloading the full media.
.. code-block:: shell
ytdl-sub --dry-run sub
.. tab-item:: Normal run
A normal run will download all files as determined by your ``presets`` and, once processing is finished, move the downloaded and processed files to your ``output_directory``.
.. code-block:: shell
ytdl-sub sub
.. tab-item:: One-time download
Sometimes you may only want to download media once, in which case adding them to your ``subscriptions.yaml`` file is unneccessary. As an example, the below code will download the same videos as our subscription file:
.. code-block:: shell
ytdl-sub dl --preset "My Favorite YouTube Channels" --overrides.subscription_name "Rick Astley" --overrides.subscription_value "https://www.youtube.com/@RickAstleyYT/videos"

21
docs/source/guides/getting_started/first_sub.rst Normal file
View file

@ -0,0 +1,21 @@
Initial Subscription
====================
Your first subscription should look similar to the below:
.. code-block:: yaml
:linenos:
__preset__:
overrides:
tv_show_directory: "/tv_shows"
"My Favorite YouTube Channels":
"Rick Astley": "https://www.youtube.com/@RickAstleyYT/videos"
The first three lines in this subscription file define where to save the downloaded files associated with all subscriptions in this file.
The fifth line in this subscription file is the ``preset``, which provides the "definitions" for the subscription as listed in :doc:`/guides/getting_started/first_config`.
The sixth line is the actual ``subscription``, named ``Rick Astley``, with a link to a :yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`, in this case a YouTube channel.

88
docs/source/guides/getting_started/index.rst Normal file
View file

@ -0,0 +1,88 @@
Getting Started
===============
Now that you've completed your install of ``ytdl-sub``, it's time to get started. This is a 3-step process:
- Create your configuration file (if the :doc:`/prebuilt_presets/index` don't fit your needs)
- Create your subscription file
- Automate starting YTDL-Sub
Prerequisite Knowledge
----------------------
.. _navigate directories: https://en.wikipedia.org/wiki/Cd_(command)
.. _YAML syntax: https://yaml.org/spec/1.2.2/#chapter-2-language-overview
In order to use ``ytdl-sub`` in any of the forms listed in these docs, you will need some basic knowledge.
Be sure that you:
☑ Can `navigate directories`_ in a command line interface (or CLI)
☑ Have a basic understanding of `YAML syntax`_
If you plan on using the headless image of ``ytdl-sub``, you:
☑ Can use ``nano`` or ``vim`` to edit OR
☑ Can mount the config directory somewhere you can open it using gui text editors
Additional useful (but not required) knowledge:
☑ Understanding how :yt-dlp:`\ ` works
Quick Overview of ``ytdl-sub``
------------------------------
``ytdl-sub`` uses two types of YAML files:
- ``config.yaml`` defines ``presets``, which are the "definitions" of your media. ``presets`` "define" how you want your media downloaded, which formats, naming conventions to follow when saving them, etc. These ``presets`` can also inherit other ``presets``, so that you can easily modify an existing ``preset``.
- ``subscriptions.yaml`` defines ``subscriptions``, which specify the media we want to recurrently download, like YouTube channels and playlists, SoundCloud artists, or any :yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`. ``subscriptions`` use ``presets`` to define how ``ytdl-sub`` should handle downloading, processing, and saving them.
When ``ytdl-sub`` is run, in its most basic form:
.. tab-set-code::
.. code-block:: shell
ytdl-sub sub
.. code-block:: powershell
ytdl-sub.exe sub
``ytdl-sub`` initially downloads all files to a defined ``working_directory``. This is a temporary storage spot for metadata and media files so that errors during processing- if they occur- don't affect your existing media library. Once all file processing is complete, your media files are moved to the ``output_directory``.
Ready to Start?
---------------
Now that you have installed ``ytdl-sub``, checked your skills, and gotten a bit of background on how ``ytdl-sub`` functions, read through the articles below to get started:
:doc:`Step 1: Initial Subscriptions <first_sub>`
:doc:`Step 2: Your First Download <first_download>`
:doc:`Step 3: Automating Downloads <automating_downloads>`
Want to go a step further?
If you want to use atypical paths or specific configuration options, check out :doc:`Basic Configuration <first_config>`
For tips on creating your own presets when the prebuilt presets aren't cutting it, check out :doc:`Advanced Configuration <advanced_configuration>`
Other docs that may be of use:
:doc:`/prebuilt_presets/index`
:doc:`examples`
.. toctree::
:hidden:
:caption: Getting Started Guide
:maxdepth: 1
first_sub
first_download
automating_downloads
first_config
advanced_configuration
examples

7
docs/source/guides/index.rst Normal file
View file

@ -0,0 +1,7 @@
Guides
======
.. toctree::
install/index
getting_started/index
development/index

43
docs/source/guides/install/agnostic.rst Normal file
View file

@ -0,0 +1,43 @@
====================
Environment Agnostic
====================
The PIP install method is not recommended; use of this method may cause unintended requirement conflicts if you have other locally installed apps that depend on ffmpeg.
PIP Install
--------------
You can install our
`PyPI package <https://pypi.org/project/ytdl-sub/>`_.
Both ffmpeg and Python 3.10 or greater are required.
.. code-block:: bash
python3 -m pip install -U ytdl-sub
Install for Development
=======================
These environment-agnostic methods of installing ``ytdl-sub`` are meant for local development of ``ytdl-sub``. If you want to contribute your changes, please read :doc:`/guides/development/index`.
Local Install
--------------
With a Python 3.10 virtual environment, you can clone and install the repo.
.. code-block:: bash
git clone https://github.com/jmbannon/ytdl-sub.git
cd ytdl-sub
pip install -e .
Local Docker Build
-------------------
Run ``make docker`` in the root directory of this repo to build the image. This
will build the python wheel and install it in the Dockerfile.
.. code-block:: bash
git clone https://github.com/jmbannon/ytdl-sub.git
cd ytdl-sub
make docker

155
docs/source/guides/install/docker.rst Normal file
View file

@ -0,0 +1,155 @@
======
Docker
======
For automating ``subscriptions.yaml`` downloads to pull new media, see :ref:`this page <guides/getting_started/automating_downloads:docker and unraid>` on how to set up a cron job in any of the docker containers.
The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install ytdl-sub on top. There are two flavors to choose from.
.. margin::
.. tip::
The recommended docker image is the GUI image.
:ref:`Docker Compose <guides/install/docker:install with docker compose>` is the recommended way of setting up a ``ytdl-sub`` docker container.
GUI Image
---------
The GUI image uses LSIO's :lsio-gh:`docker-code-server image` for its base image. More info on other code-server environment variables can be found within its documentation.
After starting, code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``.
Headless Image
--------------
The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image` for its base image. Execute the following command to access and interact with ``ytdl-sub``:
.. code-block:: bash
docker exec -u abc -it ytdl-sub /bin/bash
Install with Docker Compose
---------------------------
Docker Compose is an easy "set it and forget it" install method. Follow the instructions below to create a ``compose.yaml`` file for your chosen ``ytdl-sub`` image.
.. margin::
.. important::
Set the PUID and PGID to the UID and GID associated with the user you want to own the downloaded files. Setting these values to root UID and GID may create issues with your media managers.
.. tab-set::
.. tab-item:: GUI Image
.. code-block:: yaml
:caption: compose.yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
ports:
- 8443:8443
restart: unless-stopped
.. tab-item:: Headless Image
.. code-block:: yaml
:caption: compose.yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub:latest
container_name: ytdl-sub
environment:
- PUID=1000
- PGID=1000
- TZ=America/Los_Angeles
- DOCKER_MODS=linuxserver/mods:universal-cron
volumes:
- <path/to/ytdl-sub/config>:/config
- <path/to/tv_shows>:/tv_shows # optional
- <path/to/movies>:/movies # optional
- <path/to/music_videos>:/music_videos # optional
- <path/to/music>:/music # optional
restart: unless-stopped
Device Passthrough
~~~~~~~~~~~~~~~~~~~
For CPU or GPU passthrough, you must use either the GUI image or the headless Ubuntu image
``ghcr.io/jmbannon/ytdl-sub:ubuntu-latest``.
The docker-compose examples use the GUI image.
CPU Passthrough
^^^^^^^^^^^^^^^
.. code-block:: yaml
:emphasize-lines: 5-6
:caption: compose.yaml
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
devices:
- /dev/dri:/dev/dri # CPU passthrough
restart: unless-stopped
GPU Passthrough
^^^^^^^^^^^^^^^
.. Awe
.. code-block:: yaml
:caption: compose.yaml
:emphasize-lines: 5-13
services:
ytdl-sub:
image: ghcr.io/jmbannon/ytdl-sub-gui:latest
container_name: ytdl-sub
environment:
- ..
- NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args
- NVIDIA_VISIBLE_DEVICES=all
deploy:
resources:
reservations:
devices:
- capabilities: ["gpu"] # GPU passthrough
restart: unless-stopped
Docker CLI
----------
If you prefer to only run the container once, you can use the CLI command instead. The following command is for the gui image, and will not restart if it comes down for any reason. See `the Docker reference <https://docs.docker.com/engine/reference/run/>`_ for further information on the parameters and other options you can use.
.. code-block:: bash
docker run -d \
--name=ytdl-sub \
-e PUID=1000 \
-e PGID=1000 \
-e TZ=America/Los_Angeles \
-p 8443:8443 \
-v <path/to/ytdl-sub/config>:/config \
-v <OPTIONAL/path/to/tv_shows>:/tv_shows \
-v <OPTIONAL/path/to/movies>:/movies \
-v <OPTIONAL/path/to/music_videos>:/music_videos \
-v <OPTIONAL/path/to/music>:/music \
ghcr.io/jmbannon/ytdl-sub-gui:latest

34
docs/source/guides/install/index.rst Normal file
View file

@ -0,0 +1,34 @@
Install by Platform
===================
``ytdl-sub`` can be installed on the following platforms.
All installations require a 64-bit CPU. 32-bit is not supported.
.. margin::
.. tip::
The recommended install method of ``ytdl-sub`` is one of our :doc:`docker containers </guides/install/docker>`. For install on Unraid, check out our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>`.
:doc:`/guides/install/docker`
:doc:`/guides/install/unraid`
:doc:`/guides/install/linux`
:doc:`/guides/install/windows`
:doc:`/guides/install/agnostic`
Once you've completed your installation, please refer to the :doc:`../getting_started/index` guide for next steps
.. toctree::
:hidden:
docker
linux
unraid
windows
agnostic

50
docs/source/guides/install/linux.rst Normal file
View file

@ -0,0 +1,50 @@
=====
Linux
=====
``ytdl-sub`` should be installable using any Linux package manager, and requires ffmpeg to be installed.
.. tab-set::
.. tab-item:: Linux
.. code-block:: bash
curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub
chmod +x ytdl-sub
ytdl-sub -h
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
.. code-block:: bash
curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xf ffmpeg.tar.gz
chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg
chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe
# May need sudo / root permissions to perform
mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg
mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe
.. tab-item:: Linux ARM
.. code-block:: bash
curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub_aarch64
chmod +x ytdl-sub
ytdl-sub -h
You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
.. code-block:: bash
curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz
tar -xf ffmpeg.tar.gz
chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg
chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe
# May need sudo / root permissions to perform
mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg
mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe

3
docs/source/guides/install/unraid.rst Normal file
View file

@ -0,0 +1,3 @@
Unraid
--------------
You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>` through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_. Uses Docker under the hood.

15
docs/source/guides/install/windows.rst Normal file
View file

@ -0,0 +1,15 @@
Windows
--------------
From powershell, run:
.. code-block:: powershell
# Download ffmpeg/ffprobe dependencies from yt-dlp
curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
tar -xf ffmpeg.zip
move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe"
move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
# Download ytdl-sub
curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe
ytdl-sub.exe -h

16
docs/source/index.rst Normal file
View file

@ -0,0 +1,16 @@
ytdl-sub User Guide
===================
.. toctree::
:maxdepth: 2
:titlesonly:
introduction
guides/index
prebuilt_presets/index
usage
config_reference/index
faq/index
deprecation_notices
.. note:: The docs are heavily work-in-progress. Please bear with us while we're under construction!

43
docs/source/introduction.rst Normal file
View file

@ -0,0 +1,43 @@
=================
What is ytdl-sub?
=================
.. _yt-dlp: https://github.com/yt-dlp/yt-dlp
.. _kodi: https://github.com/xbmc/xbmc
.. _jellyfin: https://github.com/jellyfin/jellyfin
.. _plex: https://github.com/plexinc/pms-docker
.. _emby: https://github.com/plexinc/pms-docker
``ytdl-sub`` is a command-line tool that downloads media via `yt-dlp`_ and prepares it for your favorite media player (`Kodi`_, `Jellyfin`_, `Plex`_, `Emby`_, modern music players).
Visual examples
===============
.. figure:: https://user-images.githubusercontent.com/10107080/182677243-b4184e51-9780-4094-bd40-ea4ff58555d0.PNG
:alt: The Jellyfin web interface, showing the thumbnails of various YouTube shows.
Youtube channels as TV shows in Jellyfin
.. figure:: https://user-images.githubusercontent.com/10107080/182677256-43aeb029-0c3f-4648-9fd2-352b9666b262.PNG
:alt: The Jellyfin web interace, showing the thumbnails of various music videos starring the Red Hot Chili Peppers
Music videos and concerts in Jellyfin
.. figure:: https://user-images.githubusercontent.com/10107080/182677268-d1bf2ff0-9b9c-4a04-98ec-443a67ada734.png
:alt: The Kodi app interface, showing a list of artists available to watch under the "Music videos" heading
Music videos and concerts in Kodi
.. figure:: https://user-images.githubusercontent.com/10107080/182685415-06adf477-3dd3-475d-bbcd-53b0152b9f0a.PNG
:alt: The MusicBee app interface, showing a list of album artists and the thumbnails of all downloaded songs produced by the currently selected artist
SoundCloud albums and singles in MusicBee
Why ytdl-sub?
-------------
There is a lack of open-source tools to download media and generate metadata to play it in these players. Most solutions involve using multiple tools or bash scripts to achieve this. ``ytdl-sub`` aims to consolidate all of this logic into a single easy-to-use application that can run automatically once configured.
Why download instead of stream?
-------------------------------
We believe it is important to download what you like because there is no guarantee it will stay online forever. We also believe it is important to download it in such a way that it is easy to consume. Most solutions today force you to watch/listen to your downloaded content via file system or web browser. ``ytdl-sub`` aims to format downloaded content for any media player.

37
docs/source/prebuilt_presets/helpers.rst Normal file
View file

@ -0,0 +1,37 @@
==============
Helper Presets
==============
Common presets are not usable by themselves- setting one of these as the sole preset of your subscription and attempting to download will not work. But you can add these presets to quickly modify an existing preset to better suit your needs.
Best A/V Quality
----------------
Add the following preset to download the best available video and audio quality, and remux it into an MP4 container:
``best_video_quality``
Max 1080p Video
---------------
Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container:
``max_1080p``
Chunk Initial Download
----------------------
If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the following preset:
``chunk_initial_download``
It will download videos starting from the oldest one, and only download 20 at a time. You can
change this number by setting:
.. code-block:: yaml
ytdl_options:
max_downloads: 30 # Desired number to download per invocation
Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to oldest again, and stop pulling additional metadata once it reaches a video that has already been downloaded.

14
docs/source/prebuilt_presets/index.rst Normal file
View file

@ -0,0 +1,14 @@
================
Prebuilt Presets
================
``ytdl-sub`` offers a number of built-in presets using best practices for formatting
media in various players. For advanced users, you can review the prebuilt preset
definitions :doc:`here </config_reference/prebuilt_presets/index>`.
.. toctree::
:titlesonly:
helpers
tv_shows
music

3
docs/source/prebuilt_presets/music.rst Normal file
View file

@ -0,0 +1,3 @@
=============
Music Presets
=============

171
docs/source/prebuilt_presets/tv_shows.rst Normal file
View file

@ -0,0 +1,171 @@
===============
TV Show Presets
===============
Player-Specific Presets
=======================
``ytdl-sub`` provides player-specific versions of certain presets, which apply settings to optimize the downloads for that player.
The following actions are taken based on the indicated player:
Jellyfin
~~~~~~~~
* Places any season-specific poster art in the main show folder
* Generates NFO tags
Kodi
~~~~
* Everything that the Jellyfin version does
* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□``
Plex
~~~~
* :ref:`Special sanitization <config_reference/scripting/entry_variables:title_sanitized_plex>` of numbers so Plex doesn't recognize numbers that are part of the title as the episode number
* Converts all downloaded videos to the mp4 format
* Places any season-specific poster art into the season folder
----------------------------------------------
Generic Presets
===============
There are two main methods for downloading and formatting videos as a TV show.
TV Show by Date
---------------
TV Show by Date will organize something like a YouTube channel or playlist into a tv show, where seasons and episodes are organized using upload date.
Plug and Play Presets
~~~~~~~~~~~~~~~~~~~~~
You can use any of these presets in your ``subscriptions.yaml`` as a "One size fits all" solution- they should set all appropriate values. These will organize seasons by year and episodes by month, then day.
Must define ``tv_show_directory``
* ``"Kodi TV Show by Date"``
* ``"Jellyfin TV Show by Date"``
* ``"Plex TV Show by Date"``
Advanced Usage
~~~~~~~~~~~~~~
If you prefer a different organization method, you can instead apply multiple presets to your subscriptions.
You will need a base of one of the below:
* ``kodi_tv_show_by_date``
* ``jellyfin_tv_show_by_date``
* ``plex_tv_show_by_date``
And then add one of these:
* ``season_by_year__episode_by_month_day``
* ``season_by_year_month__episode_by_day``
* ``season_by_year__episode_by_month_day_reversed``
* Episode numbers are reversed, meaning more recent episodes appear at the top of a season by having a lower value.
* ``season_by_year__episode_by_download_index``
* Episodes are numbered by the download order. NOTE that this is fetched using the length of the download archive. Do not use if you intend to remove old videos.
An example of a subscription that will be played on Kodi, organized by year with the most recent episode at the top (having a lower episode number), with a genre of "Pop":
.. code-block:: yaml
:caption: subscriptions.yaml
__preset__:
overrides:
tv_show_directory: "/tv_shows"
kodi_tv_show_by_date:
season_by_year_episode_by_month_day_reversed:
= Pop:
"Rick A": "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
You can also choose to combine multiple URLs into one show. This will result in your videos being downloaded to the same folder, and the episode numbers being shared between them (so you won't have two episode 10's, for example). Note that you may :ytdl-sub-gh:`experience issues <issues/833>` if you use more than 20 URLs at this time.
.. code-block:: yaml
:caption: subscriptions.yaml
__preset__:
overrides:
tv_show_directory: "/tv_shows"
kodi_tv_show_by_date:
season_by_year_episode_by_month_day_reversed:
= Pop:
"~Rick A":
url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
url2: "https://www.youtube.com/@just.rick_6"
TV Show Collection
------------------
TV Show Collections are made up of multiple URLs, where each URL is a season.
If a video belongs to multiple URLs (i.e. a channel and a channel's playlist),
it will resolve to the bottom-most season, as defined in the subscription.
Two main use cases of a collection are:
1. Organize a YouTube channel TV show where Season 1 contains any video
not in a 'season playlist', Season 2 for 'Playlist A', Season 3 for
'Playlist B', etc.
2. Organize one or more YouTube channels/playlists, where each season
represents a separate channel/playlist.
Player Presets
~~~~~~~~~~~~~~
* ``kodi_tv_show_collection``
* ``jellyfin_tv_show_collection``
* ``plex_tv_show_collection``
Episode Formatting Presets
~~~~~~~~~~~~~~~~~~~~~~~~~~
* ``season_by_collection__episode_by_year_month_day``
* ``season_by_collection__episode_by_year_month_day_reversed``
* ``season_by_collection__episode_by_playlist_index``
* Only use playlist_index episode formatting for playlists that will be fully downloaded once and never again. Otherwise, indices can change.
* ``season_by_collection__episode_by_playlist_index_reversed``
Season Presets
~~~~~~~~~~~~~~
* ``collection_season_1``
* ``collection_season_2``
* ``collection_season_3``
* ``collection_season_4``
* ``...``
* ``collection_season_40``
Example
~~~~~~~
A preset/subscription requires specifying a player, episode formatting, and
one or more season presets, with the following override variables:
.. code-block:: yaml
rick_a_tv_show_collection:
preset:
- "jellyfin_tv_show_collection"
- "season_by_collection__episode_by_year_month_day_reversed"
- "collection_season_1"
- "collection_season_2"
overrides:
# required
tv_show_name: "Rick A"
tv_show_directory: "/path/to/youtube_shows"
collection_season_1_url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
collection_season_1_name: "All Videos"
collection_season_2_url: "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
collection_season_2_name: "Official Music Videos"
# can be modified from their default value
# tv_show_genre: "ytdl-sub"
# episode_title: "{upload_date_standardized} - {title}"
# episode_description: "{webpage_url}"

3
docs/source/requirements.txt Normal file
View file

@ -0,0 +1,3 @@
sphinx-book-theme==1.0.1
sphinx-copybutton==0.5.2
sphinx-design==0.5.0

24
docs/usage.rst → docs/source/usage.rst
View file

@ -3,7 +3,7 @@ Usage
.. code-block::
ytdl-sub [GENERAL OPTIONS] {sub,dl,view} [COMMAND OPTIONS]
ytdl-sub [GENERAL OPTIONS] {sub,dl,view} [COMMAND OPTIONS]
For Windows users, it would be ``ytdl-sub.exe``
@ -43,28 +43,28 @@ Download a single subscription in the form of CLI arguments.
.. code-block::
ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS]
ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS]
``SUBSCRIPTION ARGUMENTS`` are exactly the same as YAML arguments, but use periods (``.``) instead
of indents for specifying YAML from the CLI. For example, you can represent this subscription:
.. code-block:: yaml
rick_a:
preset:
- "tv_show"
overrides:
tv_show_name: "Rick A"
url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
rick_a:
preset:
- "tv_show"
overrides:
tv_show_name: "Rick A"
url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
Using the command:
.. code-block:: bash
ytdl-sub dl \
--preset "tv_show" \
--overrides.tv_show_name "Rick A" \
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
ytdl-sub dl \
--preset "tv_show" \
--overrides.tv_show_name "Rick A" \
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
See how to shorten commands using
`download aliases <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.

3
examples/advanced/tv_show_config.yaml
View file

@ -97,4 +97,5 @@ presets:
- "Only Recent"
overrides:
date_range: "2months"
only_recent_date_range: "2months"
only_recent_max_files: 30

5
examples/tv_show_subscriptions.yaml
View file

@ -23,7 +23,10 @@
__preset__:
overrides:
tv_show_directory: "/tv_shows" # Root folder of all ytdl-sub TV Shows
date_range: "2months" # For 'Only Recent' preset, only keep vids uploaded in this range
# For 'Only Recent' preset, only keep vids within this range and limit
only_recent_date_range: "2months"
only_recent_max_files: 30
# Choose the player you intend to use by setting the top-level key to be either:
# - Plex TV Show by Date:

6
pyproject.toml
View file

@ -5,6 +5,7 @@ force_single_line = true
[tool.black]
line_length = 100
target-version = ["py310"]
[tool.pylint.MASTER]
disable = [
@ -43,3 +44,8 @@ ignore = [
include = [
"src/*"
]
[tool.coverage.report]
exclude_also = [
"raise UNREACHABLE.*",
]

3
setup.cfg
View file

@ -27,7 +27,7 @@ package_dir =
packages=find:
install_requires =
yt-dlp==2023.10.13
yt-dlp==2023.11.16
argparse==1.4.0
colorama==0.4.6
mergedeep==1.3.4
@ -49,7 +49,6 @@ lint =
black==22.3.0
isort==5.10.1
pylint==2.13.5
pydocstyle[toml]==6.1.1
docs =
sphinx==4.5.0
sphinx-rtd-theme==1.0.0

2
src/ytdl_sub/cli/entrypoint.py
View file

@ -221,6 +221,7 @@ def main() -> List[Subscription]:
"full backup before usage. You have been warned!",
)
logger.info("Validating subscriptions...")
subscriptions = _download_subscriptions_from_yaml_files(
config=config,
subscription_paths=args.subscription_paths,
@ -230,6 +231,7 @@ def main() -> List[Subscription]:
# One-off download
elif args.subparser == "dl":
logger.info("Validating presets...")
subscriptions.append(
_download_subscription_from_cli(
config=config, dry_run=args.dry_run, extra_args=extra_args

4
src/ytdl_sub/config/config_file.py
View file

@ -6,8 +6,8 @@ from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.preset import Preset
from ytdl_sub.utils.exceptions import FileNotFoundException
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_path import FilePathTruncater
from ytdl_sub.utils.yaml import load_yaml
from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin
class ConfigFile(ConfigValidator):
@ -36,7 +36,7 @@ class ConfigFile(ConfigValidator):
ffprobe_path=self.config_options.ffprobe_path,
)
FilePathValidatorMixin.set_max_file_name_bytes(
FilePathTruncater.set_max_file_name_bytes(
max_file_name_bytes=self.config_options.file_name_max_bytes
)

13
src/ytdl_sub/config/config_validator.py
View file

@ -10,7 +10,6 @@ from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH
from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
from ytdl_sub.subscriptions.utils import SUBSCRIPTION_VALUE_CONFIG_KEY
from ytdl_sub.validators.file_path_validators import FFmpegFileValidator
from ytdl_sub.validators.file_path_validators import FFprobeFileValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@ -107,7 +106,6 @@ class ConfigOptions(StrictDictValidator):
"ffprobe_path",
"file_name_max_bytes",
"experimental",
SUBSCRIPTION_VALUE_CONFIG_KEY,
}
def __init__(self, name: str, value: Any):
@ -142,9 +140,6 @@ class ConfigOptions(StrictDictValidator):
self._file_name_max_bytes = self._validate_key(
key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES
)
self._subscription_value = self._validate_key_if_present(
key=SUBSCRIPTION_VALUE_CONFIG_KEY, validator=StringValidator
)
@property
def working_directory(self) -> str:
@ -237,14 +232,6 @@ class ConfigOptions(StrictDictValidator):
"""
return self._ffprobe_path.value
@property
def subscription_value(self) -> Optional[str]:
"""
Sets the :ref:`subscription value` for subscription
files that use this config.
"""
return self._subscription_value.value if self._subscription_value else None
class ConfigValidator(StrictDictValidator):
_optional_keys = {"configuration", "presets"}

199
src/ytdl_sub/config/overrides.py Normal file
View file

@ -0,0 +1,199 @@
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
import mergedeep
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.variables.override_variables import OverrideHelpers
from ytdl_sub.entries.variables.override_variables import OverrideVariables
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
class Overrides(DictFormatterValidator, Scriptable):
"""
Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter.
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
overrides:
output_directory: "/path/to/media"
custom_file_name: "{upload_date_standardized}.{title_sanitized}"
# Then use the override variables in the output options
output_options:
output_directory: "{output_directory}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
Override variables can contain explicit values and other variables, including both override
and source variables.
In addition, any override variable defined will automatically create a ``sanitized`` variable
for use. In the example above, ``output_directory_sanitized`` will exist and perform
sanitization on the value when used.
"""
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
dict_formatter = DictFormatterValidator(name=name, value=value)
_ = [parse(format_string) for format_string in dict_formatter.dict_with_format_strings]
def __init__(self, name, value):
DictFormatterValidator.__init__(self, name, value)
Scriptable.__init__(self)
for key in self._keys:
self.ensure_variable_name_valid(key)
self.unresolvable.add(VARIABLES.entry_metadata.variable_name)
def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool:
"""
Returns False if the variable exists as a non-override.
Raises
------
ValidationException
If the variable is already added as an override variable.
"""
try:
self.ensure_variable_name_valid(added_variable)
except ValidationException:
return False
if added_variable in self.keys:
raise self._validation_exception(
f"Override variable with name {added_variable} cannot be used since it is"
" added by a plugin."
)
return True
def ensure_variable_name_valid(self, name: str) -> None:
"""
Ensures the variable name does not collide with any entry variables or built-in functions.
"""
if not OverrideHelpers.is_valid_name(name):
override_type = "function" if name.startswith("%") else "variable"
raise self._validation_exception(
f"Override {override_type} with name {name} is invalid. Names must be"
" lower_snake_cased and begin with a letter.",
exception_class=InvalidVariableNameException,
)
if OverrideHelpers.is_entry_variable_name(name):
raise self._validation_exception(
f"Override variable with name {name} cannot be used since it is a"
" built-in ytdl-sub entry variable name.",
exception_class=InvalidVariableNameException,
)
if OverrideHelpers.is_function_name(name):
raise self._validation_exception(
f"Override function definition with name {name} cannot be used since it is"
" a built-in ytdl-sub function name.",
exception_class=InvalidVariableNameException,
)
def initial_variables(
self, unresolved_variables: Optional[Dict[str, str]] = None
) -> Dict[str, str]:
"""
Returns
-------
Variables and format strings for all Override variables + additional variables (Optional)
"""
initial_variables: Dict[str, str] = {}
mergedeep.merge(
initial_variables,
self.dict_with_format_strings,
unresolved_variables if unresolved_variables else {},
)
return ScriptUtils.add_sanitized_variables(initial_variables)
def initialize_script(
self, subscription_name: str, unresolved_variables: Set[str]
) -> "Overrides":
"""
Initialize the override script with override variables + any unresolved variables
"""
self.script.add(
ScriptUtils.add_sanitized_variables(
{OverrideVariables.subscription_name(): subscription_name}
)
)
self.script.add(
self.initial_variables(
unresolved_variables={
var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
for var_name in unresolved_variables
}
)
)
self.unresolvable.update(unresolved_variables)
self.update_script()
return self
def apply_formatter(
self,
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Dict[str, str] = None,
) -> str:
"""
Parameters
----------
formatter
Formatter to apply
entry
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
Returns
-------
The format_string after .format has been called
Raises
------
StringFormattingException
If the formatter that is trying to be resolved cannot
"""
script: Script = self.script
unresolvable: Set[str] = self.unresolvable
if entry:
script = entry.script
unresolvable = entry.unresolvable
try:
return formatter.post_process(
str(
script.resolve_once(
dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
unresolvable=unresolvable,
)["tmp_var"]
)
)
except ScriptVariableNotResolved as exc:
raise StringFormattingException(
"Tried to resolve the following script, but could not due to unresolved "
f"variables:\n {formatter.format_string}\n"
"This is most likely due to circular dependencies in variables. "
"If you think otherwise, please file a bug on GitHub and post your config. Thanks!"
) from exc

0
src/ytdl_sub/config/plugin/__init__.py Normal file
View file

36
src/ytdl_sub/config/plugin.py → src/ytdl_sub/config/plugin/plugin.py
View file

@ -7,44 +7,13 @@ from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class PluginPriority:
"""
Defines priority for plugins, 0 is highest priority
"""
# If modify_entry priority is >= to this value, run after split
MODIFY_ENTRY_AFTER_SPLIT = 10
# if post_process is >= to this value, run after file_convert
POST_PROCESS_AFTER_FILE_CONVERT = 10
MODIFY_ENTRY_FIRST = 0
def __init__(
self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5
):
self.modify_entry_metadata = modify_entry_metadata
self.modify_entry = modify_entry
self.post_process = post_process
@property
def modify_entry_after_split(self) -> bool:
"""
Returns
-------
True if the plugin should modify an entry after a potential split. False otherwise.
"""
return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
# pylint: disable=no-self-use,unused-argument
@ -53,7 +22,6 @@ class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC):
Shared code amongst all SourcePlugins (downloaders) and Plugins (post-download modification)
"""
priority: PluginPriority = PluginPriority()
plugin_options_type: Type[TOptionsValidator]
def __init__(

208
src/ytdl_sub/config/plugin/plugin_mapping.py Normal file
View file

@ -0,0 +1,208 @@
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin
from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
from ytdl_sub.plugins.file_convert import FileConvertPlugin
from ytdl_sub.plugins.filter_exclude import FilterExcludePlugin
from ytdl_sub.plugins.filter_include import FilterIncludePlugin
from ytdl_sub.plugins.format import FormatPlugin
from ytdl_sub.plugins.internal.view import ViewPlugin
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_sub.plugins.regex import RegexPlugin
from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin
from ytdl_sub.plugins.video_tags import VideoTagsPlugin
class PluginMapping:
"""
Maps plugins defined in the preset to its respective plugin class
"""
_MAPPING: Dict[str, Type[Plugin]] = {
"_view": ViewPlugin,
"audio_extract": AudioExtractPlugin,
"date_range": DateRangePlugin,
"embed_thumbnail": EmbedThumbnailPlugin,
"file_convert": FileConvertPlugin,
"format": FormatPlugin,
"match_filters": MatchFiltersPlugin,
"music_tags": MusicTagsPlugin,
"video_tags": VideoTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
"regex": RegexPlugin,
"subtitles": SubtitlesPlugin,
"chapters": ChaptersPlugin,
"split_by_chapters": SplitByChaptersPlugin,
"throttle_protection": ThrottleProtectionPlugin,
"filter_include": FilterIncludePlugin,
"filter_exclude": FilterExcludePlugin,
}
# All other plugins are added after the defined ordered ones
_ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [
ThrottleProtectionPlugin,
UrlDownloaderCollectionVariablePlugin,
SubtitlesPlugin,
FilterExcludePlugin,
FilterIncludePlugin,
# add all others
]
_ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [
UrlDownloaderThumbnailPlugin,
AudioExtractPlugin,
FileConvertPlugin,
ChaptersPlugin,
SplitByChaptersPlugin,
FilterExcludePlugin,
FilterIncludePlugin,
RegexPlugin,
# add all others
]
_ORDER_POST_PROCESS: List[Type[Plugin]] = [
AudioExtractPlugin,
FileConvertPlugin,
ChaptersPlugin,
SubtitlesPlugin,
MusicTagsPlugin,
VideoTagsPlugin,
NfoTagsPlugin,
EmbedThumbnailPlugin,
]
@classmethod
def _order_by(
cls, plugin_types: List[Type[Plugin]], operation: PluginOperation
) -> List[Type[Plugin]]:
if operation == PluginOperation.MODIFY_ENTRY_METADATA:
ordering = cls._ORDER_MODIFY_ENTRY_METADATA
elif operation == PluginOperation.MODIFY_ENTRY:
ordering = cls._ORDER_MODIFY_ENTRY
elif operation == PluginOperation.POST_PROCESS:
ordering = cls._ORDER_POST_PROCESS
else:
raise ValueError("PluginOperation does not support ordering")
ordered_plugin_operations: List[Type[Plugin]] = []
for pl_type in reversed(ordering):
for plugin_type in plugin_types:
if plugin_type == pl_type:
ordered_plugin_operations.insert(0, plugin_type)
else:
ordered_plugin_operations.append(plugin_type)
return ordered_plugin_operations
@classmethod
def order_options_by(
cls, zipped: List[Tuple[Type[Plugin], OptionsValidator]], operation: PluginOperation
) -> List[OptionsValidator]:
"""
Returns
-------
Ordered plugin options with respect to the PluginOperation.
"""
ordered_types: List[Type[Plugin]] = cls._order_by(
plugin_types=[val[0] for val in zipped], operation=operation
)
ordered_options: List[OptionsValidator] = []
for plugin_type, plugin_options in zipped:
sorted_idx = ordered_types.index(plugin_type)
ordered_options.insert(sorted_idx, plugin_options)
return ordered_options
@classmethod
def _is_modified_after_split(cls, plugin: Plugin) -> bool:
if type(plugin) not in cls._ORDER_MODIFY_ENTRY:
return True
return cls._ORDER_MODIFY_ENTRY.index(type(plugin)) > cls._ORDER_MODIFY_ENTRY.index(
SplitByChaptersPlugin
)
@classmethod
def order_plugins_by(
cls, plugins: List[Plugin], operation: PluginOperation, before_split: Optional[bool] = None
) -> List[Plugin]:
"""
Returns
-------
Ordered plugins with respect to the PluginOperation. Optionally only return plugins
before/after a split plugin.
"""
ordered_types: List[Type[Plugin]] = cls._order_by(
plugin_types=[type(plugin) for plugin in plugins], operation=operation
)
ordered_plugins: List[Plugin] = []
for plugin in plugins:
sorted_idx = ordered_types.index(type(plugin))
ordered_plugins.insert(sorted_idx, plugin)
if before_split is None:
return ordered_plugins
# Remove the split plugin if differentiating
ordered_plugins = [
plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin)
]
if before_split:
return [
plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin)
]
# before_split is False
return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)]
@classmethod
def plugins(cls) -> List[str]:
"""
Returns
-------
Available download sources
"""
return sorted(list(cls._MAPPING.keys()))
@classmethod
def get(cls, plugin: str) -> Type[Plugin]:
"""
Parameters
----------
plugin
Name of the plugin
Returns
-------
The plugin class
Raises
------
ValueError
Raised if the plugin does not exist
"""
if plugin not in cls.plugins():
raise ValueError(
f"Tried to use plugin '{plugin}' that does not exist. Available plugins: "
f"{', '.join(cls.plugins())}"
)
return cls._MAPPING[plugin]

9
src/ytdl_sub/config/plugin/plugin_operation.py Normal file
View file

@ -0,0 +1,9 @@
from enum import Enum
class PluginOperation(Enum):
ANY = -2
DOWNLOADER = -1
MODIFY_ENTRY_METADATA = 0
MODIFY_ENTRY = 1
POST_PROCESS = 2

46
src/ytdl_sub/config/plugin/preset_plugins.py Normal file
View file

@ -0,0 +1,46 @@
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.config.validators.options import TOptionsValidator
class PresetPlugins:
def __init__(self):
self.plugin_types: List[Type[Plugin]] = []
self.plugin_options: List[OptionsValidator] = []
def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins":
"""
Add a pair of plugin type and options to the list
"""
self.plugin_types.append(plugin_type)
self.plugin_options.append(plugin_options)
return self
def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]:
"""
Returns
-------
Plugin and PluginOptions zipped
"""
return list(zip(self.plugin_types, self.plugin_options))
def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
"""
Parameters
----------
plugin_type
Fetch the plugin options for this type
Returns
-------
Options of this plugin if they exit. Otherwise, return None.
"""
plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options]
if plugin_type in plugin_option_types:
return self.plugin_options[plugin_option_types.index(plugin_type)]
return None

79
src/ytdl_sub/config/plugin_mapping.py
View file

@ -1,79 +0,0 @@
from typing import Dict
from typing import List
from typing import Type
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
from ytdl_sub.plugins.file_convert import FileConvertPlugin
from ytdl_sub.plugins.format import FormatPlugin
from ytdl_sub.plugins.internal.view import ViewPlugin
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
from ytdl_sub.plugins.music_tags import MusicTagsPlugin
from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin
from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin
from ytdl_sub.plugins.regex import RegexPlugin
from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin
from ytdl_sub.plugins.video_tags import VideoTagsPlugin
class PluginMapping:
"""
Maps plugins defined in the preset to its respective plugin class
"""
_MAPPING: Dict[str, Type[Plugin]] = {
"_view": ViewPlugin,
"audio_extract": AudioExtractPlugin,
"date_range": DateRangePlugin,
"embed_thumbnail": EmbedThumbnailPlugin,
"file_convert": FileConvertPlugin,
"format": FormatPlugin,
"match_filters": MatchFiltersPlugin,
"music_tags": MusicTagsPlugin,
"video_tags": VideoTagsPlugin,
"nfo_tags": NfoTagsPlugin,
"output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin,
"regex": RegexPlugin,
"subtitles": SubtitlesPlugin,
"chapters": ChaptersPlugin,
"split_by_chapters": SplitByChaptersPlugin,
"throttle_protection": ThrottleProtectionPlugin,
}
@classmethod
def plugins(cls) -> List[str]:
"""
Returns
-------
Available download sources
"""
return sorted(list(cls._MAPPING.keys()))
@classmethod
def get(cls, plugin: str) -> Type[Plugin]:
"""
Parameters
----------
plugin
Name of the plugin
Returns
-------
The plugin class
Raises
------
ValueError
Raised if the plugin does not exist
"""
if plugin not in cls.plugins():
raise ValueError(
f"Tried to use plugin '{plugin}' that does not exist. Available plugins: "
f"{', '.join(cls.plugins())}"
)
return cls._MAPPING[plugin]

165
src/ytdl_sub/config/preset.py
View file

@ -1,39 +1,25 @@
import copy
from typing import Any
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from typing import Union
from mergedeep import mergedeep
from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin_mapping import PluginMapping
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.config.validators.variable_validation import VariableValidation
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.yaml import dump_yaml
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import StringListValidator
from ytdl_sub.validators.validators import Validator
from ytdl_sub.validators.validators import validation_exception
PRESET_KEYS = {
@ -61,44 +47,6 @@ def _parent_preset_error_message(
)
class PresetPlugins:
def __init__(self):
self.plugin_types: List[Type[Plugin]] = []
self.plugin_options: List[OptionsValidator] = []
def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins":
"""
Add a pair of plugin type and options to the list
"""
self.plugin_types.append(plugin_type)
self.plugin_options.append(plugin_options)
return self
def zipped(self) -> Iterable[Tuple[Type[Plugin], OptionsValidator]]:
"""
Returns
-------
Plugin and PluginOptions zipped
"""
return zip(self.plugin_types, self.plugin_options)
def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
"""
Parameters
----------
plugin_type
Fetch the plugin options for this type
Returns
-------
Options of this plugin if they exit. Otherwise, return None.
"""
plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options]
if plugin_type in plugin_option_types:
return self.plugin_options[plugin_option_types.index(plugin_type)]
return None
class _PresetShell(StrictDictValidator):
# Have all present keys optional since parent presets could not have all the
# required keys. They will get validated in the init after the mergedeep of dicts
@ -154,11 +102,7 @@ class Preset(_PresetShell):
validator=PluginMapping.get(plugin_name).plugin_options_type,
)
@property
def _source_variables(self) -> List[str]:
return Entry.source_variables()
def __validate_and_get_plugins(self) -> PresetPlugins:
def _validate_and_get_plugins(self) -> PresetPlugins:
preset_plugins = PresetPlugins()
for key in self._keys:
@ -172,88 +116,6 @@ class Preset(_PresetShell):
return preset_plugins
def __validate_added_variables(self):
source_variables = copy.deepcopy(self._source_variables)
# Validate added download option variables here since plugins could subsequently use them
self.downloader_options.validate_with_variables(
source_variables=source_variables,
override_variables=self.overrides.dict_with_format_strings,
)
source_variables.extend(self.downloader_options.added_source_variables())
for _, plugin_options in sorted(
self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry
):
# Validate current plugin using source + added plugin variables
plugin_options.validate_with_variables(
source_variables=source_variables,
override_variables=self.overrides.dict_with_format_strings,
)
# Extend existing source variables with ones created from this plugin
source_variables.extend(plugin_options.added_source_variables())
def __validate_override_string_formatter_validator(
self,
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
):
# Set the formatter variables to be the overrides
variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings)
# If the formatter supports source variables, set the formatter variables to include
# both source and override variables
if not isinstance(formatter_validator, OverridesStringFormatterValidator):
source_variables = {
source_var: "dummy_string"
for source_var in self._source_variables
+ self.downloader_options.added_source_variables()
}
variable_dict = dict(source_variables, **variable_dict)
# For all plugins, add in any extra added source variables
for plugin_options in self.plugins.plugin_options:
added_plugin_variables = {
source_var: "dummy_string"
for source_var in plugin_options.added_source_variables()
}
# sanity check plugin variables do not override source variables
expected_len = len(variable_dict) + len(added_plugin_variables)
variable_dict = dict(variable_dict, **added_plugin_variables)
assert (
len(variable_dict) == expected_len
), "plugin variables overwrote source variables"
_ = formatter_validator.apply_formatter(variable_dict=variable_dict)
def __recursive_preset_validate(
self,
validator: Optional[Validator] = None,
) -> None:
"""
Ensure all OverridesStringFormatterValidator's only contain variables from the overrides
and resolve.
"""
if validator is None:
validator = self
if isinstance(validator, DictValidator):
# pylint: disable=protected-access
# Usage of protected variables in other validators is fine. The reason to keep
# them protected is for readability when using them in subscriptions.
for validator_value in validator._validator_dict.values():
self.__recursive_preset_validate(validator_value)
# pylint: enable=protected-access
elif isinstance(validator, ListValidator):
for list_value in validator.list:
self.__recursive_preset_validate(list_value)
elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)):
self.__validate_override_string_formatter_validator(validator)
elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)):
for validator_value in validator.dict.values():
self.__validate_override_string_formatter_validator(validator_value)
def _get_presets_to_merge(
self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator
) -> List[Dict]:
@ -291,7 +153,7 @@ class Preset(_PresetShell):
return presets_to_merge
def __merge_parent_preset_dicts_if_present(self, config: ConfigValidator):
def _merge_parent_preset_dicts_if_present(self, config: ConfigValidator):
parent_preset_validator = self._validate_key_if_present(
key="preset", validator=StringListValidator
)
@ -314,7 +176,7 @@ class Preset(_PresetShell):
super().__init__(name=name, value=value)
# Perform the merge of parent presets before validating any keys
self.__merge_parent_preset_dicts_if_present(config=config)
self._merge_parent_preset_dicts_if_present(config=config)
self.downloader_options: MultiUrlValidator = self._validate_key(
key="download", validator=MultiUrlValidator
@ -329,13 +191,16 @@ class Preset(_PresetShell):
key="ytdl_options", validator=YTDLOptions, default={}
)
self.plugins: PresetPlugins = self._validate_and_get_plugins()
self.overrides = self._validate_key(key="overrides", validator=Overrides, default={})
self.plugins: PresetPlugins = self.__validate_and_get_plugins()
self.__validate_added_variables()
# After all options are initialized, perform a recursive post-validate that requires
# values from multiple validators
self.__recursive_preset_validate()
VariableValidation(
downloader_options=self.downloader_options,
output_options=self.output_options,
plugins=self.plugins,
).initialize_overrides(
subscription_name=self.name, overrides=self.overrides
).ensure_proper_usage()
@property
def name(self) -> str:

272
src/ytdl_sub/config/preset_options.py
View file

@ -1,97 +1,26 @@
from abc import ABC
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import TypeVar
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use
# pylint: disable=unused-argument
class OptionsValidator(Validator, ABC):
"""
Abstract class that validates options for preset sections (plugins, downloaders)
"""
def validation_exception(
self,
error_message: str | Exception,
) -> ValidationException:
"""
Parameters
----------
error_message
Error message to include in the validation exception
Returns
-------
Validation exception that points to the location in the config. To be used to throw good
validation exceptions at runtime from code outside this class.
"""
return self._validation_exception(error_message=error_message)
def added_source_variables(self) -> List[str]:
"""
If the plugin adds source variables, list them here.
Returns
-------
List of added source variables this plugin creates
"""
return []
def validate_with_variables(
self, source_variables: List[str], override_variables: Dict[str, str]
) -> None:
"""
Optional validation after init with the session's source and override variables.
Parameters
----------
source_variables
Available source variables when running the plugin
override_variables
Available override variables when running the plugin
"""
return None
TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass
# pylint: enable=no-self-use
# pylint: enable=unused-argument
class YTDLOptions(LiteralDictValidator):
"""
Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader.
Allows you to add any ytdl argument to ytdl-sub's downloader.
The argument names can differ slightly from the command-line argument names. See
`this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_
for more details.
ytdl_options should be formatted like:
:Usage:
.. code-block:: yaml
@ -123,108 +52,13 @@ class YTDLOptions(LiteralDictValidator):
# Disable for proper docstring formatting
# pylint: disable=line-too-long
class Overrides(DictFormatterValidator):
"""
Optional. This section allows you to define variables that can be used in any string formatter.
For example, if you want your file and thumbnail files to match without copy-pasting a large
format string, you can define something like:
.. code-block:: yaml
presets:
my_example_preset:
overrides:
output_directory: "/path/to/media"
custom_file_name: "{upload_year}.{upload_month_padded}.{upload_day_padded}.{title_sanitized}"
# Then use the override variables in the output options
output_options:
output_directory: "{output_directory}"
file_name: "{custom_file_name}.{ext}"
thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
Override variables can contain explicit values and other variables, including both override
and source variables.
In addition, any override variable defined will automatically create a ``sanitized`` variable
for use. In the example above, ``output_directory_sanitized`` will exist and perform
sanitization on the value when used.
"""
# pylint: enable=line-too-long
def _add_override_variable(self, key_name: str, format_string: str, sanitize: bool = False):
if sanitize:
key_name = f"{key_name}_sanitized"
format_string = sanitize_filename(format_string)
self._value[key_name] = StringFormatterValidator(
name="__should_never_fail__",
value=format_string,
)
def __init__(self, name, value):
super().__init__(name, value)
# Add sanitized overrides
for key in self._keys:
self._add_override_variable(
key_name=key,
format_string=self._value[key].format_string,
sanitize=True,
)
if SUBSCRIPTION_NAME not in self._value:
for sanitized in [True, False]:
self._add_override_variable(
key_name=SUBSCRIPTION_NAME,
format_string=self.subscription_name,
sanitize=sanitized,
)
@property
def subscription_name(self) -> str:
"""
Returns
-------
Name of the subscription
"""
return self._root_name
def apply_formatter(
self,
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Dict[str, str] = None,
) -> str:
"""
Parameters
----------
formatter
Formatter to apply
entry
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
Returns
-------
The format_string after .format has been called
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
if function_overrides:
variable_dict = dict(variable_dict, **function_overrides)
return formatter.apply_formatter(variable_dict)
class OutputOptions(StrictDictValidator):
"""
Defines where to output files and thumbnails after all post-processing has completed.
Usage:
:Usage:
.. code-block:: yaml
@ -253,6 +87,7 @@ class OutputOptions(StrictDictValidator):
"maintain_download_archive",
"keep_files_before",
"keep_files_after",
"keep_max_files",
}
@classmethod
@ -307,96 +142,133 @@ class OutputOptions(StrictDictValidator):
self._keep_files_after = self._validate_key_if_present(
"keep_files_after", StringDatetimeValidator
)
self._keep_max_files = self._validate_key_if_present(
"keep_max_files", OverridesIntegerFormatterValidator
)
if (
self._keep_files_before or self._keep_files_after
self._keep_files_before or self._keep_files_after or self._keep_max_files
) and not self.maintain_download_archive:
raise self._validation_exception(
"keep_files requires maintain_download_archive set to True"
"keep_files/keep_max requires maintain_download_archive set to True"
)
@property
def output_directory(self) -> OverridesStringFormatterValidator:
"""
Required. The output directory to store all media files downloaded.
:expected type: OverridesFormatter
:description:
The output directory to store all media files downloaded.
"""
return self._output_directory
@property
def file_name(self) -> StringFormatterValidator:
"""
Required. The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
:expected type: EntryFormatter
:description:
The file name for the media file. This can include directories such as
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
"""
return self._file_name
@property
def thumbnail_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's thumbnail image. This can include directories such
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable thumbnail writes.
"""
return self._thumbnail_name
@property
def info_json_name(self) -> Optional[StringFormatterValidator]:
"""
Optional. The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
:expected type: Optional[EntryFormatter]
:description:
The file name for the media's info json file. This can include directories such
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
directory. Can be set to empty string or `null` to disable info json writes.
"""
return self._info_json_name
@property
def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
:expected type: Optional[OverridesFormatter]
:description:
The file name to store a subscriptions download archive placed relative to
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
"""
return self._download_archive_name
@property
def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this name
first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
:expected type: Optional[OverridesFormatter]
:description:
Intended to be used if you are migrating a subscription with either a new
subscription name or output directory. It will try to load the archive file using this
name first, and fallback to ``download_archive_name``. It will always save to this file
and remove the original ``download_archive_name``.
"""
return self._migrated_download_archive_name
@property
def maintain_download_archive(self) -> bool:
"""
Optional. Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
:expected type: Optional[Boolean]
:description:
Maintains a download archive file in the output directory for a subscription.
It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
output directory.
The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
create a ytdl download-archive file when invoking a download on a subscription. This will
prevent ytdl from redownloading media already downloaded.
The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
create a ytdl download-archive file when invoking a download on a subscription. This will
prevent ytdl from redownloading media already downloaded.
Defaults to False.
Defaults to False.
"""
return self._maintain_download_archive.value
@property
def keep_files_before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``.
"""
return self._keep_files_before
@property
def keep_files_after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
``date_range`` plugin.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``.
"""
return self._keep_files_after
@property
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
"""
:expected type: Optional[OverridesFormatter]
:description:
Requires ``maintain_download_archive`` set to True.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
"""
return self._keep_max_files

0
src/ytdl_sub/config/validators/__init__.py Normal file
View file

59
src/ytdl_sub/config/validators/options.py Normal file
View file

@ -0,0 +1,59 @@
from abc import ABC
from typing import Dict
from typing import Set
from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use
# pylint: disable=unused-argument
class OptionsValidator(Validator, ABC):
"""
Abstract class that validates options for preset sections (plugins, downloaders)
"""
def validation_exception(
self,
error_message: str | Exception,
) -> ValidationException:
"""
Parameters
----------
error_message
Error message to include in the validation exception
Returns
-------
Validation exception that points to the location in the config. To be used to throw good
validation exceptions at runtime from code outside this class.
"""
return self._validation_exception(error_message=error_message)
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
"""
If the plugin modifies existing variables, define them here
"""
return {}
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
"""
If the plugin adds source variables, list them here.
"""
return {}
TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass

199
src/ytdl_sub/config/validators/variable_validation.py Normal file
View file

@ -0,0 +1,199 @@
import copy
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.entries.variables.override_variables import OverrideVariables
from ytdl_sub.script.script import Script
from ytdl_sub.validators.string_formatter_validators import validate_formatters
def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]:
dummy_variables: Dict[str, str] = {}
for var in variables:
dummy_variables[var] = ""
dummy_variables[f"{var}_sanitized"] = ""
return dummy_variables
def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
# Have the dummy override variable contain all variable deps that it uses in the string
dummy_overrides: Dict[str, str] = {}
for override_name in _override_variables(overrides):
dummy_overrides[override_name] = ""
# pylint: disable=protected-access
for variable_dependency in overrides.script._variables[override_name].variables:
dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}"
# pylint: enable=protected-access
return dummy_overrides
def _get_added_and_modified_variables(
plugins: PresetPlugins, downloader_options: MultiUrlValidator
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
for plugin_options in options:
added_variables: Set[str] = set()
modified_variables: Set[str] = set()
for plugin_added_variables in plugin_options.added_variables(
resolved_variables=set(),
unresolved_variables=set(),
plugin_op=PluginOperation.ANY,
).values():
added_variables |= set(plugin_added_variables)
for plugin_modified_variables in plugin_options.modified_variables().values():
modified_variables = plugin_modified_variables
yield plugin_options, added_variables, modified_variables
def _override_variables(overrides: Overrides) -> Set[str]:
return set(list(overrides.initial_variables().keys())) | {OverrideVariables.subscription_name()}
def _entry_variables() -> Set[str]:
return set(list(VARIABLE_SCRIPTS.keys()))
class VariableValidation:
def __init__(
self,
downloader_options: MultiUrlValidator,
output_options: OutputOptions,
plugins: PresetPlugins,
):
self.downloader_options = downloader_options
self.output_options = output_options
self.plugins = plugins
self.script: Optional[Script] = None
self.resolved_variables: Set[str] = set()
self.unresolved_variables: Set[str] = set()
def initialize_overrides(
self, subscription_name: str, overrides: Overrides
) -> "VariableValidation":
"""
Do some gymnastics to initialize the Overrides script.
"""
entry_variables = _entry_variables()
override_variables = _override_variables(overrides)
# Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable
self.resolved_variables = entry_variables | override_variables
for (
plugin_options,
added_variables,
modified_variables,
) in _get_added_and_modified_variables(
plugins=self.plugins,
downloader_options=self.downloader_options,
):
for added_variable in added_variables:
if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable):
# pylint: disable=protected-access
raise plugin_options._validation_exception(
f"Cannot use the variable name {added_variable} because it exists as a"
" built-in ytdl-sub variable name."
)
# pylint: enable=protected-access
# Set unresolved as variables that are added but do not exist as
# entry/override variables since they are created at run-time
self.unresolved_variables |= added_variables | modified_variables
# Then update resolved variables to reflect that
self.resolved_variables -= self.unresolved_variables
# Initialize overrides with unresolved variables + modified variables to throw an error.
# For modified variables, this is to prevent a resolve(update=True) to setting any
# dependencies until it has been explicitly added
overrides = overrides.initialize_script(
subscription_name=subscription_name, unresolved_variables=self.unresolved_variables
)
# copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables))
self.script.add(
variables=_add_dummy_overrides(overrides=overrides),
unresolvable=self.unresolved_variables,
)
return self
def _update_script(self) -> None:
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]:
added_variables = options.added_variables(
resolved_variables=self.resolved_variables,
unresolved_variables=self.unresolved_variables,
plugin_op=plugin_op,
).get(plugin_op, set())
modified_variables = options.modified_variables().get(plugin_op, set())
resolved_variables = added_variables | modified_variables
self.script.add(_add_dummy_variables(resolved_variables))
self.resolved_variables |= resolved_variables
self.unresolved_variables -= resolved_variables
return added_variables
def ensure_proper_usage(self) -> None:
"""
Validate variables resolve as plugins are executed, and return
a mock script which contains actualized added variables from the plugins
"""
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
# Metadata variables to be added
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
):
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
self._update_script()
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
):
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
if added:
self._update_script()
# Validate that any formatter in the plugin options can resolve
validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
validator=plugin_options,
)
validate_formatters(
script=self.script,
unresolved_variables=self.unresolved_variables,
validator=self.output_options,
)
assert not self.unresolved_variables

31
src/ytdl_sub/downloaders/info_json/info_json_downloader.py
View file

@ -6,17 +6,22 @@ from typing import Iterable
from typing import List
from typing import Optional
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import get_file_extension
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES
class InfoJsonDownloaderOptions(OptionsDictValidator):
_optional_keys = {"no-op"}
@ -97,14 +102,26 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
for download_mapping in self._enhanced_download_archive.mapping.entry_mappings.values():
entry = self._get_entry_from_download_mapping(download_mapping)
# See if prior variables exist. If so, delete them from metadata
# to avoid saving them recursively on multiple updates
prior_variables = entry.maybe_get_prior_variables()
entry.initialize_script(self.overrides).add(
{
inj: prior_variables.get(
inj.variable_name,
VARIABLE_SCRIPTS[inj.variable_name],
)
for inj in v.injected_variables()
}
)
entries.append(entry)
# Remove each entry from the live download archive since it will get re-added
# unless it is filtered
for entry in entries:
for entry in sorted(entries, key=lambda ent: ent.get(v.download_index, int)):
# Remove each entry from the live download archive since it will get re-added
# unless it is filtered
self._enhanced_download_archive.mapping.remove_entry(entry.uid)
for entry in sorted(entries, key=lambda ent: ent.download_index):
yield entry
# If the original entry file_path is no longer maintained in the new mapping, then

8
src/ytdl_sub/downloaders/source_plugin.py
View file

@ -8,10 +8,10 @@ from typing import Optional
from typing import Type
from typing import final
from ytdl_sub.config.plugin import BasePlugin
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import BasePlugin
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive

82
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -11,8 +11,7 @@ from typing import Tuple
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
@ -22,14 +21,8 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.variables.kwargs import COLLECTION_URL
from ytdl_sub.entries.variables.kwargs import COMMENTS
from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ThumbnailTypes
@ -37,6 +30,8 @@ from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES
download_logger = Logger.get(name="downloader")
@ -47,8 +42,6 @@ class URLDownloadState:
class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry=0)
def __init__(
self,
options: MultiUrlValidator,
@ -119,22 +112,18 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner.
"""
if entry.kwargs_contains(PLAYLIST_ENTRY):
if playlist_metadata := entry.get(v.playlist_metadata, dict):
self._download_parent_thumbnails(
thumbnail_list_info=collection_url.playlist_thumbnails,
entry=entry,
parent=EntryParent(
entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory
),
parent=EntryParent(playlist_metadata, working_directory=self.working_directory),
)
if entry.kwargs_contains(SOURCE_ENTRY):
if source_metadata := entry.get(v.source_metadata, dict):
self._download_parent_thumbnails(
thumbnail_list_info=collection_url.source_thumbnails,
entry=entry,
parent=EntryParent(
entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory
),
parent=EntryParent(source_metadata, working_directory=self.working_directory),
)
def modify_entry(self, entry: Entry) -> Optional[Entry]:
@ -147,17 +136,15 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
if not self.is_dry_run:
try_convert_download_thumbnail(entry=entry)
if entry.kwargs_get(COLLECTION_URL) in self._collection_url_mapping:
if (input_url := entry.get(v.ytdl_sub_input_url, str)) in self._collection_url_mapping:
self._download_url_thumbnails(
collection_url=self._collection_url_mapping[entry.kwargs(COLLECTION_URL)],
collection_url=self._collection_url_mapping[input_url],
entry=entry,
)
return entry
class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry_metadata=0)
def __init__(
self,
options: MultiUrlValidator,
@ -181,7 +168,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
"""
# COLLECTION_URL is a recent variable that may not exist for old entries when updating.
# Try to use source_webpage_url if it does not exist
entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.source_webpage_url)
entry_collection_url = entry.get(v.ytdl_sub_input_url, str)
# If the collection URL cannot find its mapping, use the last URL
collection_url = (
@ -189,7 +176,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
or list(self._collection_url_mapping.values())[-1]
)
entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings)
entry.add(collection_url.variables.dict_with_format_strings)
return entry
@ -356,7 +343,10 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
else entry.is_thumbnail_downloaded_via_ytdlp,
url=entry.webpage_url,
)
return Entry(download_entry_dict, working_directory=self.working_directory)
return Entry(
download_entry_dict,
working_directory=self.working_directory,
)
def _iterate_child_entries(
self, url_validator: UrlValidator, entries: List[Entry]
@ -395,7 +385,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
):
yield entry_child
def _download_url_metadata(self, url: str) -> Tuple[List[EntryParent], List[Entry]]:
def _download_url_metadata(
self, url: str, include_sibling_metadata: bool
) -> Tuple[List[EntryParent], List[Entry]]:
"""
Downloads only info.json files and forms EntryParent trees
"""
@ -411,9 +403,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
url=url,
entry_dicts=entry_dicts,
working_directory=self.working_directory,
include_sibling_metadata=include_sibling_metadata,
)
orphans = EntryParent.from_entry_dicts_with_no_parents(
parents=parents, entry_dicts=entry_dicts, working_directory=self.working_directory
parents=parents,
entry_dicts=entry_dicts,
working_directory=self.working_directory,
)
return parents, orphans
@ -446,7 +441,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
if not (url := self.overrides.apply_formatter(collection_url.url)):
continue
parents, orphan_entries = self._download_url_metadata(url=url)
parents, orphan_entries = self._download_url_metadata(
url=url, include_sibling_metadata=collection_url.include_sibling_metadata
)
# TODO: Encapsulate this logic into its own class
self._url_state = URLDownloadState(
@ -459,9 +456,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
for entry in self._iterate_entries(
url_validator=collection_url, parents=parents, orphans=orphan_entries
):
# Add the collection URL to the info_dict to trace where it came from
entry.add_kwargs(
{COLLECTION_URL: self.overrides.apply_formatter(collection_url.url)}
entry.initialize_script(self.overrides).add(
{v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)}
)
yield entry
@ -497,22 +493,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
return None
upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
upload_date_standardized=entry.upload_date_standardized
upload_date_standardized=entry.get(v.upload_date_standardized, str)
)
download_idx = self._enhanced_download_archive.num_entries
entry.add_kwargs(
{
# Subtitles are not downloaded in metadata run, only here, so move over
REQUESTED_SUBTITLES: download_entry.kwargs_get(REQUESTED_SUBTITLES),
# Same with sponsorblock chapters
SPONSORBLOCK_CHAPTERS: download_entry.kwargs_get(SPONSORBLOCK_CHAPTERS),
COMMENTS: download_entry.kwargs_get(COMMENTS),
# Tracks number of entries downloaded
DOWNLOAD_INDEX: download_idx,
# Tracks number of entries with the same upload date to make them unique
UPLOAD_DATE_INDEX: upload_date_idx,
}
return entry.add_injected_variables(
download_entry=download_entry,
download_idx=download_idx,
upload_date_idx=upload_date_idx,
)
return entry

37
src/ytdl_sub/downloaders/url/multi_url.py
View file

@ -1,37 +0,0 @@
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
# TODO: Remove later - keep for docstring
class MultiUrlDownloadOptions(MultiUrlValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""

25
src/ytdl_sub/downloaders/url/url.py
View file

@ -1,25 +0,0 @@
from ytdl_sub.downloaders.url.validators import UrlValidator
# TODO: Remove later - keep for docstring
class UrlDownloadOptions(UrlValidator):
"""
Downloads from a single URL supported by yt-dlp.
Usage:
.. code-block:: yaml
presets:
my_example_preset:
download:
# required
url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
# optional
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
download_reverse: True
"""

129
src/ytdl_sub/downloaders/url/validators.py
View file

@ -1,10 +1,12 @@
import copy
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.script.parser import parse
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -43,7 +45,13 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
class UrlValidator(StrictDictValidator):
_required_keys = {"url"}
_optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails", "download_reverse"}
_optional_keys = {
"variables",
"source_thumbnails",
"playlist_thumbnails",
"download_reverse",
"include_sibling_metadata",
}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
@ -72,6 +80,9 @@ class UrlValidator(StrictDictValidator):
self._download_reverse = self._validate_key(
key="download_reverse", validator=BoolValidator, default=True
)
self._include_sibling_metadata = self._validate_key(
key="include_sibling_metadata", validator=BoolValidator, default=False
)
@property
def url(self) -> OverridesStringFormatterValidator:
@ -145,6 +156,16 @@ class UrlValidator(StrictDictValidator):
"""
return self._download_reverse.value
@property
def include_sibling_metadata(self) -> bool:
"""
Optional. Whether to include sibling metadata as an entry variable, which comprises basic
metadata from all other entries (including itself) that belong to the same playlist. For
channels or large playlists, this becomes memory-intensive since you are storing
``n^2`` metadata. Defaults to False.
"""
return self._include_sibling_metadata.value
class UrlStringOrDictValidator(UrlValidator):
"""
@ -194,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]):
class MultiUrlValidator(OptionsValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.
Sets the URL(s) to download from. Can be used in many forms, including
:Single URL:
.. code-block:: yaml
download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
:Multi URL:
.. code-block:: yaml
download:
- "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
- "https://www.youtube.com/watch?v=3BFTio5296w"
:Thumbnails + Variables:
All variables must be defined for the top-most url. All subsequent URL variables can be either
overwritten or default to the top-most value.
If an entry is returned from more than one URL, it will use the variables in the bottom-most
URL.
.. code-block:: yaml
download:
# required
urls:
- url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "1"
season_name: "Uploads"
playlist_thumbnails:
- name: "poster.jpg"
uid: "avatar_uncropped"
- name: "fanart.jpg"
uid: "banner_uncropped"
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
- url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
variables:
season_index: "2"
season_name: "Playlist as Season"
playlist_thumbnails:
- name: "season{season_index}-poster.jpg"
uid: "latest_entry"
"""
@classmethod
@ -243,45 +309,26 @@ class MultiUrlValidator(OptionsValidator):
# keep for readthedocs documentation
return self._urls.list[0].variables
def added_source_variables(self) -> List[str]:
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
"""
Returns
-------
List of variables added. The first collection url always contains all the variables.
"""
return list(self._urls.list[0].variables.keys)
if plugin_op != PluginOperation.ANY:
for url in self._urls.list:
for variable_name, definition in url.variables.dict_with_format_strings.items():
used_variables = set(var.name for var in parse(definition).variables)
if unresolved := used_variables & unresolved_variables:
raise self._validation_exception(
f"variable {variable_name} cannot use the variables "
f"{', '.join(sorted(list(unresolved)))} because it depends on other"
" variables that are computed later in execution"
)
def validate_with_variables(
self, source_variables: List[str], override_variables: Dict[str, str]
) -> None:
"""
Ensures new variables added are not existing variables
"""
for source_var_name in self.added_source_variables():
if source_var_name in source_variables:
raise self._validation_exception(
f"'{source_var_name}' cannot be used as a variable name because it "
f"is an existing source variable"
)
base_variables = dict(
override_variables, **{source_var: "dummy_string" for source_var in source_variables}
)
# Apply formatting to each new source variable, ensure it resolves
for collection_url in self.urls.list:
for (
source_var_name,
source_var_formatter_str,
) in collection_url.variables.dict_with_format_strings.items():
_ = StringFormatterValidator(
name=f"{self._name}.{source_var_name}", value=source_var_formatter_str
).apply_formatter(base_variables)
# Ensure at least URL is non-empty
has_non_empty_url = False
for url_validator in self.urls.list:
has_non_empty_url |= bool(url_validator.url.apply_formatter(override_variables))
if not has_non_empty_url:
raise self._validation_exception("Must contain at least one url that is non-empty")
return {PluginOperation.DOWNLOADER: set(self._urls.list[0].variables.keys)}

2
src/ytdl_sub/downloaders/ytdl_options_builder.py
View file

@ -17,7 +17,7 @@ class YTDLOptionsBuilder:
self,
*ytdl_option_dicts: Optional[Dict],
before: bool = False,
strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE
strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE,
) -> "YTDLOptionsBuilder":
"""
Parameters

355
src/ytdl_sub/entries/base_entry.py
View file

@ -1,241 +1,22 @@
# pylint: disable=protected-access
from abc import ABC
from datetime import datetime
from pathlib import Path
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import EPOCH
from ytdl_sub.entries.variables.kwargs import EXTRACTOR
from ytdl_sub.entries.variables.kwargs import IE_KEY
from ytdl_sub.entries.variables.kwargs import TITLE
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.entries.variables.kwargs import UPLOADER
from ytdl_sub.entries.variables.kwargs import UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL
# pylint: disable=no-member
def _sanitize_plex(string: str) -> str:
out = ""
for char in string:
match char:
case "0":
out += "０"
case "1":
out += "１"
case "2":
out += "２"
case "3":
out += "３"
case "4":
out += "４"
case "5":
out += "５"
case "6":
out += "６"
case "7":
out += "７"
case "8":
out += "８"
case "9":
out += "９"
case _:
out += char
return out
class BaseEntryVariables:
"""
Source variables are ``{variables}`` that contain metadata from downloaded media.
These variables can be used with fields that expect
:class:`~ytdl_sub.validators.string_formatter_validators.StringFormatterValidator`,
but not
:class:`~ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator`.
"""
@property
def uid(self: "BaseEntry") -> str:
"""
Returns
-------
str
The entry's unique ID
"""
return str(self.kwargs(UID))
@property
def uid_sanitized(self: "BaseEntry") -> str:
"""
Returns
-------
str
The sanitized uid of the entry, which is safe to use for Unix and Windows file names.
"""
return sanitize_filename(self.uid)
@property
def uid_sanitized_plex(self: "BaseEntry") -> str:
"""
Returns
-------
str
The sanitized uid with additional sanitizing for Plex. Replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
"""
return _sanitize_plex(self.uid_sanitized)
@property
def extractor(self: "BaseEntry") -> str:
"""
Returns
-------
str
The ytdl extractor name
"""
# pylint: disable=line-too-long
# Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514
# pylint: enable=line-too-long
return self.kwargs_get(EXTRACTOR) or self.kwargs(IE_KEY)
@property
def epoch(self: "BaseEntry") -> int:
"""
Returns
-------
int
The unix epoch of when the metadata was scraped by yt-dlp.
"""
return self.kwargs(EPOCH)
@property
def epoch_date(self: "BaseEntry") -> str:
"""
Returns
-------
str
The epoch's date, in YYYYMMDD format.
"""
return datetime.utcfromtimestamp(self.epoch).strftime("%Y%m%d")
@property
def epoch_hour(self: "BaseEntry") -> str:
"""
Returns
-------
str
The epoch's hour, padded
"""
return datetime.utcfromtimestamp(self.epoch).strftime("%H")
@property
def title(self: "BaseEntry") -> str:
"""
Returns
-------
str
The title of the entry. If a title does not exist, returns its unique ID.
"""
return self.kwargs_get(TITLE, self.uid)
@property
def title_sanitized(self) -> str:
"""
Returns
-------
str
The sanitized title of the entry, which is safe to use for Unix and Windows file names.
"""
return sanitize_filename(self.title)
@property
def title_sanitized_plex(self) -> str:
"""
Returns
-------
str
The sanitized title with additional sanitizing for Plex. Replaces numbers with
fixed-width numbers so Plex does not recognize them as season or episode numbers.
"""
return _sanitize_plex(self.title_sanitized)
@property
def webpage_url(self: "BaseEntry") -> str:
"""
Returns
-------
str
The url to the webpage.
"""
return self.kwargs(WEBPAGE_URL)
@property
def info_json_ext(self) -> str:
"""
Returns
-------
str
The "info.json" extension
"""
return "info.json"
@property
def description(self: "BaseEntry") -> str:
"""
Returns
-------
str
The description if it exists. Otherwise, returns an emtpy string.
"""
return self.kwargs_get(DESCRIPTION, "")
@property
def uploader_id(self: "BaseEntry") -> str:
"""
Returns
-------
str
The uploader id if it exists, otherwise return the unique ID.
"""
return self.kwargs_get(UPLOADER_ID, self.uid)
@property
def uploader(self: "BaseEntry") -> str:
"""
Returns
-------
str
The uploader if it exists, otherwise return the uploader ID.
"""
return self.kwargs_get(UPLOADER, self.uploader_id)
@property
def uploader_url(self: "BaseEntry") -> str:
"""
Returns
-------
str
The uploader url if it exists, otherwise returns the webpage_url.
"""
return self.kwargs_get(UPLOADER_URL, self.webpage_url)
# pylint: enable=no-member
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES
TBaseEntry = TypeVar("TBaseEntry", bound="BaseEntry")
class BaseEntry(BaseEntryVariables, ABC):
class BaseEntry(ABC):
"""
Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc).
"""
@ -254,30 +35,63 @@ class BaseEntry(BaseEntryVariables, ABC):
self._working_directory = working_directory
self._kwargs = entry_dict
self._additional_variables: Dict[str, str | int] = {}
@property
def uid(self) -> str:
"""
Returns
-------
str
The entry's unique ID
"""
return str(self._kwargs[v.uid.metadata_key])
def kwargs_contains(self, key: str) -> bool:
"""Returns whether internal kwargs contains the specified key"""
return key in self._kwargs
@property
def download_archive_extractor(self) -> str:
"""
The extractor name used in yt-dlp download archives
"""
# pylint: disable=line-too-long
# Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514
# pylint: enable=line-too-long
return str(
self._kwargs_get(v.extractor_key.metadata_key)
or self._kwargs_get(v.ie_key.metadata_key)
or "NO_EXTRACTOR"
).lower()
def kwargs(self, key) -> Any:
"""Returns an internal kwarg value supplied from ytdl"""
if not self.kwargs_contains(key):
raise KeyError(f"Expected '{key}' in {self.__class__.__name__} but does not exist.")
output = self._kwargs[key]
@property
def title(self) -> str:
"""
The title of the entry. If a title does not exist, returns its unique ID.
"""
return self._kwargs_get(v.title.metadata_key, self.uid)
# Replace curly braces with unicode version to avoid variable shenanigans
if isinstance(output, str):
return output.replace("{", "｛").replace("}", "｝")
return output
@property
def webpage_url(self) -> str:
"""
The url to the webpage.
"""
return self._kwargs[v.webpage_url.metadata_key]
def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any:
@property
def info_json_ext(self) -> str:
"""The "info.json" extension"""
return "info.json"
@property
def uploader_id(self) -> str:
"""
The uploader id if it exists, otherwise return the unique ID.
"""
return self._kwargs_get(v.uploader_id.metadata_key, self.uid)
def _kwargs_get(self, key: str, default: Optional[Any] = None) -> Any:
"""
Dict get on kwargs
"""
if not self.kwargs_contains(key) or self.kwargs(key) is None:
if (out := self._kwargs.get(key)) is None:
return default
return self.kwargs(key)
return out
def working_directory(self) -> str:
"""
@ -304,31 +118,6 @@ class BaseEntry(BaseEntryVariables, ABC):
self._kwargs = dict(self._kwargs, **variables_to_add)
return self
def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry":
"""
Parameters
----------
variables_to_add
Variables to add to this entry
Returns
-------
self
Raises
------
ValueError
If a variable trying to be added already exists as a source variable
"""
for variable_name in variables_to_add.keys():
if self.kwargs_contains(variable_name):
raise ValueError(
f"Cannot add variable '{variable_name}': already exists in the kwargs"
)
self._additional_variables = dict(self._additional_variables, **variables_to_add)
return self
def get_download_info_json_name(self) -> str:
"""
Returns
@ -345,36 +134,6 @@ class BaseEntry(BaseEntryVariables, ABC):
"""
return str(Path(self.working_directory()) / self.get_download_info_json_name())
def _added_variables(self) -> Dict[str, str]:
"""
Returns
-------
Dict of variables added to this entry
"""
return self._additional_variables
@classmethod
def source_variables(cls) -> List[str]:
"""
Returns
-------
List of all source variables
"""
property_names = [prop for prop in dir(cls) if isinstance(getattr(cls, prop), property)]
return property_names
@final
def to_dict(self) -> Dict[str, str]:
"""
Returns
-------
Dictionary containing all variables
"""
source_variable_dict = {
source_var: getattr(self, source_var) for source_var in self.source_variables()
}
return dict(source_variable_dict, **self._added_variables())
@final
def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry:
"""
@ -393,7 +152,7 @@ class BaseEntry(BaseEntryVariables, ABC):
"""
entry_type: Optional[str] = None
if isinstance(entry_dict, cls):
entry_type = entry_dict.kwargs_get("_type")
entry_type = entry_dict._kwargs_get("_type")
if isinstance(entry_dict, dict):
entry_type = entry_dict.get("_type")
@ -408,7 +167,7 @@ class BaseEntry(BaseEntryVariables, ABC):
"""
entry_ext: Optional[str] = None
if isinstance(entry_dict, cls):
entry_ext = entry_dict.kwargs_get("ext")
entry_ext = entry_dict._kwargs_get("ext")
if isinstance(entry_dict, dict):
entry_ext = entry_dict.get("ext")
@ -420,4 +179,4 @@ class BaseEntry(BaseEntryVariables, ABC):
-------
extractor + uid, making this a unique hash for any entry
"""
return self.extractor + self.uid
return self.download_archive_extractor + self.uid

139
src/ytdl_sub/entries/entry.py
View file

@ -1,21 +1,109 @@
# pylint: disable=protected-access
import copy
import json
import os
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.variables.entry_variables import EntryVariables
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_types import ArrayVariable
from ytdl_sub.entries.script.variable_types import StringVariable
from ytdl_sub.entries.script.variable_types import Variable
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS
v: VariableDefinitions = VARIABLES
class Entry(EntryVariables, BaseEntry):
_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables"
ytdl_sub_chapters_from_comments = ArrayVariable(
"ytdl_sub_chapters_from_comments", definition="{ [] }"
)
ytdl_sub_split_by_chapters_parent_uid = StringVariable(
"ytdl_sub_split_by_chapters_parent_uid", definition="{ %string('') }"
)
TypeT = TypeVar("TypeT")
class Entry(BaseEntry, Scriptable):
"""
Entry object to represent a single media object returned from yt-dlp.
"""
def __init__(self, entry_dict: Dict, working_directory: str):
BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory)
Scriptable.__init__(self)
def _add_entry_kwargs_to_script(self) -> None:
# Add entry metadata, but avoid the `.add()` helper since it also adds sanitized
self.unresolvable.remove(v.entry_metadata.variable_name)
self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)})
self.update_script()
def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry":
"""
Initializes the entry script using the Overrides script, then adding
its kwargs to the entry metadata variable
"""
# Overrides contains added variables that are unresolvable, add them here
if other:
self.script = copy.deepcopy(other.script)
self.unresolvable = copy.deepcopy(other.unresolvable)
self._add_entry_kwargs_to_script()
return self
def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT:
"""
Gets a variable of an expected type. Will error if it does not exist or is not resolved.
"""
out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name)
return expected_type(out)
def try_get(self, variable: Variable, expected_type: Type[TypeT]) -> Optional[TypeT]:
"""
Gets a variable of an expected type. Returns None if it does not exist or is not resolved.
"""
try:
return self.get(variable=variable, expected_type=expected_type)
except ScriptVariableNotResolved:
return None
def add_injected_variables(
self, download_entry: "Entry", download_idx: int, upload_date_idx: int
) -> "Entry":
"""
Adds variables that get injected into the Entry script that aren't available at
metadata scrape time (only after the actual download).
"""
self.add(
{
# Tracks number of entries downloaded
v.download_index: download_idx + 1,
# Tracks number of entries with the same upload date to make them unique
v.upload_date_index: upload_date_idx + 1,
v.requested_subtitles: download_entry._kwargs_get(
v.requested_subtitles.metadata_key, []
),
v.chapters: download_entry._kwargs_get(v.chapters.metadata_key, []),
v.sponsorblock_chapters: download_entry._kwargs_get(
v.sponsorblock_chapters.metadata_key, []
),
v.comments: download_entry._kwargs_get(v.comments.metadata_key, []),
}
)
return self
@property
def ext(self) -> str:
"""
@ -23,12 +111,13 @@ class Entry(EntryVariables, BaseEntry):
This is not reflected in the entry. See if the mkv file exists and return "mkv" if so,
otherwise, return the original extension.
"""
for possible_ext in [super().ext, "mkv"]:
ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key]
for possible_ext in [ext, "mkv"]:
file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}")
if os.path.isfile(file_path):
return possible_ext
return super().ext
return ext
def get_download_file_name(self) -> str:
"""
@ -48,7 +137,7 @@ class Entry(EntryVariables, BaseEntry):
-------
The download thumbnail's file name
"""
return f"{self.uid}.{self.thumbnail_ext}"
return f"{self.uid}.{self.get(v.thumbnail_ext, str)}"
def get_download_thumbnail_path(self) -> str:
"""Returns the entry's thumbnail's file path to where it was downloaded"""
@ -59,7 +148,7 @@ class Entry(EntryVariables, BaseEntry):
The source `thumbnail` value and the actual downloaded thumbnail extension sometimes do
not match. Return the actual downloaded thumbnail path.
"""
thumbnails = self.kwargs_get("thumbnails", [])
thumbnails = self._kwargs_get("thumbnails", [])
possible_thumbnail_exts = {"jpg", "webp"} # Always check for jpg and webp thumbs
for thumbnail in thumbnails:
@ -77,7 +166,7 @@ class Entry(EntryVariables, BaseEntry):
Write the entry's _kwargs back into the info.json file as well as its source variables
"""
kwargs_dict = copy.deepcopy(self._kwargs)
kwargs_dict["ytdl_sub_entry_variables"] = self.to_dict()
kwargs_dict[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] = self.to_dict()
kwargs_json = json.dumps(kwargs_dict, ensure_ascii=False, sort_keys=True, indent=2)
with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file:
@ -119,3 +208,39 @@ class Entry(EntryVariables, BaseEntry):
break
return file_exists
def maybe_get_prior_variables(self) -> Dict[str, Any]:
"""
If variables exist in the .info.json from a prior run, delete them
from kwargs (to prevent nested writes) and return them
"""
maybe_prior_variables: Dict[str, Any] = {}
if _YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY in self._kwargs:
maybe_prior_variables = self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY]
del self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY]
return maybe_prior_variables
@final
def to_dict(self) -> Dict[str, Any]:
"""
Returns
-------
Dictionary containing all variables
"""
return self.script.resolve().as_native()
@classmethod
def create_split_entry(cls, entry: "Entry", new_uid: str) -> "Entry":
"""
Creates a copy of an entry with a new uid to use as the starting point for a split entry
"""
new_entry = copy.deepcopy(entry)
new_entry._kwargs[v.uid.metadata_key] = new_uid
new_entry.add(
{
v.uid.variable_name: new_uid,
ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid,
}
)
return new_entry

176
src/ytdl_sub/entries/entry_parent.py
View file

@ -1,55 +1,32 @@
import math
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
import mergedeep
from typing import Set
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import TBaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED
from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT
from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX
from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE
from ytdl_sub.entries.variables.kwargs import SOURCE_UID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import TITLE
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.entries.variables.kwargs import UPLOADER
from ytdl_sub.entries.variables.kwargs import UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.entries.script.variable_types import MetadataVariable
v: VariableDefinitions = VARIABLES
class ParentType:
PLAYLIST = "playlist"
SOURCE = "source"
def _sort_entries(entries: List[TBaseEntry]) -> List[TBaseEntry]:
"""Try sorting by playlist_id first, then fall back to uid"""
return sorted(entries, key=lambda ent: (ent.kwargs_get(PLAYLIST_INDEX, math.inf), ent.uid))
# pylint: disable=protected-access
class EntryParent(BaseEntry):
@classmethod
def _sort_entries(cls, entries: List[TBaseEntry]) -> List[TBaseEntry]:
"""Try sorting by playlist_id first, then fall back to uid"""
return sorted(
entries,
key=lambda ent: (ent._kwargs_get(v.playlist_index.metadata_key, math.inf), ent.uid),
)
def __init__(self, entry_dict: Dict, working_directory: str):
super().__init__(entry_dict=entry_dict, working_directory=working_directory)
self._parent_children: List["EntryParent"] = []
@ -73,87 +50,44 @@ class EntryParent(BaseEntry):
self.entry_children()
)
def _playlist_variables(self, idx: int, children: List[TBaseEntry], parent_type: str) -> Dict:
_count = self.kwargs_get(PLAYLIST_COUNT, len(children))
_index = children[idx].kwargs_get(PLAYLIST_INDEX, idx + 1)
def _sibling_entry_metadata(self) -> List[Dict[str, Any]]:
sibling_entry_metadata: List[Dict[str, Any]] = []
variable_filter: Set[MetadataVariable] = (
v.required_entry_variables() | v.default_entry_variables()
)
for entry in self.entry_children():
sibling_entry_metadata.append(
{var.metadata_key: entry._kwargs_get(var.metadata_key) for var in variable_filter}
)
return sibling_entry_metadata
if parent_type == ParentType.SOURCE:
return {SOURCE_INDEX: _index, SOURCE_COUNT: _count}
return {
SOURCE_INDEX: self.kwargs_get(SOURCE_INDEX, 1),
SOURCE_COUNT: self.kwargs_get(SOURCE_INDEX, 1),
PLAYLIST_INDEX: _index,
PLAYLIST_COUNT: _count,
}
def _parent_variables(self, parent_type: str) -> Dict:
def _(source_key: str, playlist_key: str) -> str:
return playlist_key if parent_type == ParentType.PLAYLIST else source_key
def __(key: str) -> Optional[str]:
return self.kwargs_get(key=key)
return {
_(SOURCE_ENTRY, PLAYLIST_ENTRY): self._kwargs,
_(SOURCE_TITLE, PLAYLIST_TITLE): __(TITLE),
_(SOURCE_WEBPAGE_URL, PLAYLIST_WEBPAGE_URL): __(WEBPAGE_URL),
_(SOURCE_UID, PLAYLIST_UID): __(UID),
_(SOURCE_DESCRIPTION, PLAYLIST_DESCRIPTION): __(DESCRIPTION),
_(SOURCE_UPLOADER, PLAYLIST_UPLOADER): __(UPLOADER),
_(SOURCE_UPLOADER_ID, PLAYLIST_UPLOADER_ID): __(UPLOADER_ID),
_(SOURCE_UPLOADER_URL, PLAYLIST_UPLOADER_URL): __(UPLOADER_URL),
}
def _get_entry_children_variable_list(self, variable_name: str) -> List[str | int]:
return [getattr(entry_child, variable_name) for entry_child in self.entry_children()]
def _entry_aggregate_variables(self) -> Dict:
if not self.entry_children():
return {}
return {
PLAYLIST_MAX_UPLOAD_YEAR: max(self._get_entry_children_variable_list("upload_year")),
PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED: max(
self._get_entry_children_variable_list("upload_year_truncated")
),
}
# pylint: disable=protected-access
def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent":
def _set_child_variables(
self, include_sibling_metadata: bool, parents: Optional[List["EntryParent"]] = None
) -> "EntryParent":
if parents is None:
parents = [self]
self.add_kwargs(
self._playlist_variables(idx=0, children=parents, parent_type=ParentType.SOURCE)
)
kwargs_to_add: Dict = {}
kwargs_to_add: Dict[str, Any] = {}
if include_sibling_metadata:
kwargs_to_add[v.sibling_metadata.metadata_key] = self._sibling_entry_metadata()
if len(parents) >= 1:
mergedeep.merge(kwargs_to_add, parents[-1]._parent_variables(ParentType.PLAYLIST))
kwargs_to_add[v.playlist_metadata.metadata_key] = parents[-1]._kwargs
if len(parents) >= 2:
mergedeep.merge(kwargs_to_add, parents[-2]._parent_variables(ParentType.SOURCE))
kwargs_to_add[v.source_metadata.metadata_key] = parents[-2]._kwargs
if len(parents) >= 3:
raise ValueError(
"ytdl-sub currently does support more than 3 layers of playlists/entries. "
"If you encounter this error, please file a ticket with the URLs used."
)
mergedeep.merge(kwargs_to_add, self._entry_aggregate_variables())
for idx, entry_child in enumerate(self.entry_children()):
entry_child.add_kwargs(
self._playlist_variables(
idx=idx, children=self.entry_children(), parent_type=ParentType.PLAYLIST
)
)
entry_child.add_kwargs(kwargs_to_add)
for entry_child in self.entry_children():
entry_child._kwargs = dict(entry_child._kwargs, **kwargs_to_add)
for idx, parent_child in enumerate(self.parent_children()):
parent_child.add_kwargs(
self._playlist_variables(
idx=idx, children=self.parent_children(), parent_type=ParentType.SOURCE
)
for parent_child in self.parent_children():
parent_child._set_child_variables(
include_sibling_metadata=include_sibling_metadata, parents=parents + [parent_child]
)
parent_child._set_child_variables(parents=parents + [parent_child])
return self
@ -170,8 +104,10 @@ class EntryParent(BaseEntry):
if entry_dict in self
]
self._parent_children = _sort_entries([ent for ent in entries if self.is_entry_parent(ent)])
self._entry_children = _sort_entries(
self._parent_children = self._sort_entries(
[ent for ent in entries if self.is_entry_parent(ent)]
)
self._entry_children = self._sort_entries(
[ent.to_type(Entry) for ent in entries if self.is_entry(ent)]
)
@ -190,7 +126,7 @@ class EntryParent(BaseEntry):
-------
Desired thumbnail url if it exists. None if it does not.
"""
for thumbnail in self.kwargs_get("thumbnails", []):
for thumbnail in self._kwargs_get("thumbnails", []):
if thumbnail["id"] == thumbnail_id:
return thumbnail["url"]
return None
@ -200,7 +136,7 @@ class EntryParent(BaseEntry):
if isinstance(item, dict):
playlist_id = item.get("playlist_id")
elif isinstance(item, BaseEntry):
playlist_id = item.kwargs_get("playlist_id")
playlist_id = item._kwargs_get("playlist_id")
if not playlist_id:
return False
@ -248,7 +184,11 @@ class EntryParent(BaseEntry):
@classmethod
def from_entry_dicts(
cls, url: str, entry_dicts: List[Dict], working_directory: str
cls,
url: str,
entry_dicts: List[Dict],
working_directory: str,
include_sibling_metadata: bool,
) -> List["EntryParent"]:
"""
Reads all entry dicts and builds a tree of EntryParents
@ -271,15 +211,16 @@ class EntryParent(BaseEntry):
parents = [root_parent]
for parent in parents:
parent._set_child_variables()
parent._set_child_variables(include_sibling_metadata=include_sibling_metadata)
return parents
# pylint: enable=protected-access
@classmethod
def from_entry_dicts_with_no_parents(
cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str
cls,
parents: List["EntryParent"],
entry_dicts: List[Dict],
working_directory: str,
) -> List[Entry]:
"""
Reads all entries that do not have any parents
@ -289,7 +230,10 @@ class EntryParent(BaseEntry):
return any(entry_dict in parent for parent in parents)
return [
Entry(entry_dict=entry_dict, working_directory=working_directory)
Entry(
entry_dict=entry_dict,
working_directory=working_directory,
)
for entry_dict in entry_dicts
if cls.is_entry(entry_dict) and not _in_any_parents(entry_dict)
]

0
src/ytdl_sub/entries/script/__init__.py Normal file
View file

178
src/ytdl_sub/entries/script/custom_functions.py Normal file
View file

@ -0,0 +1,178 @@
import os
import posixpath
from yt_dlp.utils import sanitize_filename
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import AnyArgument
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.file_path import FilePathTruncater
def _pad(num: int, width: int):
return str(num).zfill(width)
_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
class CustomFunctions:
@staticmethod
def legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument:
"""
ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not
interfere with its legacy variable scripting system. This function replicates that
behavior.
"""
if isinstance(value, String):
value = String(value.value.replace("{", "｛").replace("}", "｝"))
return value
@staticmethod
def to_native_filepath(filepath: String) -> String:
"""
Convert any unix-based path separators ('/') with the OS's native
separator.
"""
return String(filepath.value.replace(posixpath.sep, os.sep))
@staticmethod
def truncate_filepath_if_too_long(filepath: String) -> String:
"""
If a file-path is too long for the OS, this function will truncate it while preserving
the extension.
"""
return String(FilePathTruncater.maybe_truncate_file_path(filepath.value))
@staticmethod
def sanitize(value: AnyArgument) -> String:
"""
Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
for file/directory names on any OS.
"""
return String(sanitize_filename(str(value)))
@staticmethod
def sanitize_plex_episode(string: String) -> String:
"""
Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width
numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the
season and/or episode.
"""
sanitized_string = CustomFunctions.sanitize(string).value
out = ""
for char in sanitized_string:
match char:
case "0":
out += "０"
case "1":
out += "１"
case "2":
out += "２"
case "3":
out += "３"
case "4":
out += "４"
case "5":
out += "５"
case "6":
out += "６"
case "7":
out += "７"
case "8":
out += "８"
case "9":
out += "９"
case _:
out += char
return String(out)
@staticmethod
def to_date_metadata(yyyymmdd: String) -> Map:
"""
Takes a date in the form of YYYYMMDD and returns a Map containing:
- date (String, YYYYMMDD)
- date_standardized (String, YYYY-MM-DD)
- year (Integer)
- month (Integer)
- day (Integer)
- year_truncated (Integer, YY from YY[YY])
- month_padded (String)
- day_padded (String)
- year_truncated_reversed (Integer, 100 - year_truncated)
- month_reversed (Integer, 13 - month)
- month_reversed_padded (String)
- day_reversed (Integer, total_days_in_month + 1 - day)
- day_reversed_padded (String)
- day_of_year (Integer)
- day_of_year_padded (String, padded 3)
- day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year)
- day_of_year_reversed_padded (String, padded 3)
"""
date_str = yyyymmdd.value
if not (date_str.isnumeric() and len(date_str) == 8):
raise RuntimeException(
f"Expected input of date_metadata to be YYYYMMDD, but received {date_str}"
)
year: int = int(date_str[:4])
month_padded: str = date_str[4:6]
day_padded: str = date_str[6:8]
month: int = int(month_padded)
day: int = int(day_padded)
year_truncated: int = int(str(year)[-2:])
day_of_year: int = sum(_days_in_month[:month]) + day
total_days_in_month: int = _days_in_month[month]
total_days_in_year: int = 365
if year % 4 == 0:
total_days_in_year += 1
if month == 2:
total_days_in_month += 1
if month > 2:
day_of_year += 1
day_of_year_reversed: int = total_days_in_year + 1 - day_of_year
month_reversed: int = 13 - month
day_reversed: int = total_days_in_month + 1 - day
return Map(
{
String("date"): yyyymmdd,
String("date_standardized"): String(f"{year}-{month_padded}-{day_padded}"),
String("year"): Integer(year),
String("month"): Integer(month),
String("day"): Integer(day),
String("year_truncated"): Integer(year_truncated),
String("month_padded"): String(month_padded),
String("day_padded"): String(day_padded),
String("year_truncated_reversed"): Integer(100 - year_truncated),
String("month_reversed"): Integer(month_reversed),
String("month_reversed_padded"): String(_pad(month_reversed, width=2)),
String("day_reversed"): Integer(day_reversed),
String("day_reversed_padded"): String(_pad(day_reversed, width=2)),
String("day_of_year"): Integer(day_of_year),
String("day_of_year_padded"): String(_pad(day_of_year, width=3)),
String("day_of_year_reversed"): Integer(day_of_year_reversed),
String("day_of_year_reversed_padded"): String(_pad(day_of_year_reversed, width=3)),
}
)
@staticmethod
def register():
"""
Register Custom functions once and only once
"""
if not Functions.is_built_in("sanitize"):
Functions.register_function(CustomFunctions.legacy_bracket_safety)
Functions.register_function(CustomFunctions.truncate_filepath_if_too_long)
Functions.register_function(CustomFunctions.to_native_filepath)
Functions.register_function(CustomFunctions.sanitize)
Functions.register_function(CustomFunctions.sanitize_plex_episode)
Functions.register_function(CustomFunctions.to_date_metadata)

86
src/ytdl_sub/entries/script/function_scripts.py Normal file
View file

@ -0,0 +1,86 @@
from typing import Dict
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
v: VariableDefinitions = VARIABLES
CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = {
#############################################################################################
# SIBLING GETTER
"%extract_field_from_siblings": f"""{{
%if(
%bool({v.sibling_metadata.variable_name}),
%array_apply_fixed(
{v.sibling_metadata.variable_name},
%string($0),
%map_get
)
[]
)
}}""",
#############################################################################################
# REGEX PLUGIN
# $0 - input variable
# $1 - regex array
# $2 - defaults
# $3 - error message
"%regex_capture_inner": """{
%assert_then(
%array_reduce(
%array_apply_fixed(
%array_apply(
$1,
%regex_capture_groups
),
%array_size($2),
%lte
),
%and
),
%array_overlay(
%array(
%array_first(
%array_apply_fixed(
%array($1),
%string($0),
%regex_search
),
[]
)
),
%array_extend( ['using all defaults'], $2 ),
True
),
$3
)
}""",
"%regex_capture_many_required": """{
%assert_ne(
%regex_capture_inner(
$0, $1, ['', '', '', '', '', '', '', '', '', ''],
'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults'
),
['using all defaults', '', '', '', '', '', '', '', '', '', ''],
'When running %regex_capture_many_required, no regex strings were captured'
)
}""",
"%regex_capture_many_with_defaults": """{
%regex_capture_inner(
$0, $1, $2,
'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults'
)
}""",
"%regex_search_any": """{
%ne(
%array_at(
%regex_capture_inner(
$0, $1, [],
'When using %regex_search_many, all regex strings must contain no capture groups'
),
0
),
'using all defaults'
)
}""",
}

1160
src/ytdl_sub/entries/script/variable_definitions.py Normal file
View file

File diff suppressed because it is too large Load diff

321
src/ytdl_sub/entries/script/variable_types.py Normal file
View file

@ -0,0 +1,321 @@
from abc import ABC
from abc import abstractmethod
from dataclasses import dataclass
from typing import Dict
from typing import List
from typing import Optional
from typing import Type
from typing import TypeVar
from ytdl_sub.script.types.array import Array
from ytdl_sub.script.types.map import Map
from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import String
ENTRY_METADATA_VARIABLE_NAME = "entry_metadata"
PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata"
SOURCE_METADATA_VARIABLE_NAME = "source_metadata"
TMetadataVariable = TypeVar("TMetadataVariable", bound="MetadataVariable")
TVariable = TypeVar("TVariable", bound="Variable")
def _get(
cast: str,
metadata_variable_name: str,
metadata_key: str,
variable_name: Optional[str],
default: Optional[TVariable | str | int | Dict | List],
as_type: Type[TMetadataVariable],
) -> TMetadataVariable:
if default is None:
# TODO: assert with good error message if key DNE
out = f"%map_get({metadata_variable_name}, '{metadata_key}')"
elif isinstance(default, Variable):
args = f"{metadata_variable_name}, '{metadata_key}', {default.variable_name}"
out = f"%map_get_non_empty({args})"
elif isinstance(default, str):
out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', '{default}')"
elif isinstance(default, dict):
out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {{}})"
elif isinstance(default, list):
out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', [])"
else:
out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {default})"
return as_type(
variable_name=variable_name or metadata_key,
metadata_key=metadata_key,
definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}",
)
@dataclass(frozen=True)
class Variable(ABC):
variable_name: str
definition: str
@classmethod
@abstractmethod
def human_readable_type(cls) -> str:
"""
Script type of the variable, for documentation
"""
@dataclass(frozen=True)
class StringVariable(Variable):
@classmethod
def human_readable_type(cls) -> str:
return String.__name__
def to_sanitized_plex(self, variable_name: str) -> "StringVariable":
"""
Converts a String variable to be plex sanitized
"""
return StringVariable(
variable_name=variable_name,
definition=f"{{%sanitize_plex_episode({self.variable_name})}}",
)
def as_date_variable(self) -> "StringDateVariable":
"""
Converts a String variable to a date variable (which has metadata helpers)
"""
return StringDateVariable(
variable_name=self.variable_name,
definition=self.definition,
)
@dataclass(frozen=True)
class StringDateVariable(StringVariable):
def get_string_date_metadata(
self, date_metadata_key: str, variable_name: Optional[str] = None
) -> StringVariable:
"""
Gets a string-based date metadata variable
"""
return StringVariable(
variable_name=variable_name or date_metadata_key,
definition=f"""{{
%string(
%map_get(
%to_date_metadata({self.variable_name}),
'{date_metadata_key}'
)
)
}}""",
)
def get_integer_date_metadata(
self, date_metadata_key: str, variable_name: str
) -> "IntegerVariable":
"""
Gets an int-based date metadata variable
"""
return IntegerVariable(
variable_name=variable_name,
definition=f"""{{
%int(
%map_get(
%to_date_metadata({self.variable_name}),
'{date_metadata_key}'
)
)
}}""",
)
@dataclass(frozen=True)
class IntegerVariable(Variable):
@classmethod
def human_readable_type(cls) -> str:
return Integer.__name__
def to_padded_int(self, variable_name: str, pad: int) -> StringVariable:
"""
Pads an integer
"""
return StringVariable(
variable_name=variable_name, definition=f"{{%pad_zero({self.variable_name}, {pad})}}"
)
@dataclass(frozen=True)
class ArrayVariable(Variable):
@classmethod
def human_readable_type(cls) -> str:
return Array.__name__
@dataclass(frozen=True)
class MapVariable(Variable):
@classmethod
def human_readable_type(cls) -> str:
return Map.__name__
@dataclass(frozen=True)
class MetadataVariable(Variable, ABC):
metadata_key: str
@dataclass(frozen=True)
class MapMetadataVariable(MetadataVariable, MapVariable):
@classmethod
def from_entry(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional["MapMetadataVariable" | Dict] = None,
) -> "MapMetadataVariable":
"""
Creates a map variable from entry metadata
"""
return _get(
"map",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=MapMetadataVariable,
)
@dataclass(frozen=True)
class ArrayMetadataVariable(MetadataVariable, ArrayVariable):
@classmethod
def from_entry(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional["ArrayMetadataVariable" | List] = None,
) -> "ArrayMetadataVariable":
"""
Creates an array variable from entry metadata
"""
return _get(
"array",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=ArrayMetadataVariable,
)
@dataclass(frozen=True)
class StringMetadataVariable(MetadataVariable, StringVariable):
@classmethod
def from_entry(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional[StringVariable | str] = None,
) -> "StringMetadataVariable":
"""
Creates a string variable from entry metadata
"""
return _get(
"string",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=StringMetadataVariable,
)
@classmethod
def from_playlist(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional[StringVariable | str] = None,
) -> "StringMetadataVariable":
"""
Creates a string variable from playlist metadata
"""
return _get(
"string",
metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=StringMetadataVariable,
)
@classmethod
def from_source(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional[StringVariable | str] = None,
) -> "StringMetadataVariable":
"""
Creates a string variable from source metadata
"""
return _get(
"string",
metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=StringMetadataVariable,
)
def as_date_variable(self) -> "StringDateMetadataVariable":
"""
Converts a String variable to a date variable (which has metadata helpers)
"""
return StringDateMetadataVariable(
metadata_key=self.metadata_key,
variable_name=self.variable_name,
definition=self.definition,
)
@dataclass(frozen=True)
class StringDateMetadataVariable(StringMetadataVariable, StringDateVariable):
pass
@dataclass(frozen=True)
class IntegerMetadataVariable(MetadataVariable, IntegerVariable):
@classmethod
def from_entry(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional[IntegerVariable | int] = None,
) -> "IntegerMetadataVariable":
"""
Creates an int variable from entry metadata
"""
return _get(
"int",
metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=IntegerMetadataVariable,
)
@classmethod
def from_playlist(
cls,
metadata_key: str,
variable_name: Optional[str] = None,
default: Optional[IntegerVariable | int] = None,
) -> "IntegerMetadataVariable":
"""
Creates an int variable from playlist metadata
"""
return _get(
"int",
metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME,
metadata_key=metadata_key,
variable_name=variable_name,
default=default,
as_type=IntegerMetadataVariable,
)

857
src/ytdl_sub/entries/variables/entry_variables.py
View file

@ -1,857 +0,0 @@
from datetime import datetime
from typing import Union
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import BaseEntryVariables
from ytdl_sub.entries.variables.kwargs import CHANNEL
from ytdl_sub.entries.variables.kwargs import CHANNEL_ID
from ytdl_sub.entries.variables.kwargs import CREATOR
from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT
from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR
from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED
from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import RELEASE_DATE
from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT
from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION
from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX
from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE
from ytdl_sub.entries.variables.kwargs import SOURCE_UID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID
from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL
from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
# pylint: disable=no-member
# pylint: disable=too-many-public-methods
def pad(num: int, width: int = 2):
"""Pad integers"""
return str(num).zfill(width)
_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
Self = Union[BaseEntry, "EntryVariables"]
class EntryVariables(BaseEntryVariables):
@property
def source_title(self: Self) -> str:
"""
Returns
-------
str
Name of the source (i.e. channel with multiple playlists) if it exists, otherwise
returns its playlist_title.
"""
return self.kwargs_get(SOURCE_TITLE, self.playlist_title)
@property
def source_title_sanitized(self: Self) -> str:
"""
Returns
-------
str
The source title, sanitized
"""
return sanitize_filename(self.source_title)
@property
def source_uid(self: Self) -> str:
"""
Returns
-------
str
The source unique id if it exists, otherwise returns the playlist unique ID.
"""
return self.kwargs_get(SOURCE_UID, self.playlist_uid)
@property
def source_index(self: Self) -> int:
"""
Returns
-------
int
Source index if it exists, otherwise returns ``1``.
It is recommended to not use this unless you know the source will never add new content
(it is easy for this value to change).
"""
return self.kwargs_get(SOURCE_INDEX, self.playlist_index)
@property
def source_index_padded(self: Self) -> str:
"""
Returns
-------
int
The source index, padded.
"""
return pad(self.source_index, 2)
@property
def source_count(self: Self) -> int:
"""
Returns
-------
int
The source count if it exists, otherwise returns the playlist count.
"""
return self.kwargs_get(SOURCE_COUNT, self.playlist_count)
@property
def source_webpage_url(self: Self) -> str:
"""
Returns
-------
str
The source webpage url if it exists, otherwise returns the playlist webpage url.
"""
return self.kwargs_get(SOURCE_WEBPAGE_URL, self.playlist_webpage_url)
@property
def source_description(self: Self) -> str:
"""
Returns
-------
str
The source description if it exists, otherwise returns the playlist description.
"""
return self.kwargs_get(SOURCE_DESCRIPTION, self.playlist_description)
@property
def playlist_uid(self: Self) -> str:
"""
Returns
-------
str
The playlist unique ID if it exists, otherwise return the entry unique ID.
"""
return self.kwargs_get(PLAYLIST_UID, self.uid)
@property
def playlist_title(self: Self) -> str:
"""
Returns
-------
str
Name of its parent playlist/channel if it exists, otherwise returns its title.
"""
return self.kwargs_get(PLAYLIST_TITLE, self.title)
@property
def playlist_title_sanitized(self: Self) -> str:
"""
Returns
-------
str
The playlist name, sanitized
"""
return sanitize_filename(self.playlist_title)
@property
def playlist_index(self: Self) -> int:
"""
Returns
-------
int
Playlist index if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
"""
return self.kwargs_get(PLAYLIST_INDEX, 1)
@property
def playlist_index_reversed(self: Self) -> int:
"""
Returns
-------
int
Playlist index reversed via ``playlist_count - playlist_index + 1``
"""
return self.playlist_count - self.playlist_index + 1
@property
def playlist_index_padded(self: Self) -> str:
"""
Returns
-------
str
playlist_index padded two digits
"""
return pad(self.playlist_index, width=2)
@property
def playlist_index_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
playlist_index_reversed padded two digits
"""
return pad(self.playlist_index_reversed, width=2)
@property
def playlist_index_padded6(self: Self) -> str:
"""
Returns
-------
str
playlist_index padded six digits.
"""
return pad(self.playlist_index, width=6)
@property
def playlist_index_reversed_padded6(self: Self) -> str:
"""
Returns
-------
str
playlist_index_reversed padded six digits.
"""
return pad(self.playlist_index_reversed, width=6)
@property
def playlist_count(self: Self) -> int:
"""
Returns
-------
int
Playlist count if it exists, otherwise returns ``1``.
Note that for channels/playlists, any change (i.e. adding or removing a video) will make
this value change. Use with caution.
"""
return self.kwargs_get(PLAYLIST_COUNT, 1)
@property
def playlist_description(self: Self) -> str:
"""
Returns
-------
str
The playlist description if it exists, otherwise returns the entry's description.
"""
return self.kwargs_get(PLAYLIST_DESCRIPTION, self.description)
@property
def playlist_webpage_url(self: Self) -> str:
"""
Returns
-------
str
The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
"""
return self.kwargs_get(PLAYLIST_WEBPAGE_URL, self.webpage_url)
@property
def playlist_max_upload_year(self: Self) -> int:
"""
Returns
-------
int
Max upload_year for all entries in this entry's playlist if it exists, otherwise returns
``upload_year``
"""
# override in EntryParent
return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR, self.upload_year)
@property
def playlist_max_upload_year_truncated(self: Self) -> int:
"""
Returns
-------
int
The max playlist truncated upload year for all entries in this entry's playlist if it
exists, otherwise returns ``upload_year_truncated``.
"""
return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED, self.upload_year_truncated)
@property
def playlist_uploader_id(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader id if it exists, otherwise returns the entry uploader ID.
"""
return self.kwargs_get(PLAYLIST_UPLOADER_ID, self.uploader_id)
@property
def playlist_uploader(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader if it exists, otherwise return the entry uploader.
"""
return self.kwargs_get(PLAYLIST_UPLOADER, self.uploader)
@property
def playlist_uploader_sanitized(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader, sanitized.
"""
return sanitize_filename(self.playlist_uploader)
@property
def playlist_uploader_url(self: Self) -> str:
"""
Returns
-------
str
The playlist uploader url if it exists, otherwise returns the playlist webpage_url.
"""
return self.kwargs_get(PLAYLIST_UPLOADER_URL, self.playlist_webpage_url)
@property
def source_uploader_id(self: Self) -> str:
"""
Returns
-------
str
The source uploader id if it exists, otherwise returns the playlist_uploader_id
"""
return self.kwargs_get(SOURCE_UPLOADER_ID, self.playlist_uploader_id)
@property
def source_uploader(self: Self) -> str:
"""
Returns
-------
str
The source uploader if it exists, otherwise return the playlist_uploader
"""
return self.kwargs_get(SOURCE_UPLOADER, self.playlist_uploader)
@property
def source_uploader_url(self: Self) -> str:
"""
Returns
-------
str
The source uploader url if it exists, otherwise returns the source webpage_url.
"""
return self.kwargs_get(SOURCE_UPLOADER_URL, self.source_webpage_url)
@property
def creator(self: Self) -> str:
"""
Returns
-------
str
The creator name if it exists, otherwise returns the channel.
"""
return self.kwargs_get(CREATOR, self.channel)
@property
def creator_sanitized(self: Self) -> str:
"""
Returns
-------
str
The creator name, sanitized
"""
return sanitize_filename(self.creator)
@property
def channel(self: Self) -> str:
"""
Returns
-------
str
The channel name if it exists, otherwise returns the uploader.
"""
return self.kwargs_get(CHANNEL, self.uploader)
@property
def channel_sanitized(self: Self) -> str:
"""
Returns
-------
str
The channel name, sanitized.
"""
return sanitize_filename(self.channel)
@property
def channel_id(self: Self) -> str:
"""
Returns
-------
str
The channel id if it exists, otherwise returns the entry uploader ID.
"""
return self.kwargs_get(CHANNEL_ID, self.uploader_id)
@property
def ext(self: Self) -> str:
"""
Returns
-------
str
The downloaded entry's file extension
"""
return self.kwargs(EXT)
@property
def thumbnail_ext(self: Self) -> str:
"""
Returns
-------
str
The download entry's thumbnail extension. Will always return 'jpg'. Until there is a
need to support other image types, we always convert to jpg.
"""
return "jpg"
@property
def download_index(self: Self) -> int:
"""
Returns
-------
int
The i'th entry downloaded. NOTE that this is fetched dynamically from the download
archive.
"""
return self.kwargs_get(DOWNLOAD_INDEX, 0) + 1
@property
def download_index_padded6(self: Self) -> str:
"""
Returns
-------
str
The download_index padded six digits
"""
return pad(self.download_index, 6)
@property
def upload_date_index(self: Self) -> int:
"""
Returns
-------
int
The i'th entry downloaded with this upload date.
"""
return self.kwargs_get(UPLOAD_DATE_INDEX, 0) + 1
@property
def upload_date_index_padded(self: Self) -> str:
"""
Returns
-------
int
The upload_date_index padded two digits
"""
return pad(self.upload_date_index, 2)
@property
def upload_date_index_reversed(self: Self) -> int:
"""
Returns
-------
int
100 - upload_date_index
"""
return 100 - self.upload_date_index
@property
def upload_date_index_reversed_padded(self: Self) -> str:
"""
Returns
-------
int
The upload_date_index padded two digits
"""
return pad(self.upload_date_index_reversed, 2)
@property
def upload_date(self: Self) -> str:
"""
Returns
-------
str
The entry's uploaded date, in YYYYMMDD format. If not present, return today's date.
"""
return self.kwargs_get(UPLOAD_DATE, datetime.now().strftime("%Y%m%d"))
@property
def upload_year(self: Self) -> int:
"""
Returns
-------
int
The entry's upload year
"""
return int(self.upload_date[:4])
@property
def upload_year_truncated(self: Self) -> int:
"""
Returns
-------
int
The last two digits of the upload year, i.e. 22 in 2022
"""
return int(str(self.upload_year)[-2:])
@property
def upload_year_truncated_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
"""
return 100 - self.upload_year_truncated
@property
def upload_month_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10``
"""
return 13 - self.upload_month
@property
def upload_month_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed upload month, but padded. i.e. November returns "02"
"""
return pad(self.upload_month_reversed)
@property
def upload_month_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's upload month padded to two digits, i.e. March returns "03"
"""
return self.upload_date[4:6]
@property
def upload_day_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's upload day padded to two digits, i.e. the fifth returns "05"
"""
return self.upload_date[6:8]
@property
def upload_month(self: Self) -> int:
"""
Returns
-------
int
The upload month as an integer (no padding).
"""
return int(self.upload_month_padded.lstrip("0"))
@property
def upload_day(self: Self) -> int:
"""
Returns
-------
int
The upload day as an integer (no padding).
"""
return int(self.upload_day_padded.lstrip("0"))
@property
def upload_day_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``,
i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24``
"""
total_days_in_month = _days_in_month[self.upload_month]
if self.upload_month == 2 and self.upload_year % 4 == 0: # leap year
total_days_in_month += 1
return total_days_in_month + 1 - self.upload_day
@property
def upload_day_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed upload day, but padded. i.e. August 30th returns "02".
"""
return pad(self.upload_day_reversed)
@property
def upload_day_of_year(self: Self) -> int:
"""
Returns
-------
int
The day of the year, i.e. February 1st returns ``32``
"""
output = sum(_days_in_month[: self.upload_month]) + self.upload_day
if self.upload_month > 2 and self.upload_year % 4 == 0:
output += 1
return output
@property
def upload_day_of_year_padded(self: Self) -> str:
"""
Returns
-------
str
The upload day of year, but padded i.e. February 1st returns "032"
"""
return pad(self.upload_day_of_year, width=3)
@property
def upload_day_of_year_reversed(self: Self) -> int:
"""
Returns
-------
int
The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``,
i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
"""
total_days_in_year = 365
if self.upload_year % 4 == 0:
total_days_in_year += 1
return total_days_in_year + 1 - self.upload_day_of_year
@property
def upload_day_of_year_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed upload day of year, but padded i.e. December 31st returns "001"
"""
return pad(self.upload_day_of_year_reversed, width=3)
@property
def upload_date_standardized(self: Self) -> str:
"""
Returns
-------
str
The uploaded date formatted as YYYY-MM-DD
"""
return f"{self.upload_year}-{self.upload_month_padded}-{self.upload_day_padded}"
@property
def release_date(self: Self) -> str:
"""
Returns
-------
str
The entry's release date, in YYYYMMDD format. If not present, return the upload date.
"""
return self.kwargs_get(RELEASE_DATE, self.upload_date)
@property
def release_year(self: Self) -> int:
"""
Returns
-------
int
The entry's release year
"""
return int(self.release_date[:4])
@property
def release_year_truncated(self: Self) -> int:
"""
Returns
-------
int
The last two digits of the release year, i.e. 22 in 2022
"""
return int(str(self.release_year)[-2:])
@property
def release_year_truncated_reversed(self: Self) -> int:
"""
Returns
-------
int
The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
2022 returns ``100 - 22`` = ``78``
"""
return 100 - self.release_year_truncated
@property
def release_month_reversed(self: Self) -> int:
"""
Returns
-------
int
The release month, but reversed
using ``13 - {release_month}``, i.e. March returns ``10``
"""
return 13 - self.release_month
@property
def release_month_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed release month, but padded. i.e. November returns "02"
"""
return pad(self.release_month_reversed)
@property
def release_month_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's release month padded to two digits, i.e. March returns "03"
"""
return self.release_date[4:6]
@property
def release_day_padded(self: Self) -> str:
"""
Returns
-------
str
The entry's release day padded to two digits, i.e. the fifth returns "05"
"""
return self.release_date[6:8]
@property
def release_month(self: Self) -> int:
"""
Returns
-------
int
The release month as an integer (no padding).
"""
return int(self.release_month_padded.lstrip("0"))
@property
def release_day(self: Self) -> int:
"""
Returns
-------
int
The release day as an integer (no padding).
"""
return int(self.release_day_padded.lstrip("0"))
@property
def release_day_reversed(self: Self) -> int:
"""
Returns
-------
int
The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``,
i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24``
"""
total_days_in_month = _days_in_month[self.release_month]
if self.release_month == 2 and self.release_year % 4 == 0: # leap year
total_days_in_month += 1
return total_days_in_month + 1 - self.release_day
@property
def release_day_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed release day, but padded. i.e. August 30th returns "02".
"""
return pad(self.release_day_reversed)
@property
def release_day_of_year(self: Self) -> int:
"""
Returns
-------
int
The day of the year, i.e. February 1st returns ``32``
"""
output = sum(_days_in_month[: self.release_month]) + self.release_day
if self.release_month > 2 and self.release_year % 4 == 0:
output += 1
return output
@property
def release_day_of_year_padded(self: Self) -> str:
"""
Returns
-------
str
The release day of year, but padded i.e. February 1st returns "032"
"""
return pad(self.release_day_of_year, width=3)
@property
def release_day_of_year_reversed(self: Self) -> int:
"""
Returns
-------
int
The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``,
i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
"""
total_days_in_year = 365
if self.release_year % 4 == 0:
total_days_in_year += 1
return total_days_in_year + 1 - self.release_day_of_year
@property
def release_day_of_year_reversed_padded(self: Self) -> str:
"""
Returns
-------
str
The reversed release day of year, but padded i.e. December 31st returns "001"
"""
return pad(self.release_day_of_year_reversed, width=3)
@property
def release_date_standardized(self: Self) -> str:
"""
Returns
-------
str
The release date formatted as YYYY-MM-DD
"""
return f"{self.release_year}-{self.release_month_padded}-{self.release_day_padded}"

68
src/ytdl_sub/entries/variables/kwargs.py
View file

@ -1,68 +0,0 @@
from typing import List
class KwargKeys:
keys: List[str] = []
backend_keys: List[str] = []
def _(key: str, backend: bool = False) -> str:
if backend:
assert key not in KwargKeys.backend_keys
KwargKeys.backend_keys.append(key)
else:
assert key not in KwargKeys.keys
KwargKeys.keys.append(key)
return key
SOURCE_ENTRY = _("source_entry", backend=True)
SOURCE_INDEX = _("source_index")
SOURCE_COUNT = _("source_count")
SOURCE_TITLE = _("source_title")
SOURCE_UID = _("source_uid")
SOURCE_DESCRIPTION = _("source_description")
SOURCE_WEBPAGE_URL = _("source_webpage_url")
SOURCE_UPLOADER = _("source_uploader")
SOURCE_UPLOADER_ID = _("source_uploader_id")
SOURCE_UPLOADER_URL = _("source_uploader_url")
PLAYLIST_ENTRY = _("playlist_entry", backend=True)
PLAYLIST_WEBPAGE_URL = _("playlist_webpage_url")
PLAYLIST_INDEX = _("playlist_index")
PLAYLIST_COUNT = _("playlist_count")
PLAYLIST_MAX_UPLOAD_YEAR = _("playlist_max_upload_year")
PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED = _("playlist_max_upload_year_truncated")
PLAYLIST_TITLE = _("playlist_title")
PLAYLIST_DESCRIPTION = _("playlist_description")
PLAYLIST_UID = _("playlist_uid")
PLAYLIST_UPLOADER = _("playlist_uploader")
PLAYLIST_UPLOADER_ID = _("playlist_uploader_id")
PLAYLIST_UPLOADER_URL = _("playlist_uploader_url")
COLLECTION_URL = _("collection_url", backend=True)
DOWNLOAD_INDEX = _("download_index", backend=True)
UPLOAD_DATE_INDEX = _("upload_date_index", backend=True)
REQUESTED_SUBTITLES = _("requested_subtitles", backend=True)
CHAPTERS = _("chapters", backend=True)
YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)
YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True)
SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True)
SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True)
COMMENTS = _("comments", backend=True)
UID = _("id")
EXTRACTOR = _("extractor")
IE_KEY = _("ie_key")
EPOCH = _("epoch")
CHANNEL = _("channel")
CHANNEL_ID = _("channel_id")
CREATOR = _("creator")
EXT = _("ext")
TITLE = _("title")
DESCRIPTION = _("description")
WEBPAGE_URL = _("webpage_url")
RELEASE_DATE = _("release_date")
UPLOAD_DATE = _("upload_date")
UPLOADER = _("uploader")
UPLOADER_ID = _("uploader_id")
UPLOADER_URL = _("uploader_url")

92
src/ytdl_sub/entries/variables/override_variables.py
View file

@ -1,17 +1,22 @@
SUBSCRIPTION_NAME = "subscription_name"
SUBSCRIPTION_VALUE = "subscription_value"
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.utils.name_validation import is_valid_name
# TODO: use this
SUBSCRIPTION_ARRAY = "subscription_array"
class OverrideVariables:
@classmethod
def subscription_name(cls) -> str:
@staticmethod
def subscription_name() -> str:
"""
Name of the subscription
"""
return SUBSCRIPTION_NAME
return "subscription_name"
@classmethod
def subscription_value(cls) -> str:
@staticmethod
def subscription_value() -> str:
"""
For subscriptions in the form of
@ -21,10 +26,10 @@ class OverrideVariables:
``subscription_value`` gets set to ``https://...``.
"""
return SUBSCRIPTION_VALUE
return "subscription_value"
@classmethod
def subscription_indent_i(cls, index: int) -> str:
@staticmethod
def subscription_indent_i(index: int) -> str:
"""
For subscriptions in the form of
@ -39,8 +44,8 @@ class OverrideVariables:
"""
return f"subscription_indent_{index + 1}"
@classmethod
def subscription_value_i(cls, index: int) -> str:
@staticmethod
def subscription_value_i(index: int) -> str:
"""
For subscriptions in the form of
@ -55,3 +60,66 @@ class OverrideVariables:
``subscription_value``.
"""
return f"subscription_value_{index + 1}"
@staticmethod
def subscription_map() -> str:
"""
For subscriptions in the form of
.. code-block:: yaml
+ Subscription Name:
Music Videos:
- "https://url1.com/..."
Concerts:
- "https://url2.com/..."
Stores all the contents under the subscription name into the override variable
``subscription_map`` as a Map value. The above example is stored as:
.. code-block:: python
{
"Music Videos": [
"https://url1.com/..."
],
"Concerts: [
"https://url2.com/..."
]
}
"""
return "subscription_map"
class OverrideHelpers:
@classmethod
def is_entry_variable_name(cls, name: str) -> bool:
"""
Returns
-------
True if the name is an entry variable name. False otherwise.
"""
return name in VARIABLE_SCRIPTS
@classmethod
def is_function_name(cls, name: str) -> bool:
"""
Returns
-------
True if the name is a function name (either built-in or script). False otherwise.
"""
if name.startswith("%"):
return name in CUSTOM_FUNCTION_SCRIPTS or Functions.is_built_in(name[1:])
return False
@classmethod
def is_valid_name(cls, name: str) -> bool:
"""
Returns
-------
True if the override name itself is valid. False otherwise.
"""
if name.startswith("%"):
return is_valid_name(name=name[1:])
return is_valid_name(name=name)

40
src/ytdl_sub/plugins/audio_extract.py
View file

@ -2,11 +2,15 @@ import os.path
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
@ -14,20 +18,20 @@ from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_TYPES_EXTENSION
from ytdl_sub.validators.audo_codec_validator import AudioTypeValidator
from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES
class AudioExtractOptions(OptionsDictValidator):
"""
Extracts audio from a video file.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
audio_extract:
codec: "mp3"
quality: 128
audio_extract:
codec: "mp3"
quality: 128
"""
_required_keys = {"codec"}
@ -50,21 +54,31 @@ class AudioExtractOptions(OptionsDictValidator):
@property
def codec(self) -> str:
"""
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
opus, vorbis, wav, and best to grab the best possible format at runtime.
:expected type: String
:description:
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
opus, vorbis, wav, and best to grab the best possible format at runtime.
"""
return self._codec
@property
def quality(self) -> Optional[float]:
"""
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
:expected type: Float
:description:
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
"""
if self._quality is not None:
return self._quality.value
return None
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
"""
Possibly changes ``ext``, so do not resolve until this has run
"""
return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}}
class AudioExtractPlugin(Plugin[AudioExtractOptions]):
plugin_options_type = AudioExtractOptions
@ -125,7 +139,7 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]):
new_ext = AUDIO_CODEC_TYPES_EXTENSION_MAPPING[self.plugin_options.codec]
extracted_audio_file = entry.get_download_file_path().removesuffix(entry.ext) + new_ext
entry.add_kwargs({"ext": new_ext})
entry.add({v.ext: new_ext})
if not self.is_dry_run:
if not os.path.isfile(extracted_audio_file):

141
src/ytdl_sub/plugins/chapters.py
View file

@ -5,12 +5,15 @@ from typing import List
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import COMMENTS
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
from ytdl_sub.utils.file_handler import FileMetadata
@ -19,6 +22,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import ListValidator
v: VariableDefinitions = VARIABLES
SPONSORBLOCK_HIGHLIGHT_CATEGORIES: Set[str] = {"poi_highlight"}
SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
"sponsor",
@ -33,15 +38,11 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
def _chapters(entry: Entry) -> List[Dict]:
if entry.kwargs_contains("chapters"):
return entry.kwargs("chapters") or []
return []
return entry.get(v.chapters, list)
def _sponsorblock_chapters(entry: Entry) -> List[Dict]:
if entry.kwargs_contains("sponsorblock_chapters"):
return entry.kwargs("sponsorblock_chapters") or []
return []
return entry.get(v.sponsorblock_chapters, list)
def _contains_any_chapters(entry: Entry) -> bool:
@ -63,32 +64,29 @@ class ChaptersOptions(OptionsDictValidator):
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
chapters:
# Embedded Chapter Fields
embed_chapters: True
allow_chapters_from_comments: False
remove_chapters_regex:
- "Intro"
- "Outro"
# Sponsorblock Fields
sponsorblock_categories:
- "outro"
- "selfpromo"
- "preview"
- "interaction"
- "sponsor"
- "music_offtopic"
- "intro"
remove_sponsorblock_categories: "all"
force_key_frames: False
chapters:
# Embedded Chapter Fields
embed_chapters: True
allow_chapters_from_comments: False
remove_chapters_regex:
- "Intro"
- "Outro"
# Sponsorblock Fields
sponsorblock_categories:
- "outro"
- "selfpromo"
- "preview"
- "interaction"
- "sponsor"
- "music_offtopic"
- "intro"
remove_sponsorblock_categories: "all"
force_key_frames: False
"""
_optional_keys = {
@ -134,23 +132,29 @@ class ChaptersOptions(OptionsDictValidator):
@property
def embed_chapters(self) -> Optional[bool]:
"""
Optional. Embed chapters into the file. Defaults to True.
:expected type: Optional[Boolean]
:description:
Defaults to True. Embed chapters into the file.
"""
return self._embed_chapters
@property
def allow_chapters_from_comments(self) -> bool:
"""
Optional. If chapters do not exist in the video/description itself, attempt to scrape
comments to find the chapters. Defaults to False.
:expected type: Optional[Boolean]
:description:
Defaults to False. If chapters do not exist in the video/description itself, attempt to
scrape comments to find the chapters.
"""
return self._allow_chapters_from_comments
@property
def remove_chapters_regex(self) -> Optional[List[re.Pattern]]:
"""
Optional. List of regex patterns to match chapter titles against and remove them from the
entry.
:expected type: Optional[List[RegexString]
:description:
List of regex patterns to match chapter titles against and remove them from the
entry.
"""
if self._remove_chapters_regex:
return [validator.compiled_regex for validator in self._remove_chapters_regex.list]
@ -159,9 +163,11 @@ class ChaptersOptions(OptionsDictValidator):
@property
def sponsorblock_categories(self) -> Optional[List[str]]:
"""
Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor",
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
"poi_highlight", or "all" to include all categories.
:expected type: Optional[List[String]]
:description:
List of SponsorBlock categories to embed as chapters. Supports "sponsor",
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
"poi_highlight", or "all" to include all categories.
"""
if self._sponsorblock_categories:
category_list = [validator.value for validator in self._sponsorblock_categories.list]
@ -173,9 +179,11 @@ class ChaptersOptions(OptionsDictValidator):
@property
def remove_sponsorblock_categories(self) -> Optional[List[str]]:
"""
Optional. List of SponsorBlock categories to remove from the output file. Can only remove
categories that are specified in ``sponsorblock_categories`` or "all", which removes
everything specified in ``sponsorblock_categories``.
:expected type: Optional[List[String]]
:description:
List of SponsorBlock categories to remove from the output file. Can only remove
categories that are specified in ``sponsorblock_categories`` or "all", which removes
everything specified in ``sponsorblock_categories``.
"""
if self._remove_sponsorblock_categories:
category_list = [
@ -189,12 +197,21 @@ class ChaptersOptions(OptionsDictValidator):
@property
def force_key_frames(self) -> bool:
"""
Optional. Force keyframes at cuts when removing sections. This is slow due to needing a
re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to
False.
:expected type: Optional[Boolean]
:description:
Defaults to False. Force keyframes at cuts when removing sections. This is slow due to
needing a re-encode, but the resulting video may have fewer artifacts around the cuts.
"""
return self._force_key_frames
def added_variables(
self,
resolved_variables: Set[str],
unresolved_variables: Set[str],
plugin_op: PluginOperation,
) -> Dict[PluginOperation, Set[str]]:
return {PluginOperation.MODIFY_ENTRY: {ytdl_sub_chapters_from_comments.variable_name}}
class ChaptersPlugin(Plugin[ChaptersOptions]):
plugin_options_type = ChaptersOptions
@ -300,27 +317,33 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
-------
entry
"""
chapters = Chapters.from_empty()
has_chapters_from_comments = False
# If there are no embedded chapters, and comment chapters are allowed...
if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments:
chapters = Chapters.from_empty()
# Try to get chapters from comments
for comment in entry.kwargs_get(COMMENTS, []):
for comment in entry.get(v.comments, list):
chapters = Chapters.from_string(comment.get("text", ""))
if chapters.contains_any_chapters():
break
# If some are actually found, add a special kwarg and embed them
if chapters.contains_any_chapters():
entry.add_kwargs({YTDL_SUB_CUSTOM_CHAPTERS: chapters.to_file_metadata_dict()})
has_chapters_from_comments = True
entry.add({ytdl_sub_chapters_from_comments: chapters.to_yt_dlp_chapter_metadata()})
if not self.is_dry_run:
set_ffmpeg_metadata_chapters(
file_path=entry.get_download_file_path(),
chapters=chapters,
file_duration_sec=entry.kwargs("duration"),
file_duration_sec=entry.get(v.duration, int),
)
if not has_chapters_from_comments:
entry.add({ytdl_sub_chapters_from_comments: []})
return entry
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
@ -334,12 +357,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
-------
FileMetadata outlining which chapters/SponsorBlock segments got removed
"""
if custom_chapters_metadata := entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS):
title: str = "Chapters from comments"
return FileMetadata.from_dict(
value_dict=custom_chapters_metadata,
title=title,
sort_dict=False, # timestamps + titles are already sorted
if custom_chapters := entry.get(ytdl_sub_chapters_from_comments, list):
return Chapters.from_yt_dlp_chapters(custom_chapters).to_file_metadata(
title="Chapters from comments"
)
if self.plugin_options.embed_chapters:
@ -356,9 +376,10 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
if removed_sponsorblock:
metadata_dict["Removed SponsorBlock Category Count(s)"] = removed_sponsorblock
# TODO: check if file actually has embedded chapters
return FileMetadata.from_dict(
value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False
)
# If the entry wasn't split on embedded chapters, report it in the file metadata
if not entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str):
return FileMetadata.from_dict(
value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False
)
return None

32
src/ytdl_sub/plugins/date_range.py
View file

@ -2,8 +2,8 @@ from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.utils.datetime import to_date_str
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
@ -11,16 +11,24 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator
class DateRangeOptions(OptionsDictValidator):
"""
Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs:
Usage:
.. code-block:: Markdown
A string in the format YYYYMMDD or
(now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this.
Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest
granularity possible.
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
date_range:
before: "now"
after: "today-2weeks"
date_range:
before: "now"
after: "today-2weeks"
"""
_optional_keys = {"before", "after"}
@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator):
@property
def before(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Only download videos before this datetime.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
"""
return self._before
@property
def after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Only download videos after this datetime.
:expected type: Optional[OverridesFormatter]
:description:
Only download videos before this datetime.
"""
return self._after

12
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -3,9 +3,8 @@ from typing import Optional
import mediafile
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_handler import FileHandler
@ -21,19 +20,16 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
"""
Whether to embed thumbnails to the audio/video file or not.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
embed_thumbnail: True
embed_thumbnail: True
"""
class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
plugin_options_type = EmbedThumbnailOptions
priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT)
@property
def _embed_thumbnail(self) -> bool:

108
src/ytdl_sub/plugins/file_convert.py
View file

@ -2,12 +2,15 @@ import os
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import EXT
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.ffmpeg import FFMPEG
@ -16,6 +19,9 @@ from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.audo_codec_validator import FileTypeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
v: VariableDefinitions = VARIABLES
class FileConvertWithValidator(StringSelectValidator):
@ -26,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator):
"""
Converts video files from one extension to another.
Usage:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mp4"
file_convert:
convert_to: "mp4"
Supports custom ffmpeg conversions:
Also supports custom ffmpeg conversions:
:Usage:
.. code-block:: yaml
presets:
my_example_preset:
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
ffmpeg_post_process_args: >
-bitexact
-vcodec copy
-acodec copy
-scodec mov_text
file_convert:
convert_to: "mkv"
convert_with: "ffmpeg"
ffmpeg_post_process_args: >
-bitexact
-vcodec copy
-acodec copy
-scodec mov_text
"""
_required_keys = {"convert_to"}
@ -83,45 +87,61 @@ class FileConvertOptions(OptionsDictValidator):
@property
def convert_to(self) -> str:
"""
Convert to a desired file type. Supports:
:expected type: String
:description:
Convert to a desired file type. Supports
* Video: avi, flv, mkv, mov, mp4, webm
* Audio: aac, flac, mp3, m4a, opus, vorbis, wav
- Video: avi, flv, mkv, mov, mp4, webm
- Audio: aac, flac, mp3, m4a, opus, vorbis, wav
"""
return self._convert_to
@property
def convert_with(self) -> Optional[str]:
"""
Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
:expected type: Optional[String]
:description:
Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
"""
return self._convert_with
@property
def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]:
"""
Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the
form of:
:expected type: Optional[OverridesFormatter]
:description:
ffmpeg args to post-process an entry file with. The args will be inserted in the
form of
.. code-block:: bash
``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``.
ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.
The output file will use the extension specified in ``convert_to``. Post-processing args
can still be set with ``convert_with`` set to ``yt-dlp``.
"""
return self._ffmpeg_post_process_args
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}}
class FileConvertPlugin(Plugin[FileConvertOptions]):
plugin_options_type = FileConvertOptions
# Perform this after regex
priority: PluginPriority = PluginPriority(
modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 1
)
def __init__(
self,
options: FileConvertOptions,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
options=options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
# Lookup of entry id to what it was converted from for logging
self._converted_from_lookup: Dict[str, str] = {}
def ytdl_options(self) -> Optional[Dict]:
"""
@ -193,18 +213,14 @@ class FileConvertPlugin(Plugin[FileConvertOptions]):
"file_convert ffmpeg_post_process_args did not produce an output file"
)
FileHandler.delete(input_video_file_path)
FileHandler.move(tmp_output_file, converted_video_file_path)
FileHandler.delete(tmp_output_file)
FileHandler.delete(input_video_file_path)
if original_ext != new_ext:
entry.add_kwargs(
{
"__converted_from": original_ext,
}
)
self._converted_from_lookup[entry.ytdl_uid()] = original_ext
entry.add_kwargs({EXT: new_ext})
entry.add({v.ext: new_ext})
return entry
@ -212,7 +228,7 @@ class FileConvertPlugin(Plugin[FileConvertOptions]):
"""
Add metadata about conversion if it happened
"""
if converted_from := entry.kwargs_get("__converted_from"):
if converted_from := self._converted_from_lookup.get(entry.ytdl_uid()):
return FileMetadata(f"Converted from {converted_from}")
return None

70
src/ytdl_sub/plugins/filter_exclude.py Normal file
View file

@ -0,0 +1,70 @@
import json
from typing import Dict
from typing import Optional
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("conditional")
class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
"""
Applies a conditional OR on any number of filters comprised of either variables or scripts.
If any filter evaluates to True, the entry will be excluded.
:Usage:
.. code-block:: yaml
filter_exclude:
- >-
{ %contains( %lower(title), '#short' ) }
- >-
{ %contains( %lower(description), '#short' ) }
"""
class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
plugin_options_type = FilterExcludeOptions
def __init__(
self,
options: FilterExcludeOptions,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
options=options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
self._evaluated_map: Dict[str, bool] = {}
def modify_entry(self, entry: Entry) -> Optional[Entry]:
# Already evaluated in modify_entry_metadata, do not recompute
if entry.ytdl_uid() in self._evaluated_map:
return entry
for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
if bool(out):
return None
return entry
def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
try:
output_entry = self.modify_entry(entry=entry)
except StringFormattingException:
# If filtering fails at the metadata stage, try again w/no catch in modify_entry
return entry
self._evaluated_map[entry.ytdl_uid()] = True
return output_entry

78
src/ytdl_sub/plugins/filter_include.py Normal file
View file

@ -0,0 +1,78 @@
import json
from typing import Dict
from typing import Optional
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger = Logger.get("conditional")
class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
"""
Applies a conditional AND on any number of filters comprised of either variables or scripts.
If all filters evaluate to True, the entry will be included.
:Usage:
.. code-block:: yaml
filter_include:
- >-
{description}
- >-
{
%regex_search_any(
title,
[
"Full Episode",
"FULL",
]
)
}
"""
class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
plugin_options_type = FilterIncludeOptions
def __init__(
self,
options: FilterIncludeOptions,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
options=options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
self._evaluated_map: Dict[str, bool] = {}
def modify_entry(self, entry: Entry) -> Optional[Entry]:
# Already evaluated in modify_entry_metadata, do not recompute
if entry.ytdl_uid() in self._evaluated_map:
return entry
for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
if not bool(out):
return None
return entry
def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
try:
output_entry = self.modify_entry(entry=entry)
except StringFormattingException:
# If filtering fails at the metadata stage, try again w/no catch in modify_entry
return entry
self._evaluated_map[entry.ytdl_uid()] = True
return output_entry

8
src/ytdl_sub/plugins/format.py
View file

@ -1,8 +1,8 @@
from typing import Dict
from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.validators.validators import StringValidator
@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator):
.. code-block:: yaml
presets:
my_example_preset:
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
format: "(bv*[height<=1080]+bestaudio/best[height<=1080])"
"""
def __init__(self, name, value):

8
src/ytdl_sub/plugins/internal/view.py
View file

@ -1,10 +1,9 @@
import copy
from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -20,7 +19,6 @@ class ViewOptions(OptionsDictValidator):
class ViewPlugin(Plugin[ViewOptions]):
plugin_options_type = ViewOptions
priority: PluginPriority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT)
_MAX_LINE_WIDTH: int = 80

Some files were not shown because too many files have changed in this diff Show more