Merge branch 'master' into j/inspect
This commit is contained in:
Jesse Bannon
commit
14a4aeac70
213 changed files with 5062 additions and 5621 deletions
8
Makefile
8
Makefile
|
|
@ -36,12 +36,12 @@ endif
|
|||
all: check_lint docs docker docker_ubuntu docker_gui
|
||||
|
||||
lint:
|
||||
python3 -m isort .
|
||||
python3 -m black .
|
||||
python3 -m ruff format .
|
||||
python3 -m ruff check --fix .
|
||||
python3 -m pylint src
|
||||
check_lint:
|
||||
isort . --check-only --diff \
|
||||
&& black . --check \
|
||||
ruff format --check . \
|
||||
&& ruff check . \
|
||||
&& pylint src/
|
||||
wheel: clean
|
||||
$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"" > src/ytdl_sub/__init__.py)
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ __preset__:
|
|||
|
||||
# Pass any arg directly to yt-dlp's Python API
|
||||
ytdl_options:
|
||||
cookiefile: "/config/cookie.txt"
|
||||
cookiefile: "/config/ytdl-sub-configs/cookie.txt"
|
||||
|
||||
###################################################################
|
||||
# TV Show Presets. Can replace Plex with Plex/Jellyfin/Emby/Kodi
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ if [ "$CRON_SCHEDULE" != "" ] ; then
|
|||
|
||||
# create cron script wrapper
|
||||
echo '#!/bin/bash' > "$CRON_WRAPPER_SCRIPT"
|
||||
# Echo commands for easier user debugging:
|
||||
echo "set -x" >> "$CRON_WRAPPER_SCRIPT"
|
||||
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "$CRON_WRAPPER_SCRIPT"
|
||||
echo "cd \"$DEFAULT_WORKSPACE\"" >> "$CRON_WRAPPER_SCRIPT"
|
||||
echo ". \"$CRON_SCRIPT\" >> \"$LOGS_TO_STDOUT\" 2>&1" >> "$CRON_WRAPPER_SCRIPT"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,15 @@
|
|||
echo "Beginning cron job..."
|
||||
|
||||
# Place your ytdl-sub command(s) here.
|
||||
# This script is executed in the same relative path as this file.
|
||||
#
|
||||
# This script is executed in the same directory as this file which also contains the
|
||||
# default `./config.yaml` and `./subscriptions.yaml`, so you don't need to use the
|
||||
# `--config` CLI option or pass a `SUBPATH` to the `$ ytdl-sub sub` sub-command.
|
||||
#
|
||||
# Test your configuration and subscriptions carefully before automating downloads to
|
||||
# prevent triggering throttles or bans:
|
||||
#
|
||||
# https://ytdl-sub.readthedocs.io/en/latest/guides/getting_started/downloading.html
|
||||
#
|
||||
# Once you've tested your configuration and you're ready to download entries unattended,
|
||||
# remove the next line and un-comment the following line:
|
||||
echo "WARNING: Read /config/ytdl-sub-configs/cron and modify to automate downloads."
|
||||
# ytdl-sub sub
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ __preset__:
|
|||
|
||||
# Pass any arg directly to yt-dlp's Python API
|
||||
# ytdl_options:
|
||||
# cookiefile: "/config/cookie.txt"
|
||||
# cookiefile: "/config/ytdl-sub-configs/cookie.txt"
|
||||
|
||||
###################################################################
|
||||
# Subscriptions nested under this will use the
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
||||
|
||||
project = "ytdl-sub"
|
||||
copyright = "2024, Jesse Bannon"
|
||||
copyright = "2026, Jesse Bannon"
|
||||
author = "Jesse Bannon"
|
||||
release = ""
|
||||
|
||||
|
|
@ -15,10 +15,8 @@ release = ""
|
|||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
||||
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.autosectionlabel",
|
||||
"sphinx.ext.extlinks",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx_copybutton",
|
||||
"sphinx_design",
|
||||
]
|
||||
|
|
@ -70,19 +68,3 @@ extlinks = {
|
|||
"lsio-gh": ("https://github.com/linuxserver/%s", "%s image"),
|
||||
"ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"),
|
||||
}
|
||||
|
||||
# -- Options for autodoc ----------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
|
||||
|
||||
# Automatically extract typehints when specified and place them in
|
||||
# descriptions of the relevant function/method.
|
||||
autodoc_default_options = {
|
||||
"autodoc_typehints_format": "short",
|
||||
"autodoc_class_signature": "separated",
|
||||
"add_module_names": False,
|
||||
# "add_class_names": False,
|
||||
}
|
||||
|
||||
python_use_unqualified_type_names = True
|
||||
napoleon_numpy_docstring = True
|
||||
napoleon_use_rtype = False
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
==================
|
||||
..
|
||||
WARNING: This RST file is generated from docstrings in:
|
||||
The respective function docstrings within ytdl_sub/config/config_validator.py
|
||||
In order to make a change to this file, edit the respective docstring
|
||||
and run `make docs`. This will automatically sync the Python RST-based
|
||||
docstrings into this file. If the docstrings and RST file are out of sync,
|
||||
it will fail TestDocGen tests in GitHub CI.
|
||||
|
||||
Configuration File
|
||||
==================
|
||||
|
||||
ytdl-sub is configured using a ``config.yaml`` file.
|
||||
|
||||
The ``config.yaml`` is made up of two sections:
|
||||
|
|
@ -11,113 +17,118 @@ The ``config.yaml`` is made up of two sections:
|
|||
configuration:
|
||||
presets:
|
||||
|
||||
You can jump to any section and subsection of the config using the navigation section to
|
||||
the left.
|
||||
|
||||
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
|
||||
If you wish to represent paths like Windows, you will need to
|
||||
``C:\\double\\bashslash\\paths`` in order to escape the backslash character.
|
||||
|
||||
|
||||
configuration
|
||||
-------------
|
||||
|
||||
The ``configuration`` section contains app-wide configs applied to all presets and
|
||||
subscriptions.
|
||||
|
||||
.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
|
||||
:members:
|
||||
:member-order: bysource
|
||||
:exclude-members: subscription_value, persist_logs, experimental
|
||||
|
||||
persist_logs
|
||||
~~~~~~~~~~~~
|
||||
|
||||
Without this key, ``ytdl-sub`` only prints output to it's ``stdout`` and ``stderr``. If
|
||||
your configuration includes the ``persist_logs:`` key, then ``ytdl-sub`` also writes log
|
||||
files to disk.
|
||||
|
||||
.. warning::
|
||||
|
||||
The log files grow rapidly if ``keep_successful_logs:`` is ``true``, the default, and
|
||||
may fill up disk space. Set ``keep_successful_logs: false`` or prune the log files
|
||||
regularly.
|
||||
|
||||
For example:
|
||||
If you prefer to use a Windows backslash, note that it must have
|
||||
``C:\\double\\bashslash\\paths`` in order to escape the backslash character. This is due
|
||||
to it being a YAML escape character.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
configuration:
|
||||
dl_aliases:
|
||||
mv: "--preset music_video"
|
||||
u: "--download.url"
|
||||
|
||||
experimental:
|
||||
enable_update_with_info_json: True
|
||||
|
||||
ffmpeg_path: "/usr/bin/ffmpeg"
|
||||
ffprobe_path: "/usr/bin/ffprobe"
|
||||
|
||||
file_name_max_bytes: 255
|
||||
lock_directory: "/tmp"
|
||||
|
||||
persist_logs:
|
||||
logs_directory: "/path/to/log/directory"
|
||||
keep_successful_logs: True
|
||||
logs_directory: "/var/log/ytdl-sub-logs"
|
||||
|
||||
.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
|
||||
:members:
|
||||
:member-order: bysource
|
||||
umask: "022"
|
||||
working_directory: ".ytdl-sub-working-directory"
|
||||
|
||||
dl_aliases
|
||||
----------
|
||||
.. _dl_aliases:
|
||||
|
||||
presets
|
||||
-------
|
||||
|
||||
Each key under ``presets:`` defines a `formula` for how to format downloaded media and
|
||||
metadata. The key is the name of the preset and the value is a mapping that defines the
|
||||
preset.
|
||||
|
||||
.. note::
|
||||
|
||||
The ``presets:`` key at the top of the configuration file contains multiple
|
||||
user-defined presets, but *each preset* itself may include a ``preset:`` key that
|
||||
defines *that preset's* base presets. For example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
Foo Preset:
|
||||
preset:
|
||||
- "Jellyfin TV Show by Date"
|
||||
- "Only Recent"
|
||||
|
||||
preset
|
||||
~~~~~~
|
||||
|
||||
Presets support inheritance by defining one or more parent presets:
|
||||
Alias definitions to shorten :ref:`dl arguments <usage:Download Options>`. For example,
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
presets:
|
||||
custom_preset:
|
||||
...
|
||||
parent_preset:
|
||||
...
|
||||
child_preset:
|
||||
preset:
|
||||
- "parent_preset"
|
||||
configuration:
|
||||
dl_aliases:
|
||||
mv: "--preset music_video"
|
||||
u: "--download.url"
|
||||
|
||||
In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
|
||||
Use parent presets where possible to reduce duplicate yaml definitions.
|
||||
Simplifies
|
||||
|
||||
Presets also support inheritance from multiple presets:
|
||||
.. code-block:: bash
|
||||
|
||||
.. code-block:: yaml
|
||||
ytdl-sub dl --preset "Jellyfin Music Videos" --download.url "youtube.com/watch?v=a1b2c3"
|
||||
|
||||
child_preset:
|
||||
preset:
|
||||
- "custom_preset"
|
||||
- "parent_preset"
|
||||
to
|
||||
|
||||
In this example, ``child_preset`` will inherit all fields from ``custom_preset`` and
|
||||
``parent_preset`` in that order. The bottom-most preset has the highest priority. More
|
||||
specifically, presets are merged using `mergedeep`_ via `a TYPESAFE_ADDITIVE merge`_,
|
||||
which means:
|
||||
.. code-block:: bash
|
||||
|
||||
- if two conflicting keys arent lists or mappings, overwrite the higher priority one
|
||||
- otherwise, combine then re-evaluate
|
||||
ytdl-sub dl --mv --u "youtube.com/watch?v=a1b2c3"
|
||||
|
||||
If you are only inheriting from one preset, using a single string instead of a list is
|
||||
valid, for example ``preset: "parent_preset"``, but we recommend always using a list for
|
||||
consistent readability between presets.
|
||||
experimental
|
||||
------------
|
||||
Experimental flags reside under the ``experimental`` key.
|
||||
|
||||
.. _`mergedeep`:
|
||||
https://mergedeep.readthedocs.io/en/latest/
|
||||
.. _`a TYPESAFE_ADDITIVE merge`:
|
||||
https://mergedeep.readthedocs.io/en/latest/index.html#merge-strategies
|
||||
``enable_update_with_info_json``
|
||||
|
||||
Enables modifying subscription files using info.json files using the argument
|
||||
``--update-with-info-json``. This feature is still being tested and has the ability to
|
||||
destroy files. Ensure you have a full backup before usage. You have been warned!
|
||||
|
||||
ffmpeg_path
|
||||
-----------
|
||||
Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux,
|
||||
``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows.
|
||||
|
||||
ffprobe_path
|
||||
------------
|
||||
Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux,
|
||||
``./ffprobe.exe`` in the same directory as ytdl-sub for Windows.
|
||||
|
||||
file_name_max_bytes
|
||||
-------------------
|
||||
Max file name size in bytes. Most OS's typically default to 255 bytes.
|
||||
|
||||
lock_directory
|
||||
--------------
|
||||
The directory to temporarily store file locks, which prevents multiple instances
|
||||
of ``ytdl-sub`` from running. Note that file locks do not work on
|
||||
network-mounted directories. Ensure that this directory resides on the host
|
||||
machine. Defaults to ``/tmp``.
|
||||
|
||||
persist_logs
|
||||
------------
|
||||
By default, no logs are persisted. Specifying this key will enable persisted logs. The following
|
||||
options are available.
|
||||
|
||||
``keep_successful_logs``
|
||||
|
||||
Defaults to ``True``. When this key is ``False``, only write log files for failed
|
||||
subscriptions.
|
||||
|
||||
``logs_directory``
|
||||
|
||||
Required field. Write log files to this directory with names like
|
||||
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
|
||||
|
||||
umask
|
||||
-----
|
||||
Umask in octal format to apply to every created file. Defaults to ``022``.
|
||||
|
||||
working_directory
|
||||
-----------------
|
||||
The directory to temporarily store downloaded files before moving them into their final
|
||||
directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory
|
||||
that ytdl-sub is invoked from.
|
||||
|
||||
Presets
|
||||
=======
|
||||
Custom presets are defined in this section. Refer to the
|
||||
:ref:`Getting Started Guide<guides/getting_started/first_config:Basic Configuration>`
|
||||
on how to configure.
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ Extracts audio from a video file.
|
|||
The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
|
||||
opus, vorbis, wav, and best to grab the best possible format at runtime.
|
||||
|
||||
|
||||
``enable``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -37,7 +36,6 @@ Extracts audio from a video file.
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``quality``
|
||||
|
||||
:expected type: Float
|
||||
|
|
@ -45,7 +43,6 @@ Extracts audio from a video file.
|
|||
Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
|
||||
(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
chapters
|
||||
|
|
@ -84,14 +81,12 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
|||
Defaults to False. If chapters do not exist in the video/description itself, attempt to
|
||||
scrape comments to find the chapters.
|
||||
|
||||
|
||||
``embed_chapters``
|
||||
|
||||
:expected type: Optional[Boolean]
|
||||
:description:
|
||||
Defaults to True. Embed chapters into the file.
|
||||
|
||||
|
||||
``enable``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -100,7 +95,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``force_key_frames``
|
||||
|
||||
:expected type: Optional[Boolean]
|
||||
|
|
@ -108,7 +102,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
|||
Defaults to False. Force keyframes at cuts when removing sections. This is slow due to
|
||||
needing a re-encode, but the resulting video may have fewer artifacts around the cuts.
|
||||
|
||||
|
||||
``remove_chapters_regex``
|
||||
|
||||
:expected type: Optional[List[RegexString]
|
||||
|
|
@ -116,7 +109,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
|||
List of regex patterns to match chapter titles against and remove them from the
|
||||
entry.
|
||||
|
||||
|
||||
``remove_sponsorblock_categories``
|
||||
|
||||
:expected type: Optional[List[String]]
|
||||
|
|
@ -125,7 +117,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
|||
categories that are specified in ``sponsorblock_categories`` or "all", which removes
|
||||
everything specified in ``sponsorblock_categories``.
|
||||
|
||||
|
||||
``sponsorblock_categories``
|
||||
|
||||
:expected type: Optional[List[String]]
|
||||
|
|
@ -134,7 +125,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
|
|||
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
|
||||
"poi_highlight", or "all" to include all categories.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
date_range
|
||||
|
|
@ -169,14 +159,12 @@ intended download files.
|
|||
:description:
|
||||
Only download videos after or on this datetime, inclusive.
|
||||
|
||||
|
||||
``before``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
:description:
|
||||
Only download videos only before this datetime, not inclusive.
|
||||
|
||||
|
||||
``breaks``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -184,7 +172,6 @@ intended download files.
|
|||
Toggle to enable breaking subsequent metadata downloads if an entry's upload date
|
||||
is out of range. Defaults to True.
|
||||
|
||||
|
||||
``enable``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -193,7 +180,6 @@ intended download files.
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``type``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -201,7 +187,6 @@ intended download files.
|
|||
Which type of date to use. Must be either ``upload_date`` or ``release_date``.
|
||||
Defaults to ``upload_date``.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
download
|
||||
|
|
@ -305,7 +290,6 @@ Also supports custom ffmpeg conversions:
|
|||
- Video: avi, flv, mkv, mov, mp4, webm
|
||||
- Audio: aac, flac, mp3, m4a, opus, vorbis, wav
|
||||
|
||||
|
||||
``convert_with``
|
||||
|
||||
:expected type: Optional[String]
|
||||
|
|
@ -314,7 +298,6 @@ Also supports custom ffmpeg conversions:
|
|||
yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
|
||||
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
|
||||
|
||||
|
||||
``enable``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -323,7 +306,6 @@ Also supports custom ffmpeg conversions:
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``ffmpeg_post_process_args``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -336,7 +318,6 @@ Also supports custom ffmpeg conversions:
|
|||
The output file will use the extension specified in ``convert_to``. Post-processing args
|
||||
can still be set with ``convert_with`` set to ``yt-dlp``.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
filter_exclude
|
||||
|
|
@ -472,7 +453,6 @@ with a ``.nfo`` extension. You can add any values into the NFO.
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``kodi_safe``
|
||||
|
||||
:expected type: OverridesBooleanFormatterValidator
|
||||
|
|
@ -481,14 +461,12 @@ with a ``.nfo`` extension. You can add any values into the NFO.
|
|||
emojis and some foreign language characters. Setting this to True will replace those
|
||||
characters with '□'.
|
||||
|
||||
|
||||
``nfo_name``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
:description:
|
||||
The NFO file name.
|
||||
|
||||
|
||||
``nfo_root``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
|
|
@ -501,7 +479,6 @@ with a ``.nfo`` extension. You can add any values into the NFO.
|
|||
<episodedetails>
|
||||
</episodedetails>
|
||||
|
||||
|
||||
``tags``
|
||||
|
||||
:expected type: NfoTags
|
||||
|
|
@ -538,7 +515,6 @@ with a ``.nfo`` extension. You can add any values into the NFO.
|
|||
<genre>Comedy</genre>
|
||||
<genre>Drama</genre>
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
output_directory_nfo_tags
|
||||
|
|
@ -570,7 +546,6 @@ Usage:
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``kodi_safe``
|
||||
|
||||
:expected type: OverridesBooleanFormatterValidator
|
||||
|
|
@ -579,14 +554,12 @@ Usage:
|
|||
emojis and some foreign language characters. Setting this to True will replace those
|
||||
characters with '□'.
|
||||
|
||||
|
||||
``nfo_name``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
:description:
|
||||
The NFO file name.
|
||||
|
||||
|
||||
``nfo_root``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
|
|
@ -599,7 +572,6 @@ Usage:
|
|||
<tvshow>
|
||||
</tvshow>
|
||||
|
||||
|
||||
``tags``
|
||||
|
||||
:expected type: NfoTags
|
||||
|
|
@ -634,7 +606,6 @@ Usage:
|
|||
<genre>Comedy</genre>
|
||||
<genre>Drama</genre>
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
output_options
|
||||
|
|
@ -669,7 +640,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
The file name to store a subscriptions download archive placed relative to
|
||||
the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
|
||||
|
||||
|
||||
``file_name``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
|
|
@ -677,7 +647,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
The file name for the media file. This can include directories such as
|
||||
``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
|
||||
|
||||
|
||||
``info_json_name``
|
||||
|
||||
:expected type: Optional[EntryFormatter]
|
||||
|
|
@ -686,7 +655,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
|
||||
directory. Can be set to empty string or `null` to disable info json writes.
|
||||
|
||||
|
||||
``keep_files_after``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -698,7 +666,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
files after ``19000101``, which implies all files. Can be used in conjunction with
|
||||
``keep_max_files``.
|
||||
|
||||
|
||||
``keep_files_before``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -710,7 +677,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
files before ``now``, which implies all files. Can be used in conjunction with
|
||||
``keep_max_files``.
|
||||
|
||||
|
||||
``keep_files_date_eval``
|
||||
|
||||
:expected type: str
|
||||
|
|
@ -720,7 +686,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
perform evaluation for keep_files_before/after and keep_max_files. Defaults
|
||||
to the entry's upload_date_standardized variable.
|
||||
|
||||
|
||||
``keep_max_files``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -730,7 +695,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
|
||||
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
|
||||
|
||||
|
||||
``maintain_download_archive``
|
||||
|
||||
:expected type: Optional[Boolean]
|
||||
|
|
@ -745,7 +709,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
|
||||
Defaults to False.
|
||||
|
||||
|
||||
``migrated_download_archive_name``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -755,14 +718,12 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
name first, and fallback to ``download_archive_name``. It will always save to this file
|
||||
and remove the original ``download_archive_name``.
|
||||
|
||||
|
||||
``output_directory``
|
||||
|
||||
:expected type: OverridesFormatter
|
||||
:description:
|
||||
The output directory to store all media files downloaded.
|
||||
|
||||
|
||||
``preserve_mtime``
|
||||
|
||||
:expected type: Optional[Boolean]
|
||||
|
|
@ -771,7 +732,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
When True, sets the file's mtime to match the video's upload_date from
|
||||
yt-dlp metadata. Defaults to False.
|
||||
|
||||
|
||||
``thumbnail_name``
|
||||
|
||||
:expected type: Optional[EntryFormatter]
|
||||
|
|
@ -780,7 +740,6 @@ Defines where to output files and thumbnails after all post-processing has compl
|
|||
as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
|
||||
directory. Can be set to empty string or `null` to disable thumbnail writes.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
overrides
|
||||
|
|
@ -845,7 +804,6 @@ used with no modifications.
|
|||
If a file has no chapters and is set to "pass", then ``chapter_title`` is
|
||||
set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
square_thumbnail
|
||||
|
|
@ -892,7 +850,6 @@ Usage:
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``kodi_safe``
|
||||
|
||||
:expected type: OverridesBooleanFormatterValidator
|
||||
|
|
@ -901,14 +858,12 @@ Usage:
|
|||
emojis and some foreign language characters. Setting this to True will replace those
|
||||
characters with '□'.
|
||||
|
||||
|
||||
``nfo_name``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
:description:
|
||||
The NFO file name.
|
||||
|
||||
|
||||
``nfo_root``
|
||||
|
||||
:expected type: EntryFormatter
|
||||
|
|
@ -921,7 +876,6 @@ Usage:
|
|||
<season>
|
||||
</season>
|
||||
|
||||
|
||||
``tags``
|
||||
|
||||
:expected type: NfoTags
|
||||
|
|
@ -935,7 +889,6 @@ Usage:
|
|||
<title>My custom season name!</title>
|
||||
</season>
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
subtitles
|
||||
|
|
@ -963,7 +916,6 @@ It will set the respective language to the correct subtitle file.
|
|||
:description:
|
||||
Defaults to False. Whether to allow auto generated subtitles.
|
||||
|
||||
|
||||
``embed_subtitles``
|
||||
|
||||
:expected type: Optional[Boolean]
|
||||
|
|
@ -971,7 +923,6 @@ It will set the respective language to the correct subtitle file.
|
|||
Defaults to False. Whether to embed the subtitles into the video file. Note that
|
||||
webm files can only embed "vtt" subtitle types.
|
||||
|
||||
|
||||
``enable``
|
||||
|
||||
:expected type: Optional[OverridesFormatter]
|
||||
|
|
@ -980,7 +931,6 @@ It will set the respective language to the correct subtitle file.
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``languages``
|
||||
|
||||
:expected type: Optional[List[String]]
|
||||
|
|
@ -988,7 +938,6 @@ It will set the respective language to the correct subtitle file.
|
|||
Language code(s) to download for subtitles. Supports a single or list of multiple
|
||||
language codes. Defaults to only "en".
|
||||
|
||||
|
||||
``languages_required``
|
||||
|
||||
:expected type: Optional[List[String]]
|
||||
|
|
@ -996,7 +945,6 @@ It will set the respective language to the correct subtitle file.
|
|||
Language code(s) that are required to be present for downloads to continue. If missing,
|
||||
ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles.
|
||||
|
||||
|
||||
``subtitles_name``
|
||||
|
||||
:expected type: Optional[EntryFormatter]
|
||||
|
|
@ -1006,14 +954,12 @@ It will set the respective language to the correct subtitle file.
|
|||
and will be placed in the output directory. ``lang`` is dynamic since you can download
|
||||
multiple subtitles. It will set the respective language to the correct subtitle file.
|
||||
|
||||
|
||||
``subtitles_type``
|
||||
|
||||
:expected type: Optional[String]
|
||||
:description:
|
||||
Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc".
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
throttle_protection
|
||||
|
|
@ -1054,14 +1000,12 @@ Range min and max values support static override variables within their definiti
|
|||
this field can be set using an override variable to easily toggle whether this plugin
|
||||
is enabled or not via Boolean.
|
||||
|
||||
|
||||
``max_downloads_per_subscription``
|
||||
|
||||
:expected type: Optional[Range]
|
||||
:description:
|
||||
Number of downloads to perform per subscription.
|
||||
|
||||
|
||||
``sleep_per_download_s``
|
||||
|
||||
:expected type: Optional[Range]
|
||||
|
|
@ -1069,7 +1013,6 @@ Range min and max values support static override variables within their definiti
|
|||
Number in seconds to sleep between each download. Does not include time it takes for
|
||||
ytdl-sub to perform post-processing.
|
||||
|
||||
|
||||
``sleep_per_request_s``
|
||||
|
||||
:expected type: Optional[Range]
|
||||
|
|
@ -1079,14 +1022,12 @@ Range min and max values support static override variables within their definiti
|
|||
download for the entry. Also, yt-dlp only supports a single value at this time for this,
|
||||
so will always use the max value.
|
||||
|
||||
|
||||
``sleep_per_subscription_s``
|
||||
|
||||
:expected type: Optional[Range]
|
||||
:description:
|
||||
Number in seconds to sleep between each subscription.
|
||||
|
||||
|
||||
``subscription_download_probability``
|
||||
|
||||
:expected type: Optional[Float]
|
||||
|
|
@ -1095,7 +1036,6 @@ Range min and max values support static override variables within their definiti
|
|||
recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
|
||||
guaranteed over time to eventually download the subscription.
|
||||
|
||||
|
||||
----------------------------------------------------------------------------------------------------
|
||||
|
||||
video_tags
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ supply a cookies file path.
|
|||
|
||||
# Directly set plugin options:
|
||||
ytdl_options:
|
||||
cookiefile: "/config/cookie.txt"
|
||||
cookiefile: "/config/ytdl-sub-configs/cookie.txt"
|
||||
|
||||
Layout
|
||||
------
|
||||
|
|
|
|||
|
|
@ -1,51 +1,80 @@
|
|||
Automating Downloads
|
||||
====================
|
||||
Automating
|
||||
==========
|
||||
|
||||
:ref:`Guide for Docker and Unraid Containers <guides/getting_started/automating_downloads:docker and unraid>`
|
||||
Automate downloading your subscriptions by running the :ref:`'sub' sub-command
|
||||
<usage:subscriptions options>` periodically. There are various tools that can run
|
||||
commands on a schedule you may use any of them that work with your installation
|
||||
method. Most users use `cron`_ in `Docker containers <docker and unraid_>`_.
|
||||
|
||||
:ref:`Guide for Linux <guides/getting_started/automating_downloads:linux>`
|
||||
|
||||
:ref:`Guide for Windows <guides/getting_started/automating_downloads:windows>`
|
||||
|
||||
.. _cron scheduling syntax: https://crontab.guru/#0_*/6_*_*_*
|
||||
|
||||
|
||||
.. _docker-unraid-setup:
|
||||
|
||||
Docker and Unraid
|
||||
-----------------
|
||||
|
||||
Cron is preconfigured in every ytdl-sub docker container. Enable by adding the following
|
||||
ENV variables to your docker setup.
|
||||
:doc:`The 'ytdl-sub' Docker container images <../install/docker>` provide optional cron
|
||||
support. Enable cron support by setting `a cron schedule`_ in the ``CRON_SCHEDULE``
|
||||
environment variable:
|
||||
|
||||
.. code-block:: yaml
|
||||
:caption: ./compose.yaml
|
||||
:emphasize-lines: 4
|
||||
|
||||
services:
|
||||
ytdl-sub:
|
||||
environment:
|
||||
- CRON_SCHEDULE="0 */6 * * *"
|
||||
- CRON_RUN_ON_START=false
|
||||
CRON_SCHEDULE: "0 */6 * * *"
|
||||
# WARNING: See "Getting Started" -> "Automating" docs regarding throttles/bans:
|
||||
# CRON_RUN_ON_START: false
|
||||
|
||||
Then recreate the container to apply the change and start it to generate the default
|
||||
``/config/ytdl-sub-configs/cron`` script. Read the comments in that script and edit as
|
||||
appropriate.
|
||||
|
||||
- ``CRON_SCHEDULE`` follows the standard `cron scheduling syntax`_. The above value will
|
||||
run the script once every 6 hours.
|
||||
- ``CRON_RUN_ON_START`` toggles whether to run your cron script on container start in
|
||||
addition to the cron schedule.
|
||||
The container cron wrapper script will write output from the cron job to
|
||||
``/config/ytdl-sub-configs/.cron.log``. The default image ``ENTRYPOINT`` will ``$ tail
|
||||
...`` that file so you can monitor the cron job in the container's output and thus also
|
||||
in the Docker logs.
|
||||
|
||||
The cron script will reside in the main directory with the file name ``cron``. Cron
|
||||
logs should show when viewing the Docker logs.
|
||||
You may also set the ``CRON_RUN_ON_START`` environment variable to ``true`` to have the
|
||||
image run your cron script whenever the container starts in addition to the cron
|
||||
schedule.
|
||||
|
||||
.. warning::
|
||||
|
||||
Using ``CRON_RUN_ON_START`` may cause your cron script to run too often and may
|
||||
trigger throttles and bans. When enabled, your cron script will run *whenever* the
|
||||
container starts including when the host reboots, when ``# dockerd`` restarts such as
|
||||
when upgrading Docker itself, when a new image is pulled, when something applies
|
||||
Compose changes, etc.. This may result in running ``ytdl-sub`` right before or after
|
||||
the next cron scheduled run.
|
||||
|
||||
|
||||
.. _linux-setup:
|
||||
|
||||
Linux
|
||||
-----
|
||||
Must configure crontab manually, like so:
|
||||
Linux, Mac OS X, BSD, or other UNIX's
|
||||
-------------------------------------
|
||||
|
||||
For installations on systems already running ``# crond``, you can also use cron to run
|
||||
``ytdl-sub`` periodically. Write a script to run ``ytdl-sub`` in the cron job. Be sure
|
||||
the script changes to the same directory as your configuration and uses the full path to
|
||||
``ytdl-sub``:
|
||||
|
||||
.. code-block:: shell
|
||||
:caption: ~/.local/bin/ytdl-sub-cron
|
||||
:emphasize-lines: 2,3
|
||||
|
||||
crontab -e
|
||||
0 */6 * * * /config/run_cron
|
||||
#!/bin/bash
|
||||
cd "~/.config/ytdl-sub/"
|
||||
~/.local/bin/ytdl-sub --dry-run sub -o '--ytdl_options.max_downloads 3' |&
|
||||
tee -a "~/.local/state/ytdl-sub/.cron.log"
|
||||
|
||||
Then tell ``# crond`` when to run the script:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
echo "0 */6 * * * ${HOME}/.local/bin/ytdl-sub-cron" | crontab "-"
|
||||
|
||||
Remove the ``--dry-run`` and ``-o ...`` CLI options from your cron script when you've
|
||||
tested your configuration and you're ready to download entries unattended.
|
||||
|
||||
|
||||
.. _windows-setup:
|
||||
|
|
@ -53,17 +82,51 @@ Must configure crontab manually, like so:
|
|||
Windows
|
||||
-------
|
||||
|
||||
To be tested (please contact code owner or join the discord server if you can test this
|
||||
out for us)
|
||||
For most Windows users, the best way to run commands periodically is `the Task
|
||||
Scheduler`_:
|
||||
|
||||
.. code-block:: powershell
|
||||
.. attention::
|
||||
|
||||
ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml
|
||||
These instructions are untested. Use at your own risk. If you use them, whether they
|
||||
work or not, please let us know how it went in `a support post in Discord`_ or `a new
|
||||
GitHub issue`_.
|
||||
|
||||
#. Open the Task Scheduler app.
|
||||
|
||||
#. Click ``Create Basic Task`` at the top of the right sidebar.
|
||||
|
||||
#. Set all the fields as appropriate until you get to the ``Action``...
|
||||
|
||||
#. For the ``Action``, select ``Start a program``...
|
||||
|
||||
#. Click ``Browse...`` to the installed ``ytdl-sub.exe`` executable...
|
||||
|
||||
#. Add CLI arguments to ``Add arguments (optional):``, for example ``--dry-run sub -o
|
||||
'--ytdl_options.max_downloads 3'``...
|
||||
|
||||
#. Set ``Start in (optional):`` to the directory containing your configuration.
|
||||
|
||||
#. Finish the rest of the ``Create Basic Task`` wizard.
|
||||
|
||||
|
||||
Next Steps
|
||||
----------
|
||||
|
||||
Once you have a significant quantity of subscriptions or have use cases not served using
|
||||
:doc:`YAML keys and the special characters <./subscriptions>`, it's time to start
|
||||
:doc:`defining your own custom presets <./first_config>`.
|
||||
At this point, ``ytdl-sub`` should run periodically and keep your subscriptions current
|
||||
in your media library without your intervention. As your :doc:`subscriptions file
|
||||
<./subscriptions>` grows or you discover new use cases, it becomes worth while to
|
||||
simplify things by :doc:`defining your own custom presets <./first_config>`.
|
||||
|
||||
|
||||
|
||||
.. _`cron`:
|
||||
https://en.wikipedia.org/wiki/Cron
|
||||
.. _`a cron schedule`:
|
||||
https://crontab.cronhub.io/
|
||||
|
||||
.. _`the Task Scheduler`:
|
||||
https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-start-page
|
||||
.. _`a support post in Discord`:
|
||||
https://discord.com/channels/994270357957648404/1084886228266127460
|
||||
.. _`a new GitHub issue`:
|
||||
https://github.com/jmbannon/ytdl-sub/issues/new
|
||||
|
|
|
|||
|
|
@ -3,13 +3,33 @@ Basic Configuration
|
|||
|
||||
A configuration file serves two purposes:
|
||||
|
||||
1. Set advanced functionality that is not specifiable in a subscription file, such as
|
||||
working directory location. These are set underneath ``configuration``.
|
||||
2. Create custom presets, which can drastically simplify your subscription file. These
|
||||
are defined underneath ``presets``. Presets are intended to be applicable and
|
||||
reusable across multiple subscriptions.
|
||||
1. Set application-level functionality that is not specifiable in a subscription file.
|
||||
|
||||
Below is a common configuration:
|
||||
.. note::
|
||||
|
||||
ytdl-sub does not require a configuration file. However,
|
||||
certain application settings may be desirable for tweak, such as setting
|
||||
``working_directory`` to make ytdl-sub perform the initial download
|
||||
to an SSD drive.
|
||||
|
||||
2. Create custom presets.
|
||||
|
||||
.. note::
|
||||
|
||||
In the prior Initial Subscription examples, we leveraged the prebuilt preset
|
||||
``Jellyfin TV Show by Date``. This preset is entirely built using the same
|
||||
YAML configuration system offered to users by using a configuration file.
|
||||
|
||||
The following section attempts to demystify and explain how to...
|
||||
|
||||
- Set an application setting
|
||||
- Know whether or not custom presets are actually needed
|
||||
- How to create a custom preset
|
||||
- How to use a custom preset on subscriptions
|
||||
|
||||
-------------
|
||||
|
||||
how this works, and show-case how
|
||||
|
||||
.. code-block:: yaml
|
||||
:linenos:
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ use no preset at all and will just run ``yt-dlp`` without any customization or p
|
|||
processing. The subscriptions file has special support for :ref:`overriding the presets
|
||||
of all subscriptions in the file <config_reference/subscription_yaml:file preset>`. The
|
||||
configuration file supports :ref:`a few special options
|
||||
<config_reference/config_yaml:configuration>` that are not about defining presets. See
|
||||
<config_reference/config_yaml:Configuration File>` that are not about defining presets. See
|
||||
:doc:`the reference documentation <../../config_reference/index>` for technically
|
||||
complete details, but for almost all of the use cases served by ``ytdl-sub``, the above
|
||||
is accurate and representative.
|
||||
|
|
|
|||
|
|
@ -38,6 +38,13 @@ For example::
|
|||
|
||||
$ docker compose run --rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub" ytdl-sub sub
|
||||
|
||||
.. note::
|
||||
|
||||
In `the recommended GUI image <gui image_>`_, the ``DEFAULT_WORKSPACE`` directory is
|
||||
``/config/ytdl-sub-configs/`` which is used throughout the documentation and
|
||||
examples. In the headless images, that directory is just ``/config/``, so substitute
|
||||
that path if using a headless image.
|
||||
|
||||
|
||||
Install with Docker Compose
|
||||
---------------------------
|
||||
|
|
|
|||
|
|
@ -87,8 +87,7 @@ Using the command:
|
|||
--overrides.tv_show_name "Rick A" \
|
||||
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
|
||||
|
||||
See how to shorten commands using `download aliases
|
||||
<https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
|
||||
See how to shorten commands using :ref:`download aliases <config_reference/config_yaml:dl_aliases>`.
|
||||
|
||||
|
||||
View Options
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ presets:
|
|||
# ytdl_options lets you pass any arg into yt-dlp's Python API
|
||||
ytdl_options:
|
||||
# Set the cookie file
|
||||
# cookiefile: "/config/youtube_cookies.txt"
|
||||
# cookiefile: "/config/ytdl-sub-configs/youtube_cookies.txt"
|
||||
|
||||
# For YouTube, get English metadata if multiple languages are present
|
||||
extractor_args:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ classifiers = [
|
|||
"Programming Language :: Python :: 3.11",
|
||||
]
|
||||
dependencies = [
|
||||
"yt-dlp[default]==2025.12.8",
|
||||
"yt-dlp[default]==2026.3.3",
|
||||
"colorama~=0.4",
|
||||
"mergedeep~=1.3",
|
||||
"mediafile~=0.12",
|
||||
|
|
@ -43,13 +43,12 @@ where = ["src"]
|
|||
[project.optional-dependencies]
|
||||
test = [
|
||||
"coverage[toml]>=6.3,<8.0",
|
||||
"pytest>=7.2,<9.0",
|
||||
"pytest>=7.2,<10.0",
|
||||
"pytest-rerunfailures>=14,<17",
|
||||
]
|
||||
lint = [
|
||||
"black==24.10.0",
|
||||
"isort==7.0.0",
|
||||
"pylint==4.0.1",
|
||||
"pylint==4.0.5",
|
||||
"ruff==0.15.5",
|
||||
]
|
||||
docs = [
|
||||
"sphinx>=7,<10",
|
||||
|
|
@ -66,15 +65,6 @@ build = [
|
|||
[project.scripts]
|
||||
ytdl-sub = "ytdl_sub.main:main"
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
line_length = 100
|
||||
force_single_line = true
|
||||
|
||||
[tool.black]
|
||||
line_length = 100
|
||||
target-version = ["py310"]
|
||||
|
||||
[tool.pylint.MASTER]
|
||||
disable = [
|
||||
"C0115", # Missing class docstring
|
||||
|
|
@ -100,3 +90,27 @@ include = [
|
|||
exclude_also = [
|
||||
"raise UNREACHABLE.*",
|
||||
]
|
||||
|
||||
# ruff
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
indent-width = 4
|
||||
|
||||
# Assume Python 3.10
|
||||
target-version = "py310"
|
||||
|
||||
[tool.ruff.lint]
|
||||
extend-select = ["I"]
|
||||
|
||||
[tool.ruff.format]
|
||||
# Like Black, use double quotes for strings.
|
||||
quote-style = "double"
|
||||
|
||||
# Like Black, indent with spaces, rather than tabs.
|
||||
indent-style = "space"
|
||||
|
||||
# Like Black, respect magic trailing commas.
|
||||
skip-magic-trailing-comma = false
|
||||
|
||||
# Like Black, automatically detect the appropriate line ending.
|
||||
line-ending = "auto"
|
||||
|
|
@ -1,26 +1,24 @@
|
|||
import gc
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from yt_dlp.utils import sanitize_filename
|
||||
|
||||
from ytdl_sub.cli.output_summary import output_summary
|
||||
from ytdl_sub.cli.output_transaction_log import _maybe_validate_transaction_log_file
|
||||
from ytdl_sub.cli.output_transaction_log import output_transaction_log
|
||||
from ytdl_sub.cli.output_transaction_log import (
|
||||
_maybe_validate_transaction_log_file,
|
||||
output_transaction_log,
|
||||
)
|
||||
from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
|
||||
from ytdl_sub.cli.parsers.dl import DownloadArgsParser
|
||||
from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME
|
||||
from ytdl_sub.cli.parsers.main import InspectArguments
|
||||
from ytdl_sub.cli.parsers.main import parser
|
||||
from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME, parser, InspectArguments
|
||||
from ytdl_sub.config.config_file import ConfigFile
|
||||
from ytdl_sub.subscriptions.subscription import Subscription
|
||||
from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled, ValidationException
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_lock import working_directory_lock
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
|
|
@ -78,6 +76,7 @@ def _download_subscriptions_from_yaml_files(
|
|||
subscription_override_dict: Dict,
|
||||
update_with_info_json: bool,
|
||||
dry_run: bool,
|
||||
shuffle: bool,
|
||||
) -> List[Subscription]:
|
||||
"""
|
||||
Downloads all subscriptions from one or many subscription yaml files.
|
||||
|
|
@ -94,6 +93,8 @@ def _download_subscriptions_from_yaml_files(
|
|||
Whether to actually download or update using existing info json
|
||||
dry_run
|
||||
Whether to dry run or not
|
||||
shuffle
|
||||
Whether to shuffle the subscription download order
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -115,6 +116,10 @@ def _download_subscriptions_from_yaml_files(
|
|||
subscription_override_dict=subscription_override_dict,
|
||||
)
|
||||
|
||||
if shuffle:
|
||||
logger.info("Shuffling subscriptions")
|
||||
random.shuffle(subscriptions)
|
||||
|
||||
for subscription in subscriptions:
|
||||
with subscription.exception_handling():
|
||||
logger.info(
|
||||
|
|
@ -320,6 +325,7 @@ def main() -> List[Subscription]:
|
|||
subscription_override_dict=subscription_override_dict,
|
||||
update_with_info_json=args.update_with_info_json,
|
||||
dry_run=args.dry_run,
|
||||
shuffle=args.shuffle,
|
||||
)
|
||||
|
||||
# One-off download
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
from ytdl_sub.subscriptions.subscription import Subscription
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ create_parser = yt_dlp.options.create_parser
|
|||
|
||||
|
||||
def parse_patched_options(opts):
|
||||
|
||||
patched_parser = create_parser()
|
||||
patched_parser.defaults.update(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
import hashlib
|
||||
import re
|
||||
import shlex
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Tuple
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from mergedeep import mergedeep
|
||||
|
||||
|
|
|
|||
|
|
@ -173,6 +173,10 @@ class SubArguments:
|
|||
short="-o",
|
||||
long="--dl-override",
|
||||
)
|
||||
SHUFFLE = CLIArgument(
|
||||
short="-sh",
|
||||
long="--shuffle",
|
||||
)
|
||||
|
||||
|
||||
subscription_parser = subparsers.add_parser("sub")
|
||||
|
|
@ -198,6 +202,13 @@ subscription_parser.add_argument(
|
|||
help="override all subscription config values using `dl` syntax, "
|
||||
"i.e. --dl-override='--ytdl_options.max_downloads 3'",
|
||||
)
|
||||
subscription_parser.add_argument(
|
||||
SubArguments.SHUFFLE.short,
|
||||
SubArguments.SHUFFLE.long,
|
||||
action="store_true",
|
||||
help="shuffle subscription order when downloading",
|
||||
default=False,
|
||||
)
|
||||
|
||||
###################################################################################################
|
||||
# DOWNLOAD PARSER
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import os
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Any, Dict
|
||||
|
||||
from ytdl_sub.config.config_validator import ConfigValidator
|
||||
from ytdl_sub.config.preset import Preset
|
||||
|
|
|
|||
|
|
@ -1,29 +1,34 @@
|
|||
import os
|
||||
import posixpath
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from mergedeep import mergedeep
|
||||
from yt_dlp.utils import datetime_from_str
|
||||
|
||||
from ytdl_sub.config.defaults import DEFAULT_FFMPEG_PATH
|
||||
from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH
|
||||
from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY
|
||||
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
|
||||
from ytdl_sub.config.defaults import (
|
||||
DEFAULT_FFMPEG_PATH,
|
||||
DEFAULT_FFPROBE_PATH,
|
||||
DEFAULT_LOCK_DIRECTORY,
|
||||
MAX_FILE_NAME_BYTES,
|
||||
)
|
||||
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
|
||||
from ytdl_sub.utils.exceptions import SubscriptionPermissionError
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.validators.file_path_validators import FFmpegFileValidator
|
||||
from ytdl_sub.validators.file_path_validators import FFprobeFileValidator
|
||||
from ytdl_sub.validators.file_path_validators import FFmpegFileValidator, FFprobeFileValidator
|
||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
from ytdl_sub.validators.validators import IntValidator
|
||||
from ytdl_sub.validators.validators import LiteralDictValidator
|
||||
from ytdl_sub.validators.validators import StringValidator
|
||||
from ytdl_sub.validators.validators import (
|
||||
BoolValidator,
|
||||
IntValidator,
|
||||
LiteralDictValidator,
|
||||
StringValidator,
|
||||
)
|
||||
|
||||
|
||||
class ExperimentalValidator(StrictDictValidator):
|
||||
"""
|
||||
Experimental flags reside under the ``experimental`` key.
|
||||
"""
|
||||
|
||||
_optional_keys = {"enable_update_with_info_json"}
|
||||
_allow_extra_keys = True
|
||||
|
||||
|
|
@ -45,6 +50,11 @@ class ExperimentalValidator(StrictDictValidator):
|
|||
|
||||
|
||||
class PersistLogsValidator(StrictDictValidator):
|
||||
"""
|
||||
By default, no logs are persisted. Specifying this key will enable persisted logs. The following
|
||||
options are available.
|
||||
"""
|
||||
|
||||
_required_keys = {"logs_directory"}
|
||||
_optional_keys = {"keep_logs_after", "keep_successful_logs"}
|
||||
|
||||
|
|
@ -69,8 +79,8 @@ class PersistLogsValidator(StrictDictValidator):
|
|||
@property
|
||||
def logs_directory(self) -> str:
|
||||
"""
|
||||
Write log files to this directory with names like
|
||||
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. (required)
|
||||
Required field. Write log files to this directory with names like
|
||||
``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
|
||||
"""
|
||||
return self._logs_directory.value
|
||||
|
||||
|
|
@ -95,15 +105,54 @@ class PersistLogsValidator(StrictDictValidator):
|
|||
@property
|
||||
def keep_successful_logs(self) -> bool:
|
||||
"""
|
||||
If the ``persist_logs:`` key is in the configuration, then ``ytdl-sub`` *always*
|
||||
writes log files for the subscription both for successful downloads and when it
|
||||
encounters an error while downloading. When this key is ``False``, only write
|
||||
log files for errors. (default ``True``)
|
||||
Defaults to ``True``. When this key is ``False``, only write log files for failed
|
||||
subscriptions.
|
||||
"""
|
||||
return self._keep_successful_logs.value
|
||||
|
||||
|
||||
class ConfigOptions(StrictDictValidator):
|
||||
"""
|
||||
ytdl-sub is configured using a ``config.yaml`` file.
|
||||
|
||||
The ``config.yaml`` is made up of two sections:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
configuration:
|
||||
presets:
|
||||
|
||||
|
||||
Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
|
||||
If you prefer to use a Windows backslash, note that it must have
|
||||
``C:\\\\double\\\\bashslash\\\\paths`` in order to escape the backslash character. This is due
|
||||
to it being a YAML escape character.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
configuration:
|
||||
dl_aliases:
|
||||
mv: "--preset music_video"
|
||||
u: "--download.url"
|
||||
|
||||
experimental:
|
||||
enable_update_with_info_json: True
|
||||
|
||||
ffmpeg_path: "/usr/bin/ffmpeg"
|
||||
ffprobe_path: "/usr/bin/ffprobe"
|
||||
|
||||
file_name_max_bytes: 255
|
||||
lock_directory: "/tmp"
|
||||
|
||||
persist_logs:
|
||||
keep_successful_logs: True
|
||||
logs_directory: "/var/log/ytdl-sub-logs"
|
||||
|
||||
umask: "022"
|
||||
working_directory: ".ytdl-sub-working-directory"
|
||||
|
||||
"""
|
||||
|
||||
_optional_keys = {
|
||||
"working_directory",
|
||||
"umask",
|
||||
|
|
@ -159,7 +208,8 @@ class ConfigOptions(StrictDictValidator):
|
|||
def working_directory(self) -> str:
|
||||
"""
|
||||
The directory to temporarily store downloaded files before moving them into their final
|
||||
directory. (default ``./.ytdl-sub-working-directory``)
|
||||
directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory
|
||||
that ytdl-sub is invoked from.
|
||||
"""
|
||||
# Expands tildas to actual paths, use native os sep
|
||||
return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep))
|
||||
|
|
@ -167,7 +217,7 @@ class ConfigOptions(StrictDictValidator):
|
|||
@property
|
||||
def umask(self) -> Optional[str]:
|
||||
"""
|
||||
Umask in octal format to apply to every created file. (default ``022``)
|
||||
Umask in octal format to apply to every created file. Defaults to ``022``.
|
||||
"""
|
||||
return self._umask.value
|
||||
|
||||
|
|
@ -176,7 +226,7 @@ class ConfigOptions(StrictDictValidator):
|
|||
"""
|
||||
.. _dl_aliases:
|
||||
|
||||
Alias definitions to shorten ``ytdl-sub dl`` arguments. For example,
|
||||
Alias definitions to shorten :ref:`dl arguments <usage:Download Options>`. For example,
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
|
|
@ -228,23 +278,23 @@ class ConfigOptions(StrictDictValidator):
|
|||
The directory to temporarily store file locks, which prevents multiple instances
|
||||
of ``ytdl-sub`` from running. Note that file locks do not work on
|
||||
network-mounted directories. Ensure that this directory resides on the host
|
||||
machine. (default ``/tmp``)
|
||||
machine. Defaults to ``/tmp``.
|
||||
"""
|
||||
return self._lock_directory.value
|
||||
|
||||
@property
|
||||
def ffmpeg_path(self) -> str:
|
||||
"""
|
||||
Path to ffmpeg executable. (default ``/usr/bin/ffmpeg`` for Linux,
|
||||
``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows)
|
||||
Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux,
|
||||
``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows.
|
||||
"""
|
||||
return self._ffmpeg_path.value
|
||||
|
||||
@property
|
||||
def ffprobe_path(self) -> str:
|
||||
"""
|
||||
Path to ffprobe executable. (default ``/usr/bin/ffprobe`` for Linux,
|
||||
``./ffprobe.exe`` in the same directory as ytdl-sub for Windows)
|
||||
Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux,
|
||||
``./ffprobe.exe`` in the same directory as ytdl-sub for Windows.
|
||||
"""
|
||||
return self._ffprobe_path.value
|
||||
|
||||
|
|
|
|||
|
|
@ -1,28 +1,30 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Iterable
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Any, Dict, Iterable, Optional, Set, Type, TypeVar
|
||||
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
|
||||
from ytdl_sub.entries.variables.override_variables import OverrideHelpers
|
||||
from ytdl_sub.entries.variables.override_variables import (
|
||||
REQUIRED_OVERRIDE_VARIABLE_NAMES,
|
||||
OverrideHelpers,
|
||||
)
|
||||
from ytdl_sub.script.parser import parse
|
||||
from ytdl_sub.script.script import Script
|
||||
from ytdl_sub.script.types.function import BuiltInFunction
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import Resolvable, String
|
||||
from ytdl_sub.script.types.syntax_tree import SyntaxTree
|
||||
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
|
||||
from ytdl_sub.utils.exceptions import InvalidVariableNameException
|
||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.exceptions import (
|
||||
InvalidVariableNameException,
|
||||
StringFormattingException,
|
||||
ValidationException,
|
||||
)
|
||||
from ytdl_sub.utils.script import ScriptUtils
|
||||
from ytdl_sub.utils.scriptable import Scriptable
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import (
|
||||
StringFormatterValidator,
|
||||
UnstructuredDictFormatterValidator,
|
||||
)
|
||||
|
||||
ExpectedT = TypeVar("ExpectedT")
|
||||
|
||||
|
||||
class Overrides(UnstructuredDictFormatterValidator, Scriptable):
|
||||
|
|
@ -207,7 +209,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
|
|||
formatter: StringFormatterValidator,
|
||||
entry: Optional[Entry] = None,
|
||||
function_overrides: Optional[Dict[str, str]] = None,
|
||||
) -> str:
|
||||
expected_type: Type[ExpectedT] = str,
|
||||
) -> ExpectedT:
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
|
|
@ -217,6 +220,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
|
|||
Optional. Entry to add source variables to the formatter
|
||||
function_overrides
|
||||
Optional. Explicit values to override the overrides themselves and source variables
|
||||
expected_type
|
||||
The expected type that should return. Defaults to string.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -227,42 +232,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
|
|||
StringFormattingException
|
||||
If the formatter that is trying to be resolved cannot
|
||||
"""
|
||||
return formatter.post_process(
|
||||
str(
|
||||
self._apply_to_resolvable(
|
||||
formatter=formatter, entry=entry, function_overrides=function_overrides
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def apply_overrides_formatter_to_native(
|
||||
self,
|
||||
formatter: OverridesStringFormatterValidator,
|
||||
function_overrides: Optional[Dict[str, str]] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
formatter
|
||||
Overrides formatter to apply
|
||||
function_overrides
|
||||
Optional. Explicit values to override the overrides themselves and source variables
|
||||
|
||||
Returns
|
||||
-------
|
||||
The native python form of the resolved variable
|
||||
"""
|
||||
return formatter.post_process_native(
|
||||
out = formatter.post_process(
|
||||
self._apply_to_resolvable(
|
||||
formatter=formatter, entry=None, function_overrides=function_overrides
|
||||
formatter=formatter, entry=entry, function_overrides=function_overrides
|
||||
).native
|
||||
)
|
||||
|
||||
def evaluate_boolean(
|
||||
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Apply a formatter, and evaluate it to a boolean
|
||||
"""
|
||||
output = self.apply_formatter(formatter=formatter, entry=entry)
|
||||
return ScriptUtils.bool_formatter_output(output)
|
||||
if not isinstance(out, expected_type):
|
||||
raise StringFormattingException(
|
||||
f"Expected type {expected_type.__name__}, but received '{out.__class__.__name__}'"
|
||||
)
|
||||
|
||||
return out
|
||||
|
|
|
|||
|
|
@ -1,20 +1,15 @@
|
|||
from abc import ABC
|
||||
from abc import abstractmethod
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import cached_property
|
||||
from typing import Dict
|
||||
from typing import Generic
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Tuple
|
||||
from typing import Type
|
||||
from typing import Dict, Generic, List, Optional, Tuple, Type
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.validators.options import OptionsValidatorT
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
from ytdl_sub.config.validators.options import OptionsValidatorT, ToggleableOptionsDictValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import (
|
||||
DownloadArchiver,
|
||||
EnhancedDownloadArchive,
|
||||
)
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
|
|
@ -48,7 +43,7 @@ class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC):
|
|||
Returns True if enabled, False if disabled.
|
||||
"""
|
||||
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
|
||||
return self.overrides.evaluate_boolean(self.plugin_options.enable)
|
||||
return self.overrides.apply_formatter(self.plugin_options.enable, expected_type=bool)
|
||||
return True
|
||||
|
||||
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Tuple
|
||||
from typing import Type
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin import SplitPlugin
|
||||
from ytdl_sub.config.plugin.plugin import Plugin, SplitPlugin
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin
|
||||
from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin
|
||||
from ytdl_sub.downloaders.url.downloader import (
|
||||
UrlDownloaderCollectionVariablePlugin,
|
||||
UrlDownloaderThumbnailPlugin,
|
||||
)
|
||||
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
|
||||
from ytdl_sub.plugins.chapters import ChaptersPlugin
|
||||
from ytdl_sub.plugins.date_range import DateRangePlugin
|
||||
|
|
|
|||
|
|
@ -1,13 +1,7 @@
|
|||
from typing import Iterable
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
from typing import Type
|
||||
from typing import Iterable, List, Optional, Set, Tuple, Type
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.config.validators.options import OptionsValidatorT
|
||||
from ytdl_sub.config.validators.options import OptionsValidator, OptionsValidatorT
|
||||
|
||||
|
||||
class PresetPlugins:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
import copy
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Set
|
||||
from typing import Any, Dict, List, Set
|
||||
|
||||
from mergedeep import mergedeep
|
||||
|
||||
|
|
@ -10,17 +7,14 @@ from ytdl_sub.config.config_validator import ConfigValidator
|
|||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
|
||||
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
|
||||
from ytdl_sub.config.preset_options import OutputOptions
|
||||
from ytdl_sub.config.preset_options import YTDLOptions
|
||||
from ytdl_sub.config.preset_options import OutputOptions, YTDLOptions
|
||||
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
||||
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
|
||||
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
|
||||
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES, PUBLISHED_PRESET_NAMES
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.utils.yaml import dump_yaml
|
||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.validators.validators import StringListValidator
|
||||
from ytdl_sub.validators.validators import validation_exception
|
||||
from ytdl_sub.validators.validators import StringListValidator, validation_exception
|
||||
|
||||
PRESET_KEYS = {
|
||||
"preset",
|
||||
|
|
@ -55,6 +49,12 @@ class _PresetShell(StrictDictValidator):
|
|||
|
||||
|
||||
class Preset(_PresetShell):
|
||||
"""
|
||||
Custom presets are defined in this section. Refer to the
|
||||
:ref:`Getting Started Guide<guides/getting_started/first_config:Basic Configuration>`
|
||||
on how to configure.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None:
|
||||
"""
|
||||
|
|
@ -255,11 +255,18 @@ class Preset(_PresetShell):
|
|||
"""
|
||||
return cls(config=config, name=preset_name, value=preset_dict)
|
||||
|
||||
@property
|
||||
def yaml(self) -> str:
|
||||
def yaml(self, subscription_only: bool) -> str:
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
subscription_only:
|
||||
Only include the subscription contents, not the surrounding boiler-plate.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Preset in YAML format
|
||||
"""
|
||||
if subscription_only:
|
||||
return dump_yaml(self._value)
|
||||
|
||||
return dump_yaml({"presets": {self._name: self._value}})
|
||||
|
|
|
|||
|
|
@ -1,24 +1,22 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Any, Dict, Optional, Set
|
||||
|
||||
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
|
||||
from ytdl_sub.utils.exceptions import SubscriptionPermissionError
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.exceptions import SubscriptionPermissionError, ValidationException
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
|
||||
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
|
||||
from ytdl_sub.validators.file_path_validators import (
|
||||
OverridesStringFormatterFilePathValidator,
|
||||
StringFormatterFileNameValidator,
|
||||
)
|
||||
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import (
|
||||
OverridesIntegerFormatterValidator,
|
||||
OverridesStringFormatterValidator,
|
||||
StandardizedDateValidator,
|
||||
StringFormatterValidator,
|
||||
UnstructuredOverridesDictFormatterValidator,
|
||||
)
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
|
|
@ -63,7 +61,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
|
|||
native python.
|
||||
"""
|
||||
out = {
|
||||
key: overrides.apply_overrides_formatter_to_native(val)
|
||||
key: overrides.apply_formatter(val, expected_type=object)
|
||||
for key, val in self.dict.items()
|
||||
}
|
||||
if "cookiefile" in out:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
from abc import ABC
|
||||
from typing import Dict
|
||||
from typing import Set
|
||||
from typing import TypeVar
|
||||
from typing import Dict, Set, TypeVar
|
||||
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
|
|
@ -61,9 +59,9 @@ class ToggleableOptionsDictValidator(OptionsDictValidator):
|
|||
_optional_keys = {"enable"}
|
||||
|
||||
def __init__(self, name, value):
|
||||
assert (
|
||||
"enable" in self._optional_keys
|
||||
), f"{self.__class__.__name__} does not have enable as an optional field"
|
||||
assert "enable" in self._optional_keys, (
|
||||
f"{self.__class__.__name__} does not have enable as an optional field"
|
||||
)
|
||||
super().__init__(name, value)
|
||||
|
||||
self._enable = self._validate_key(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Dict
|
||||
from typing import Dict, List, Set
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
|
||||
|
|
@ -7,80 +7,163 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
|
|||
from ytdl_sub.config.preset_options import OutputOptions
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
||||
from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES, VARIABLES
|
||||
from ytdl_sub.script.script import Script
|
||||
from ytdl_sub.script.utils.name_validation import is_function
|
||||
from ytdl_sub.validators.string_formatter_validators import validate_formatters
|
||||
|
||||
|
||||
class ResolutionLevel:
|
||||
ORIGINAL = 0
|
||||
FILL = 1
|
||||
RESOLVE = 2
|
||||
INTERNAL = 3
|
||||
|
||||
@classmethod
|
||||
def name_of(cls, resolution_level: int) -> str:
|
||||
"""
|
||||
Name of the resolution level.
|
||||
"""
|
||||
if resolution_level == cls.ORIGINAL:
|
||||
return "original"
|
||||
if resolution_level == cls.FILL:
|
||||
return "fill"
|
||||
if resolution_level == cls.RESOLVE:
|
||||
return "resolve"
|
||||
if resolution_level == cls.INTERNAL:
|
||||
return "internal"
|
||||
raise ValueError("Invalid resolution level")
|
||||
|
||||
@classmethod
|
||||
def all(cls) -> List[int]:
|
||||
"""
|
||||
All possible resolution levels.
|
||||
"""
|
||||
return [cls.ORIGINAL, cls.FILL, cls.RESOLVE, cls.INTERNAL]
|
||||
|
||||
|
||||
class VariableValidation:
|
||||
def _get_resolve_partial_filter(self) -> Set[str]:
|
||||
# Exclude sanitized variables from partial validation. This lessens the work
|
||||
# and prevents double-evaluation, which can lead to bad behavior like double-prints.
|
||||
return {
|
||||
name
|
||||
for name in self.script.variable_names
|
||||
if name not in self.unresolved_variables and not name.endswith("_sanitized")
|
||||
}
|
||||
|
||||
def _apply_resolution_level(self) -> None:
|
||||
if self._resolution_level == ResolutionLevel.FILL:
|
||||
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
|
||||
# Only partial resolve definitions that are already resolved
|
||||
self.unresolved_variables |= {
|
||||
name
|
||||
for name in self.overrides.keys
|
||||
if not is_function(name) and not self.script.definition_of(name).maybe_resolvable
|
||||
}
|
||||
elif self._resolution_level == ResolutionLevel.RESOLVE:
|
||||
# Partial resolve everything, but not including internal variables
|
||||
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
|
||||
elif self._resolution_level == ResolutionLevel.INTERNAL:
|
||||
# Partial resolve everything including internal variables
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Invalid resolution level for validation")
|
||||
|
||||
self.script = self.script.resolve_partial(
|
||||
unresolvable=self.unresolved_variables,
|
||||
output_filter=self._get_resolve_partial_filter(),
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
overrides: Overrides,
|
||||
downloader_options: MultiUrlValidator,
|
||||
output_options: OutputOptions,
|
||||
plugins: PresetPlugins,
|
||||
resolution_level: int = ResolutionLevel.RESOLVE,
|
||||
):
|
||||
self.overrides = overrides
|
||||
self.downloader_options = downloader_options
|
||||
self.output_options = output_options
|
||||
self.plugins = plugins
|
||||
|
||||
self.script = self.overrides.script
|
||||
self.unresolved_variables = self.plugins.get_all_variables(
|
||||
self.script: Script = self.overrides.script
|
||||
self.unresolved_variables = (
|
||||
self.plugins.get_all_variables(
|
||||
additional_options=[self.output_options, self.downloader_options]
|
||||
)
|
||||
| UNRESOLVED_VARIABLES
|
||||
)
|
||||
self.unresolved_runtime_variables = self.plugins.get_all_variables(
|
||||
additional_options=[self.output_options, self.downloader_options]
|
||||
)
|
||||
self._resolution_level = resolution_level
|
||||
|
||||
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
|
||||
self._apply_resolution_level()
|
||||
|
||||
def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
|
||||
"""
|
||||
Add dummy variables for script validation
|
||||
"""
|
||||
added_variables = options.added_variables(
|
||||
unresolved_variables=self.unresolved_variables,
|
||||
unresolved_variables=self.unresolved_runtime_variables,
|
||||
).get(plugin_op, set())
|
||||
modified_variables = options.modified_variables().get(plugin_op, set())
|
||||
|
||||
self.unresolved_variables -= added_variables | modified_variables
|
||||
self.unresolved_runtime_variables -= added_variables | modified_variables
|
||||
|
||||
def ensure_proper_usage(self) -> Dict:
|
||||
def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict:
|
||||
"""
|
||||
Validate variables resolve as plugins are executed, and return
|
||||
a mock script which contains actualized added variables from the plugins
|
||||
"""
|
||||
|
||||
resolved_subscription: Dict = {}
|
||||
|
||||
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
|
||||
self._add_runtime_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
|
||||
|
||||
# Always add output options first
|
||||
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options)
|
||||
self._add_runtime_variables(
|
||||
PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options
|
||||
)
|
||||
|
||||
# Metadata variables to be added
|
||||
for plugin_options in PluginMapping.order_options_by(
|
||||
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
|
||||
):
|
||||
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
|
||||
self._add_runtime_variables(
|
||||
PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options
|
||||
)
|
||||
|
||||
for plugin_options in PluginMapping.order_options_by(
|
||||
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
|
||||
):
|
||||
self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
|
||||
self._add_runtime_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
|
||||
|
||||
# Validate that any formatter in the plugin options can resolve
|
||||
resolved_subscription |= validate_formatters(
|
||||
script=self.script,
|
||||
unresolved_variables=self.unresolved_variables,
|
||||
unresolved_runtime_variables=self.unresolved_runtime_variables,
|
||||
validator=plugin_options,
|
||||
partial_resolve_formatters=partial_resolve_formatters,
|
||||
)
|
||||
|
||||
resolved_subscription |= validate_formatters(
|
||||
script=self.script,
|
||||
unresolved_variables=self.unresolved_variables,
|
||||
unresolved_runtime_variables=self.unresolved_runtime_variables,
|
||||
validator=self.output_options,
|
||||
partial_resolve_formatters=partial_resolve_formatters,
|
||||
)
|
||||
|
||||
# TODO: make this a function
|
||||
raw_download_output = validate_formatters(
|
||||
script=self.script,
|
||||
unresolved_variables=self.unresolved_variables,
|
||||
unresolved_runtime_variables=self.unresolved_runtime_variables,
|
||||
validator=self.downloader_options.urls,
|
||||
partial_resolve_formatters=partial_resolve_formatters,
|
||||
)
|
||||
resolved_subscription["download"] = []
|
||||
for url_output in raw_download_output["download"]:
|
||||
|
|
|
|||
|
|
@ -1,24 +1,24 @@
|
|||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
from typing import Iterable
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
||||
from ytdl_sub.downloaders.source_plugin import SourcePlugin
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import (
|
||||
VARIABLE_SCRIPTS,
|
||||
VARIABLES,
|
||||
VariableDefinitions,
|
||||
)
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import get_file_extension
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
from ytdl_sub.utils.file_handler import FileHandler, get_file_extension
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import (
|
||||
DownloadMapping,
|
||||
EnhancedDownloadArchive,
|
||||
)
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,9 @@
|
|||
import abc
|
||||
from abc import ABC
|
||||
from typing import Dict
|
||||
from typing import Generic
|
||||
from typing import Iterable
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import final
|
||||
from typing import Dict, Generic, Iterable, List, Optional, Type, final
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin import BasePlugin
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin import BasePlugin, Plugin
|
||||
from ytdl_sub.config.validators.options import OptionsValidatorT
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
|
|
|
|||
|
|
@ -1,33 +1,29 @@
|
|||
import contextlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
from typing import Iterable
|
||||
from typing import Iterator
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple
|
||||
|
||||
from yt_dlp.utils import RejectedVideoReached
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.downloaders.source_plugin import SourcePlugin
|
||||
from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
|
||||
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
||||
from ytdl_sub.downloaders.url.validators import UrlThumbnailListValidator
|
||||
from ytdl_sub.downloaders.url.validators import UrlValidator
|
||||
from ytdl_sub.downloaders.source_plugin import SourcePlugin, SourcePluginExtension
|
||||
from ytdl_sub.downloaders.url.validators import (
|
||||
MultiUrlValidator,
|
||||
UrlThumbnailListValidator,
|
||||
UrlValidator,
|
||||
)
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.downloaders.ytdlp import YTDLP
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.entry_parent import EntryParent
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.utils.thumbnail import ThumbnailTypes
|
||||
from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail
|
||||
from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
|
||||
from ytdl_sub.utils.thumbnail import (
|
||||
ThumbnailTypes,
|
||||
download_and_convert_url_thumbnail,
|
||||
try_convert_download_thumbnail,
|
||||
)
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
|
@ -52,12 +48,12 @@ class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator])
|
|||
|
||||
if 0 <= input_url_idx < len(self.plugin_options.urls.list):
|
||||
validator = self.plugin_options.urls.list[input_url_idx]
|
||||
if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
|
||||
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
|
||||
return validator
|
||||
|
||||
# Match the first validator based on the URL, if one exists
|
||||
for validator in self.plugin_options.urls.list:
|
||||
if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
|
||||
if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
|
||||
return validator
|
||||
|
||||
# Return the first validator if none exist
|
||||
|
|
@ -97,7 +93,6 @@ class UrlDownloaderThumbnailPlugin(UrlDownloaderBasePluginExtension):
|
|||
|
||||
# If latest entry, always update the thumbnail on each entry
|
||||
if thumbnail_id == ThumbnailTypes.LATEST_ENTRY:
|
||||
|
||||
# always save in dry-run even if it doesn't exist...
|
||||
if self.is_dry_run or entry.is_thumbnail_downloaded():
|
||||
self.save_file(
|
||||
|
|
@ -382,7 +377,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
|||
entries_to_iter: List[Optional[Entry]] = entries
|
||||
|
||||
indices = list(range(len(entries_to_iter)))
|
||||
if self.overrides.evaluate_boolean(validator.download_reverse):
|
||||
if self.overrides.apply_formatter(validator.download_reverse, expected_type=bool):
|
||||
indices = reversed(indices)
|
||||
|
||||
for idx in indices:
|
||||
|
|
@ -461,8 +456,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
|||
ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides)
|
||||
)
|
||||
|
||||
include_sibling_metadata = self.overrides.evaluate_boolean(
|
||||
validator.include_sibling_metadata
|
||||
include_sibling_metadata = self.overrides.apply_formatter(
|
||||
validator.include_sibling_metadata, expected_type=bool
|
||||
)
|
||||
|
||||
parents, orphan_entries = self._download_url_metadata(
|
||||
|
|
@ -487,11 +482,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
|||
# download the bottom-most urls first since they are top-priority
|
||||
for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
|
||||
# URLs can be empty. If they are, then skip
|
||||
if not (urls := self.overrides.apply_overrides_formatter_to_native(url_validator.url)):
|
||||
if not (urls := self.overrides.apply_formatter(url_validator.url, expected_type=list)):
|
||||
continue
|
||||
|
||||
assert isinstance(urls, list)
|
||||
|
||||
for url in reversed(urls):
|
||||
assert isinstance(url, str)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.preset_options import YTDLOptions
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.script.parser import parse
|
||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import (
|
||||
DictFormatterValidator,
|
||||
OverridesBooleanFormatterValidator,
|
||||
OverridesStringFormatterValidator,
|
||||
StringFormatterValidator,
|
||||
)
|
||||
from ytdl_sub.validators.validators import ListValidator
|
||||
|
||||
|
||||
|
|
@ -44,7 +43,7 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
|
|||
|
||||
|
||||
class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator):
|
||||
def post_process_native(self, resolved: Any) -> Any:
|
||||
def post_process(self, resolved: Any) -> List[str]:
|
||||
if isinstance(resolved, str):
|
||||
return [resolved]
|
||||
if isinstance(resolved, list):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import copy
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
import mergedeep
|
||||
|
||||
|
|
|
|||
|
|
@ -5,15 +5,10 @@ import os
|
|||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
import yt_dlp as ytdl
|
||||
from yt_dlp.utils import ExistingVideoReached
|
||||
from yt_dlp.utils import MaxDownloadsReached
|
||||
from yt_dlp.utils import RejectedVideoReached
|
||||
from yt_dlp.utils import ExistingVideoReached, MaxDownloadsReached, RejectedVideoReached
|
||||
|
||||
from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener
|
||||
from ytdl_sub.utils.exceptions import FileNotDownloadedException
|
||||
|
|
|
|||
|
|
@ -1,18 +1,11 @@
|
|||
# pylint: disable=protected-access
|
||||
from abc import ABC
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import final
|
||||
from typing import Any, Dict, Optional, Type, TypeVar, final
|
||||
|
||||
from yt_dlp.utils import LazyList
|
||||
from yt_dlp.utils import sanitize_filename
|
||||
from yt_dlp.utils import LazyList, sanitize_filename
|
||||
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
|
|||
|
|
@ -3,24 +3,15 @@ import copy
|
|||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import final
|
||||
from typing import Any, Dict, Optional, Type, TypeVar, final
|
||||
|
||||
from ytdl_sub.entries.base_entry import BaseEntry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_types import ArrayVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringVariable
|
||||
from ytdl_sub.entries.script.variable_types import Variable
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_types import ArrayVariable, StringVariable, Variable
|
||||
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
|
||||
from ytdl_sub.utils.script import ScriptUtils
|
||||
from ytdl_sub.utils.scriptable import Scriptable
|
||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
|
||||
from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS
|
||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS, VIDEO_CODEC_EXTS
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,10 @@
|
|||
import math
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ytdl_sub.entries.base_entry import BaseEntry
|
||||
from ytdl_sub.entries.base_entry import BaseEntryT
|
||||
from ytdl_sub.entries.base_entry import BaseEntry, BaseEntryT
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_types import MetadataVariable
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
|
|
|||
|
|
@ -5,10 +5,7 @@ from yt_dlp.utils import sanitize_filename
|
|||
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.types.map import Map
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgument
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument, Integer, ReturnableArgument, String
|
||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||
from ytdl_sub.utils.file_path import FilePathTruncater
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from typing import Dict
|
||||
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
from abc import ABC
|
||||
from functools import cache
|
||||
from functools import cached_property
|
||||
from typing import Dict
|
||||
from typing import Set
|
||||
from functools import cache, cached_property
|
||||
from typing import Dict, Set
|
||||
|
||||
from ytdl_sub.entries.script.custom_functions import CustomFunctions
|
||||
from ytdl_sub.entries.script.variable_types import ArrayMetadataVariable
|
||||
from ytdl_sub.entries.script.variable_types import IntegerMetadataVariable
|
||||
from ytdl_sub.entries.script.variable_types import IntegerVariable
|
||||
from ytdl_sub.entries.script.variable_types import MapMetadataVariable
|
||||
from ytdl_sub.entries.script.variable_types import MapVariable
|
||||
from ytdl_sub.entries.script.variable_types import MetadataVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringDateMetadataVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringDateVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringMetadataVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringVariable
|
||||
from ytdl_sub.entries.script.variable_types import Variable
|
||||
from ytdl_sub.entries.script.variable_types import (
|
||||
ArrayMetadataVariable,
|
||||
IntegerMetadataVariable,
|
||||
IntegerVariable,
|
||||
MapMetadataVariable,
|
||||
MapVariable,
|
||||
MetadataVariable,
|
||||
StringDateMetadataVariable,
|
||||
StringDateVariable,
|
||||
StringMetadataVariable,
|
||||
StringVariable,
|
||||
Variable,
|
||||
)
|
||||
|
||||
# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion
|
||||
# pylint: disable=no-member
|
||||
|
|
@ -1135,6 +1135,16 @@ class VariableDefinitions(
|
|||
]
|
||||
}
|
||||
|
||||
@cache
|
||||
def variable_names(self, include_sanitized: bool):
|
||||
"""
|
||||
Returns all variable names, and can include sanitized.
|
||||
"""
|
||||
var_names: Set[str] = self.scripts().keys()
|
||||
if include_sanitized:
|
||||
var_names |= {f"{name}_sanitized" for name in var_names}
|
||||
return var_names
|
||||
|
||||
@cache
|
||||
def injected_variables(self) -> Set[MetadataVariable]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,17 +1,10 @@
|
|||
from abc import ABC
|
||||
from abc import abstractmethod
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import Dict, List, Optional, Type, TypeVar
|
||||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.map import Map
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import Boolean, Integer, String
|
||||
|
||||
ENTRY_METADATA_VARIABLE_NAME = "entry_metadata"
|
||||
PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
from typing import Dict
|
||||
from typing import Set
|
||||
from typing import Dict, Set
|
||||
|
||||
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS
|
||||
from ytdl_sub.entries.script.variable_types import ArrayVariable
|
||||
from ytdl_sub.entries.script.variable_types import BooleanVariable
|
||||
from ytdl_sub.entries.script.variable_types import MapVariable
|
||||
from ytdl_sub.entries.script.variable_types import StringVariable
|
||||
from ytdl_sub.entries.script.variable_types import Variable
|
||||
from ytdl_sub.entries.script.variable_types import (
|
||||
ArrayVariable,
|
||||
BooleanVariable,
|
||||
MapVariable,
|
||||
StringVariable,
|
||||
Variable,
|
||||
)
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.utils.name_validation import is_valid_name
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,19 @@
|
|||
import os.path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Any, Dict, Optional, Set
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.exceptions import FileNotDownloadedException
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
|
||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_TYPES_EXTENSION_MAPPING
|
||||
from ytdl_sub.validators.audo_codec_validator import AudioTypeValidator
|
||||
from ytdl_sub.validators.audo_codec_validator import (
|
||||
AUDIO_CODEC_EXTS,
|
||||
AUDIO_CODEC_TYPES_EXTENSION_MAPPING,
|
||||
AudioTypeValidator,
|
||||
)
|
||||
from ytdl_sub.validators.validators import FloatValidator
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
|
|
|||
|
|
@ -1,26 +1,23 @@
|
|||
import collections
|
||||
import re
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Dict, List, Optional, Set
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
|
||||
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.entry import (
|
||||
Entry,
|
||||
ytdl_sub_chapters_from_comments,
|
||||
ytdl_sub_split_by_chapters_parent_uid,
|
||||
)
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.chapters import Chapters
|
||||
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.validators.regex_validator import RegexListValidator
|
||||
from ytdl_sub.validators.string_select_validator import StringSelectValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
from ytdl_sub.validators.validators import ListValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator, ListValidator
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
from typing import List, Optional, Set, Tuple
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
|
|
@ -116,7 +113,7 @@ class DateRangePlugin(Plugin[DateRangeOptions]):
|
|||
date_validator=self.plugin_options.after, overrides=self.overrides
|
||||
)
|
||||
after_filter = f"{date_type} >= {after_str}"
|
||||
if self.overrides.evaluate_boolean(self.plugin_options.breaks):
|
||||
if self.overrides.apply_formatter(self.plugin_options.breaks, expected_type=bool):
|
||||
breaking_match_filters.append(after_filter)
|
||||
else:
|
||||
match_filters.append(after_filter)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
import mediafile
|
||||
|
||||
|
|
@ -7,8 +6,7 @@ from ytdl_sub.config.plugin.plugin import Plugin
|
|||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.ffmpeg import FFMPEG
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
|
||||
|
|
@ -33,7 +31,7 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
|
|||
|
||||
@property
|
||||
def _embed_thumbnail(self) -> bool:
|
||||
return self.overrides.evaluate_boolean(self.plugin_options)
|
||||
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
|
||||
|
||||
@classmethod
|
||||
def _embed_video_thumbnail(cls, entry: Entry) -> None:
|
||||
|
|
|
|||
|
|
@ -1,22 +1,16 @@
|
|||
import os
|
||||
from subprocess import CalledProcessError
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Any, Dict, Optional, Set
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.utils.exceptions import FileNotDownloadedException
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.exceptions import FileNotDownloadedException, ValidationException
|
||||
from ytdl_sub.utils.ffmpeg import FFMPEG
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata
|
||||
from ytdl_sub.validators.audo_codec_validator import FileTypeValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.string_select_validator import StringSelectValidator
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
|
|
@ -7,13 +6,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
|
|||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
|
||||
from ytdl_sub.validators.validators import ListValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
logger = Logger.get("filter-exclude")
|
||||
|
||||
|
||||
class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
|
||||
class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
|
||||
"""
|
||||
Applies a conditional OR on any number of filters comprised of either variables or scripts.
|
||||
If any filter evaluates to True, the entry will be excluded.
|
||||
|
|
@ -29,6 +29,8 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
|
|||
{ %contains( %lower(description), '#short' ) }
|
||||
"""
|
||||
|
||||
_inner_list_type = BooleanFormatterValidator
|
||||
|
||||
|
||||
class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
|
||||
plugin_options_type = FilterExcludeOptions
|
||||
|
|
@ -52,7 +54,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
|
|||
return entry
|
||||
|
||||
for formatter in self.plugin_options.list:
|
||||
should_exclude = self.overrides.evaluate_boolean(formatter=formatter, entry=entry)
|
||||
should_exclude = self.overrides.apply_formatter(
|
||||
formatter=formatter, entry=entry, expected_type=bool
|
||||
)
|
||||
|
||||
if should_exclude:
|
||||
logger.info(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
|
|
@ -7,14 +6,14 @@ from ytdl_sub.config.validators.options import OptionsValidator
|
|||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.utils.script import ScriptUtils
|
||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
|
||||
from ytdl_sub.validators.validators import ListValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
logger = Logger.get("filter-include")
|
||||
|
||||
|
||||
class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
|
||||
class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
|
||||
"""
|
||||
Applies a conditional AND on any number of filters comprised of either variables or scripts.
|
||||
If all filters evaluate to True, the entry will be included.
|
||||
|
|
@ -38,6 +37,8 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
|
|||
}
|
||||
"""
|
||||
|
||||
_inner_list_type = BooleanFormatterValidator
|
||||
|
||||
|
||||
class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
|
||||
plugin_options_type = FilterIncludeOptions
|
||||
|
|
@ -61,8 +62,8 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
|
|||
return entry
|
||||
|
||||
for formatter in self.plugin_options.list:
|
||||
should_exclude = ScriptUtils.bool_formatter_output(
|
||||
self.overrides.apply_formatter(formatter=formatter, entry=entry)
|
||||
should_exclude = self.overrides.apply_formatter(
|
||||
formatter=formatter, entry=entry, expected_type=bool
|
||||
)
|
||||
if not should_exclude:
|
||||
logger.info(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import copy
|
||||
from typing import Any
|
||||
from typing import List
|
||||
from typing import Tuple
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
|
|
|
|||
|
|
@ -1,22 +1,21 @@
|
|||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import mediafile
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
|
||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import (
|
||||
ListFormatterValidator,
|
||||
StringFormatterValidator,
|
||||
)
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
@ -147,8 +146,7 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
|
|||
else:
|
||||
if len(tag_value) > 1:
|
||||
logger.warning(
|
||||
"Music tag '%s' does not support lists. "
|
||||
"Only setting the first element",
|
||||
"Music tag '%s' does not support lists. Only setting the first element",
|
||||
tag_name,
|
||||
)
|
||||
setattr(audio_file, tag_name, tag_value[0])
|
||||
|
|
|
|||
|
|
@ -2,24 +2,25 @@ import os
|
|||
from abc import ABC
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.xml import XmlElement
|
||||
from ytdl_sub.utils.xml import to_max_3_byte_utf8_dict
|
||||
from ytdl_sub.utils.xml import to_max_3_byte_utf8_string
|
||||
from ytdl_sub.utils.xml import to_xml
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata
|
||||
from ytdl_sub.utils.xml import (
|
||||
XmlElement,
|
||||
to_max_3_byte_utf8_dict,
|
||||
to_max_3_byte_utf8_string,
|
||||
to_xml,
|
||||
)
|
||||
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
|
||||
from ytdl_sub.validators.nfo_validators import NfoTagsValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import (
|
||||
DictFormatterValidator,
|
||||
OverridesBooleanFormatterValidator,
|
||||
StringFormatterValidator,
|
||||
)
|
||||
|
||||
|
||||
class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
|
||||
|
|
@ -140,7 +141,7 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC):
|
|||
if not nfo_tags:
|
||||
return
|
||||
|
||||
if self.overrides.evaluate_boolean(self.plugin_options.kodi_safe):
|
||||
if self.overrides.apply_formatter(self.plugin_options.kodi_safe, expected_type=bool):
|
||||
nfo_root = to_max_3_byte_utf8_string(nfo_root)
|
||||
nfo_tags = {
|
||||
to_max_3_byte_utf8_string(key): [
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
|
||||
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
|
||||
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
|
||||
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator, SharedNfoTagsOptions, SharedNfoTagsPlugin
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
|
|
|
|||
|
|
@ -1,23 +1,14 @@
|
|||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import SplitPlugin
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import OptionsDictValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.utils.chapters import Chapters
|
||||
from ytdl_sub.utils.chapters import Timestamp
|
||||
from ytdl_sub.entries.entry import Entry, ytdl_sub_split_by_chapters_parent_uid
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.chapters import Chapters, Timestamp
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.ffmpeg import FFMPEG
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata
|
||||
from ytdl_sub.validators.string_select_validator import StringSelectValidator
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.validators.options import OptionsValidator
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.utils.ffmpeg import FFMPEG
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
|
||||
|
||||
|
|
@ -31,7 +29,7 @@ class SquareThumbnailPlugin(Plugin[SquareThumbnailOptions]):
|
|||
|
||||
@property
|
||||
def _square_thumbnail(self) -> bool:
|
||||
return self.overrides.evaluate_boolean(self.plugin_options)
|
||||
return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
|
||||
|
||||
@classmethod
|
||||
def _convert_to_square_thumbnail(cls, entry: Entry) -> None:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator
|
||||
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions
|
||||
from ytdl_sub.plugins.nfo_tags import SharedNfoTagsPlugin
|
||||
from ytdl_sub.plugins.nfo_tags import NfoTagsValidator, SharedNfoTagsOptions, SharedNfoTagsPlugin
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,28 +1,21 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
|
||||
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.string_select_validator import StringSelectValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
from ytdl_sub.validators.validators import StringListValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator, StringListValidator
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
import random
|
||||
import time
|
||||
from abc import ABC
|
||||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import Dict, Optional, Type, TypeVar
|
||||
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
|
|
@ -13,8 +10,10 @@ from ytdl_sub.entries.entry import Entry
|
|||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import FloatFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesFloatFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import (
|
||||
FloatFormatterValidator,
|
||||
OverridesFloatFormatterValidator,
|
||||
)
|
||||
from ytdl_sub.validators.validators import ProbabilityValidator
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
||||
|
|
@ -42,8 +41,8 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC):
|
|||
)
|
||||
|
||||
def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
|
||||
actualized_min = float(overrides.apply_formatter(self._min, entry=entry))
|
||||
actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
|
||||
actualized_min = overrides.apply_formatter(self._min, entry=entry, expected_type=float)
|
||||
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float)
|
||||
|
||||
if actualized_min < 0:
|
||||
raise self._validation_exception(
|
||||
|
|
@ -70,7 +69,7 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC):
|
|||
-------
|
||||
Max possible value
|
||||
"""
|
||||
actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
|
||||
actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float)
|
||||
if actualized_max < 0:
|
||||
raise self._validation_exception(
|
||||
f"max must be greater than zero, received {actualized_max}"
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]):
|
|||
Tags the entry's audio file using values defined in the metadata options
|
||||
"""
|
||||
tags_to_write: Dict[str, str] = {}
|
||||
for tag_name, tag_formatter in self.plugin_options.dict.items():
|
||||
for tag_name, tag_formatter in sorted(self.plugin_options.dict.items()):
|
||||
tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry)
|
||||
tags_to_write[tag_name] = tag_value
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import os
|
||||
import pathlib
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Set
|
||||
from typing import Any, Dict, Set
|
||||
|
||||
import mergedeep
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from ytdl_sub.prebuilt_presets import PrebuiltPresets
|
||||
from ytdl_sub.prebuilt_presets import get_prebuilt_preset_package_name
|
||||
from ytdl_sub.prebuilt_presets import PrebuiltPresets, get_prebuilt_preset_package_name
|
||||
|
||||
PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ presets:
|
|||
urls:
|
||||
# The first URL will be all the artist's tracks.
|
||||
# Treat these as singles - an album with a single track
|
||||
- url: "{url}/tracks"
|
||||
- url: "{url}"
|
||||
include_sibling_metadata: False
|
||||
variables:
|
||||
sc_track_album: "{title}"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from ytdl_sub.prebuilt_presets import PrebuiltPresets
|
||||
from ytdl_sub.prebuilt_presets import get_prebuilt_preset_package_name
|
||||
from ytdl_sub.prebuilt_presets import PrebuiltPresets, get_prebuilt_preset_package_name
|
||||
|
||||
PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from ytdl_sub.prebuilt_presets import PrebuiltPresets
|
||||
from ytdl_sub.prebuilt_presets import get_prebuilt_preset_package_name
|
||||
from ytdl_sub.prebuilt_presets import PrebuiltPresets, get_prebuilt_preset_package_name
|
||||
|
||||
PREBUILT_PRESET_PACKAGE_NAME = get_prebuilt_preset_package_name(__file__)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from typing import Callable
|
||||
from typing import Dict
|
||||
from typing import Callable, Dict
|
||||
|
||||
from ytdl_sub.script.functions.array_functions import ArrayFunctions
|
||||
from ytdl_sub.script.functions.boolean_functions import BooleanFunctions
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
import itertools
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import Lambda
|
||||
from ytdl_sub.script.types.resolvable import LambdaReduce
|
||||
from ytdl_sub.script.types.resolvable import LambdaTwo
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
from ytdl_sub.script.utils.exceptions import ArrayValueDoesNotExist
|
||||
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
AnyArgument,
|
||||
Boolean,
|
||||
Integer,
|
||||
Lambda,
|
||||
LambdaReduce,
|
||||
LambdaTwo,
|
||||
Resolvable,
|
||||
)
|
||||
from ytdl_sub.script.utils.exceptions import (
|
||||
UNREACHABLE,
|
||||
ArrayValueDoesNotExist,
|
||||
FunctionRuntimeException,
|
||||
)
|
||||
|
||||
|
||||
class ArrayFunctions:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.map import Map
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Float
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument, Boolean, Float, Integer, String
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
from typing import Union
|
||||
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgumentA
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgumentB
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
AnyArgument,
|
||||
Boolean,
|
||||
ReturnableArgumentA,
|
||||
ReturnableArgumentB,
|
||||
)
|
||||
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from datetime import datetime
|
||||
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import Integer, String
|
||||
|
||||
|
||||
class DateFunctions:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgument
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument, ReturnableArgument, String
|
||||
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ from typing import Any
|
|||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.map import Map
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Float
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
AnyArgument,
|
||||
Boolean,
|
||||
Float,
|
||||
Integer,
|
||||
Resolvable,
|
||||
String,
|
||||
)
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
from typing import Dict
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.map import Map
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Hashable
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import LambdaThree
|
||||
from ytdl_sub.script.types.resolvable import LambdaTwo
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||
from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException
|
||||
from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
AnyArgument,
|
||||
Boolean,
|
||||
Hashable,
|
||||
Integer,
|
||||
LambdaThree,
|
||||
LambdaTwo,
|
||||
String,
|
||||
)
|
||||
from ytdl_sub.script.utils.exceptions import (
|
||||
FunctionRuntimeException,
|
||||
KeyDoesNotExistRuntimeException,
|
||||
KeyNotHashableRuntimeException,
|
||||
)
|
||||
|
||||
|
||||
class MapFunctions:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,7 @@ import math
|
|||
from typing import Optional
|
||||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Float
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import Numeric
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument, Float, Integer, Numeric
|
||||
|
||||
|
||||
def _to_numeric(value: int | float) -> Numeric:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
from typing import Optional
|
||||
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgument
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument, Integer, ReturnableArgument
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
|
||||
logger = Logger.get(name="preset")
|
||||
|
|
|
|||
|
|
@ -1,15 +1,9 @@
|
|||
import re
|
||||
from typing import AnyStr
|
||||
from typing import List
|
||||
from typing import Match
|
||||
from typing import Optional
|
||||
from typing import AnyStr, List, Match, Optional
|
||||
|
||||
from ytdl_sub.script.functions.array_functions import ArrayFunctions
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Float
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import Boolean, Float, Integer, String
|
||||
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,7 @@
|
|||
from typing import Optional
|
||||
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Float
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import Numeric
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import AnyArgument, Boolean, Float, Integer, Numeric, String
|
||||
|
||||
|
||||
class StringFunctions:
|
||||
|
|
|
|||
|
|
@ -1,39 +1,31 @@
|
|||
import json
|
||||
from enum import Enum
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Dict, List, Optional, Set
|
||||
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.types.array import UnresolvedArray
|
||||
from ytdl_sub.script.types.function import Argument
|
||||
from ytdl_sub.script.types.function import BuiltInFunction
|
||||
from ytdl_sub.script.types.function import CustomFunction
|
||||
from ytdl_sub.script.types.function import Function
|
||||
from ytdl_sub.script.types.function import Argument, BuiltInFunction, CustomFunction, Function
|
||||
from ytdl_sub.script.types.map import UnresolvedMap
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import Float
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import Lambda
|
||||
from ytdl_sub.script.types.resolvable import NonHashable
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import Boolean, Float, Integer, Lambda, NonHashable, String
|
||||
from ytdl_sub.script.types.syntax_tree import SyntaxTree
|
||||
from ytdl_sub.script.types.variable import FunctionArgument
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.variable import FunctionArgument, Variable
|
||||
from ytdl_sub.script.utils.exception_formatters import ParserExceptionFormatter
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
from ytdl_sub.script.utils.exceptions import CycleDetected
|
||||
from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist
|
||||
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
|
||||
from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName
|
||||
from ytdl_sub.script.utils.exceptions import InvalidSyntaxException
|
||||
from ytdl_sub.script.utils.exceptions import InvalidVariableName
|
||||
from ytdl_sub.script.utils.exceptions import UserException
|
||||
from ytdl_sub.script.utils.exceptions import VariableDoesNotExist
|
||||
from ytdl_sub.script.utils.name_validation import is_function
|
||||
from ytdl_sub.script.utils.name_validation import to_function_name
|
||||
from ytdl_sub.script.utils.name_validation import validate_variable_name
|
||||
from ytdl_sub.script.utils.exceptions import (
|
||||
UNREACHABLE,
|
||||
CycleDetected,
|
||||
FunctionDoesNotExist,
|
||||
IncompatibleFunctionArguments,
|
||||
InvalidCustomFunctionArgumentName,
|
||||
InvalidSyntaxException,
|
||||
InvalidVariableName,
|
||||
UserException,
|
||||
VariableDoesNotExist,
|
||||
)
|
||||
from ytdl_sub.script.utils.name_validation import (
|
||||
is_function,
|
||||
to_function_name,
|
||||
validate_variable_name,
|
||||
)
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
# pylint: disable=too-many-branches
|
||||
|
|
@ -580,7 +572,6 @@ class _Parser:
|
|||
return True
|
||||
|
||||
def _parse(self) -> SyntaxTree:
|
||||
|
||||
while ch := self._read():
|
||||
continue_parse = self._parse_main_loop(ch)
|
||||
if not continue_parse:
|
||||
|
|
|
|||
|
|
@ -1,32 +1,28 @@
|
|||
# pylint: disable=missing-raises-doc
|
||||
import copy
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional, Set
|
||||
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.parser import parse
|
||||
from ytdl_sub.script.script_output import ScriptOutput
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
|
||||
from ytdl_sub.script.types.resolvable import Lambda
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree
|
||||
from ytdl_sub.script.types.syntax_tree import SyntaxTree
|
||||
from ytdl_sub.script.types.variable import FunctionArgument
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.resolvable import Argument, BuiltInFunctionType, Lambda, Resolvable
|
||||
from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree, SyntaxTree
|
||||
from ytdl_sub.script.types.variable import FunctionArgument, Variable
|
||||
from ytdl_sub.script.types.variable_dependency import VariableDependency
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
from ytdl_sub.script.utils.exceptions import CycleDetected
|
||||
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
|
||||
from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments
|
||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
|
||||
from ytdl_sub.script.utils.name_validation import is_function
|
||||
from ytdl_sub.script.utils.name_validation import to_function_definition_name
|
||||
from ytdl_sub.script.utils.name_validation import to_function_name
|
||||
from ytdl_sub.script.utils.name_validation import validate_variable_name
|
||||
from ytdl_sub.script.utils.exceptions import (
|
||||
UNREACHABLE,
|
||||
CycleDetected,
|
||||
IncompatibleFunctionArguments,
|
||||
InvalidCustomFunctionArguments,
|
||||
RuntimeException,
|
||||
ScriptVariableNotResolved,
|
||||
)
|
||||
from ytdl_sub.script.utils.name_validation import (
|
||||
is_function,
|
||||
to_function_definition_name,
|
||||
to_function_name,
|
||||
validate_variable_name,
|
||||
)
|
||||
from ytdl_sub.script.utils.type_checking import FunctionSpec
|
||||
|
||||
|
||||
|
|
@ -38,59 +34,71 @@ class Script:
|
|||
``{ %custom_function: syntax }``
|
||||
"""
|
||||
|
||||
def _ensure_no_cycle(
|
||||
def _throw_cycle_error(
|
||||
self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree]
|
||||
):
|
||||
if dep not in definitions:
|
||||
return # does not exist, will throw downstream in parser
|
||||
type_name, pre = (
|
||||
("custom functions", "%") if definitions is self._functions else ("variables", "")
|
||||
)
|
||||
cycle_deps = [name] + deps + [dep]
|
||||
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
|
||||
|
||||
if name in deps + [dep]:
|
||||
type_name, pre = (
|
||||
("custom functions", "%") if definitions is self._functions else ("variables", "")
|
||||
)
|
||||
cycle_deps = [name] + deps + [dep]
|
||||
cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps])
|
||||
|
||||
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}")
|
||||
raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}")
|
||||
|
||||
def _traverse_variable_dependencies(
|
||||
self,
|
||||
variable_name: str,
|
||||
variable_dependency: SyntaxTree,
|
||||
deps: List[str],
|
||||
ensured: Dict[str, Set[str]],
|
||||
) -> None:
|
||||
for dep in variable_dependency.variables:
|
||||
self._ensure_no_cycle(
|
||||
name=variable_name, dep=dep.name, deps=deps, definitions=self._variables
|
||||
)
|
||||
if variable_name == dep.name:
|
||||
self._throw_cycle_error(
|
||||
name=variable_name, dep=dep.name, deps=deps, definitions=self._variables
|
||||
)
|
||||
|
||||
if dep.name in ensured[variable_name]:
|
||||
continue
|
||||
|
||||
self._traverse_variable_dependencies(
|
||||
variable_name=variable_name,
|
||||
variable_dependency=self._variables[dep.name],
|
||||
deps=deps + [dep.name],
|
||||
ensured=ensured,
|
||||
)
|
||||
ensured[variable_name].add(dep.name)
|
||||
|
||||
for custom_func in variable_dependency.custom_function_dependencies(
|
||||
custom_function_definitions=self._functions
|
||||
):
|
||||
for dep in self._functions[custom_func.name].variables:
|
||||
self._ensure_no_cycle(
|
||||
name=variable_name,
|
||||
dep=dep.name,
|
||||
deps=deps + [custom_func.definition_name()],
|
||||
definitions=self._variables,
|
||||
)
|
||||
if variable_name == dep.name:
|
||||
self._throw_cycle_error(
|
||||
name=variable_name,
|
||||
dep=dep.name,
|
||||
deps=deps + [custom_func.definition_name()],
|
||||
definitions=self._variables,
|
||||
)
|
||||
if dep.name in ensured[variable_name]:
|
||||
continue
|
||||
|
||||
self._traverse_variable_dependencies(
|
||||
variable_name=variable_name,
|
||||
variable_dependency=self._variables[dep.name],
|
||||
deps=deps + [custom_func.definition_name(), dep.name],
|
||||
ensured=ensured,
|
||||
)
|
||||
ensured[variable_name].add(dep.name)
|
||||
|
||||
def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]):
|
||||
ensured: Dict[str, Set[str]] = defaultdict(set)
|
||||
for variable_name, variable_definition in variables.items():
|
||||
self._traverse_variable_dependencies(
|
||||
variable_name=variable_name,
|
||||
variable_dependency=variable_definition,
|
||||
deps=[],
|
||||
ensured=ensured,
|
||||
)
|
||||
|
||||
def _traverse_custom_function_dependencies(
|
||||
|
|
@ -100,9 +108,10 @@ class Script:
|
|||
deps: List[str],
|
||||
) -> None:
|
||||
for dep in custom_function_dependency.custom_functions:
|
||||
self._ensure_no_cycle(
|
||||
name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions
|
||||
)
|
||||
if custom_function_name == dep.name:
|
||||
self._throw_cycle_error(
|
||||
name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions
|
||||
)
|
||||
self._traverse_custom_function_dependencies(
|
||||
custom_function_name=custom_function_name,
|
||||
custom_function_dependency=self._functions[dep.name],
|
||||
|
|
@ -679,6 +688,18 @@ class Script:
|
|||
|
||||
raise RuntimeException(f"Tried to get unresolved variable {variable_name}")
|
||||
|
||||
def definition_of(self, name: str) -> SyntaxTree:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
The definition of the variable or function.
|
||||
"""
|
||||
if name.startswith("%") and name[1:] in self._functions:
|
||||
return self._functions[name[1:]]
|
||||
if name in self._variables:
|
||||
return self._variables[name]
|
||||
raise RuntimeException(f"Tried to get non-existent definition with name {name}")
|
||||
|
||||
@property
|
||||
def variable_names(self) -> Set[str]:
|
||||
"""
|
||||
|
|
@ -699,10 +720,56 @@ class Script:
|
|||
"""
|
||||
return set(to_function_definition_name(name) for name in self._functions.keys())
|
||||
|
||||
def resolve_partial(
|
||||
def _resolve_partial_loop(
|
||||
self,
|
||||
output_filter: Optional[Set[str]],
|
||||
resolved: Dict[Variable, Resolvable],
|
||||
unresolved: Dict[Variable, Argument],
|
||||
unresolvable: Optional[Set[str]],
|
||||
):
|
||||
to_partially_resolve: Set[Variable] = (
|
||||
{Variable(name) for name in output_filter} if output_filter else set(unresolved.keys())
|
||||
)
|
||||
|
||||
partially_resolved = True
|
||||
while partially_resolved:
|
||||
partially_resolved = False
|
||||
|
||||
for variable in list(to_partially_resolve):
|
||||
definition = unresolved[variable]
|
||||
maybe_resolved = definition
|
||||
|
||||
if isinstance(definition, Variable) and definition.name not in unresolvable:
|
||||
if definition in resolved:
|
||||
maybe_resolved = resolved[definition]
|
||||
elif definition in unresolved:
|
||||
maybe_resolved = unresolved[definition]
|
||||
else:
|
||||
raise UNREACHABLE
|
||||
elif isinstance(definition, VariableDependency):
|
||||
maybe_resolved = definition.partial_resolve(
|
||||
resolved_variables=resolved,
|
||||
unresolved_variables=unresolved,
|
||||
custom_functions=self._functions,
|
||||
)
|
||||
|
||||
if isinstance(maybe_resolved, Resolvable):
|
||||
resolved[variable] = maybe_resolved
|
||||
del unresolved[variable]
|
||||
to_partially_resolve.remove(variable)
|
||||
partially_resolved = True
|
||||
else:
|
||||
unresolved[variable] = maybe_resolved
|
||||
|
||||
# If the definition changed, then the script changed
|
||||
# which means we can iterate again
|
||||
partially_resolved |= definition != maybe_resolved
|
||||
|
||||
def _resolve_partial(
|
||||
self,
|
||||
unresolvable: Optional[Set[str]] = None,
|
||||
) -> "Script":
|
||||
output_filter: Optional[Set[str]] = None,
|
||||
) -> Dict[str, SyntaxTree]:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -717,40 +784,54 @@ class Script:
|
|||
if name not in unresolvable
|
||||
}
|
||||
|
||||
partially_resolved = True
|
||||
while partially_resolved:
|
||||
|
||||
partially_resolved = False
|
||||
|
||||
for variable in list(unresolved.keys()):
|
||||
definition = unresolved[variable]
|
||||
|
||||
maybe_resolved = definition
|
||||
if isinstance(definition, Variable) and definition.name not in unresolvable:
|
||||
maybe_resolved = resolved.get(definition, unresolved[definition])
|
||||
elif isinstance(definition, VariableDependency):
|
||||
maybe_resolved = definition.partial_resolve(
|
||||
resolved_variables=resolved,
|
||||
unresolved_variables=unresolved,
|
||||
custom_functions=self._functions,
|
||||
)
|
||||
|
||||
if isinstance(maybe_resolved, Resolvable):
|
||||
resolved[variable] = maybe_resolved
|
||||
del unresolved[variable]
|
||||
partially_resolved = True
|
||||
else:
|
||||
unresolved[variable] = maybe_resolved
|
||||
|
||||
# If the definition changed, then the script changed
|
||||
# which means we can iterate again
|
||||
partially_resolved |= definition != maybe_resolved
|
||||
|
||||
return copy.deepcopy(self).add_parsed(
|
||||
{var_name: self._variables[var_name] for var_name in unresolvable}
|
||||
| {
|
||||
var.name: ResolvedSyntaxTree(ast=[definition])
|
||||
for var, definition in resolved.items()
|
||||
}
|
||||
| {var.name: SyntaxTree(ast=[definition]) for var, definition in unresolved.items()}
|
||||
self._resolve_partial_loop(
|
||||
output_filter=output_filter,
|
||||
resolved=resolved,
|
||||
unresolved=unresolved,
|
||||
unresolvable=unresolvable,
|
||||
)
|
||||
|
||||
if output_filter:
|
||||
out: Dict[str, SyntaxTree] = {}
|
||||
for name in output_filter:
|
||||
variable_name = Variable(name)
|
||||
if variable_name in resolved:
|
||||
out[name] = ResolvedSyntaxTree(ast=[resolved[variable_name]])
|
||||
else:
|
||||
out[name] = SyntaxTree(ast=[unresolved[variable_name]])
|
||||
|
||||
return out
|
||||
|
||||
return {
|
||||
var.name: ResolvedSyntaxTree(ast=[definition]) for var, definition in resolved.items()
|
||||
} | {var.name: SyntaxTree(ast=[definition]) for var, definition in unresolved.items()}
|
||||
|
||||
def resolve_partial(
|
||||
self,
|
||||
unresolvable: Optional[Set[str]] = None,
|
||||
output_filter: Optional[Set[str]] = None,
|
||||
) -> "Script":
|
||||
"""
|
||||
Updates the internal script to resolve as much as possible.
|
||||
"""
|
||||
out = self._resolve_partial(unresolvable=unresolvable, output_filter=output_filter)
|
||||
for var_name, definition in out.items():
|
||||
self._variables[var_name] = definition
|
||||
|
||||
return self
|
||||
|
||||
def resolve_partial_once(
|
||||
self, variable_definitions: Dict[str, SyntaxTree], unresolvable: Optional[Set[str]] = None
|
||||
) -> Dict[str, SyntaxTree]:
|
||||
"""
|
||||
Partially resolves the input variable definitions as much as possible.
|
||||
"""
|
||||
try:
|
||||
self.add_parsed(variable_definitions)
|
||||
return self._resolve_partial(
|
||||
unresolvable=unresolvable,
|
||||
output_filter=set(list(variable_definitions.keys())),
|
||||
)
|
||||
finally:
|
||||
for name in variable_definitions.keys():
|
||||
self._variables.pop(name, None)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import Any, Dict
|
||||
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
from abc import ABC
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Type
|
||||
from typing import Any, Dict, List, Type
|
||||
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import FutureResolvable
|
||||
from ytdl_sub.script.types.resolvable import NonHashable
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import ResolvableToJson
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
Argument,
|
||||
FutureResolvable,
|
||||
NonHashable,
|
||||
Resolvable,
|
||||
ResolvableToJson,
|
||||
)
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.variable_dependency import VariableDependency
|
||||
|
||||
|
|
|
|||
|
|
@ -1,38 +1,34 @@
|
|||
import functools
|
||||
from abc import ABC
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import Union
|
||||
from typing import Callable, Dict, List, Optional, Type, Union
|
||||
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.types.array import Array
|
||||
from ytdl_sub.script.types.array import UnresolvedArray
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import Boolean
|
||||
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
|
||||
from ytdl_sub.script.types.resolvable import FunctionType
|
||||
from ytdl_sub.script.types.resolvable import FutureResolvable
|
||||
from ytdl_sub.script.types.resolvable import Integer
|
||||
from ytdl_sub.script.types.resolvable import Lambda
|
||||
from ytdl_sub.script.types.resolvable import NamedCustomFunction
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgument
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgumentA
|
||||
from ytdl_sub.script.types.resolvable import ReturnableArgumentB
|
||||
from ytdl_sub.script.types.variable import FunctionArgument
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.array import Array, UnresolvedArray
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
Argument,
|
||||
Boolean,
|
||||
BuiltInFunctionType,
|
||||
FunctionType,
|
||||
FutureResolvable,
|
||||
Integer,
|
||||
Lambda,
|
||||
NamedCustomFunction,
|
||||
Resolvable,
|
||||
ReturnableArgument,
|
||||
ReturnableArgumentA,
|
||||
ReturnableArgumentB,
|
||||
)
|
||||
from ytdl_sub.script.types.variable import FunctionArgument, Variable
|
||||
from ytdl_sub.script.types.variable_dependency import VariableDependency
|
||||
from ytdl_sub.script.utils.exception_formatters import FunctionArgumentsExceptionFormatter
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||
from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
|
||||
from ytdl_sub.script.utils.type_checking import FunctionSpec
|
||||
from ytdl_sub.script.utils.type_checking import is_union
|
||||
from ytdl_sub.script.utils.exceptions import (
|
||||
UNREACHABLE,
|
||||
FunctionRuntimeException,
|
||||
RuntimeException,
|
||||
UserThrownRuntimeError,
|
||||
)
|
||||
from ytdl_sub.script.utils.type_checking import FunctionSpec, is_union
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -371,13 +367,6 @@ class BuiltInFunction(Function, BuiltInFunctionType):
|
|||
If the conditional partially resolvable enough to warrant evaluation,
|
||||
perform it here.
|
||||
"""
|
||||
if self.is_subset_of(
|
||||
variables=resolved_variables, custom_function_definitions=custom_functions
|
||||
):
|
||||
return self.resolve(
|
||||
resolved_variables=resolved_variables,
|
||||
custom_functions=custom_functions,
|
||||
)
|
||||
|
||||
if self.name == "if":
|
||||
maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve(
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
import itertools
|
||||
from abc import ABC
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Type
|
||||
from typing import Any, Dict, List, Type
|
||||
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import FutureResolvable
|
||||
from ytdl_sub.script.types.resolvable import Hashable
|
||||
from ytdl_sub.script.types.resolvable import NonHashable
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import ResolvableToJson
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
Argument,
|
||||
FutureResolvable,
|
||||
Hashable,
|
||||
NonHashable,
|
||||
Resolvable,
|
||||
ResolvableToJson,
|
||||
)
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.variable_dependency import VariableDependency
|
||||
from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException
|
||||
|
|
|
|||
|
|
@ -1,12 +1,7 @@
|
|||
import json
|
||||
from abc import ABC
|
||||
from abc import abstractmethod
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Generic
|
||||
from typing import List
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import Any, Generic, List, Type, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
NumericT = TypeVar("NumericT", bound=int | float)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,8 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from ytdl_sub.script.types.function import BuiltInFunction
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import String
|
||||
from ytdl_sub.script.types.resolvable import Argument, Resolvable, String
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.variable_dependency import VariableDependency
|
||||
|
||||
|
|
@ -55,6 +51,10 @@ class SyntaxTree(VariableDependency):
|
|||
custom_functions=custom_functions,
|
||||
)
|
||||
|
||||
# If no arguments, must be empty string
|
||||
if len(maybe_resolvable_values) == 0:
|
||||
return String(value="")
|
||||
|
||||
# Mimic the above resolve behavior
|
||||
if len(maybe_resolvable_values) > 1:
|
||||
return BuiltInFunction(name="concat", args=maybe_resolvable_values)
|
||||
|
|
|
|||
|
|
@ -1,24 +1,17 @@
|
|||
from abc import ABC
|
||||
from abc import abstractmethod
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict
|
||||
from typing import Iterable
|
||||
from typing import List
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import final
|
||||
from typing import Dict, Iterable, List, Set, Tuple, Type, TypeVar, final
|
||||
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
|
||||
from ytdl_sub.script.types.resolvable import FunctionType
|
||||
from ytdl_sub.script.types.resolvable import Lambda
|
||||
from ytdl_sub.script.types.resolvable import NamedCustomFunction
|
||||
from ytdl_sub.script.types.resolvable import ParsedCustomFunction
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.variable import FunctionArgument
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
Argument,
|
||||
BuiltInFunctionType,
|
||||
FunctionType,
|
||||
Lambda,
|
||||
NamedCustomFunction,
|
||||
ParsedCustomFunction,
|
||||
Resolvable,
|
||||
)
|
||||
from ytdl_sub.script.types.variable import FunctionArgument, Variable
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
|
||||
TypeT = TypeVar("TypeT")
|
||||
|
|
@ -44,7 +37,7 @@ class VariableDependency(ABC):
|
|||
output.append(arg)
|
||||
elif instance and isinstance(arg, ttype):
|
||||
output.append(arg)
|
||||
elif type(arg) == ttype: # pylint: disable=unidiomatic-typecheck
|
||||
elif type(arg) is ttype: # pylint: disable=unidiomatic-typecheck
|
||||
output.append(arg)
|
||||
|
||||
if isinstance(arg, VariableDependency):
|
||||
|
|
@ -279,8 +272,11 @@ class VariableDependency(ABC):
|
|||
if not isinstance(maybe_resolvable_args[-1], Resolvable):
|
||||
is_resolvable = False
|
||||
elif isinstance(arg, Variable):
|
||||
if arg not in resolved_variables:
|
||||
if arg in resolved_variables:
|
||||
maybe_resolvable_args[-1] = resolved_variables[arg]
|
||||
else:
|
||||
is_resolvable = False
|
||||
# Could be un unresolvable
|
||||
if arg in unresolved_variables:
|
||||
maybe_resolvable_args[-1] = unresolved_variables[arg]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
import sys
|
||||
from typing import List
|
||||
from typing import TypeVar
|
||||
from typing import List, TypeVar
|
||||
|
||||
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
|
||||
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments
|
||||
from ytdl_sub.script.utils.exceptions import UserException
|
||||
from ytdl_sub.script.utils.type_checking import FunctionSpec
|
||||
from ytdl_sub.script.utils.type_checking import is_union
|
||||
from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments, UserException
|
||||
from ytdl_sub.script.utils.type_checking import FunctionSpec, is_union
|
||||
|
||||
UserExceptionT = TypeVar("UserExceptionT", bound=UserException)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import re
|
||||
|
||||
from ytdl_sub.script.functions import Functions
|
||||
from ytdl_sub.script.utils.exceptions import InvalidFunctionName
|
||||
from ytdl_sub.script.utils.exceptions import InvalidVariableName
|
||||
from ytdl_sub.script.utils.exceptions import InvalidFunctionName, InvalidVariableName
|
||||
|
||||
_NAME_REGEX_VALIDATOR = re.compile(r"^[a-z][a-z0-9_]*$")
|
||||
|
||||
|
|
|
|||
|
|
@ -3,24 +3,20 @@ import inspect
|
|||
from dataclasses import dataclass
|
||||
from inspect import FullArgSpec
|
||||
from types import NoneType
|
||||
from typing import Callable
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import Union
|
||||
from typing import get_origin
|
||||
from typing import Callable, List, Optional, Type, TypeVar, Union, get_origin
|
||||
|
||||
from ytdl_sub.script.types.resolvable import Argument
|
||||
from ytdl_sub.script.types.resolvable import BuiltInFunctionType
|
||||
from ytdl_sub.script.types.resolvable import FutureResolvable
|
||||
from ytdl_sub.script.types.resolvable import Lambda
|
||||
from ytdl_sub.script.types.resolvable import LambdaReduce
|
||||
from ytdl_sub.script.types.resolvable import LambdaThree
|
||||
from ytdl_sub.script.types.resolvable import LambdaTwo
|
||||
from ytdl_sub.script.types.resolvable import NamedCustomFunction
|
||||
from ytdl_sub.script.types.resolvable import NamedType
|
||||
from ytdl_sub.script.types.resolvable import Resolvable
|
||||
from ytdl_sub.script.types.resolvable import (
|
||||
Argument,
|
||||
BuiltInFunctionType,
|
||||
FutureResolvable,
|
||||
Lambda,
|
||||
LambdaReduce,
|
||||
LambdaThree,
|
||||
LambdaTwo,
|
||||
NamedCustomFunction,
|
||||
NamedType,
|
||||
Resolvable,
|
||||
)
|
||||
from ytdl_sub.script.types.variable import Variable
|
||||
from ytdl_sub.script.utils.exceptions import UNREACHABLE
|
||||
|
||||
|
|
|
|||
|
|
@ -6,14 +6,12 @@ from ytdl_sub.config.config_validator import ConfigOptions
|
|||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
|
||||
from ytdl_sub.config.preset import Preset
|
||||
from ytdl_sub.config.preset_options import OutputOptions
|
||||
from ytdl_sub.config.preset_options import YTDLOptions
|
||||
from ytdl_sub.config.validators.variable_validation import VariableValidation
|
||||
from ytdl_sub.config.preset_options import OutputOptions, YTDLOptions
|
||||
from ytdl_sub.config.validators.variable_validation import ResolutionLevel, VariableValidation
|
||||
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
|
||||
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
|
||||
from ytdl_sub.utils.exceptions import SubscriptionPermissionError
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileHandlerTransactionLog
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
from ytdl_sub.utils.yaml import dump_yaml
|
||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||
|
|
@ -79,7 +77,7 @@ class BaseSubscription(ABC):
|
|||
)
|
||||
|
||||
# Validate after adding the subscription name
|
||||
self._validated_dict = VariableValidation(
|
||||
_ = VariableValidation(
|
||||
overrides=self.overrides,
|
||||
downloader_options=self.downloader_options,
|
||||
output_options=self.output_options,
|
||||
|
|
@ -254,12 +252,22 @@ class BaseSubscription(ABC):
|
|||
-------
|
||||
Subscription in yaml format
|
||||
"""
|
||||
return self._preset_options.yaml
|
||||
return self._preset_options.yaml(subscription_only=False)
|
||||
|
||||
def resolved_yaml(self) -> str:
|
||||
def resolved_yaml(self, resolution_level: int = ResolutionLevel.RESOLVE) -> str:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Human-readable, condensed YAML definition of the subscription.
|
||||
"""
|
||||
return dump_yaml(self._validated_dict)
|
||||
if resolution_level == ResolutionLevel.ORIGINAL:
|
||||
return self._preset_options.yaml(subscription_only=True)
|
||||
|
||||
out = VariableValidation(
|
||||
overrides=self.overrides,
|
||||
downloader_options=self.downloader_options,
|
||||
output_options=self.output_options,
|
||||
plugins=self.plugins,
|
||||
resolution_level=resolution_level,
|
||||
).ensure_proper_usage(partial_resolve_formatters=True)
|
||||
return dump_yaml(out)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
import copy
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from mergedeep import mergedeep
|
||||
|
||||
|
|
@ -143,7 +140,6 @@ class Subscription(SubscriptionDownload):
|
|||
}
|
||||
|
||||
for subscription_name, subscription_object in subscriptions_dicts.items():
|
||||
|
||||
# Hard-override subscriptions here
|
||||
mergedeep.merge(
|
||||
subscription_object,
|
||||
|
|
|
|||
|
|
@ -4,15 +4,15 @@ import os
|
|||
import shutil
|
||||
from abc import ABC
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
from ytdl_sub.config.plugin.plugin import Plugin
|
||||
from ytdl_sub.config.plugin.plugin import SplitPlugin
|
||||
from ytdl_sub.config.plugin.plugin import Plugin, SplitPlugin
|
||||
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
|
||||
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
|
||||
from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader
|
||||
from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions
|
||||
from ytdl_sub.downloaders.info_json.info_json_downloader import (
|
||||
InfoJsonDownloader,
|
||||
InfoJsonDownloaderOptions,
|
||||
)
|
||||
from ytdl_sub.downloaders.source_plugin import SourcePlugin
|
||||
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
|
||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||
|
|
@ -22,9 +22,7 @@ from ytdl_sub.subscriptions.base_subscription import BaseSubscription
|
|||
from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions
|
||||
from ytdl_sub.utils.datetime import to_date_range
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
from ytdl_sub.utils.file_handler import FileHandler
|
||||
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
from ytdl_sub.utils.file_handler import FileHandler, FileHandlerTransactionLog, FileMetadata
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
|
||||
logger: logging.Logger = Logger.get()
|
||||
|
|
@ -155,8 +153,8 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
|||
keep_max_files: Optional[int] = None
|
||||
if self.output_options.keep_max_files:
|
||||
# validated it can be cast to int within the validator
|
||||
keep_max_files = int(
|
||||
self.overrides.apply_formatter(self.output_options.keep_max_files)
|
||||
keep_max_files = self.overrides.apply_formatter(
|
||||
self.output_options.keep_max_files, expected_type=int
|
||||
)
|
||||
|
||||
if date_range_to_keep or self.output_options.keep_max_files is not None:
|
||||
|
|
|
|||
|
|
@ -1,22 +1,20 @@
|
|||
import copy
|
||||
import dataclasses
|
||||
from abc import ABC
|
||||
from abc import abstractmethod
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import final
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Optional, final
|
||||
|
||||
from ytdl_sub.config.config_file import ConfigFile
|
||||
from ytdl_sub.config.overrides import Overrides
|
||||
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
|
||||
from ytdl_sub.utils.script import ScriptUtils
|
||||
from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
|
||||
from ytdl_sub.validators.validators import DictValidator
|
||||
from ytdl_sub.validators.validators import LiteralDictValidator
|
||||
from ytdl_sub.validators.validators import StringListValidator
|
||||
from ytdl_sub.validators.validators import StringValidator
|
||||
from ytdl_sub.validators.validators import Validator
|
||||
from ytdl_sub.validators.validators import (
|
||||
DictValidator,
|
||||
LiteralDictValidator,
|
||||
StringListValidator,
|
||||
StringValidator,
|
||||
Validator,
|
||||
)
|
||||
|
||||
|
||||
class SubscriptionOutput(Validator, ABC):
|
||||
|
|
|
|||
|
|
@ -1,9 +1,5 @@
|
|||
from pathlib import Path
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Type
|
||||
from typing import TypeVar
|
||||
from typing import Dict, List, Optional, Type, TypeVar
|
||||
|
||||
from yt_dlp import match_filter_func
|
||||
|
||||
|
|
@ -15,9 +11,7 @@ from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
|
|||
from ytdl_sub.plugins.chapters import ChaptersPlugin
|
||||
from ytdl_sub.plugins.file_convert import FileConvertPlugin
|
||||
from ytdl_sub.plugins.format import FormatPlugin
|
||||
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
|
||||
from ytdl_sub.plugins.match_filters import combine_filters
|
||||
from ytdl_sub.plugins.match_filters import default_filters
|
||||
from ytdl_sub.plugins.match_filters import MatchFiltersPlugin, combine_filters, default_filters
|
||||
from ytdl_sub.plugins.subtitles import SubtitlesPlugin
|
||||
from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin
|
||||
from ytdl_sub.utils.ffmpeg import FFMPEG
|
||||
|
|
@ -94,8 +88,8 @@ class SubscriptionYTDLOptions:
|
|||
self._enhanced_download_archive.working_ytdl_file_path
|
||||
)
|
||||
if self._preset.output_options.keep_max_files:
|
||||
keep_max_files = int(
|
||||
self._overrides.apply_formatter(self._preset.output_options.keep_max_files)
|
||||
keep_max_files = self._overrides.apply_formatter(
|
||||
self._preset.output_options.keep_max_files, expected_type=int
|
||||
)
|
||||
if keep_max_files > 0:
|
||||
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ import threading
|
|||
import time
|
||||
from json import JSONDecodeError
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Optional, Set
|
||||
|
||||
from ytdl_sub.utils.logger import Logger
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
import re
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Tuple
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from ytdl_sub.entries.entry import Entry
|
||||
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||
from ytdl_sub.entries.entry import Entry, ytdl_sub_chapters_from_comments
|
||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES, VariableDefinitions
|
||||
from ytdl_sub.utils.file_handler import FileMetadata
|
||||
|
||||
v: VariableDefinitions = VARIABLES
|
||||
|
||||
|
||||
class Timestamp:
|
||||
|
||||
# Captures the following formats:
|
||||
# 0:00 title
|
||||
# 00:00 title
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import subprocess
|
||||
import tempfile
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from ytdl_sub.utils.chapters import Chapters
|
||||
from ytdl_sub.utils.exceptions import ValidationException
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue