diff --git a/.gitignore b/.gitignore index e18f90d3..5e9f1aad 100644 --- a/.gitignore +++ b/.gitignore @@ -146,8 +146,11 @@ docker/testing/volumes .local/ .ytdl-sub-working-directory +.ytdl-sub-lock ffmpeg.exe ffprobe.exe -tools/docgen/out \ No newline at end of file +tools/docgen/out + +prof/ diff --git a/README.md b/README.md index 3ba8ab5f..d9203837 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ __preset__: # Pass any arg directly to yt-dlp's Python API ytdl_options: - cookiefile: "/config/cookie.txt" + cookiefile: "/config/ytdl-sub-configs/cookie.txt" ################################################################### # TV Show Presets. Can replace Plex with Plex/Jellyfin/Emby/Kodi diff --git a/docker/Dockerfile b/docker/Dockerfile index 558f75dc..adc3e443 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,8 +46,10 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \ phantomjs --version && \ cd -; \ fi && \ - # Install ytdl-sub, ensure it is installed properly - python3 -m pip install --break-system-packages --no-cache-dir ytdl_sub-*.whl && \ + # Configure pip globally + echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \ + # Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly + python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \ ytdl-sub -h && \ # Delete unneeded packages after install rm ytdl_sub-*.whl && \ diff --git a/docker/Dockerfile.gui b/docker/Dockerfile.gui index 11a31d6c..e78266ce 100644 --- a/docker/Dockerfile.gui +++ b/docker/Dockerfile.gui @@ -65,8 +65,10 @@ RUN mkdir -p /config && \ # Install Deno, required for YouTube downloads curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \ deno --help && \ - # Install ytdl-sub, ensure it is installed properly - python3 -m pip install --no-cache-dir --break-system-packages ytdl_sub-*.whl && \ + # Configure pip globally + echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \ + # Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly + python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \ ytdl-sub -h && \ # Delete unneeded packages after install rm ytdl_sub-*.whl && \ diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index b2de4019..5ba76866 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -68,8 +68,10 @@ RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \ # Install Deno, required for YouTube downloads curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \ deno --help && \ - # Install ytdl-sub, ensure it is installed properly - python3 -m pip install --no-cache-dir --break-system-packages ytdl_sub-*.whl && \ + # Configure pip globally + echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \ + # Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly + python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \ ytdl-sub -h && \ # Delete unneeded packages after install rm ytdl_sub-*.whl && \ diff --git a/docker/root/custom-cont-init.d/defaults b/docker/root/custom-cont-init.d/defaults old mode 100644 new mode 100755 index 0dcbc36d..af6b8350 --- a/docker/root/custom-cont-init.d/defaults +++ b/docker/root/custom-cont-init.d/defaults @@ -17,25 +17,25 @@ echo "Starting ytdl-sub..." echo "alias ls='ls --color=auto'" > /config/.bashrc && \ echo "cd ." >> /config/.bashrc +# always create empty cron log file on start +echo "" > "$LOGS_TO_STDOUT" + # permissions chown -R ${PUID:-abc}:${PGID:-abc} \ /config -# always create empty cron log file on start -echo "" > "$LOGS_TO_STDOUT" - # update command reference: # https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip if [ "$UPDATE_YT_DLP_ON_START" == "stable" ] ; then echo "UPDATE_YT_DLP_ON_START is set to stable, attempting to update to a new stable version of yt-dlp if it exists." - python3 -m pip install -U "yt-dlp[default]" --break-system-packages --root-user-action=ignore + python3 -m pip install -U "yt-dlp[default]" elif [ "$UPDATE_YT_DLP_ON_START" == "nightly" ] ; then echo "UPDATE_YT_DLP_ON_START is set to nightly, attempting to update to the latest nightly version of yt-dlp." - python3 -m pip install -U --pre "yt-dlp[default]" --root-user-action=ignore + python3 -m pip install -U --pre "yt-dlp[default]" elif [ "$UPDATE_YT_DLP_ON_START" == "master" ] ; then echo "UPDATE_YT_DLP_ON_START is set to master, pulling yt-dlp's latest commit for install." - python3 -m pip install -U pip hatchling wheel --root-user-action=ignore - python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz" --root-user-action=ignore + python3 -m pip install -U pip hatchling wheel + python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz" else echo "UPDATE_YT_DLP_ON_START is not set, using packaged version." fi @@ -47,6 +47,8 @@ if [ "$CRON_SCHEDULE" != "" ] ; then # create cron script wrapper echo '#!/bin/bash' > "$CRON_WRAPPER_SCRIPT" + # Echo commands for easier user debugging: + echo "set -x" >> "$CRON_WRAPPER_SCRIPT" echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "$CRON_WRAPPER_SCRIPT" echo "cd \"$DEFAULT_WORKSPACE\"" >> "$CRON_WRAPPER_SCRIPT" echo ". \"$CRON_SCRIPT\" >> \"$LOGS_TO_STDOUT\" 2>&1" >> "$CRON_WRAPPER_SCRIPT" diff --git a/docker/root/defaults/cron b/docker/root/defaults/cron index 6fdbea71..0deb3568 100644 --- a/docker/root/defaults/cron +++ b/docker/root/defaults/cron @@ -1,4 +1,15 @@ -echo "Beginning cron job..." - # Place your ytdl-sub command(s) here. -# This script is executed in the same relative path as this file. \ No newline at end of file +# +# This script is executed in the same directory as this file which also contains the +# default `./config.yaml` and `./subscriptions.yaml`, so you don't need to use the +# `--config` CLI option or pass a `SUBPATH` to the `$ ytdl-sub sub` sub-command. +# +# Test your configuration and subscriptions carefully before automating downloads to +# prevent triggering throttles or bans: +# +# https://ytdl-sub.readthedocs.io/en/latest/guides/getting_started/downloading.html +# +# Once you've tested your configuration and you're ready to download entries unattended, +# remove the next line and un-comment the following line: +echo "WARNING: Read /config/ytdl-sub-configs/cron and modify to automate downloads." +# ytdl-sub sub diff --git a/docker/root/defaults/subscriptions.yaml b/docker/root/defaults/subscriptions.yaml index 98535e27..4fd02a39 100644 --- a/docker/root/defaults/subscriptions.yaml +++ b/docker/root/defaults/subscriptions.yaml @@ -20,7 +20,7 @@ __preset__: # Pass any arg directly to yt-dlp's Python API # ytdl_options: - # cookiefile: "/config/cookie.txt" + # cookiefile: "/config/ytdl-sub-configs/cookie.txt" ################################################################### # Subscriptions nested under this will use the diff --git a/docs/source/conf.py b/docs/source/conf.py index f754815a..c226b589 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -7,7 +7,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = "ytdl-sub" -copyright = "2024, Jesse Bannon" +copyright = "2026, Jesse Bannon" author = "Jesse Bannon" release = "" @@ -15,10 +15,8 @@ release = "" # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "sphinx.ext.autodoc", "sphinx.ext.autosectionlabel", "sphinx.ext.extlinks", - "sphinx.ext.napoleon", "sphinx_copybutton", "sphinx_design", ] @@ -70,19 +68,3 @@ extlinks = { "lsio-gh": ("https://github.com/linuxserver/%s", "%s image"), "ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"), } - -# -- Options for autodoc ---------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration - -# Automatically extract typehints when specified and place them in -# descriptions of the relevant function/method. -autodoc_default_options = { - "autodoc_typehints_format": "short", - "autodoc_class_signature": "separated", - "add_module_names": False, - # "add_class_names": False, -} - -python_use_unqualified_type_names = True -napoleon_numpy_docstring = True -napoleon_use_rtype = False diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst index 6955c386..ccfa901b 100644 --- a/docs/source/config_reference/config_yaml.rst +++ b/docs/source/config_reference/config_yaml.rst @@ -1,7 +1,13 @@ -================== +.. + WARNING: This RST file is generated from docstrings in: + The respective function docstrings within ytdl_sub/config/config_validator.py + In order to make a change to this file, edit the respective docstring + and run `make docs`. This will automatically sync the Python RST-based + docstrings into this file. If the docstrings and RST file are out of sync, + it will fail TestDocGen tests in GitHub CI. + Configuration File ================== - ytdl-sub is configured using a ``config.yaml`` file. The ``config.yaml`` is made up of two sections: @@ -11,113 +17,118 @@ The ``config.yaml`` is made up of two sections: configuration: presets: -You can jump to any section and subsection of the config using the navigation section to -the left. Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. -If you wish to represent paths like Windows, you will need to -``C:\\double\\bashslash\\paths`` in order to escape the backslash character. - - -configuration -------------- - -The ``configuration`` section contains app-wide configs applied to all presets and -subscriptions. - -.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions() - :members: - :member-order: bysource - :exclude-members: subscription_value, persist_logs, experimental - -persist_logs -~~~~~~~~~~~~ - -Without this key, ``ytdl-sub`` only prints output to it's ``stdout`` and ``stderr``. If -your configuration includes the ``persist_logs:`` key, then ``ytdl-sub`` also writes log -files to disk. - -.. warning:: - - The log files grow rapidly if ``keep_successful_logs:`` is ``true``, the default, and - may fill up disk space. Set ``keep_successful_logs: false`` or prune the log files - regularly. - -For example: +If you prefer to use a Windows backslash, note that it must have +``C:\\double\\bashslash\\paths`` in order to escape the backslash character. This is due +to it being a YAML escape character. .. code-block:: yaml configuration: + dl_aliases: + mv: "--preset music_video" + u: "--download.url" + + experimental: + enable_update_with_info_json: True + + ffmpeg_path: "/usr/bin/ffmpeg" + ffprobe_path: "/usr/bin/ffprobe" + + file_name_max_bytes: 255 + lock_directory: "/tmp" + persist_logs: - logs_directory: "/path/to/log/directory" + keep_successful_logs: True + logs_directory: "/var/log/ytdl-sub-logs" -.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator() - :members: - :member-order: bysource + umask: "022" + working_directory: ".ytdl-sub-working-directory" +dl_aliases +---------- +.. _dl_aliases: -presets -------- - -Each key under ``presets:`` defines a `formula` for how to format downloaded media and -metadata. The key is the name of the preset and the value is a mapping that defines the -preset. - -.. note:: - - The ``presets:`` key at the top of the configuration file contains multiple - user-defined presets, but *each preset* itself may include a ``preset:`` key that - defines *that preset's* base presets. For example: - - .. code-block:: yaml - - presets: - Foo Preset: - preset: - - "Jellyfin TV Show by Date" - - "Only Recent" - -preset -~~~~~~ - -Presets support inheritance by defining one or more parent presets: +Alias definitions to shorten :ref:`dl arguments `. For example, .. code-block:: yaml - presets: - custom_preset: - ... - parent_preset: - ... - child_preset: - preset: - - "parent_preset" + configuration: + dl_aliases: + mv: "--preset music_video" + u: "--download.url" -In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``. -Use parent presets where possible to reduce duplicate yaml definitions. +Simplifies -Presets also support inheritance from multiple presets: +.. code-block:: bash -.. code-block:: yaml + ytdl-sub dl --preset "Jellyfin Music Videos" --download.url "youtube.com/watch?v=a1b2c3" - child_preset: - preset: - - "custom_preset" - - "parent_preset" +to -In this example, ``child_preset`` will inherit all fields from ``custom_preset`` and -``parent_preset`` in that order. The bottom-most preset has the highest priority. More -specifically, presets are merged using `mergedeep`_ via `a TYPESAFE_ADDITIVE merge`_, -which means: +.. code-block:: bash -- if two conflicting keys arent lists or mappings, overwrite the higher priority one -- otherwise, combine then re-evaluate + ytdl-sub dl --mv --u "youtube.com/watch?v=a1b2c3" -If you are only inheriting from one preset, using a single string instead of a list is -valid, for example ``preset: "parent_preset"``, but we recommend always using a list for -consistent readability between presets. +experimental +------------ +Experimental flags reside under the ``experimental`` key. -.. _`mergedeep`: - https://mergedeep.readthedocs.io/en/latest/ -.. _`a TYPESAFE_ADDITIVE merge`: - https://mergedeep.readthedocs.io/en/latest/index.html#merge-strategies +``enable_update_with_info_json`` + +Enables modifying subscription files using info.json files using the argument +``--update-with-info-json``. This feature is still being tested and has the ability to +destroy files. Ensure you have a full backup before usage. You have been warned! + +ffmpeg_path +----------- +Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux, +``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows. + +ffprobe_path +------------ +Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux, +``./ffprobe.exe`` in the same directory as ytdl-sub for Windows. + +file_name_max_bytes +------------------- +Max file name size in bytes. Most OS's typically default to 255 bytes. + +lock_directory +-------------- +The directory to temporarily store file locks, which prevents multiple instances +of ``ytdl-sub`` from running. Note that file locks do not work on +network-mounted directories. Ensure that this directory resides on the host +machine. Defaults to ``/tmp``. + +persist_logs +------------ +By default, no logs are persisted. Specifying this key will enable persisted logs. The following +options are available. + +``keep_successful_logs`` + +Defaults to ``True``. When this key is ``False``, only write log files for failed +subscriptions. + +``logs_directory`` + +Required field. Write log files to this directory with names like +``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. + +umask +----- +Umask in octal format to apply to every created file. Defaults to ``022``. + +working_directory +----------------- +The directory to temporarily store downloaded files before moving them into their final +directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory +that ytdl-sub is invoked from. + +Presets +======= +Custom presets are defined in this section. Refer to the +:ref:`Getting Started Guide` +on how to configure. diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 74fbe988..75496323 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -28,7 +28,6 @@ Extracts audio from a video file. The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, opus, vorbis, wav, and best to grab the best possible format at runtime. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -37,7 +36,6 @@ Extracts audio from a video file. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``quality`` :expected type: Float @@ -45,7 +43,6 @@ Extracts audio from a video file. Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. - ---------------------------------------------------------------------------------------------------- chapters @@ -84,14 +81,12 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to False. If chapters do not exist in the video/description itself, attempt to scrape comments to find the chapters. - ``embed_chapters`` :expected type: Optional[Boolean] :description: Defaults to True. Embed chapters into the file. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -100,7 +95,6 @@ chapters and remove specific ones. Can also remove chapters using regex. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``force_key_frames`` :expected type: Optional[Boolean] @@ -108,7 +102,6 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to False. Force keyframes at cuts when removing sections. This is slow due to needing a re-encode, but the resulting video may have fewer artifacts around the cuts. - ``remove_chapters_regex`` :expected type: Optional[List[RegexString] @@ -116,7 +109,6 @@ chapters and remove specific ones. Can also remove chapters using regex. List of regex patterns to match chapter titles against and remove them from the entry. - ``remove_sponsorblock_categories`` :expected type: Optional[List[String]] @@ -125,7 +117,6 @@ chapters and remove specific ones. Can also remove chapters using regex. categories that are specified in ``sponsorblock_categories`` or "all", which removes everything specified in ``sponsorblock_categories``. - ``sponsorblock_categories`` :expected type: Optional[List[String]] @@ -134,7 +125,6 @@ chapters and remove specific ones. Can also remove chapters using regex. "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", "poi_highlight", or "all" to include all categories. - ---------------------------------------------------------------------------------------------------- date_range @@ -169,14 +159,12 @@ intended download files. :description: Only download videos after or on this datetime, inclusive. - ``before`` :expected type: Optional[OverridesFormatter] :description: Only download videos only before this datetime, not inclusive. - ``breaks`` :expected type: Optional[OverridesFormatter] @@ -184,7 +172,6 @@ intended download files. Toggle to enable breaking subsequent metadata downloads if an entry's upload date is out of range. Defaults to True. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -193,7 +180,6 @@ intended download files. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``type`` :expected type: Optional[OverridesFormatter] @@ -201,7 +187,6 @@ intended download files. Which type of date to use. Must be either ``upload_date`` or ``release_date``. Defaults to ``upload_date``. - ---------------------------------------------------------------------------------------------------- download @@ -305,7 +290,6 @@ Also supports custom ffmpeg conversions: - Video: avi, flv, mkv, mov, mp4, webm - Audio: aac, flac, mp3, m4a, opus, vorbis, wav - ``convert_with`` :expected type: Optional[String] @@ -314,7 +298,6 @@ Also supports custom ffmpeg conversions: yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -323,7 +306,6 @@ Also supports custom ffmpeg conversions: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``ffmpeg_post_process_args`` :expected type: Optional[OverridesFormatter] @@ -336,7 +318,6 @@ Also supports custom ffmpeg conversions: The output file will use the extension specified in ``convert_to``. Post-processing args can still be set with ``convert_with`` set to ``yt-dlp``. - ---------------------------------------------------------------------------------------------------- filter_exclude @@ -472,7 +453,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: OverridesBooleanFormatterValidator @@ -481,14 +461,12 @@ with a ``.nfo`` extension. You can add any values into the NFO. emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -501,7 +479,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. - ``tags`` :expected type: NfoTags @@ -538,7 +515,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. Comedy Drama - ---------------------------------------------------------------------------------------------------- output_directory_nfo_tags @@ -570,7 +546,6 @@ Usage: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: OverridesBooleanFormatterValidator @@ -579,14 +554,12 @@ Usage: emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -599,7 +572,6 @@ Usage: - ``tags`` :expected type: NfoTags @@ -634,7 +606,6 @@ Usage: Comedy Drama - ---------------------------------------------------------------------------------------------------- output_options @@ -669,7 +640,6 @@ Defines where to output files and thumbnails after all post-processing has compl The file name to store a subscriptions download archive placed relative to the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` - ``file_name`` :expected type: EntryFormatter @@ -677,7 +647,6 @@ Defines where to output files and thumbnails after all post-processing has compl The file name for the media file. This can include directories such as ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. - ``info_json_name`` :expected type: Optional[EntryFormatter] @@ -686,7 +655,6 @@ Defines where to output files and thumbnails after all post-processing has compl as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output directory. Can be set to empty string or `null` to disable info json writes. - ``keep_files_after`` :expected type: Optional[OverridesFormatter] @@ -698,7 +666,6 @@ Defines where to output files and thumbnails after all post-processing has compl files after ``19000101``, which implies all files. Can be used in conjunction with ``keep_max_files``. - ``keep_files_before`` :expected type: Optional[OverridesFormatter] @@ -710,7 +677,6 @@ Defines where to output files and thumbnails after all post-processing has compl files before ``now``, which implies all files. Can be used in conjunction with ``keep_max_files``. - ``keep_files_date_eval`` :expected type: str @@ -720,7 +686,6 @@ Defines where to output files and thumbnails after all post-processing has compl perform evaluation for keep_files_before/after and keep_max_files. Defaults to the entry's upload_date_standardized variable. - ``keep_max_files`` :expected type: Optional[OverridesFormatter] @@ -730,7 +695,6 @@ Defines where to output files and thumbnails after all post-processing has compl Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. - ``maintain_download_archive`` :expected type: Optional[Boolean] @@ -745,7 +709,6 @@ Defines where to output files and thumbnails after all post-processing has compl Defaults to False. - ``migrated_download_archive_name`` :expected type: Optional[OverridesFormatter] @@ -755,13 +718,19 @@ Defines where to output files and thumbnails after all post-processing has compl name first, and fallback to ``download_archive_name``. It will always save to this file and remove the original ``download_archive_name``. - ``output_directory`` :expected type: OverridesFormatter :description: The output directory to store all media files downloaded. +``preserve_mtime`` + +:expected type: Optional[Boolean] +:description: + Preserve the video's original upload time as the file modification time. + When True, sets the file's mtime to match the video's upload_date from + yt-dlp metadata. Defaults to False. ``thumbnail_name`` @@ -771,7 +740,6 @@ Defines where to output files and thumbnails after all post-processing has compl as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output directory. Can be set to empty string or `null` to disable thumbnail writes. - ---------------------------------------------------------------------------------------------------- overrides @@ -836,7 +804,6 @@ used with no modifications. If a file has no chapters and is set to "pass", then ``chapter_title`` is set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. - ---------------------------------------------------------------------------------------------------- square_thumbnail @@ -883,7 +850,6 @@ Usage: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: OverridesBooleanFormatterValidator @@ -892,14 +858,12 @@ Usage: emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -912,7 +876,6 @@ Usage: - ``tags`` :expected type: NfoTags @@ -926,7 +889,6 @@ Usage: My custom season name! - ---------------------------------------------------------------------------------------------------- subtitles @@ -954,7 +916,6 @@ It will set the respective language to the correct subtitle file. :description: Defaults to False. Whether to allow auto generated subtitles. - ``embed_subtitles`` :expected type: Optional[Boolean] @@ -962,7 +923,6 @@ It will set the respective language to the correct subtitle file. Defaults to False. Whether to embed the subtitles into the video file. Note that webm files can only embed "vtt" subtitle types. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -971,7 +931,6 @@ It will set the respective language to the correct subtitle file. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``languages`` :expected type: Optional[List[String]] @@ -979,7 +938,6 @@ It will set the respective language to the correct subtitle file. Language code(s) to download for subtitles. Supports a single or list of multiple language codes. Defaults to only "en". - ``languages_required`` :expected type: Optional[List[String]] @@ -987,7 +945,6 @@ It will set the respective language to the correct subtitle file. Language code(s) that are required to be present for downloads to continue. If missing, ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles. - ``subtitles_name`` :expected type: Optional[EntryFormatter] @@ -997,14 +954,12 @@ It will set the respective language to the correct subtitle file. and will be placed in the output directory. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - ``subtitles_type`` :expected type: Optional[String] :description: Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". - ---------------------------------------------------------------------------------------------------- throttle_protection @@ -1045,14 +1000,12 @@ Range min and max values support static override variables within their definiti this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``max_downloads_per_subscription`` :expected type: Optional[Range] :description: Number of downloads to perform per subscription. - ``sleep_per_download_s`` :expected type: Optional[Range] @@ -1060,7 +1013,6 @@ Range min and max values support static override variables within their definiti Number in seconds to sleep between each download. Does not include time it takes for ytdl-sub to perform post-processing. - ``sleep_per_request_s`` :expected type: Optional[Range] @@ -1070,14 +1022,12 @@ Range min and max values support static override variables within their definiti download for the entry. Also, yt-dlp only supports a single value at this time for this, so will always use the max value. - ``sleep_per_subscription_s`` :expected type: Optional[Range] :description: Number in seconds to sleep between each subscription. - ``subscription_download_probability`` :expected type: Optional[Float] @@ -1086,7 +1036,6 @@ Range min and max values support static override variables within their definiti recommended to set if you run ytdl-sub in a cron-job, that way you are statistically guaranteed over time to eventually download the subscription. - ---------------------------------------------------------------------------------------------------- video_tags diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index d6064e0f..e2a200ef 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -521,6 +521,13 @@ pow :description: ``**`` operator. Returns the exponential of the base and exponent value. +range +~~~~~ +:spec: ``range(end: Integer, start: Optional[Integer], step: Optional[Integer]) -> Array`` + +:description: + Returns the desired range of Integers in the form of an Array. + sub ~~~ :spec: ``sub(values: Numeric, ...) -> Numeric`` diff --git a/docs/source/config_reference/subscription_yaml.rst b/docs/source/config_reference/subscription_yaml.rst index 298456f8..5d1ad0fb 100644 --- a/docs/source/config_reference/subscription_yaml.rst +++ b/docs/source/config_reference/subscription_yaml.rst @@ -31,7 +31,7 @@ supply a cookies file path. # Directly set plugin options: ytdl_options: - cookiefile: "/config/cookie.txt" + cookiefile: "/config/ytdl-sub-configs/cookie.txt" Layout ------ diff --git a/docs/source/deprecation_notices.rst b/docs/source/deprecation_notices.rst index 9fc4b48d..a75240c2 100644 --- a/docs/source/deprecation_notices.rst +++ b/docs/source/deprecation_notices.rst @@ -1,6 +1,15 @@ Deprecation Notices =================== +Dec 2025 +-------- + +Override variables names can no longer be plugin names, to avoid the common pitfall of +defining a plugin underneath ``overrides``. + +In the past, there was usage of a ``date_range`` override variable in a few example configs +that complimented the ``Only Recent`` preset. This overrride variable usage needs to be +replaced with ``only_recent_date_range``. Sep 2024 -------- diff --git a/docs/source/guides/getting_started/automating_downloads.rst b/docs/source/guides/getting_started/automating_downloads.rst index 80e86902..094a5f1f 100644 --- a/docs/source/guides/getting_started/automating_downloads.rst +++ b/docs/source/guides/getting_started/automating_downloads.rst @@ -1,51 +1,80 @@ -Automating Downloads -==================== +Automating +========== -:ref:`Guide for Docker and Unraid Containers ` +Automate downloading your subscriptions by running the :ref:`'sub' sub-command +` periodically. There are various tools that can run +commands on a schedule you may use any of them that work with your installation +method. Most users use `cron`_ in `Docker containers `_. -:ref:`Guide for Linux ` - -:ref:`Guide for Windows ` - -.. _cron scheduling syntax: https://crontab.guru/#0_*/6_*_*_* - - -.. _docker-unraid-setup: Docker and Unraid ----------------- -Cron is preconfigured in every ytdl-sub docker container. Enable by adding the following -ENV variables to your docker setup. +:doc:`The 'ytdl-sub' Docker container images <../install/docker>` provide optional cron +support. Enable cron support by setting `a cron schedule`_ in the ``CRON_SCHEDULE`` +environment variable: .. code-block:: yaml + :caption: ./compose.yaml + :emphasize-lines: 4 services: ytdl-sub: environment: - - CRON_SCHEDULE="0 */6 * * *" - - CRON_RUN_ON_START=false + CRON_SCHEDULE: "0 */6 * * *" + # WARNING: See "Getting Started" -> "Automating" docs regarding throttles/bans: + # CRON_RUN_ON_START: false +Then recreate the container to apply the change and start it to generate the default +``/config/ytdl-sub-configs/cron`` script. Read the comments in that script and edit as +appropriate. -- ``CRON_SCHEDULE`` follows the standard `cron scheduling syntax`_. The above value will - run the script once every 6 hours. -- ``CRON_RUN_ON_START`` toggles whether to run your cron script on container start in - addition to the cron schedule. +The container cron wrapper script will write output from the cron job to +``/config/ytdl-sub-configs/.cron.log``. The default image ``ENTRYPOINT`` will ``$ tail +...`` that file so you can monitor the cron job in the container's output and thus also +in the Docker logs. -The cron script will reside in the main directory with the file name ``cron``. Cron -logs should show when viewing the Docker logs. +You may also set the ``CRON_RUN_ON_START`` environment variable to ``true`` to have the +image run your cron script whenever the container starts in addition to the cron +schedule. + +.. warning:: + + Using ``CRON_RUN_ON_START`` may cause your cron script to run too often and may + trigger throttles and bans. When enabled, your cron script will run *whenever* the + container starts including when the host reboots, when ``# dockerd`` restarts such as + when upgrading Docker itself, when a new image is pulled, when something applies + Compose changes, etc.. This may result in running ``ytdl-sub`` right before or after + the next cron scheduled run. .. _linux-setup: -Linux ------ -Must configure crontab manually, like so: +Linux, Mac OS X, BSD, or other UNIX's +------------------------------------- + +For installations on systems already running ``# crond``, you can also use cron to run +``ytdl-sub`` periodically. Write a script to run ``ytdl-sub`` in the cron job. Be sure +the script changes to the same directory as your configuration and uses the full path to +``ytdl-sub``: .. code-block:: shell + :caption: ~/.local/bin/ytdl-sub-cron + :emphasize-lines: 2,3 - crontab -e - 0 */6 * * * /config/run_cron + #!/bin/bash + cd "~/.config/ytdl-sub/" + ~/.local/bin/ytdl-sub --dry-run sub -o '--ytdl_options.max_downloads 3' |& + tee -a "~/.local/state/ytdl-sub/.cron.log" + +Then tell ``# crond`` when to run the script: + +.. code-block:: console + + echo "0 */6 * * * ${HOME}/.local/bin/ytdl-sub-cron" | crontab "-" + +Remove the ``--dry-run`` and ``-o ...`` CLI options from your cron script when you've +tested your configuration and you're ready to download entries unattended. .. _windows-setup: @@ -53,17 +82,51 @@ Must configure crontab manually, like so: Windows ------- -To be tested (please contact code owner or join the discord server if you can test this -out for us) +For most Windows users, the best way to run commands periodically is `the Task +Scheduler`_: -.. code-block:: powershell +.. attention:: - ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml + These instructions are untested. Use at your own risk. If you use them, whether they + work or not, please let us know how it went in `a support post in Discord`_ or `a new + GitHub issue`_. + +#. Open the Task Scheduler app. + +#. Click ``Create Basic Task`` at the top of the right sidebar. + +#. Set all the fields as appropriate until you get to the ``Action``... + +#. For the ``Action``, select ``Start a program``... + +#. Click ``Browse...`` to the installed ``ytdl-sub.exe`` executable... + +#. Add CLI arguments to ``Add arguments (optional):``, for example ``--dry-run sub -o + '--ytdl_options.max_downloads 3'``... + +#. Set ``Start in (optional):`` to the directory containing your configuration. + +#. Finish the rest of the ``Create Basic Task`` wizard. Next Steps ---------- -Once you have a significant quantity of subscriptions or have use cases not served using -:doc:`YAML keys and the special characters <./subscriptions>`, it's time to start -:doc:`defining your own custom presets <./first_config>`. +At this point, ``ytdl-sub`` should run periodically and keep your subscriptions current +in your media library without your intervention. As your :doc:`subscriptions file +<./subscriptions>` grows or you discover new use cases, it becomes worth while to +simplify things by :doc:`defining your own custom presets <./first_config>`. + + + +.. _`cron`: + https://en.wikipedia.org/wiki/Cron +.. _`a cron schedule`: + https://crontab.cronhub.io/ + +.. _`the Task Scheduler`: + https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-start-page +.. _`a support post in Discord`: + https://discord.com/channels/994270357957648404/1084886228266127460 +.. _`a new GitHub issue`: + https://github.com/jmbannon/ytdl-sub/issues/new diff --git a/docs/source/guides/getting_started/first_config.rst b/docs/source/guides/getting_started/first_config.rst index 34dabdb2..0f98b84e 100644 --- a/docs/source/guides/getting_started/first_config.rst +++ b/docs/source/guides/getting_started/first_config.rst @@ -3,13 +3,33 @@ Basic Configuration A configuration file serves two purposes: -1. Set advanced functionality that is not specifiable in a subscription file, such as - working directory location. These are set underneath ``configuration``. -2. Create custom presets, which can drastically simplify your subscription file. These - are defined underneath ``presets``. Presets are intended to be applicable and - reusable across multiple subscriptions. +1. Set application-level functionality that is not specifiable in a subscription file. -Below is a common configuration: + .. note:: + + ytdl-sub does not require a configuration file. However, + certain application settings may be desirable for tweak, such as setting + ``working_directory`` to make ytdl-sub perform the initial download + to an SSD drive. + +2. Create custom presets. + + .. note:: + + In the prior Initial Subscription examples, we leveraged the prebuilt preset + ``Jellyfin TV Show by Date``. This preset is entirely built using the same + YAML configuration system offered to users by using a configuration file. + +The following section attempts to demystify and explain how to... + +- Set an application setting +- Know whether or not custom presets are actually needed +- How to create a custom preset +- How to use a custom preset on subscriptions + +------------- + +how this works, and show-case how .. code-block:: yaml :linenos: diff --git a/docs/source/guides/getting_started/index.rst b/docs/source/guides/getting_started/index.rst index 1288380d..5c04e47f 100644 --- a/docs/source/guides/getting_started/index.rst +++ b/docs/source/guides/getting_started/index.rst @@ -134,7 +134,7 @@ use no preset at all and will just run ``yt-dlp`` without any customization or p processing. The subscriptions file has special support for :ref:`overriding the presets of all subscriptions in the file `. The configuration file supports :ref:`a few special options -` that are not about defining presets. See +` that are not about defining presets. See :doc:`the reference documentation <../../config_reference/index>` for technically complete details, but for almost all of the use cases served by ``ytdl-sub``, the above is accurate and representative. diff --git a/docs/source/guides/install/docker.rst b/docs/source/guides/install/docker.rst index aa7bfd5f..93df8119 100644 --- a/docs/source/guides/install/docker.rst +++ b/docs/source/guides/install/docker.rst @@ -38,6 +38,13 @@ For example:: $ docker compose run --rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub" ytdl-sub sub +.. note:: + + In `the recommended GUI image `_, the ``DEFAULT_WORKSPACE`` directory is + ``/config/ytdl-sub-configs/`` which is used throughout the documentation and + examples. In the headless images, that directory is just ``/config/``, so substitute + that path if using a headless image. + Install with Docker Compose --------------------------- diff --git a/docs/source/prebuilt_presets/tv_shows.rst b/docs/source/prebuilt_presets/tv_shows.rst index 0d4a038c..6dfce04a 100644 --- a/docs/source/prebuilt_presets/tv_shows.rst +++ b/docs/source/prebuilt_presets/tv_shows.rst @@ -160,6 +160,8 @@ Two main use cases of a collection are: 2. Organize one or more YouTube channels/playlists, where each season represents a separate channel/playlist. +Today, ytdl-supports up to 40 seasons with 11 URLs per season. + Example ~~~~~~~ @@ -185,6 +187,29 @@ Must define ``tv_show_directory``. Available presets: s02_name: "Covers" s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W" +Other notable features include: + +* TV show poster info is pulled from the first URL in s01. +* Duplicate videos in different URLs (channel /videos vs playlist) will not download twice. + + * The video will attributed to the season with the highest number. +* Individual seasons support both single and multi URL. +* s00 is supported for specials. + +.. code-block:: yaml + + "~Beyond the Guitar": + s00_name: "Specials" + s00_url: + - "https://www.youtube.com/watch?v=vXzguOdulAI" + - "https://www.youtube.com/watch?v=IGwYDvaGAz0" + s01_name: "Videos" + s01_url: + - "https://www.youtube.com/c/BeyondTheGuitar" + - "https://www.youtube.com/@BeyondTheGuitarAcademy" + s02_name: "Covers" + s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W" + Advanced Usage ~~~~~~~~~~~~~~ diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 113e14f8..cb5b827e 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -87,8 +87,7 @@ Using the command: --overrides.tv_show_name "Rick A" \ --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" -See how to shorten commands using `download aliases -`_. +See how to shorten commands using :ref:`download aliases `. View Options @@ -105,3 +104,11 @@ Preview the source variables for a given URL. Helpful to create new subscription -sc, --split-chapters View source variables after splitting by chapters + +CLI to SUB Options +------------------ +Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments. + +.. code-block:: + + ytdl-sub cli-to-sub [YT-DLP ARGS] diff --git a/examples/advanced/tv_show_config.yaml b/examples/advanced/tv_show_config.yaml index d2019b9a..4b6c5db7 100644 --- a/examples/advanced/tv_show_config.yaml +++ b/examples/advanced/tv_show_config.yaml @@ -69,7 +69,7 @@ presets: # ytdl_options lets you pass any arg into yt-dlp's Python API ytdl_options: # Set the cookie file - # cookiefile: "/config/youtube_cookies.txt" + # cookiefile: "/config/ytdl-sub-configs/youtube_cookies.txt" # For YouTube, get English metadata if multiple languages are present extractor_args: diff --git a/examples/advanced/tv_show_subscriptions.yaml b/examples/advanced/tv_show_subscriptions.yaml index 52e1728c..f0b18dfa 100644 --- a/examples/advanced/tv_show_subscriptions.yaml +++ b/examples/advanced/tv_show_subscriptions.yaml @@ -24,6 +24,6 @@ TV Show Only Recent: # to set only for that subscriptions "~BBC News": url: "https://www.youtube.com/@BBCNews" # use url2, url3, ... for multi-url in this form - date_range: "2weeks" + only_recent_date_range: "2weeks" "Frontline PBS": "https://www.youtube.com/@frontline" "Whitehouse": "https://www.bitchute.com/channel/zWsYVmCOu4JA/" # Supports non-YT sites \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8f1d08b3..31b3158d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "yt-dlp[default]==2025.10.14", + "yt-dlp[default]==2026.3.3", "colorama~=0.4", "mergedeep~=1.3", "mediafile~=0.12", @@ -48,12 +48,12 @@ test = [ ] lint = [ "black==24.10.0", - "isort==6.1.0", - "pylint==3.3.8", + "isort==7.0.0", + "pylint==4.0.1", "ruff==0.14.0", ] docs = [ - "sphinx>=7,<9", + "sphinx>=7,<10", "sphinx-rtd-theme>=2,<4", "sphinx-book-theme~=1.0", "sphinx-copybutton~=0.5", diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index eace6173..29f4f482 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -1,5 +1,6 @@ import gc import os +import random import sys from datetime import datetime from pathlib import Path @@ -12,6 +13,7 @@ from yt_dlp.utils import sanitize_filename from ytdl_sub.cli.output_summary import output_summary from ytdl_sub.cli.output_transaction_log import _maybe_validate_transaction_log_file from ytdl_sub.cli.output_transaction_log import output_transaction_log +from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub from ytdl_sub.cli.parsers.dl import DownloadArgsParser from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME from ytdl_sub.cli.parsers.main import parser @@ -74,6 +76,7 @@ def _download_subscriptions_from_yaml_files( subscription_override_dict: Dict, update_with_info_json: bool, dry_run: bool, + shuffle: bool, ) -> List[Subscription]: """ Downloads all subscriptions from one or many subscription yaml files. @@ -90,6 +93,8 @@ def _download_subscriptions_from_yaml_files( Whether to actually download or update using existing info json dry_run Whether to dry run or not + shuffle + Whether to shuffle the subscription download order Returns ------- @@ -111,6 +116,10 @@ def _download_subscriptions_from_yaml_files( subscription_override_dict=subscription_override_dict, ) + if shuffle: + logger.info("Shuffling subscriptions") + random.shuffle(subscriptions) + for subscription in subscriptions: with subscription.exception_handling(): logger.info( @@ -206,6 +215,10 @@ def main() -> List[Subscription]: args, extra_args = parser.parse_known_args() + if args.subparser == "cli-to-sub": + print_cli_to_sub(args=extra_args) + return [] + # Load the config if args.config: config = ConfigFile.from_file_path(args.config) @@ -248,6 +261,7 @@ def main() -> List[Subscription]: subscription_override_dict=subscription_override_dict, update_with_info_json=args.update_with_info_json, dry_run=args.dry_run, + shuffle=args.shuffle, ) # One-off download @@ -263,7 +277,7 @@ def main() -> List[Subscription]: _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters) ) else: - raise ValidationException("Must provide one of the commands: sub, dl, view") + raise ValidationException("Must provide one of the commands: sub, dl, view, cli-to-sub") if not args.suppress_transaction_log: output_transaction_log( diff --git a/src/ytdl_sub/cli/parsers/cli_to_sub.py b/src/ytdl_sub/cli/parsers/cli_to_sub.py new file mode 100644 index 00000000..d4190042 --- /dev/null +++ b/src/ytdl_sub/cli/parsers/cli_to_sub.py @@ -0,0 +1,64 @@ +from typing import List + +import yt_dlp +import yt_dlp.options + +from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.yaml import dump_yaml + +logger = Logger.get() + +# pylint: disable=missing-function-docstring + +############################################################## +# --- BEGIN ---- +# Copy of https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py + +create_parser = yt_dlp.options.create_parser + + +def parse_patched_options(opts): + + patched_parser = create_parser() + patched_parser.defaults.update( + { + "ignoreerrors": False, + "retries": 0, + "fragment_retries": 0, + "extract_flat": False, + "concat_playlist": "never", + "update_self": False, + } + ) + yt_dlp.options.create_parser = lambda: patched_parser + try: + return yt_dlp.parse_options(opts) + finally: + yt_dlp.options.create_parser = create_parser + + +default_opts = parse_patched_options([]).ydl_opts + + +def cli_to_api(opts, cli_defaults=False): + opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts + + diff = {k: v for k, v in opts.items() if default_opts[k] != v} + if "postprocessors" in diff: + diff["postprocessors"] = [ + pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"] + ] + return diff + + +# --- END ---- +############################################################## + + +def print_cli_to_sub(args: List[str]) -> None: + api_args = cli_to_api(args) + if not api_args: + logger.info("Does not resolve to any yt-dlp args") + return + + print(dump_yaml({"ytdl_options": api_args})) diff --git a/src/ytdl_sub/cli/parsers/main.py b/src/ytdl_sub/cli/parsers/main.py index daa4d5ea..eb9b49ca 100644 --- a/src/ytdl_sub/cli/parsers/main.py +++ b/src/ytdl_sub/cli/parsers/main.py @@ -172,6 +172,10 @@ class SubArguments: short="-o", long="--dl-override", ) + SHUFFLE = CLIArgument( + short="-sh", + long="--shuffle", + ) subscription_parser = subparsers.add_parser("sub") @@ -197,6 +201,13 @@ subscription_parser.add_argument( help="override all subscription config values using `dl` syntax, " "i.e. --dl-override='--ytdl_options.max_downloads 3'", ) +subscription_parser.add_argument( + SubArguments.SHUFFLE.short, + SubArguments.SHUFFLE.long, + action="store_true", + help="shuffle subscription order when downloading", + default=False, +) ################################################################################################### # DOWNLOAD PARSER @@ -221,3 +232,7 @@ view_parser.add_argument( help="View source variables after splitting by chapters", ) view_parser.add_argument("url", help="URL to view source variables for") + +################################################################################################### +# CLI-TO-SUB PARSER +cli_to_sub_parser = subparsers.add_parser("cli-to-sub") diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py index b5d234a9..0dc2c1e0 100644 --- a/src/ytdl_sub/config/config_validator.py +++ b/src/ytdl_sub/config/config_validator.py @@ -12,6 +12,8 @@ from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS +from ytdl_sub.utils.exceptions import SubscriptionPermissionError +from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.validators.file_path_validators import FFmpegFileValidator from ytdl_sub.validators.file_path_validators import FFprobeFileValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -22,6 +24,10 @@ from ytdl_sub.validators.validators import StringValidator class ExperimentalValidator(StrictDictValidator): + """ + Experimental flags reside under the ``experimental`` key. + """ + _optional_keys = {"enable_update_with_info_json"} _allow_extra_keys = True @@ -43,6 +49,11 @@ class ExperimentalValidator(StrictDictValidator): class PersistLogsValidator(StrictDictValidator): + """ + By default, no logs are persisted. Specifying this key will enable persisted logs. The following + options are available. + """ + _required_keys = {"logs_directory"} _optional_keys = {"keep_logs_after", "keep_successful_logs"} @@ -67,8 +78,8 @@ class PersistLogsValidator(StrictDictValidator): @property def logs_directory(self) -> str: """ - Write log files to this directory with names like - ``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. (required) + Required field. Write log files to this directory with names like + ``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. """ return self._logs_directory.value @@ -93,15 +104,54 @@ class PersistLogsValidator(StrictDictValidator): @property def keep_successful_logs(self) -> bool: """ - If the ``persist_logs:`` key is in the configuration, then ``ytdl-sub`` *always* - writes log files for the subscription both for successful downloads and when it - encounters an error while downloading. When this key is ``False``, only write - log files for errors. (default ``True``) + Defaults to ``True``. When this key is ``False``, only write log files for failed + subscriptions. """ return self._keep_successful_logs.value class ConfigOptions(StrictDictValidator): + """ + ytdl-sub is configured using a ``config.yaml`` file. + + The ``config.yaml`` is made up of two sections: + + .. code-block:: yaml + + configuration: + presets: + + + Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. + If you prefer to use a Windows backslash, note that it must have + ``C:\\\\double\\\\bashslash\\\\paths`` in order to escape the backslash character. This is due + to it being a YAML escape character. + + .. code-block:: yaml + + configuration: + dl_aliases: + mv: "--preset music_video" + u: "--download.url" + + experimental: + enable_update_with_info_json: True + + ffmpeg_path: "/usr/bin/ffmpeg" + ffprobe_path: "/usr/bin/ffprobe" + + file_name_max_bytes: 255 + lock_directory: "/tmp" + + persist_logs: + keep_successful_logs: True + logs_directory: "/var/log/ytdl-sub-logs" + + umask: "022" + working_directory: ".ytdl-sub-working-directory" + + """ + _optional_keys = { "working_directory", "umask", @@ -147,11 +197,18 @@ class ConfigOptions(StrictDictValidator): key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES ) + if not FileHandler.is_path_writable(self.working_directory): + raise SubscriptionPermissionError( + "ytdl-sub does not have permissions to the working directory: " + f"{self.working_directory}" + ) + @property def working_directory(self) -> str: """ The directory to temporarily store downloaded files before moving them into their final - directory. (default ``./.ytdl-sub-working-directory``) + directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory + that ytdl-sub is invoked from. """ # Expands tildas to actual paths, use native os sep return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep)) @@ -159,7 +216,7 @@ class ConfigOptions(StrictDictValidator): @property def umask(self) -> Optional[str]: """ - Umask in octal format to apply to every created file. (default ``022``) + Umask in octal format to apply to every created file. Defaults to ``022``. """ return self._umask.value @@ -168,7 +225,7 @@ class ConfigOptions(StrictDictValidator): """ .. _dl_aliases: - Alias definitions to shorten ``ytdl-sub dl`` arguments. For example, + Alias definitions to shorten :ref:`dl arguments `. For example, .. code-block:: yaml @@ -220,23 +277,23 @@ class ConfigOptions(StrictDictValidator): The directory to temporarily store file locks, which prevents multiple instances of ``ytdl-sub`` from running. Note that file locks do not work on network-mounted directories. Ensure that this directory resides on the host - machine. (default ``/tmp``) + machine. Defaults to ``/tmp``. """ return self._lock_directory.value @property def ffmpeg_path(self) -> str: """ - Path to ffmpeg executable. (default ``/usr/bin/ffmpeg`` for Linux, - ``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows) + Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux, + ``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows. """ return self._ffmpeg_path.value @property def ffprobe_path(self) -> str: """ - Path to ffprobe executable. (default ``/usr/bin/ffprobe`` for Linux, - ``./ffprobe.exe`` in the same directory as ytdl-sub for Windows) + Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux, + ``./ffprobe.exe`` in the same directory as ytdl-sub for Windows. """ return self._ffprobe_path.value diff --git a/src/ytdl_sub/config/defaults.py b/src/ytdl_sub/config/defaults.py index cf1f3096..e0746e1e 100644 --- a/src/ytdl_sub/config/defaults.py +++ b/src/ytdl_sub/config/defaults.py @@ -2,6 +2,8 @@ import os from ytdl_sub.utils.system import IS_WINDOWS +# pylint: disable=invalid-name + def _existing_path(*paths: str) -> str: """ diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 8ed6990c..f1011fcf 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -1,9 +1,10 @@ from typing import Any from typing import Dict +from typing import Iterable from typing import Optional from typing import Set - -import mergedeep +from typing import Type +from typing import TypeVar from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES @@ -11,17 +12,21 @@ from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARI from ytdl_sub.entries.variables.override_variables import OverrideHelpers from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script +from ytdl_sub.script.types.function import BuiltInFunction from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.utils.exceptions import InvalidVariableNameException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.scriptable import Scriptable -from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator +ExpectedT = TypeVar("ExpectedT") + class Overrides(UnstructuredDictFormatterValidator, Scriptable): """ @@ -88,6 +93,24 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): return True + def ensure_variable_names_not_a_plugin(self, plugin_names: Iterable[str]) -> None: + """ + Throws an error if an override variable or function has the same name as a + preset key. This is to avoid confusion when accidentally defining things in + overrides that are meant to be in the preset. + """ + for name in self.keys: + if name.startswith("%"): + name = name[1:] + + if name in plugin_names: + raise self._validation_exception( + f"Override variable with name {name} cannot be used since it is" + " the name of a plugin. Perhaps you meant to define it as a plugin? If so," + " indent it left to make it at the same level as overrides.", + exception_class=InvalidVariableNameException, + ) + def ensure_variable_name_valid(self, name: str) -> None: """ Ensures the variable name does not collide with any entry variables or built-in functions. @@ -115,29 +138,35 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): ) def initial_variables( - self, unresolved_variables: Optional[Dict[str, str]] = None - ) -> Dict[str, str]: + self, unresolved_variables: Optional[Dict[str, SyntaxTree]] = None + ) -> Dict[str, SyntaxTree]: """ Returns ------- Variables and format strings for all Override variables + additional variables (Optional) """ - initial_variables: Dict[str, str] = {} - mergedeep.merge( - initial_variables, - self.dict_with_format_strings, - unresolved_variables if unresolved_variables else {}, - ) - return ScriptUtils.add_sanitized_variables(initial_variables) + initial_variables: Dict[str, SyntaxTree] = self.dict_with_parsed_format_strings + if unresolved_variables: + initial_variables |= unresolved_variables + return ScriptUtils.add_sanitized_parsed_variables(initial_variables) def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": """ Initialize the override script with any unresolved variables """ - self.script.add( + self.script.add_parsed( self.initial_variables( unresolved_variables={ - var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" + var_name: SyntaxTree( + ast=[ + BuiltInFunction( + name="throw", + args=[ + String(f"Plugin variable {var_name} has not been created yet") + ], + ) + ] + ) for var_name in unresolved_variables } ) @@ -158,10 +187,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): script = entry.script unresolvable = entry.unresolvable + # Update the script internally so long as we are not supplying overrides + # that could alter the script with one-off state + update = function_overrides is None + try: return script.resolve_once( dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), unresolvable=unresolvable, + update=update, )["tmp_var"] except ScriptVariableNotResolved as exc: raise StringFormattingException( @@ -176,7 +210,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): formatter: StringFormatterValidator, entry: Optional[Entry] = None, function_overrides: Optional[Dict[str, str]] = None, - ) -> str: + expected_type: Type[ExpectedT] = str, + ) -> ExpectedT: """ Parameters ---------- @@ -186,6 +221,8 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): Optional. Entry to add source variables to the formatter function_overrides Optional. Explicit values to override the overrides themselves and source variables + expected_type + The expected type that should return. Defaults to string. Returns ------- @@ -196,37 +233,15 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): StringFormattingException If the formatter that is trying to be resolved cannot """ - return formatter.post_process( - str( - self._apply_to_resolvable( - formatter=formatter, entry=entry, function_overrides=function_overrides - ) - ) + out = formatter.post_process( + self._apply_to_resolvable( + formatter=formatter, entry=entry, function_overrides=function_overrides + ).native ) - def apply_overrides_formatter_to_native( - self, - formatter: OverridesStringFormatterValidator, - ) -> Any: - """ - Parameters - ---------- - formatter - Overrides formatter to apply + if not isinstance(out, expected_type): + raise StringFormattingException( + f"Expected type {expected_type.__name__}, but received '{out.__class__.__name__}'" + ) - Returns - ------- - The native python form of the resolved variable - """ - return self._apply_to_resolvable( - formatter=formatter, entry=None, function_overrides=None - ).native - - def evaluate_boolean( - self, formatter: StringFormatterValidator, entry: Optional[Entry] = None - ) -> bool: - """ - Apply a formatter, and evaluate it to a boolean - """ - output = self.apply_formatter(formatter=formatter, entry=entry) - return ScriptUtils.bool_formatter_output(output) + return out diff --git a/src/ytdl_sub/config/plugin/plugin.py b/src/ytdl_sub/config/plugin/plugin.py index 0cd02b53..23232c48 100644 --- a/src/ytdl_sub/config/plugin/plugin.py +++ b/src/ytdl_sub/config/plugin/plugin.py @@ -48,7 +48,7 @@ class Plugin(BasePlugin[OptionsValidatorT], Generic[OptionsValidatorT], ABC): Returns True if enabled, False if disabled. """ if isinstance(self.plugin_options, ToggleableOptionsDictValidator): - return self.overrides.evaluate_boolean(self.plugin_options.enable) + return self.overrides.apply_formatter(self.plugin_options.enable, expected_type=bool) return True def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: diff --git a/src/ytdl_sub/config/plugin/preset_plugins.py b/src/ytdl_sub/config/plugin/preset_plugins.py index 7eda8a9d..61305a37 100644 --- a/src/ytdl_sub/config/plugin/preset_plugins.py +++ b/src/ytdl_sub/config/plugin/preset_plugins.py @@ -1,5 +1,7 @@ +from typing import Iterable from typing import List from typing import Optional +from typing import Set from typing import Tuple from typing import Type @@ -44,3 +46,34 @@ class PresetPlugins: if plugin_type in plugin_option_types: return self.plugin_options[plugin_option_types.index(plugin_type)] return None + + def get_added_and_modified_variables( + self, additional_options: List[OptionsValidator] + ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: + """ + Iterates and returns the plugin options, added variables, modified variables + """ + for plugin_options in self.plugin_options + additional_options: + added_variables: Set[str] = set() + modified_variables: Set[str] = set() + + for plugin_added_variables in plugin_options.added_variables( + unresolved_variables=set(), + ).values(): + added_variables |= set(plugin_added_variables) + + for plugin_modified_variables in plugin_options.modified_variables().values(): + modified_variables = plugin_modified_variables + + yield plugin_options, added_variables, modified_variables + + def get_all_variables(self, additional_options: List[OptionsValidator]) -> Set[str]: + """ + Returns set of all added and modified variables' names. + """ + all_variables: Set[str] = set() + for _, added, modified in self.get_added_and_modified_variables(additional_options): + all_variables.update(added) + all_variables.update(modified) + + return all_variables diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 914ebc77..2c8ad348 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -2,6 +2,7 @@ import copy from typing import Any from typing import Dict from typing import List +from typing import Set from mergedeep import mergedeep @@ -11,7 +12,6 @@ from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions -from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES @@ -55,6 +55,12 @@ class _PresetShell(StrictDictValidator): class Preset(_PresetShell): + """ + Custom presets are defined in this section. Refer to the + :ref:`Getting Started Guide` + on how to configure. + """ + @classmethod def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None: """ @@ -172,6 +178,37 @@ class Preset(_PresetShell): mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE) ) + def _initialize_overrides_script(self, overrides: Overrides) -> Overrides: + """ + Do some gymnastics to initialize the Overrides script. + """ + unresolved_variables: Set[str] = set() + + for ( + plugin_options, + added_variables, + modified_variables, + ) in self.plugins.get_added_and_modified_variables( + additional_options=[self.downloader_options, self.output_options] + ): + for added_variable in added_variables: + if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable): + # pylint: disable=protected-access + raise plugin_options._validation_exception( + f"Cannot use the variable name {added_variable} because it exists as a" + " built-in ytdl-sub variable name." + ) + # pylint: enable=protected-access + + # Set unresolved as variables that are added but do not exist as + # entry/override variables since they are created at run-time + unresolved_variables |= added_variables | modified_variables + + # Initialize overrides with unresolved variables + modified variables to throw an error. + # For modified variables, this is to prevent a resolve(update=True) to setting any + # dependencies until it has been explicitly added + return overrides.initialize_script(unresolved_variables=unresolved_variables) + def __init__(self, config: ConfigValidator, name: str, value: Any): super().__init__(name=name, value=value) @@ -192,13 +229,10 @@ class Preset(_PresetShell): ) self.plugins: PresetPlugins = self._validate_and_get_plugins() - self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) - - VariableValidation( - downloader_options=self.downloader_options, - output_options=self.output_options, - plugins=self.plugins, - ).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage() + self.overrides = self._initialize_overrides_script( + overrides=self._validate_key(key="overrides", validator=Overrides, default={}) + ) + self.overrides.ensure_variable_names_not_a_plugin(plugin_names=PRESET_KEYS) @property def name(self) -> str: @@ -227,11 +261,18 @@ class Preset(_PresetShell): """ return cls(config=config, name=preset_name, value=preset_dict) - @property - def yaml(self) -> str: + def yaml(self, subscription_only: bool) -> str: """ + Parameters + ---------- + subscription_only: + Only include the subscription contents, not the surrounding boiler-plate. + Returns ------- Preset in YAML format """ + if subscription_only: + return dump_yaml(self._value) + return dump_yaml({"presets": {self._name: self._value}}) diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index a16476ed..27ad8a92 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -8,6 +8,9 @@ from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.script.variable_definitions import VARIABLES as v +from ytdl_sub.utils.exceptions import SubscriptionPermissionError +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator @@ -57,12 +60,24 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator): def to_native_dict(self, overrides: Overrides) -> Dict: """ Materializes the entire ytdl-options dict from OverrideStringFormatters into - native python + native python. """ - return { - key: overrides.apply_overrides_formatter_to_native(val) + out = { + key: overrides.apply_formatter(val, expected_type=object) for key, val in self.dict.items() } + if "cookiefile" in out: + if not FileHandler.is_file_existent(out["cookiefile"]): + raise ValidationException( + f"Specified cookiefile {out['cookiefile']} but it does not exist as a file." + ) + + if not FileHandler.is_file_readable(out["cookiefile"]): + raise SubscriptionPermissionError( + f"Cannot read cookiefile {out['cookiefile']} due to permissions issue." + ) + + return out # Disable for proper docstring formatting @@ -107,6 +122,7 @@ class OutputOptions(OptionsDictValidator): "keep_max_files", "download_archive_standardized_date", "keep_files_date_eval", + "preserve_mtime", } @classmethod @@ -170,6 +186,10 @@ class OutputOptions(OptionsDictValidator): default=f"{{{v.upload_date_standardized.variable_name}}}", ) + self._preserve_mtime = self._validate_key_if_present( + key="preserve_mtime", validator=BoolValidator, default=False + ) + if ( self._keep_files_before or self._keep_files_after or self._keep_max_files ) and not self.maintain_download_archive: @@ -309,6 +329,17 @@ class OutputOptions(OptionsDictValidator): """ return self._keep_max_files + @property + def preserve_mtime(self) -> bool: + """ + :expected type: Optional[Boolean] + :description: + Preserve the video's original upload time as the file modification time. + When True, sets the file's mtime to match the video's upload_date from + yt-dlp metadata. Defaults to False. + """ + return self._preserve_mtime.value + def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]: return { # PluginOperation.MODIFY_ENTRY_METADATA: { diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index e2d5e85d..941d228e 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -1,10 +1,6 @@ -import copy from typing import Dict -from typing import Iterable from typing import List -from typing import Optional from typing import Set -from typing import Tuple from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_mapping import PluginMapping @@ -13,203 +9,188 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES +from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES +from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.script.script import Script -from ytdl_sub.script.script import _is_function -from ytdl_sub.utils.scriptable import BASE_SCRIPT -from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string +from ytdl_sub.script.utils.name_validation import is_function +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import validate_formatters -# Entry variables to mock during validation -_DUMMY_ENTRY_VARIABLES: Dict[str, str] = { - name: to_variable_dependency_format_string( - # pylint: disable=protected-access - script=BASE_SCRIPT, - parsed_format_string=BASE_SCRIPT._variables[name], - # pylint: enable=protected-access - ) - for name in BASE_SCRIPT.variable_names -} +class ResolutionLevel: + ORIGINAL = 0 + FILL = 1 + RESOLVE = 2 + INTERNAL = 3 -def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: - dummy_variables: Dict[str, str] = {} - for var in variables: - dummy_variables[var] = "" - dummy_variables[f"{var}_sanitized"] = "" + @classmethod + def name_of(cls, resolution_level: int) -> str: + """ + Name of the resolution level. + """ + if resolution_level == cls.ORIGINAL: + return "original" + if resolution_level == cls.FILL: + return "fill" + if resolution_level == cls.RESOLVE: + return "resolve" + if resolution_level == cls.INTERNAL: + return "internal" + raise ValueError("Invalid resolution level") - return dummy_variables - - -def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: - # Have the dummy override variable contain all variable deps that it uses in the string - dummy_overrides: Dict[str, str] = {} - for override_name in _override_variables(overrides): - if _is_function(override_name): - continue - - # pylint: disable=protected-access - dummy_overrides[override_name] = to_variable_dependency_format_string( - script=overrides.script, parsed_format_string=overrides.script._variables[override_name] - ) - # pylint: enable=protected-access - return dummy_overrides - - -def _get_added_and_modified_variables( - plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions -) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: - """ - Iterates and returns the plugin options, added variables, modified variables - """ - options: List[OptionsValidator] = plugins.plugin_options - options.append(downloader_options) - options.append(output_options) - - for plugin_options in options: - added_variables: Set[str] = set() - modified_variables: Set[str] = set() - - for plugin_added_variables in plugin_options.added_variables( - unresolved_variables=set(), - ).values(): - added_variables |= set(plugin_added_variables) - - for plugin_modified_variables in plugin_options.modified_variables().values(): - modified_variables = plugin_modified_variables - - yield plugin_options, added_variables, modified_variables - - -def _override_variables(overrides: Overrides) -> Set[str]: - return set(list(overrides.initial_variables().keys())) + @classmethod + def all(cls) -> List[int]: + """ + All possible resolution levels. + """ + return [cls.ORIGINAL, cls.FILL, cls.RESOLVE, cls.INTERNAL] class VariableValidation: + + def _get_resolve_partial_filter(self) -> Set[str]: + # Exclude sanitized variables from partial validation. This lessens the work + # and prevents double-evaluation, which can lead to bad behavior like double-prints. + return { + name + for name in self.script.variable_names + if name not in self.unresolved_variables and not name.endswith("_sanitized") + } + + def _apply_resolution_level(self) -> None: + if self._resolution_level == ResolutionLevel.FILL: + self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True) + # Only partial resolve definitions that are already resolved + self.unresolved_variables |= { + name + for name in self.overrides.keys + if not is_function(name) and not self.script.definition_of(name).maybe_resolvable + } + elif self._resolution_level == ResolutionLevel.RESOLVE: + # Partial resolve everything, but not including internal variables + self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True) + elif self._resolution_level == ResolutionLevel.INTERNAL: + # Partial resolve everything including internal variables + pass + else: + raise ValueError("Invalid resolution level for validation") + + self.script = self.script.resolve_partial( + unresolvable=self.unresolved_variables, + output_filter=self._get_resolve_partial_filter(), + ) + def __init__( self, + overrides: Overrides, downloader_options: MultiUrlValidator, output_options: OutputOptions, plugins: PresetPlugins, + resolution_level: int = ResolutionLevel.RESOLVE, ): + self.overrides = overrides self.downloader_options = downloader_options self.output_options = output_options self.plugins = plugins - self.script: Optional[Script] = None - self.resolved_variables: Set[str] = set() - self.unresolved_variables: Set[str] = set() - - def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation": - """ - Do some gymnastics to initialize the Overrides script. - """ - override_variables = set(list(overrides.initial_variables().keys())) - - # Set resolved variables as all entry + override variables - # at this point to generate every possible added/modified variable - self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables - plugin_variables: Set[str] = set() - - for ( - plugin_options, - added_variables, - modified_variables, - ) in _get_added_and_modified_variables( - plugins=self.plugins, - downloader_options=self.downloader_options, - output_options=self.output_options, - ): - - for added_variable in added_variables: - if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable): - # pylint: disable=protected-access - raise plugin_options._validation_exception( - f"Cannot use the variable name {added_variable} because it exists as a" - " built-in ytdl-sub variable name." - ) - # pylint: enable=protected-access - - # Set unresolved as variables that are added but do not exist as - # entry/override variables since they are created at run-time - self.unresolved_variables |= added_variables | modified_variables - plugin_variables |= added_variables | modified_variables - - # Then update resolved variables to reflect that - self.resolved_variables -= self.unresolved_variables - - # Initialize overrides with unresolved variables + modified variables to throw an error. - # For modified variables, this is to prevent a resolve(update=True) to setting any - # dependencies until it has been explicitly added - overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables) - - # copy the script and mock entry variables - self.script = copy.deepcopy(overrides.script) - self.script.add( - variables=_add_dummy_overrides(overrides=overrides) - | _add_dummy_variables(variables=plugin_variables) - | _DUMMY_ENTRY_VARIABLES + self.script: Script = self.overrides.script + self.unresolved_variables = ( + self.plugins.get_all_variables( + additional_options=[self.output_options, self.downloader_options] + ) + | UNRESOLVED_VARIABLES ) + self.unresolved_runtime_variables = self.plugins.get_all_variables( + additional_options=[self.output_options, self.downloader_options] + ) + self._resolution_level = resolution_level - return self + self._apply_resolution_level() - def _update_script(self) -> None: - _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) - - def _add_subscription_override_variables(self) -> None: - """ - Add dummy subscription variables for script validation - """ - self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES - - def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None: + def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None: """ Add dummy variables for script validation """ added_variables = options.added_variables( - unresolved_variables=self.unresolved_variables, + unresolved_variables=self.unresolved_runtime_variables, ).get(plugin_op, set()) modified_variables = options.modified_variables().get(plugin_op, set()) - resolved_variables = added_variables | modified_variables + self.unresolved_runtime_variables -= added_variables | modified_variables - self.resolved_variables |= resolved_variables - self.unresolved_variables -= resolved_variables - - def ensure_proper_usage(self) -> None: + def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict: """ Validate variables resolve as plugins are executed, and return a mock script which contains actualized added variables from the plugins """ + resolved_subscription: Dict = {} - self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) - self._add_subscription_override_variables() + self._add_runtime_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) # Always add output options first - self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options) + self._add_runtime_variables( + PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options + ) # Metadata variables to be added for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA ): - self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) + self._add_runtime_variables( + PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options + ) for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY ): - self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) + self._add_runtime_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) # Validate that any formatter in the plugin options can resolve - validate_formatters( + resolved_subscription |= validate_formatters( script=self.script, unresolved_variables=self.unresolved_variables, + unresolved_runtime_variables=self.unresolved_runtime_variables, validator=plugin_options, + partial_resolve_formatters=partial_resolve_formatters, ) - validate_formatters( + resolved_subscription |= validate_formatters( script=self.script, unresolved_variables=self.unresolved_variables, + unresolved_runtime_variables=self.unresolved_runtime_variables, validator=self.output_options, + partial_resolve_formatters=partial_resolve_formatters, ) - assert not self.unresolved_variables + # TODO: make this a function + raw_download_output = validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + unresolved_runtime_variables=self.unresolved_runtime_variables, + validator=self.downloader_options.urls, + partial_resolve_formatters=partial_resolve_formatters, + ) + resolved_subscription["download"] = [] + for url_output in raw_download_output["download"]: + if isinstance(url_output["url"], list): + url_output["url"] = [url for url in url_output["url"] if bool(url)] + + if url_output["url"]: + resolved_subscription["download"].append(url_output) + + # TODO: make function + resolved_subscription["overrides"] = {} + for name in self.overrides.keys: + value = self.script.definition_of(name) + if name in self.script.function_names: + # Keep custom functions as-is + resolved_subscription["overrides"][name] = self.overrides.dict_with_format_strings[ + name + ] + elif resolved := value.maybe_resolvable: + resolved_subscription["overrides"][name] = resolved.native + else: + resolved_subscription["overrides"][name] = ScriptUtils.to_native_script(value) + + assert not self.unresolved_runtime_variables + return resolved_subscription diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index ef5fbd5a..c94d6910 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -52,12 +52,12 @@ class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator]) if 0 <= input_url_idx < len(self.plugin_options.urls.list): validator = self.plugin_options.urls.list[input_url_idx] - if self.overrides.apply_formatter(validator.url) == entry_input_url: + if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list): return validator # Match the first validator based on the URL, if one exists for validator in self.plugin_options.urls.list: - if self.overrides.apply_formatter(validator.url) == entry_input_url: + if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list): return validator # Return the first validator if none exist @@ -382,7 +382,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): entries_to_iter: List[Optional[Entry]] = entries indices = list(range(len(entries_to_iter))) - if self.overrides.evaluate_boolean(validator.download_reverse): + if self.overrides.apply_formatter(validator.download_reverse, expected_type=bool): indices = reversed(indices) for idx in indices: @@ -461,8 +461,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides) ) - include_sibling_metadata = self.overrides.evaluate_boolean( - validator.include_sibling_metadata + include_sibling_metadata = self.overrides.apply_formatter( + validator.include_sibling_metadata, expected_type=bool ) parents, orphan_entries = self._download_url_metadata( @@ -487,19 +487,25 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): # download the bottom-most urls first since they are top-priority for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))): # URLs can be empty. If they are, then skip - if not (url := self.overrides.apply_formatter(url_validator.url)): + if not (urls := self.overrides.apply_formatter(url_validator.url, expected_type=list)): continue - for entry in self._download_metadata(url=url, validator=url_validator): - entry.initialize_script(self.overrides).add( - { - v.ytdl_sub_input_url: url, - v.ytdl_sub_input_url_index: idx, - v.ytdl_sub_input_url_count: len(self.collection.urls.list), - } - ) + for url in reversed(urls): + assert isinstance(url, str) - yield entry + if not url: + continue + + for entry in self._download_metadata(url=url, validator=url_validator): + entry.initialize_script(self.overrides).add( + { + v.ytdl_sub_input_url: url, + v.ytdl_sub_input_url_index: idx, + v.ytdl_sub_input_url_count: len(self.collection.urls.list), + } + ) + + yield entry def download(self, entry: Entry) -> Optional[Entry]: """ diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 4504eccf..b2010ebc 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -1,5 +1,6 @@ from typing import Any from typing import Dict +from typing import List from typing import Optional from typing import Set @@ -21,7 +22,7 @@ class UrlThumbnailValidator(StrictDictValidator): def __init__(self, name, value): super().__init__(name, value) - self._name = self._validate_key(key="name", validator=StringFormatterValidator) + self._thumb_name = self._validate_key(key="name", validator=StringFormatterValidator) self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator) @property @@ -29,7 +30,7 @@ class UrlThumbnailValidator(StrictDictValidator): """ File name for the thumbnail """ - return self._name + return self._thumb_name @property def uid(self) -> OverridesStringFormatterValidator: @@ -43,6 +44,19 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]): _inner_list_type = UrlThumbnailValidator +class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator): + def post_process(self, resolved: Any) -> List[str]: + if isinstance(resolved, str): + return [resolved] + if isinstance(resolved, list): + for value in resolved: + if not isinstance(value, str): + raise self._validation_exception("Must be a string or an array of strings.") + return resolved + + raise self._validation_exception("Must be a string or an array of strings.") + + class UrlValidator(StrictDictValidator): _required_keys = {"url"} _optional_keys = { @@ -68,7 +82,7 @@ class UrlValidator(StrictDictValidator): super().__init__(name, value) # TODO: url validate using yt-dlp IE - self._url = self._validate_key(key="url", validator=OverridesStringFormatterValidator) + self._url = self._validate_key(key="url", validator=OverridesOneOrManyUrlValidator) self._variables = self._validate_key_if_present( key="variables", validator=DictFormatterValidator, default={} ) diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 67505473..97fb7759 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1135,6 +1135,16 @@ class VariableDefinitions( ] } + @cache + def variable_names(self, include_sanitized: bool): + """ + Returns all variable names, and can include sanitized. + """ + var_names: Set[str] = self.scripts().keys() + if include_sanitized: + var_names |= {f"{name}_sanitized" for name in var_names} + return var_names + @cache def injected_variables(self) -> Set[MetadataVariable]: """ diff --git a/src/ytdl_sub/entries/script/variable_types.py b/src/ytdl_sub/entries/script/variable_types.py index f406baf6..bbf994e9 100644 --- a/src/ytdl_sub/entries/script/variable_types.py +++ b/src/ytdl_sub/entries/script/variable_types.py @@ -22,7 +22,6 @@ VariableT = TypeVar("VariableT", bound="Variable") def _get( - cast: str, metadata_variable_name: str, metadata_key: str, variable_name: Optional[str], @@ -47,7 +46,7 @@ def _get( return as_type( variable_name=variable_name or metadata_key, metadata_key=metadata_key, - definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}", + definition=f"{{ {out} }}", ) @@ -182,7 +181,6 @@ class MapMetadataVariable(MetadataVariable, MapVariable): Creates a map variable from entry metadata """ return _get( - "map", metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, @@ -204,7 +202,6 @@ class ArrayMetadataVariable(MetadataVariable, ArrayVariable): Creates an array variable from entry metadata """ return _get( - "array", metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, @@ -226,7 +223,6 @@ class StringMetadataVariable(MetadataVariable, StringVariable): Creates a string variable from entry metadata """ return _get( - "string", metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, @@ -245,7 +241,6 @@ class StringMetadataVariable(MetadataVariable, StringVariable): Creates a string variable from playlist metadata """ return _get( - "string", metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, @@ -264,7 +259,6 @@ class StringMetadataVariable(MetadataVariable, StringVariable): Creates a string variable from source metadata """ return _get( - "string", metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, @@ -301,7 +295,6 @@ class IntegerMetadataVariable(MetadataVariable, IntegerVariable): Creates an int variable from entry metadata """ return _get( - "int", metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, @@ -320,7 +313,6 @@ class IntegerMetadataVariable(MetadataVariable, IntegerVariable): Creates an int variable from playlist metadata """ return _get( - "int", metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, metadata_key=metadata_key, variable_name=variable_name, diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index e9b3da07..8dd2081c 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -11,9 +11,6 @@ from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.functions import Functions from ytdl_sub.script.utils.name_validation import is_valid_name -# TODO: use this -SUBSCRIPTION_ARRAY = "subscription_array" - class SubscriptionVariables: @staticmethod @@ -163,7 +160,7 @@ class OverrideHelpers: True if the override name itself is valid. False otherwise. """ if name.startswith("%"): - return is_valid_name(name=name[1:]) + name = name[1:] return is_valid_name(name=name) diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 478cf938..0cba6f89 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -116,7 +116,7 @@ class DateRangePlugin(Plugin[DateRangeOptions]): date_validator=self.plugin_options.after, overrides=self.overrides ) after_filter = f"{date_type} >= {after_str}" - if self.overrides.evaluate_boolean(self.plugin_options.breaks): + if self.overrides.apply_formatter(self.plugin_options.breaks, expected_type=bool): breaking_match_filters.append(after_filter) else: match_filters.append(after_filter) diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index e7d19b8b..4ce2b144 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -33,7 +33,7 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]): @property def _embed_thumbnail(self) -> bool: - return self.overrides.evaluate_boolean(self.plugin_options) + return self.overrides.apply_formatter(self.plugin_options, expected_type=bool) @classmethod def _embed_video_thumbnail(cls, entry: Entry) -> None: diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index 7d256998..d4adcc86 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -7,13 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.logger import Logger -from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator +from ytdl_sub.validators.validators import ListValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get("filter-exclude") -class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): +class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator): """ Applies a conditional OR on any number of filters comprised of either variables or scripts. If any filter evaluates to True, the entry will be excluded. @@ -29,6 +30,8 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): { %contains( %lower(description), '#short' ) } """ + _inner_list_type = BooleanFormatterValidator + class FilterExcludePlugin(Plugin[FilterExcludeOptions]): plugin_options_type = FilterExcludeOptions @@ -52,7 +55,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]): return entry for formatter in self.plugin_options.list: - should_exclude = self.overrides.evaluate_boolean(formatter=formatter, entry=entry) + should_exclude = self.overrides.apply_formatter( + formatter=formatter, entry=entry, expected_type=bool + ) if should_exclude: logger.info( diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index c2bc5217..5002dfdb 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -7,14 +7,14 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.logger import Logger -from ytdl_sub.utils.script import ScriptUtils -from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator +from ytdl_sub.validators.validators import ListValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get("filter-include") -class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): +class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator): """ Applies a conditional AND on any number of filters comprised of either variables or scripts. If all filters evaluate to True, the entry will be included. @@ -38,6 +38,8 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): } """ + _inner_list_type = BooleanFormatterValidator + class FilterIncludePlugin(Plugin[FilterIncludeOptions]): plugin_options_type = FilterIncludeOptions @@ -61,8 +63,8 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]): return entry for formatter in self.plugin_options.list: - should_exclude = ScriptUtils.bool_formatter_output( - self.overrides.apply_formatter(formatter=formatter, entry=entry) + should_exclude = self.overrides.apply_formatter( + formatter=formatter, entry=entry, expected_type=bool ) if not should_exclude: logger.info( diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 24c07c72..802dab63 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -140,7 +140,7 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC): if not nfo_tags: return - if self.overrides.evaluate_boolean(self.plugin_options.kodi_safe): + if self.overrides.apply_formatter(self.plugin_options.kodi_safe, expected_type=bool): nfo_root = to_max_3_byte_utf8_string(nfo_root) nfo_tags = { to_max_3_byte_utf8_string(key): [ diff --git a/src/ytdl_sub/plugins/square_thumbnail.py b/src/ytdl_sub/plugins/square_thumbnail.py index 030f5a09..dc5db0b7 100644 --- a/src/ytdl_sub/plugins/square_thumbnail.py +++ b/src/ytdl_sub/plugins/square_thumbnail.py @@ -31,7 +31,7 @@ class SquareThumbnailPlugin(Plugin[SquareThumbnailOptions]): @property def _square_thumbnail(self) -> bool: - return self.overrides.evaluate_boolean(self.plugin_options) + return self.overrides.apply_formatter(self.plugin_options, expected_type=bool) @classmethod def _convert_to_square_thumbnail(cls, entry: Entry) -> None: diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 7acad676..463d1ace 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -42,8 +42,8 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC): ) def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float: - actualized_min = float(overrides.apply_formatter(self._min, entry=entry)) - actualized_max = float(overrides.apply_formatter(self._max, entry=entry)) + actualized_min = overrides.apply_formatter(self._min, entry=entry, expected_type=float) + actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float) if actualized_min < 0: raise self._validation_exception( @@ -70,7 +70,7 @@ class _RandomizedRangeValidator(StrictDictValidator, ABC): ------- Max possible value """ - actualized_max = float(overrides.apply_formatter(self._max, entry=entry)) + actualized_max = overrides.apply_formatter(self._max, entry=entry, expected_type=float) if actualized_max < 0: raise self._validation_exception( f"max must be greater than zero, received {actualized_max}" diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 16b7d949..00c5d460 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -34,7 +34,7 @@ class VideoTagsPlugin(Plugin[VideoTagsOptions]): Tags the entry's audio file using values defined in the metadata options """ tags_to_write: Dict[str, str] = {} - for tag_name, tag_formatter in self.plugin_options.dict.items(): + for tag_name, tag_formatter in sorted(self.plugin_options.dict.items()): tag_value = self.overrides.apply_formatter(formatter=tag_formatter, entry=entry) tags_to_write[tag_name] = tag_value diff --git a/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml index 676862a3..0bf420c3 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml @@ -11,8 +11,7 @@ presets: # Set the default date_range to 2 months overrides: - date_range: "2months" # keep for legacy-reasons - only_recent_date_range: "{date_range}" + only_recent_date_range: "2months" ############################################################################# # Only Recent diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml index 5c4439e7..7bd4d3f5 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml @@ -14,7 +14,7 @@ presets: # _multi_url: download: - - url: "{url}" + - url: "{ %array_at(urls, 0) }" playlist_thumbnails: - name: "{avatar_uncropped_thumbnail_file_name}" uid: "avatar_uncropped" @@ -27,301 +27,8 @@ presets: uid: "banner_uncropped" include_sibling_metadata: "{include_sibling_metadata}" webpage_url: "{modified_webpage_url}" - - url: "{url2}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url3}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url4}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url5}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url6}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url7}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url8}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url9}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url10}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url11}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url12}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url13}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url14}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url15}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url16}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url17}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url18}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url19}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url20}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url21}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url22}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url23}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url24}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url25}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url26}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url27}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url28}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url29}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url30}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url31}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url32}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url33}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url34}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url35}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url36}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url37}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url38}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url39}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url40}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url41}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url42}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url43}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url44}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url45}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url46}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url47}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url48}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url49}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url50}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url51}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url52}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url53}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url54}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url55}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url56}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url57}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url58}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url59}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url60}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url61}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url62}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url63}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url64}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url65}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url66}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url67}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url68}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url69}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url70}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url71}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url72}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url73}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url74}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url75}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url76}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url77}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url78}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url79}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url80}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url81}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url82}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url83}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url84}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url85}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url86}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url87}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url88}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url89}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url90}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url91}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url92}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url93}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url94}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url95}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url96}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url97}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url98}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url99}" - include_sibling_metadata: "{include_sibling_metadata}" - webpage_url: "{modified_webpage_url}" - - url: "{url100}" + + - url: "{ %array_slice(urls, 1) }" include_sibling_metadata: "{include_sibling_metadata}" webpage_url: "{modified_webpage_url}" overrides: @@ -330,208 +37,124 @@ presets: include_sibling_metadata: False modified_webpage_url: "{webpage_url}" - subscription_array: "{ [] }" + subscription_array: >- + { + [ + url, url2, url3, url4, url5, url6, url7, url8, url9, url10, + url11, url12, url13, url14, url15, url16, url17, url18, url19, url20, + url21, url22, url23, url24, url25, url26, url27, url28, url29, url30, + url31, url32, url33, url34, url35, url36, url37, url38, url39, url40, + url41, url42, url43, url44, url45, url46, url47, url48, url49, url50, + url51, url52, url53, url54, url55, url56, url57, url58, url59, url60, + url61, url62, url63, url64, url65, url66, url67, url68, url69, url70, + url71, url72, url73, url74, url75, url76, url77, url78, url79, url80, + url81, url82, url83, url84, url85, url86, url87, url88, url89, url90, + url91, url92, url93, url94, url95, url96, url97, url98, url99, url100 + ] + } + urls: "{subscription_array}" + subscription_value: "" - subscription_value_2: "" - subscription_value_3: "" - subscription_value_4: "" - subscription_value_5: "" - subscription_value_6: "" - subscription_value_7: "" - subscription_value_8: "" - subscription_value_9: "" - subscription_value_10: "" - subscription_value_11: "" - subscription_value_12: "" - subscription_value_13: "" - subscription_value_14: "" - subscription_value_15: "" - subscription_value_16: "" - subscription_value_17: "" - subscription_value_18: "" - subscription_value_19: "" - subscription_value_20: "" - subscription_value_21: "" - subscription_value_22: "" - subscription_value_23: "" - subscription_value_24: "" - subscription_value_25: "" - subscription_value_26: "" - subscription_value_27: "" - subscription_value_28: "" - subscription_value_29: "" - subscription_value_30: "" - subscription_value_31: "" - subscription_value_32: "" - subscription_value_33: "" - subscription_value_34: "" - subscription_value_35: "" - subscription_value_36: "" - subscription_value_37: "" - subscription_value_38: "" - subscription_value_39: "" - subscription_value_40: "" - subscription_value_41: "" - subscription_value_42: "" - subscription_value_43: "" - subscription_value_44: "" - subscription_value_45: "" - subscription_value_46: "" - subscription_value_47: "" - subscription_value_48: "" - subscription_value_49: "" - subscription_value_50: "" - subscription_value_51: "" - subscription_value_52: "" - subscription_value_53: "" - subscription_value_54: "" - subscription_value_55: "" - subscription_value_56: "" - subscription_value_57: "" - subscription_value_58: "" - subscription_value_59: "" - subscription_value_60: "" - subscription_value_61: "" - subscription_value_62: "" - subscription_value_63: "" - subscription_value_64: "" - subscription_value_65: "" - subscription_value_66: "" - subscription_value_67: "" - subscription_value_68: "" - subscription_value_69: "" - subscription_value_70: "" - subscription_value_71: "" - subscription_value_72: "" - subscription_value_73: "" - subscription_value_74: "" - subscription_value_75: "" - subscription_value_76: "" - subscription_value_77: "" - subscription_value_78: "" - subscription_value_79: "" - subscription_value_80: "" - subscription_value_81: "" - subscription_value_82: "" - subscription_value_83: "" - subscription_value_84: "" - subscription_value_85: "" - subscription_value_86: "" - subscription_value_87: "" - subscription_value_88: "" - subscription_value_89: "" - subscription_value_90: "" - subscription_value_91: "" - subscription_value_92: "" - subscription_value_93: "" - subscription_value_94: "" - subscription_value_95: "" - subscription_value_96: "" - subscription_value_97: "" - subscription_value_98: "" - subscription_value_99: "" - subscription_value_100: "" - url: "{subscription_value}" - url2: "{subscription_value_2}" - url3: "{subscription_value_3}" - url4: "{subscription_value_4}" - url5: "{subscription_value_5}" - url6: "{subscription_value_6}" - url7: "{subscription_value_7}" - url8: "{subscription_value_8}" - url9: "{subscription_value_9}" - url10: "{subscription_value_10}" - url11: "{subscription_value_11}" - url12: "{subscription_value_12}" - url13: "{subscription_value_13}" - url14: "{subscription_value_14}" - url15: "{subscription_value_15}" - url16: "{subscription_value_16}" - url17: "{subscription_value_17}" - url18: "{subscription_value_18}" - url19: "{subscription_value_19}" - url20: "{subscription_value_20}" - url21: "{subscription_value_21}" - url22: "{subscription_value_22}" - url23: "{subscription_value_23}" - url24: "{subscription_value_24}" - url25: "{subscription_value_25}" - url26: "{subscription_value_26}" - url27: "{subscription_value_27}" - url28: "{subscription_value_28}" - url29: "{subscription_value_29}" - url30: "{subscription_value_30}" - url31: "{subscription_value_31}" - url32: "{subscription_value_32}" - url33: "{subscription_value_33}" - url34: "{subscription_value_34}" - url35: "{subscription_value_35}" - url36: "{subscription_value_36}" - url37: "{subscription_value_37}" - url38: "{subscription_value_38}" - url39: "{subscription_value_39}" - url40: "{subscription_value_40}" - url41: "{subscription_value_41}" - url42: "{subscription_value_42}" - url43: "{subscription_value_43}" - url44: "{subscription_value_44}" - url45: "{subscription_value_45}" - url46: "{subscription_value_46}" - url47: "{subscription_value_47}" - url48: "{subscription_value_48}" - url49: "{subscription_value_49}" - url50: "{subscription_value_50}" - url51: "{subscription_value_51}" - url52: "{subscription_value_52}" - url53: "{subscription_value_53}" - url54: "{subscription_value_54}" - url55: "{subscription_value_55}" - url56: "{subscription_value_56}" - url57: "{subscription_value_57}" - url58: "{subscription_value_58}" - url59: "{subscription_value_59}" - url60: "{subscription_value_60}" - url61: "{subscription_value_61}" - url62: "{subscription_value_62}" - url63: "{subscription_value_63}" - url64: "{subscription_value_64}" - url65: "{subscription_value_65}" - url66: "{subscription_value_66}" - url67: "{subscription_value_67}" - url68: "{subscription_value_68}" - url69: "{subscription_value_69}" - url70: "{subscription_value_70}" - url71: "{subscription_value_71}" - url72: "{subscription_value_72}" - url73: "{subscription_value_73}" - url74: "{subscription_value_74}" - url75: "{subscription_value_75}" - url76: "{subscription_value_76}" - url77: "{subscription_value_77}" - url78: "{subscription_value_78}" - url79: "{subscription_value_79}" - url80: "{subscription_value_80}" - url81: "{subscription_value_81}" - url82: "{subscription_value_82}" - url83: "{subscription_value_83}" - url84: "{subscription_value_84}" - url85: "{subscription_value_85}" - url86: "{subscription_value_86}" - url87: "{subscription_value_87}" - url88: "{subscription_value_88}" - url89: "{subscription_value_89}" - url90: "{subscription_value_90}" - url91: "{subscription_value_91}" - url92: "{subscription_value_92}" - url93: "{subscription_value_93}" - url94: "{subscription_value_94}" - url95: "{subscription_value_95}" - url96: "{subscription_value_96}" - url97: "{subscription_value_97}" - url98: "{subscription_value_98}" - url99: "{subscription_value_99}" - url100: "{subscription_value_100}" + url2: "" + url3: "" + url4: "" + url5: "" + url6: "" + url7: "" + url8: "" + url9: "" + url10: "" + url11: "" + url12: "" + url13: "" + url14: "" + url15: "" + url16: "" + url17: "" + url18: "" + url19: "" + url20: "" + url21: "" + url22: "" + url23: "" + url24: "" + url25: "" + url26: "" + url27: "" + url28: "" + url29: "" + url30: "" + url31: "" + url32: "" + url33: "" + url34: "" + url35: "" + url36: "" + url37: "" + url38: "" + url39: "" + url40: "" + url41: "" + url42: "" + url43: "" + url44: "" + url45: "" + url46: "" + url47: "" + url48: "" + url49: "" + url50: "" + url51: "" + url52: "" + url53: "" + url54: "" + url55: "" + url56: "" + url57: "" + url58: "" + url59: "" + url60: "" + url61: "" + url62: "" + url63: "" + url64: "" + url65: "" + url66: "" + url67: "" + url68: "" + url69: "" + url70: "" + url71: "" + url72: "" + url73: "" + url74: "" + url75: "" + url76: "" + url77: "" + url78: "" + url79: "" + url80: "" + url81: "" + url82: "" + url83: "" + url84: "" + url85: "" + url86: "" + url87: "" + url88: "" + url89: "" + url90: "" + url91: "" + url92: "" + url93: "" + url94: "" + url95: "" + url96: "" + url97: "" + url98: "" + url99: "" + url100: "" # multi-url with bilateral scraping built into it via @@ -542,502 +165,7 @@ presets: - "_url_bilateral_overrides" download: - - url: "{%bilateral_url(url) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url2) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url3) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url4) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url5) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url6) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url7) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url8) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url9) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url10) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url11) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url12) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url13) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url14) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url15) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url16) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url17) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url18) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url19) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url20) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url21) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url22) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url23) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url24) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url25) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url26) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url27) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url28) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url29) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url30) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url31) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url32) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url33) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url34) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url35) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url36) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url37) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url38) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url39) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url40) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url41) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url42) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url43) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url44) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url45) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url46) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url47) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url48) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url49) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url50) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url51) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url52) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url53) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url54) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url55) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url56) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url57) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url58) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url59) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url60) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url61) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url62) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url63) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url64) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url65) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url66) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url67) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url68) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url69) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url70) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url71) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url72) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url73) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url74) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url75) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url76) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url77) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url78) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url79) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url80) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url81) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url82) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url83) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url84) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url85) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url86) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url87) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url88) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url89) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url90) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url91) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url92) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url93) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url94) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url95) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url96) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url97) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url98) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{ %bilateral_url(url99) }" - download_reverse: False - ytdl_options: - playlist_items: "-1:0:-1" - webpage_url: "{modified_webpage_url}" - - url: "{%bilateral_url(url100) }" + - url: "{ %array_apply(urls, %bilateral_url) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url_categorized.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url_categorized.yaml index b212f640..80e041a7 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url_categorized.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url_categorized.yaml @@ -108,103 +108,4 @@ presets: } subscription_map: "{ {} }" - url: "{ %get_url_i(1) }" - url2: "{ %get_url_i(2) }" - url3: "{ %get_url_i(3) }" - url4: "{ %get_url_i(4) }" - url5: "{ %get_url_i(5) }" - url6: "{ %get_url_i(6) }" - url7: "{ %get_url_i(7) }" - url8: "{ %get_url_i(8) }" - url9: "{ %get_url_i(9) }" - url10: "{ %get_url_i(10) }" - url11: "{ %get_url_i(11) }" - url12: "{ %get_url_i(12) }" - url13: "{ %get_url_i(13) }" - url14: "{ %get_url_i(14) }" - url15: "{ %get_url_i(15) }" - url16: "{ %get_url_i(16) }" - url17: "{ %get_url_i(17) }" - url18: "{ %get_url_i(18) }" - url19: "{ %get_url_i(19) }" - url20: "{ %get_url_i(20) }" - url21: "{ %get_url_i(21) }" - url22: "{ %get_url_i(22) }" - url23: "{ %get_url_i(23) }" - url24: "{ %get_url_i(24) }" - url25: "{ %get_url_i(25) }" - url26: "{ %get_url_i(26) }" - url27: "{ %get_url_i(27) }" - url28: "{ %get_url_i(28) }" - url29: "{ %get_url_i(29) }" - url30: "{ %get_url_i(30) }" - url31: "{ %get_url_i(31) }" - url32: "{ %get_url_i(32) }" - url33: "{ %get_url_i(33) }" - url34: "{ %get_url_i(34) }" - url35: "{ %get_url_i(35) }" - url36: "{ %get_url_i(36) }" - url37: "{ %get_url_i(37) }" - url38: "{ %get_url_i(38) }" - url39: "{ %get_url_i(39) }" - url40: "{ %get_url_i(40) }" - url41: "{ %get_url_i(41) }" - url42: "{ %get_url_i(42) }" - url43: "{ %get_url_i(43) }" - url44: "{ %get_url_i(44) }" - url45: "{ %get_url_i(45) }" - url46: "{ %get_url_i(46) }" - url47: "{ %get_url_i(47) }" - url48: "{ %get_url_i(48) }" - url49: "{ %get_url_i(49) }" - url50: "{ %get_url_i(50) }" - url51: "{ %get_url_i(51) }" - url52: "{ %get_url_i(52) }" - url53: "{ %get_url_i(53) }" - url54: "{ %get_url_i(54) }" - url55: "{ %get_url_i(55) }" - url56: "{ %get_url_i(56) }" - url57: "{ %get_url_i(57) }" - url58: "{ %get_url_i(58) }" - url59: "{ %get_url_i(59) }" - url60: "{ %get_url_i(60) }" - url61: "{ %get_url_i(61) }" - url62: "{ %get_url_i(62) }" - url63: "{ %get_url_i(63) }" - url64: "{ %get_url_i(64) }" - url65: "{ %get_url_i(65) }" - url66: "{ %get_url_i(66) }" - url67: "{ %get_url_i(67) }" - url68: "{ %get_url_i(68) }" - url69: "{ %get_url_i(69) }" - url70: "{ %get_url_i(70) }" - url71: "{ %get_url_i(71) }" - url72: "{ %get_url_i(72) }" - url73: "{ %get_url_i(73) }" - url74: "{ %get_url_i(74) }" - url75: "{ %get_url_i(75) }" - url76: "{ %get_url_i(76) }" - url77: "{ %get_url_i(77) }" - url78: "{ %get_url_i(78) }" - url79: "{ %get_url_i(79) }" - url80: "{ %get_url_i(80) }" - url81: "{ %get_url_i(81) }" - url82: "{ %get_url_i(82) }" - url83: "{ %get_url_i(83) }" - url84: "{ %get_url_i(84) }" - url85: "{ %get_url_i(85) }" - url86: "{ %get_url_i(86) }" - url87: "{ %get_url_i(87) }" - url88: "{ %get_url_i(88) }" - url89: "{ %get_url_i(89) }" - url90: "{ %get_url_i(90) }" - url91: "{ %get_url_i(91) }" - url92: "{ %get_url_i(92) }" - url93: "{ %get_url_i(93) }" - url94: "{ %get_url_i(94) }" - url95: "{ %get_url_i(95) }" - url96: "{ %get_url_i(96) }" - url97: "{ %get_url_i(97) }" - url98: "{ %get_url_i(98) }" - url99: "{ %get_url_i(99) }" - url100: "{ %get_url_i(100) }" \ No newline at end of file + urls: "{ %array_apply(%range(100, 1), %get_url_i) }" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml b/src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml index 32eb964a..966be5f8 100644 --- a/src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml +++ b/src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml @@ -8,7 +8,7 @@ presets: urls: # The first URL will be all the artist's tracks. # Treat these as singles - an album with a single track - - url: "{url}/tracks" + - url: "{url}" include_sibling_metadata: False variables: sc_track_album: "{title}" diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index 70c0fd9d..c8a78a7a 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -124,7 +124,7 @@ presets: - "_tv_show_collection_asserts" download: - - url: "{collection_season_1_url}" + - url: "{ %array_at( %get_season_urls(collection_season_1_url), 0) }" variables: collection_season_number: "1" collection_season_name: "{collection_season_1_name}" @@ -142,318 +142,493 @@ presets: uid: "avatar_uncropped" - name: "{tv_show_fanart_file_name}" uid: "banner_uncropped" + - url: "{ %array_slice( %get_season_urls(collection_season_1_url), 1) }" + variables: + collection_season_number: "1" + collection_season_name: "{collection_season_1_name}" - - url: "{collection_season_2_url}" + - url: "{ %array_at( %get_season_urls(collection_season_2_url), 0) }" variables: collection_season_number: "2" collection_season_name: "{collection_season_2_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_2_url), 1) }" + variables: + collection_season_number: "2" + collection_season_name: "{collection_season_2_name}" - - url: "{collection_season_3_url}" + - url: "{ %array_at( %get_season_urls(collection_season_3_url), 0) }" variables: collection_season_number: "3" collection_season_name: "{collection_season_3_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_3_url), 1) }" + variables: + collection_season_number: "3" + collection_season_name: "{collection_season_3_name}" - - url: "{collection_season_4_url}" + - url: "{ %array_at( %get_season_urls(collection_season_4_url), 0) }" variables: collection_season_number: "4" collection_season_name: "{collection_season_4_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_4_url), 1) }" + variables: + collection_season_number: "4" + collection_season_name: "{collection_season_4_name}" - - url: "{collection_season_5_url}" + - url: "{ %array_at( %get_season_urls(collection_season_5_url), 0) }" variables: collection_season_number: "5" collection_season_name: "{collection_season_5_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_5_url), 1) }" + variables: + collection_season_number: "5" + collection_season_name: "{collection_season_5_name}" - - url: "{collection_season_6_url}" + - url: "{ %array_at( %get_season_urls(collection_season_6_url), 0) }" variables: collection_season_number: "6" collection_season_name: "{collection_season_6_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_6_url), 1) }" + variables: + collection_season_number: "6" + collection_season_name: "{collection_season_6_name}" - - url: "{collection_season_7_url}" + - url: "{ %array_at( %get_season_urls(collection_season_7_url), 0) }" variables: collection_season_number: "7" collection_season_name: "{collection_season_7_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_7_url), 1) }" + variables: + collection_season_number: "7" + collection_season_name: "{collection_season_7_name}" - - url: "{collection_season_8_url}" + - url: "{ %array_at( %get_season_urls(collection_season_8_url), 0) }" variables: collection_season_number: "8" collection_season_name: "{collection_season_8_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_8_url), 1) }" + variables: + collection_season_number: "8" + collection_season_name: "{collection_season_8_name}" - - url: "{collection_season_9_url}" + - url: "{ %array_at( %get_season_urls(collection_season_9_url), 0) }" variables: collection_season_number: "9" collection_season_name: "{collection_season_9_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_9_url), 1) }" + variables: + collection_season_number: "9" + collection_season_name: "{collection_season_9_name}" - - url: "{collection_season_10_url}" + - url: "{ %array_at( %get_season_urls(collection_season_10_url), 0) }" variables: collection_season_number: "10" collection_season_name: "{collection_season_10_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_10_url), 1) }" + variables: + collection_season_number: "10" + collection_season_name: "{collection_season_10_name}" - - url: "{collection_season_11_url}" + - url: "{ %array_at( %get_season_urls(collection_season_11_url), 0) }" variables: collection_season_number: "11" collection_season_name: "{collection_season_11_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_11_url), 1) }" + variables: + collection_season_number: "11" + collection_season_name: "{collection_season_11_name}" - - url: "{collection_season_12_url}" + - url: "{ %array_at( %get_season_urls(collection_season_12_url), 0) }" variables: collection_season_number: "12" collection_season_name: "{collection_season_12_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_12_url), 1) }" + variables: + collection_season_number: "12" + collection_season_name: "{collection_season_12_name}" - - url: "{collection_season_13_url}" + - url: "{ %array_at( %get_season_urls(collection_season_13_url), 0) }" variables: collection_season_number: "13" collection_season_name: "{collection_season_13_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_13_url), 1) }" + variables: + collection_season_number: "13" + collection_season_name: "{collection_season_13_name}" - - url: "{collection_season_14_url}" + - url: "{ %array_at( %get_season_urls(collection_season_14_url), 0) }" variables: collection_season_number: "14" collection_season_name: "{collection_season_14_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_14_url), 1) }" + variables: + collection_season_number: "14" + collection_season_name: "{collection_season_14_name}" - - url: "{collection_season_15_url}" + - url: "{ %array_at( %get_season_urls(collection_season_15_url), 0) }" variables: collection_season_number: "15" collection_season_name: "{collection_season_15_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_15_url), 1) }" + variables: + collection_season_number: "15" + collection_season_name: "{collection_season_15_name}" - - url: "{collection_season_16_url}" + - url: "{ %array_at( %get_season_urls(collection_season_16_url), 0) }" variables: collection_season_number: "16" collection_season_name: "{collection_season_16_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_16_url), 1) }" + variables: + collection_season_number: "16" + collection_season_name: "{collection_season_16_name}" - - url: "{collection_season_17_url}" + - url: "{ %array_at( %get_season_urls(collection_season_17_url), 0) }" variables: collection_season_number: "17" collection_season_name: "{collection_season_17_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_17_url), 1) }" + variables: + collection_season_number: "17" + collection_season_name: "{collection_season_17_name}" - - url: "{collection_season_18_url}" + - url: "{ %array_at( %get_season_urls(collection_season_18_url), 0) }" variables: collection_season_number: "18" collection_season_name: "{collection_season_18_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_18_url), 1) }" + variables: + collection_season_number: "18" + collection_season_name: "{collection_season_18_name}" - - url: "{collection_season_19_url}" + - url: "{ %array_at( %get_season_urls(collection_season_19_url), 0) }" variables: collection_season_number: "19" collection_season_name: "{collection_season_19_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_19_url), 1) }" + variables: + collection_season_number: "19" + collection_season_name: "{collection_season_19_name}" - - url: "{collection_season_20_url}" + - url: "{ %array_at( %get_season_urls(collection_season_20_url), 0) }" variables: collection_season_number: "20" collection_season_name: "{collection_season_20_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_20_url), 1) }" + variables: + collection_season_number: "20" + collection_season_name: "{collection_season_20_name}" - - url: "{collection_season_21_url}" + - url: "{ %array_at( %get_season_urls(collection_season_21_url), 0) }" variables: collection_season_number: "21" collection_season_name: "{collection_season_21_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_21_url), 1) }" + variables: + collection_season_number: "21" + collection_season_name: "{collection_season_21_name}" - - url: "{collection_season_22_url}" + - url: "{ %array_at( %get_season_urls(collection_season_22_url), 0) }" variables: collection_season_number: "22" collection_season_name: "{collection_season_22_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_22_url), 1) }" + variables: + collection_season_number: "22" + collection_season_name: "{collection_season_22_name}" - - url: "{collection_season_23_url}" + - url: "{ %array_at( %get_season_urls(collection_season_23_url), 0) }" variables: collection_season_number: "23" collection_season_name: "{collection_season_23_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_23_url), 1) }" + variables: + collection_season_number: "23" + collection_season_name: "{collection_season_23_name}" - - url: "{collection_season_24_url}" + - url: "{ %array_at( %get_season_urls(collection_season_24_url), 0) }" variables: collection_season_number: "24" collection_season_name: "{collection_season_24_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_24_url), 1) }" + variables: + collection_season_number: "24" + collection_season_name: "{collection_season_24_name}" - - url: "{collection_season_25_url}" + - url: "{ %array_at( %get_season_urls(collection_season_25_url), 0) }" variables: collection_season_number: "25" collection_season_name: "{collection_season_25_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_25_url), 1) }" + variables: + collection_season_number: "25" + collection_season_name: "{collection_season_25_name}" - - url: "{collection_season_26_url}" + - url: "{ %array_at( %get_season_urls(collection_season_26_url), 0) }" variables: collection_season_number: "26" collection_season_name: "{collection_season_26_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_26_url), 1) }" + variables: + collection_season_number: "26" + collection_season_name: "{collection_season_26_name}" - - url: "{collection_season_27_url}" + - url: "{ %array_at( %get_season_urls(collection_season_27_url), 0) }" variables: collection_season_number: "27" collection_season_name: "{collection_season_27_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_27_url), 1) }" + variables: + collection_season_number: "27" + collection_season_name: "{collection_season_27_name}" - - url: "{collection_season_28_url}" + - url: "{ %array_at( %get_season_urls(collection_season_28_url), 0) }" variables: collection_season_number: "28" collection_season_name: "{collection_season_28_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_28_url), 1) }" + variables: + collection_season_number: "28" + collection_season_name: "{collection_season_28_name}" - - url: "{collection_season_29_url}" + - url: "{ %array_at( %get_season_urls(collection_season_29_url), 0) }" variables: collection_season_number: "29" collection_season_name: "{collection_season_29_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_29_url), 1) }" + variables: + collection_season_number: "29" + collection_season_name: "{collection_season_29_name}" - - url: "{collection_season_30_url}" + - url: "{ %array_at( %get_season_urls(collection_season_30_url), 0) }" variables: collection_season_number: "30" collection_season_name: "{collection_season_30_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_30_url), 1) }" + variables: + collection_season_number: "30" + collection_season_name: "{collection_season_30_name}" - - url: "{collection_season_31_url}" + - url: "{ %array_at( %get_season_urls(collection_season_31_url), 0) }" variables: collection_season_number: "31" collection_season_name: "{collection_season_31_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_31_url), 1) }" + variables: + collection_season_number: "31" + collection_season_name: "{collection_season_31_name}" - - url: "{collection_season_32_url}" + - url: "{ %array_at( %get_season_urls(collection_season_32_url), 0) }" variables: collection_season_number: "32" collection_season_name: "{collection_season_32_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_32_url), 1) }" + variables: + collection_season_number: "32" + collection_season_name: "{collection_season_32_name}" - - url: "{collection_season_33_url}" + - url: "{ %array_at( %get_season_urls(collection_season_33_url), 0) }" variables: collection_season_number: "33" collection_season_name: "{collection_season_33_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_33_url), 1) }" + variables: + collection_season_number: "33" + collection_season_name: "{collection_season_33_name}" - - url: "{collection_season_34_url}" + - url: "{ %array_at( %get_season_urls(collection_season_34_url), 0) }" variables: collection_season_number: "34" collection_season_name: "{collection_season_34_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_34_url), 1) }" + variables: + collection_season_number: "34" + collection_season_name: "{collection_season_34_name}" - - url: "{collection_season_35_url}" + - url: "{ %array_at( %get_season_urls(collection_season_35_url), 0) }" variables: collection_season_number: "35" collection_season_name: "{collection_season_35_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_35_url), 1) }" + variables: + collection_season_number: "35" + collection_season_name: "{collection_season_35_name}" - - url: "{collection_season_36_url}" + - url: "{ %array_at( %get_season_urls(collection_season_36_url), 0) }" variables: collection_season_number: "36" collection_season_name: "{collection_season_36_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_36_url), 1) }" + variables: + collection_season_number: "36" + collection_season_name: "{collection_season_36_name}" - - url: "{collection_season_37_url}" + - url: "{ %array_at( %get_season_urls(collection_season_37_url), 0) }" variables: collection_season_number: "37" collection_season_name: "{collection_season_37_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_37_url), 1) }" + variables: + collection_season_number: "37" + collection_season_name: "{collection_season_37_name}" - - url: "{collection_season_38_url}" + - url: "{ %array_at( %get_season_urls(collection_season_38_url), 0) }" variables: collection_season_number: "38" collection_season_name: "{collection_season_38_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_38_url), 1) }" + variables: + collection_season_number: "38" + collection_season_name: "{collection_season_38_name}" - - url: "{collection_season_39_url}" + - url: "{ %array_at( %get_season_urls(collection_season_39_url), 0) }" variables: collection_season_number: "39" collection_season_name: "{collection_season_39_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_39_url), 1) }" + variables: + collection_season_number: "39" + collection_season_name: "{collection_season_39_name}" - - url: "{collection_season_40_url}" + - url: "{ %array_at( %get_season_urls(collection_season_40_url), 0) }" variables: collection_season_number: "40" collection_season_name: "{collection_season_40_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_40_url), 1) }" + variables: + collection_season_number: "40" + collection_season_name: "{collection_season_40_name}" + + + # Place season 0 at end + - url: "{ %array_at( %get_season_urls(collection_season_0_url), 0) }" + variables: + collection_season_number: "0" + collection_season_name: "{collection_season_0_name}" + playlist_thumbnails: + - name: "{season_poster_file_name}" + uid: "latest_entry" + - url: "{ %array_slice( %get_season_urls(collection_season_0_url), 1) }" + variables: + collection_season_number: "0" + collection_season_name: "{collection_season_0_name}" + output_directory_nfo_tags: tags: @@ -625,6 +800,7 @@ presets: collection_season_38_name: "{s38_name}" collection_season_39_name: "{s39_name}" collection_season_40_name: "{s40_name}" + collection_season_0_name: "{s00_name}" # Legacy url variable collection_season_1_url: "{s01_url}" @@ -667,6 +843,7 @@ presets: collection_season_38_url: "{s38_url}" collection_season_39_url: "{s39_url}" collection_season_40_url: "{s40_url}" + collection_season_0_url: "{s00_url}" s01_name: "" s02_name: "" @@ -708,6 +885,7 @@ presets: s38_name: "" s39_name: "" s40_name: "" + s00_name: "" s01_url: "" s02_url: "" @@ -749,13 +927,22 @@ presets: s38_url: "" s39_url: "" s40_url: "" + s00_url: "" + + "%get_season_urls": >- + { %if( %is_array( $0 ), $0, [ $0 ] ) } + + # $0 - season url variable + # $1 - get the i'th url from the array + "%get_season_url": >- + { %array_at( %get_season_urls($0), $1, "" ) } _tv_show_collection_bilateral: preset: - "_url_bilateral_overrides" download: - - url: "{ %bilateral_url(collection_season_1_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_1_url), %bilateral_url) }" variables: collection_season_number: "1" collection_season_name: "{collection_season_1_name}" @@ -763,7 +950,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_2_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_2_url), %bilateral_url) }" variables: collection_season_number: "2" collection_season_name: "{collection_season_2_name}" @@ -771,7 +958,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_3_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_3_url), %bilateral_url) }" variables: collection_season_number: "3" collection_season_name: "{collection_season_3_name}" @@ -779,7 +966,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_4_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_4_url), %bilateral_url) }" variables: collection_season_number: "4" collection_season_name: "{collection_season_4_name}" @@ -787,7 +974,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_5_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_5_url), %bilateral_url) }" variables: collection_season_number: "5" collection_season_name: "{collection_season_5_name}" @@ -795,7 +982,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_6_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_6_url), %bilateral_url) }" variables: collection_season_number: "6" collection_season_name: "{collection_season_6_name}" @@ -803,7 +990,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_7_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_7_url), %bilateral_url) }" variables: collection_season_number: "7" collection_season_name: "{collection_season_7_name}" @@ -811,7 +998,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_8_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_8_url), %bilateral_url) }" variables: collection_season_number: "8" collection_season_name: "{collection_season_8_name}" @@ -819,7 +1006,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_9_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_9_url), %bilateral_url) }" variables: collection_season_number: "9" collection_season_name: "{collection_season_9_name}" @@ -827,7 +1014,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_10_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_10_url), %bilateral_url) }" variables: collection_season_number: "10" collection_season_name: "{collection_season_10_name}" @@ -835,7 +1022,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_11_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_11_url), %bilateral_url) }" variables: collection_season_number: "11" collection_season_name: "{collection_season_11_name}" @@ -843,7 +1030,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_12_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_12_url), %bilateral_url) }" variables: collection_season_number: "12" collection_season_name: "{collection_season_12_name}" @@ -851,7 +1038,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_13_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_13_url), %bilateral_url) }" variables: collection_season_number: "13" collection_season_name: "{collection_season_13_name}" @@ -859,7 +1046,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_14_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_14_url), %bilateral_url) }" variables: collection_season_number: "14" collection_season_name: "{collection_season_14_name}" @@ -867,7 +1054,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_15_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_15_url), %bilateral_url) }" variables: collection_season_number: "15" collection_season_name: "{collection_season_15_name}" @@ -875,7 +1062,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_16_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_16_url), %bilateral_url) }" variables: collection_season_number: "16" collection_season_name: "{collection_season_16_name}" @@ -883,7 +1070,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_17_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_17_url), %bilateral_url) }" variables: collection_season_number: "17" collection_season_name: "{collection_season_17_name}" @@ -891,7 +1078,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_18_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_18_url), %bilateral_url) }" variables: collection_season_number: "18" collection_season_name: "{collection_season_18_name}" @@ -899,7 +1086,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_19_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_19_url), %bilateral_url) }" variables: collection_season_number: "19" collection_season_name: "{collection_season_19_name}" @@ -907,7 +1094,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_20_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_20_url), %bilateral_url) }" variables: collection_season_number: "20" collection_season_name: "{collection_season_20_name}" @@ -915,7 +1102,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_21_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_21_url), %bilateral_url) }" variables: collection_season_number: "21" collection_season_name: "{collection_season_21_name}" @@ -923,7 +1110,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_22_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_22_url), %bilateral_url) }" variables: collection_season_number: "22" collection_season_name: "{collection_season_22_name}" @@ -931,7 +1118,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_23_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_23_url), %bilateral_url) }" variables: collection_season_number: "23" collection_season_name: "{collection_season_23_name}" @@ -939,7 +1126,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_24_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_24_url), %bilateral_url) }" variables: collection_season_number: "24" collection_season_name: "{collection_season_24_name}" @@ -947,7 +1134,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_25_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_25_url), %bilateral_url) }" variables: collection_season_number: "25" collection_season_name: "{collection_season_25_name}" @@ -955,7 +1142,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_26_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_26_url), %bilateral_url) }" variables: collection_season_number: "26" collection_season_name: "{collection_season_26_name}" @@ -963,7 +1150,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_27_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_27_url), %bilateral_url) }" variables: collection_season_number: "27" collection_season_name: "{collection_season_27_name}" @@ -971,7 +1158,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_28_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_28_url), %bilateral_url) }" variables: collection_season_number: "28" collection_season_name: "{collection_season_28_name}" @@ -979,7 +1166,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_29_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_29_url), %bilateral_url) }" variables: collection_season_number: "29" collection_season_name: "{collection_season_29_name}" @@ -987,7 +1174,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_30_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_30_url), %bilateral_url) }" variables: collection_season_number: "30" collection_season_name: "{collection_season_30_name}" @@ -995,7 +1182,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_31_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_31_url), %bilateral_url) }" variables: collection_season_number: "31" collection_season_name: "{collection_season_31_name}" @@ -1003,7 +1190,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_32_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_32_url), %bilateral_url) }" variables: collection_season_number: "32" collection_season_name: "{collection_season_32_name}" @@ -1011,7 +1198,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_33_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_33_url), %bilateral_url) }" variables: collection_season_number: "33" collection_season_name: "{collection_season_33_name}" @@ -1019,7 +1206,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_34_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_34_url), %bilateral_url) }" variables: collection_season_number: "34" collection_season_name: "{collection_season_34_name}" @@ -1027,7 +1214,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_35_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_35_url), %bilateral_url) }" variables: collection_season_number: "35" collection_season_name: "{collection_season_35_name}" @@ -1035,7 +1222,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_36_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_36_url), %bilateral_url) }" variables: collection_season_number: "36" collection_season_name: "{collection_season_36_name}" @@ -1043,7 +1230,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_37_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_37_url), %bilateral_url) }" variables: collection_season_number: "37" collection_season_name: "{collection_season_37_name}" @@ -1051,7 +1238,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_38_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_38_url), %bilateral_url) }" variables: collection_season_number: "38" collection_season_name: "{collection_season_38_name}" @@ -1059,7 +1246,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_39_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_39_url), %bilateral_url) }" variables: collection_season_number: "39" collection_season_name: "{collection_season_39_name}" @@ -1067,7 +1254,7 @@ presets: ytdl_options: playlist_items: "-1:0:-1" - - url: "{ %bilateral_url(collection_season_40_url) }" + - url: "{ %array_apply( %get_season_urls(collection_season_40_url), %bilateral_url) }" variables: collection_season_number: "40" collection_season_name: "{collection_season_40_name}" @@ -1075,6 +1262,15 @@ presets: ytdl_options: playlist_items: "-1:0:-1" + # Season 0 at end (to download first) + - url: "{ %array_apply( %get_season_urls(collection_season_0_url), %bilateral_url) }" + variables: + collection_season_number: "0" + collection_season_name: "{collection_season_0_name}" + download_reverse: False + ytdl_options: + playlist_items: "-1:0:-1" + _tv_show_collection_asserts: overrides: url: "" diff --git a/src/ytdl_sub/script/functions/numeric_functions.py b/src/ytdl_sub/script/functions/numeric_functions.py index b25374f9..003c8837 100644 --- a/src/ytdl_sub/script/functions/numeric_functions.py +++ b/src/ytdl_sub/script/functions/numeric_functions.py @@ -1,5 +1,7 @@ import math +from typing import Optional +from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import Integer @@ -96,3 +98,18 @@ class NumericFunctions: Returns min of all values. """ return _to_numeric(min(val.value for val in values)) + + @staticmethod + def range( + end: Integer, start: Optional[Integer] = None, step: Optional[Integer] = None + ) -> Array: + """ + :description: + Returns the desired range of Integers in the form of an Array. + """ + if start is None: + start = Integer(0) + if step is None: + step = Integer(1) + + return Array(value=[Integer(idx) for idx in range(start.value, end.value, step.value)]) diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py index 737d6ffa..73dbb3f4 100644 --- a/src/ytdl_sub/script/parser.py +++ b/src/ytdl_sub/script/parser.py @@ -31,6 +31,8 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException from ytdl_sub.script.utils.exceptions import InvalidVariableName from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.exceptions import VariableDoesNotExist +from ytdl_sub.script.utils.name_validation import is_function +from ytdl_sub.script.utils.name_validation import to_function_name from ytdl_sub.script.utils.name_validation import validate_variable_name # pylint: disable=invalid-name @@ -144,6 +146,9 @@ class _Parser: ): self._text = text self._name = name + if name and is_function(name): + self._name = to_function_name(name) + self._custom_function_names = custom_function_names self._variable_names = variable_names self._pos = 0 @@ -605,7 +610,7 @@ def parse( name=name, custom_function_names=custom_function_names, variable_names=variable_names, - ).ast + ).ast.maybe_resolvable_casted() # pylint: enable=invalid-name diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index ab506318..41c3f1be 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -1,4 +1,5 @@ # pylint: disable=missing-raises-doc +from collections import defaultdict from typing import Dict from typing import List from typing import Optional @@ -7,40 +8,28 @@ from typing import Set from ytdl_sub.script.functions import Functions from ytdl_sub.script.parser import parse from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Argument from ytdl_sub.script.types.resolvable import BuiltInFunctionType from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.variable import FunctionArgument from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency from ytdl_sub.script.utils.exceptions import UNREACHABLE from ytdl_sub.script.utils.exceptions import CycleDetected from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.script.utils.name_validation import is_function +from ytdl_sub.script.utils.name_validation import to_function_definition_name +from ytdl_sub.script.utils.name_validation import to_function_name from ytdl_sub.script.utils.name_validation import validate_variable_name from ytdl_sub.script.utils.type_checking import FunctionSpec -def _is_function(override_name: str): - return override_name.startswith("%") - - -def _function_name(function_key: str) -> str: - """ - Drop the % in %custom_function - """ - return function_key[1:] - - -def _to_function_definition_name(function_key: str) -> str: - """ - Add % in %custom_function - """ - return f"%{function_key}" - - class Script: """ Takes a dictionary of both @@ -49,43 +38,71 @@ class Script: ``{ %custom_function: syntax }`` """ - def _ensure_no_cycle( + def _throw_cycle_error( self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree] ): - if dep not in definitions: - return # does not exist, will throw downstream in parser + type_name, pre = ( + ("custom functions", "%") if definitions is self._functions else ("variables", "") + ) + cycle_deps = [name] + deps + [dep] + cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps]) - if name in deps + [dep]: - type_name, pre = ( - ("custom functions", "%") if definitions is self._functions else ("variables", "") - ) - cycle_deps = [name] + deps + [dep] - cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps]) - - raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}") + raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}") def _traverse_variable_dependencies( self, variable_name: str, variable_dependency: SyntaxTree, deps: List[str], + ensured: Dict[str, Set[str]], ) -> None: for dep in variable_dependency.variables: - self._ensure_no_cycle( - name=variable_name, dep=dep.name, deps=deps, definitions=self._variables - ) + if variable_name == dep.name: + self._throw_cycle_error( + name=variable_name, dep=dep.name, deps=deps, definitions=self._variables + ) + + if dep.name in ensured[variable_name]: + continue + self._traverse_variable_dependencies( variable_name=variable_name, variable_dependency=self._variables[dep.name], deps=deps + [dep.name], + ensured=ensured, ) + ensured[variable_name].add(dep.name) + + for custom_func in variable_dependency.custom_function_dependencies( + custom_function_definitions=self._functions + ): + for dep in self._functions[custom_func.name].variables: + if variable_name == dep.name: + self._throw_cycle_error( + name=variable_name, + dep=dep.name, + deps=deps + [custom_func.definition_name()], + definitions=self._variables, + ) + if dep.name in ensured[variable_name]: + continue + + self._traverse_variable_dependencies( + variable_name=variable_name, + variable_dependency=self._variables[dep.name], + deps=deps + [custom_func.definition_name(), dep.name], + ensured=ensured, + ) + ensured[variable_name].add(dep.name) def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]): + ensured: Dict[str, Set[str]] = defaultdict(set) for variable_name, variable_definition in variables.items(): self._traverse_variable_dependencies( variable_name=variable_name, variable_dependency=variable_definition, deps=[], + ensured=ensured, ) def _traverse_custom_function_dependencies( @@ -95,9 +112,10 @@ class Script: deps: List[str], ) -> None: for dep in custom_function_dependency.custom_functions: - self._ensure_no_cycle( - name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions - ) + if custom_function_name == dep.name: + self._throw_cycle_error( + name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions + ) self._traverse_custom_function_dependencies( custom_function_name=custom_function_name, custom_function_dependency=self._functions[dep.name], @@ -240,23 +258,23 @@ class Script: def __init__(self, script: Dict[str, str]): function_names: Set[str] = { - _function_name(name) for name in script.keys() if _is_function(name) + to_function_name(name) for name in script.keys() if is_function(name) } variable_names: Set[str] = { - validate_variable_name(name) for name in script.keys() if not _is_function(name) + validate_variable_name(name) for name in script.keys() if not is_function(name) } self._functions: Dict[str, SyntaxTree] = { # custom_function_name must be passed to properly type custom function # arguments uniquely if they're nested (i.e. $0 to $custom_func___0) - _function_name(function_key): parse( + to_function_name(function_key): parse( text=function_value, - name=_function_name(function_key), + name=to_function_name(function_key), custom_function_names=function_names, variable_names=variable_names, ) for function_key, function_value in script.items() - if _is_function(function_key) + if is_function(function_key) } self._variables: Dict[str, SyntaxTree] = { @@ -267,13 +285,13 @@ class Script: variable_names=variable_names, ) for variable_key, variable_value in script.items() - if not _is_function(variable_key) + if not is_function(variable_key) } self._validate() def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None: for variable_name, resolved in resolved_variables.items(): - self._variables[variable_name] = SyntaxTree(ast=[resolved]) + self._variables[variable_name] = ResolvedSyntaxTree(ast=[resolved]) def _recursive_get_unresolved_output_filter_variables( self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable] @@ -302,14 +320,21 @@ class Script: unresolvable=unresolvable, ) + for lambda_func in current_var.lambdas: + if lambda_func.value in self._functions: + subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables( + current_var=self._functions[lambda_func.value], + subset_to_resolve=subset_to_resolve, + unresolvable=unresolvable, + ) + return subset_to_resolve def _get_unresolved_output_filter( self, - unresolved: Dict[Variable, SyntaxTree], output_filter: Set[str], unresolvable: Set[Variable], - ) -> Dict[Variable, SyntaxTree]: + ) -> Set[str]: """ When an output filter is applied, only a subset of variables that the filter depends on need to be resolved. @@ -330,7 +355,7 @@ class Script: unresolvable=unresolvable, ) - return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve} + return subset_to_resolve def _resolve( self, @@ -366,18 +391,21 @@ class Script: unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})} unresolved_filter = set(resolved.keys()).union(unresolvable) - unresolved: Dict[Variable, SyntaxTree] = { - Variable(name): ast - for name, ast in self._variables.items() - if Variable(name) not in unresolved_filter - } if output_filter: - unresolved = self._get_unresolved_output_filter( - unresolved=unresolved, - output_filter=output_filter, - unresolvable=unresolvable, - ) + unresolved = { + Variable(name): self._variables[name] + for name in self._get_unresolved_output_filter( + output_filter=output_filter, + unresolvable=unresolvable, + ) + } + else: + unresolved = { + Variable(name): ast + for name, ast in self._variables.items() + if Variable(name) not in unresolved_filter + } while unresolved: unresolved_count: int = len(unresolved) @@ -398,8 +426,8 @@ class Script: # Otherwise, if it has dependencies that are all resolved, then # resolve the definition - elif not definition.is_subset_of( - variables=resolved.keys(), custom_function_definitions=self._functions + elif definition.is_subset_of( + variables=resolved, custom_function_definitions=self._functions ): resolved[variable] = unresolved[variable].resolve( resolved_variables=resolved, @@ -482,12 +510,12 @@ class Script: added_variables_to_validate: Set[str] = set() functions_to_add = { - _function_name(name): definition + to_function_name(name): definition for name, definition in variables.items() - if _is_function(name) + if is_function(name) } variables_to_add = { - name: definition for name, definition in variables.items() if not _is_function(name) + name: definition for name, definition in variables.items() if not is_function(name) } custom_function_names = set(self._functions.keys()) | functions_to_add.keys() @@ -517,11 +545,52 @@ class Script: return self + def add_parsed(self, variables: Dict[str, SyntaxTree]) -> "Script": + """ + Adds already parsed, new variables to the script. + + Parameters + ---------- + variables + Mapping containing variable name to definition. + + Returns + ------- + Script + self + """ + added_variables_to_validate: Set[str] = set() + + functions_to_add = { + to_function_name(name): definition + for name, definition in variables.items() + if is_function(name) + } + variables_to_add = { + name: definition for name, definition in variables.items() if not is_function(name) + } + + for definitions in [functions_to_add, variables_to_add]: + for name, parsed in definitions.items(): + if parsed.maybe_resolvable is None: + added_variables_to_validate.add(name) + + if name in functions_to_add: + self._functions[name] = parsed + else: + self._variables[name] = parsed + + if added_variables_to_validate: + self._validate(added_variables=added_variables_to_validate) + + return self + def resolve_once( self, variable_definitions: Dict[str, str], resolved: Optional[Dict[str, Resolvable]] = None, unresolvable: Optional[Set[str]] = None, + update: bool = False, ) -> Dict[str, Resolvable]: """ Given a new set of variable definitions, resolve them using the Script, but do not @@ -536,6 +605,8 @@ class Script: unresolvable Optional. Unresolvable variables that will be ignored in resolution, including all variables with a dependency to them. + update + Whether to update the script's state with resolved variables. Defaults to False. Returns ------- @@ -548,11 +619,51 @@ class Script: pre_resolved=resolved, unresolvable=unresolvable, output_filter=set(list(variable_definitions.keys())), + update=update, ).output finally: for name in variable_definitions.keys(): - if name in self._variables: - del self._variables[name] + self._variables.pop(name, None) + + def resolve_once_parsed( + self, + variable_definitions: Dict[str, SyntaxTree], + resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + update: bool = False, + ) -> Dict[str, Resolvable]: + """ + Given a new set of variable definitions, resolve them using the Script, but do not + add them to the Script itself. + + Parameters + ---------- + variable_definitions + Variables to resolve, but not store in the Script + resolved + Optional. Pre-resolved variables that should be used instead of what is in the script. + unresolvable + Optional. Unresolvable variables that will be ignored in resolution, including all + variables with a dependency to them. + update + Whether to update the script's state with resolved variables. Defaults to False. + + Returns + ------- + Dict[str, Resolvable] + Dict containing the variable names to their resolved values. + """ + try: + self.add_parsed(variable_definitions) + return self._resolve( + pre_resolved=resolved, + unresolvable=unresolvable, + output_filter=set(list(variable_definitions.keys())), + update=update, + ).output + finally: + for name in variable_definitions.keys(): + self._variables.pop(name, None) def get(self, variable_name: str) -> Resolvable: """ @@ -581,6 +692,18 @@ class Script: raise RuntimeException(f"Tried to get unresolved variable {variable_name}") + def definition_of(self, name: str) -> SyntaxTree: + """ + Returns + ------- + The definition of the variable or function. + """ + if name.startswith("%") and name[1:] in self._functions: + return self._functions[name[1:]] + if name in self._variables: + return self._variables[name] + raise RuntimeException(f"Tried to get non-existent definition with name {name}") + @property def variable_names(self) -> Set[str]: """ @@ -599,4 +722,121 @@ class Script: Set[str] Names of all functions within the Script. """ - return set(_to_function_definition_name(name) for name in self._functions.keys()) + return set(to_function_definition_name(name) for name in self._functions.keys()) + + def _resolve_partial_loop( + self, + output_filter: Optional[Set[str]], + resolved: Dict[Variable, Resolvable], + unresolved: Dict[Variable, Argument], + unresolvable: Optional[Set[str]], + ): + to_partially_resolve: Set[Variable] = ( + {Variable(name) for name in output_filter} if output_filter else set(unresolved.keys()) + ) + + partially_resolved = True + while partially_resolved: + + partially_resolved = False + + for variable in list(to_partially_resolve): + definition = unresolved[variable] + maybe_resolved = definition + + if isinstance(definition, Variable) and definition.name not in unresolvable: + if definition in resolved: + maybe_resolved = resolved[definition] + elif definition in unresolved: + maybe_resolved = unresolved[definition] + else: + raise UNREACHABLE + elif isinstance(definition, VariableDependency): + maybe_resolved = definition.partial_resolve( + resolved_variables=resolved, + unresolved_variables=unresolved, + custom_functions=self._functions, + ) + + if isinstance(maybe_resolved, Resolvable): + resolved[variable] = maybe_resolved + del unresolved[variable] + to_partially_resolve.remove(variable) + partially_resolved = True + else: + unresolved[variable] = maybe_resolved + + # If the definition changed, then the script changed + # which means we can iterate again + partially_resolved |= definition != maybe_resolved + + def _resolve_partial( + self, + unresolvable: Optional[Set[str]] = None, + output_filter: Optional[Set[str]] = None, + ) -> Dict[str, SyntaxTree]: + """ + Returns + ------- + New (deep-copied) script that resolves inner variables as much + as possible. + """ + unresolvable: Set[str] = unresolvable or {} + resolved: Dict[Variable, Resolvable] = {} + unresolved: Dict[Variable, Argument] = { + Variable(name): definition + for name, definition in self._variables.items() + if name not in unresolvable + } + + self._resolve_partial_loop( + output_filter=output_filter, + resolved=resolved, + unresolved=unresolved, + unresolvable=unresolvable, + ) + + if output_filter: + out: Dict[str, SyntaxTree] = {} + for name in output_filter: + variable_name = Variable(name) + if variable_name in resolved: + out[name] = ResolvedSyntaxTree(ast=[resolved[variable_name]]) + else: + out[name] = SyntaxTree(ast=[unresolved[variable_name]]) + + return out + + return { + var.name: ResolvedSyntaxTree(ast=[definition]) for var, definition in resolved.items() + } | {var.name: SyntaxTree(ast=[definition]) for var, definition in unresolved.items()} + + def resolve_partial( + self, + unresolvable: Optional[Set[str]] = None, + output_filter: Optional[Set[str]] = None, + ) -> "Script": + """ + Updates the internal script to resolve as much as possible. + """ + out = self._resolve_partial(unresolvable=unresolvable, output_filter=output_filter) + for var_name, definition in out.items(): + self._variables[var_name] = definition + + return self + + def resolve_partial_once( + self, variable_definitions: Dict[str, SyntaxTree], unresolvable: Optional[Set[str]] = None + ) -> Dict[str, SyntaxTree]: + """ + Partially resolves the input variable definitions as much as possible. + """ + try: + self.add_parsed(variable_definitions) + return self._resolve_partial( + unresolvable=unresolvable, + output_filter=set(list(variable_definitions.keys())), + ) + finally: + for name in variable_definitions.keys(): + self._variables.pop(name, None) diff --git a/src/ytdl_sub/script/types/array.py b/src/ytdl_sub/script/types/array.py index 8adcfe50..204697fb 100644 --- a/src/ytdl_sub/script/types/array.py +++ b/src/ytdl_sub/script/types/array.py @@ -28,7 +28,7 @@ class UnresolvedArray(_Array, VariableDependency, FutureResolvable): value: List[Argument] @property - def _iterable_arguments(self) -> List[Argument]: + def iterable_arguments(self) -> List[Argument]: return self.value def resolve( @@ -47,6 +47,27 @@ class UnresolvedArray(_Array, VariableDependency, FutureResolvable): ] ) + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, VariableDependency], + ) -> Argument | Resolvable: + maybe_resolvable_values, is_resolvable = VariableDependency.try_partial_resolve( + args=self.value, + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + out = UnresolvedArray(value=maybe_resolvable_values) + if is_resolvable: + return out.resolve( + resolved_variables=resolved_variables, custom_functions=custom_functions + ) + + return out + def future_resolvable_type(self) -> Type[Resolvable]: return Array diff --git a/src/ytdl_sub/script/types/function.py b/src/ytdl_sub/script/types/function.py index 7ecd1999..bf64108b 100644 --- a/src/ytdl_sub/script/types/function.py +++ b/src/ytdl_sub/script/types/function.py @@ -1,10 +1,10 @@ -import copy import functools from abc import ABC from dataclasses import dataclass from typing import Callable from typing import Dict from typing import List +from typing import Optional from typing import Type from typing import Union @@ -12,9 +12,11 @@ from ytdl_sub.script.functions import Functions from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import UnresolvedArray from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import BuiltInFunctionType from ytdl_sub.script.types.resolvable import FunctionType from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import NamedCustomFunction from ytdl_sub.script.types.resolvable import Resolvable @@ -36,7 +38,7 @@ from ytdl_sub.script.utils.type_checking import is_union @dataclass(frozen=True) class Function(FunctionType, VariableDependency, ABC): @property - def _iterable_arguments(self) -> List[Argument]: + def iterable_arguments(self) -> List[Argument]: return self.args @@ -58,27 +60,79 @@ class CustomFunction(Function, NamedCustomFunction): # Should be validated in the Script raise UNREACHABLE - resolved_variables_with_args = copy.deepcopy(resolved_variables) + function_args: List[FunctionArgument] = [] for i, arg in enumerate(resolved_args): function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name) - if function_arg in resolved_variables_with_args: + if function_arg in resolved_variables: # function args should always be unique since they are only defined once # in the custom function as %custom_function_name___idx # and returned as a set from each custom function. raise UNREACHABLE - resolved_variables_with_args[function_arg] = arg + resolved_variables[function_arg] = arg + function_args.append(function_arg) - return custom_functions[self.name].resolve( - resolved_variables=resolved_variables_with_args, + out = custom_functions[self.name].resolve( + resolved_variables=resolved_variables, custom_functions=custom_functions, ) + for function_arg in function_args: + del resolved_variables[function_arg] + + return out + # Implies the custom function does not exist. This should have # been checked in the parser with raise UNREACHABLE + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, VariableDependency], + ) -> Argument | Resolvable: + maybe_resolvable_args, _ = VariableDependency.try_partial_resolve( + args=self.args, + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + for i in range(len(self.args)): + function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name) + function_value = maybe_resolvable_args[i] + + if isinstance(function_value, Resolvable): + resolved_variables[function_arg] = function_value + else: + unresolved_variables[function_arg] = function_value + + assert len(custom_functions[self.name].iterable_arguments) == 1 + custom_function_definition = custom_functions[self.name].iterable_arguments[0] + + maybe_resolvable_custom_function, is_resolvable = VariableDependency.try_partial_resolve( + args=[custom_function_definition], + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + for i in range(len(self.args)): + function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name) + + if isinstance(maybe_resolvable_args[i], Resolvable): + del resolved_variables[function_arg] + else: + del unresolved_variables[function_arg] + + if is_resolvable: + return maybe_resolvable_custom_function[0] + + # Did not resolve custom function arguments, do not proceed + return CustomFunction(name=self.name, args=maybe_resolvable_args) + class BuiltInFunction(Function, BuiltInFunctionType): def validate_args(self) -> "BuiltInFunction": @@ -307,5 +361,126 @@ class BuiltInFunction(Function, BuiltInFunctionType): f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" ) from exc + def _partial_resolve_conditional( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, "VariableDependency"], + ): + """ + If the conditional partially resolvable enough to warrant evaluation, + perform it here. + """ + + if self.name == "if": + maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve( + args=[self.args[0]], + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + if is_resolvable: + boolean_output = maybe_resolvable_arg[0] + assert isinstance(boolean_output, Boolean) + return self.args[1] if boolean_output.native else self.args[2] + + if self.name == "elif": + for idx in range(0, len(self.args), 2): + maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve( + args=[self.args[idx]], + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + if is_resolvable: + boolean_output = maybe_resolvable_arg[0] + assert isinstance(boolean_output, Boolean) + if boolean_output.native: + return self.args[idx + 1] + else: + break + + if self.name == "assert_then": + maybe_resolvable_arg, is_resolvable = VariableDependency.try_partial_resolve( + args=[self.args[0]], + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + if is_resolvable: + boolean_output = maybe_resolvable_arg[0] + assert isinstance(boolean_output, Boolean) + if boolean_output.native: + return self.args[1] + + return self + + def _try_optimized_partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, "VariableDependency"], + ) -> Optional[Argument]: + """ + If a function has enough (but not all) resolved parameters to warrant a return, + perform it here. + """ + if self.name == "array_at": + if ( + isinstance(self.args[0], UnresolvedArray) + and isinstance(self.args[1], Integer) + and len(self.args[0].value) >= self.args[1].value + ): + maybe_resolvable_values, _ = VariableDependency.try_partial_resolve( + args=[self.args[0].value[self.args[1].value]], + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + return maybe_resolvable_values[0] + + return None + + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, VariableDependency], + ) -> Argument | Resolvable: + conditional_return_args = self.function_spec.conditional_arg_indices( + num_input_args=len(self.args) + ) + + if conditional_return_args: + return self._partial_resolve_conditional( + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + if partial_resolved := self._try_optimized_partial_resolve( + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ): + return partial_resolved + + maybe_resolvable_values, is_resolvable = VariableDependency.try_partial_resolve( + args=self.args, + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + out = BuiltInFunction(name=self.name, args=maybe_resolvable_values) + if is_resolvable: + return out.resolve( + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + return out + def __hash__(self): return hash((self.name, *self.args)) diff --git a/src/ytdl_sub/script/types/map.py b/src/ytdl_sub/script/types/map.py index 0fca1cbb..d5e2eb27 100644 --- a/src/ytdl_sub/script/types/map.py +++ b/src/ytdl_sub/script/types/map.py @@ -31,7 +31,7 @@ class UnresolvedMap(_Map, VariableDependency, FutureResolvable): value: Dict[Argument, Argument] @property - def _iterable_arguments(self) -> List[Argument]: + def iterable_arguments(self) -> List[Argument]: return list(itertools.chain(*self.value.items())) def resolve( @@ -55,6 +55,35 @@ class UnresolvedMap(_Map, VariableDependency, FutureResolvable): return Map(output) + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, VariableDependency], + ) -> Argument | Resolvable: + maybe_resolvable_keys, is_keys_resolvable = VariableDependency.try_partial_resolve( + args=self.value.keys(), + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + maybe_resolvable_values, is_values_resolvable = VariableDependency.try_partial_resolve( + args=self.value.values(), + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + out = UnresolvedMap(value=dict(zip(maybe_resolvable_keys, maybe_resolvable_values))) + if is_keys_resolvable and is_values_resolvable: + return out.resolve( + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + return out + def future_resolvable_type(self) -> Type[Resolvable]: return Map diff --git a/src/ytdl_sub/script/types/resolvable.py b/src/ytdl_sub/script/types/resolvable.py index c042b1f5..ad9ff300 100644 --- a/src/ytdl_sub/script/types/resolvable.py +++ b/src/ytdl_sub/script/types/resolvable.py @@ -193,6 +193,14 @@ class NamedCustomFunction(NamedArgument, ABC): class ParsedCustomFunction(NamedCustomFunction): num_input_args: int + def definition_name(self) -> str: + """ + Returns + ------- + The function definition name, including the % + """ + return f"%{self.name}" + @dataclass(frozen=True) class FunctionType(NamedArgument, ABC): diff --git a/src/ytdl_sub/script/types/syntax_tree.py b/src/ytdl_sub/script/types/syntax_tree.py index d058852c..e914f396 100644 --- a/src/ytdl_sub/script/types/syntax_tree.py +++ b/src/ytdl_sub/script/types/syntax_tree.py @@ -3,6 +3,7 @@ from typing import Dict from typing import List from typing import Optional +from ytdl_sub.script.types.function import BuiltInFunction from ytdl_sub.script.types.resolvable import Argument from ytdl_sub.script.types.resolvable import Resolvable from ytdl_sub.script.types.resolvable import String @@ -15,7 +16,7 @@ class SyntaxTree(VariableDependency): ast: List[Argument] @property - def _iterable_arguments(self) -> List[Argument]: + def iterable_arguments(self) -> List[Argument]: return self.ast def resolve( @@ -40,6 +41,30 @@ class SyntaxTree(VariableDependency): # Otherwise, to concat multiple resolved outputs, we must concat as strings return String("".join([str(res) for res in resolved])) + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, VariableDependency], + ) -> Argument | Resolvable: + # Ensure this does not get returned as a SyntaxTree since nesting them is not supported. + maybe_resolvable_values, _ = VariableDependency.try_partial_resolve( + args=self.ast, + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + # If no arguments, must be empty string + if len(maybe_resolvable_values) == 0: + return String(value="") + + # Mimic the above resolve behavior + if len(maybe_resolvable_values) > 1: + return BuiltInFunction(name="concat", args=maybe_resolvable_values) + + return maybe_resolvable_values[0] + @property def maybe_resolvable(self) -> Optional[Resolvable]: """ @@ -47,6 +72,40 @@ class SyntaxTree(VariableDependency): ------- A resolvable if the AST contains a single type that is resolvable. None otherwise. """ - if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable): - return self.ast[0] return None + + def maybe_resolvable_casted(self) -> "SyntaxTree": + """ + Returns + ------- + Optimized SyntaxTree if its deemed resolvable + """ + if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable): + return ResolvedSyntaxTree(self.ast) + return self + + +@dataclass(frozen=True) +class ResolvedSyntaxTree(SyntaxTree): + """ + SyntaxTree with optimized helper functions if it's known to be resolved. + """ + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, VariableDependency], + ) -> Resolvable: + return self.ast[0] + + @property + def maybe_resolvable(self) -> Optional[Resolvable]: + return self.ast[0] + + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, VariableDependency], + ) -> Argument | Resolvable: + return self.ast[0] diff --git a/src/ytdl_sub/script/types/variable_dependency.py b/src/ytdl_sub/script/types/variable_dependency.py index eb5f646e..a185a0af 100644 --- a/src/ytdl_sub/script/types/variable_dependency.py +++ b/src/ytdl_sub/script/types/variable_dependency.py @@ -5,6 +5,7 @@ from typing import Dict from typing import Iterable from typing import List from typing import Set +from typing import Tuple from typing import Type from typing import TypeVar from typing import final @@ -27,7 +28,7 @@ TypeT = TypeVar("TypeT") class VariableDependency(ABC): @property @abstractmethod - def _iterable_arguments(self) -> List[Argument]: + def iterable_arguments(self) -> List[Argument]: """ Returns ------- @@ -38,7 +39,7 @@ class VariableDependency(ABC): self, ttype: Type[TypeT], subclass: bool = False, instance: bool = True ) -> List[TypeT]: output: List[TypeT] = [] - for arg in self._iterable_arguments: + for arg in self.iterable_arguments: if subclass and issubclass(type(arg), ttype): output.append(arg) elif instance and isinstance(arg, ttype): @@ -104,7 +105,7 @@ class VariableDependency(ABC): All CustomFunctions that this depends on. """ output: Set[ParsedCustomFunction] = set() - for arg in self._iterable_arguments: + for arg in self.iterable_arguments: if isinstance(arg, NamedCustomFunction): if not isinstance(arg, FunctionType): # A NamedCustomFunction should also always be a FunctionType @@ -138,6 +139,28 @@ class VariableDependency(ABC): Resolved value """ + @abstractmethod + def partial_resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, "VariableDependency"], + ) -> Argument | Resolvable: + """ + Parameters + ---------- + resolved_variables + Lookup of variables that have been resolved + unresolved_variables + Lookup of variables that have not been resolved + custom_functions + Lookup of any custom functions that have been parsed + + Returns + ------- + Either a fully resolved value or partially resolved value of the same type. + """ + @classmethod def _resolve_argument_type( cls, @@ -159,10 +182,35 @@ class VariableDependency(ABC): raise UNREACHABLE + @final + def custom_function_dependencies( + self, custom_function_definitions: Dict[str, "VariableDependency"] + ) -> Set[ParsedCustomFunction]: + """ + Parameters + ---------- + custom_function_definitions + Definition of all currently existing custom functions. Needed to check whether + a lambda function's input function is custom or not. + + Returns + ------- + All custom function dependencies + """ + custom_functions = self.custom_functions + for lambda_func in self.lambdas: + if lambda_func.value in custom_function_definitions: + custom_functions.add( + ParsedCustomFunction( + name=lambda_func.value, num_input_args=lambda_func.num_input_args() + ) + ) + return custom_functions + @final def is_subset_of( self, - variables: Iterable[Variable], + variables: Dict[Variable, Resolvable], custom_function_definitions: Dict[str, "VariableDependency"], ) -> bool: """ @@ -170,13 +218,14 @@ class VariableDependency(ABC): ------- True if it contains all input variables as a dependency. False otherwise. """ - for custom_function in self.custom_functions: - if custom_function_definitions[custom_function.name].is_subset_of( + # If there are lambdas, see if they are custom functions. If so, check them + for custom_function in self.custom_function_dependencies(custom_function_definitions): + if not custom_function_definitions[custom_function.name].is_subset_of( variables=variables, custom_function_definitions=custom_function_definitions ): - return True + return False - return not self.variables.issubset(variables) + return all(var in variables for var in self.variables) @final def contains( @@ -189,9 +238,53 @@ class VariableDependency(ABC): ------- True if it contains any of the input variables. False otherwise. """ - for custom_function in self.custom_functions: + # If there are lambdas, see if they are custom functions. If so, check them + for custom_function in self.custom_function_dependencies(custom_function_definitions): if custom_function_definitions[custom_function.name].contains( variables=variables, custom_function_definitions=custom_function_definitions ): return True return len(self.variables.intersection(variables)) > 0 + + @classmethod + def try_partial_resolve( + cls, + args: Iterable[Argument], + resolved_variables: Dict[Variable, Resolvable], + unresolved_variables: Dict[Variable, Argument], + custom_functions: Dict[str, "VariableDependency"], + ) -> Tuple[List[Argument], bool]: + """ + Attempts to resolve a list of arguments. Returns a tuple of them post partially resolved, + and a boolean indicating whether all of them are fully resolved. + """ + maybe_resolvable_args: List[Argument] = [] + is_resolvable = True + for arg in args: + maybe_resolvable_args.append(arg) + + if isinstance(arg, Lambda) and arg.value in custom_functions: + if not custom_functions[arg.value].is_subset_of( + variables=resolved_variables, + custom_function_definitions=custom_functions, + ): + is_resolvable = False + elif isinstance(arg, VariableDependency): + maybe_resolvable_args[-1] = arg.partial_resolve( + resolved_variables=resolved_variables, + unresolved_variables=unresolved_variables, + custom_functions=custom_functions, + ) + + if not isinstance(maybe_resolvable_args[-1], Resolvable): + is_resolvable = False + elif isinstance(arg, Variable): + if arg in resolved_variables: + maybe_resolvable_args[-1] = resolved_variables[arg] + else: + is_resolvable = False + # Could be un unresolvable + if arg in unresolved_variables: + maybe_resolvable_args[-1] = unresolved_variables[arg] + + return maybe_resolvable_args, is_resolvable diff --git a/src/ytdl_sub/script/utils/name_validation.py b/src/ytdl_sub/script/utils/name_validation.py index de187ec8..5b2597ce 100644 --- a/src/ytdl_sub/script/utils/name_validation.py +++ b/src/ytdl_sub/script/utils/name_validation.py @@ -58,3 +58,24 @@ def validate_custom_function_name(custom_function_name: str) -> None: f"Custom function name '%{custom_function_name}' is invalid:" " The name is used by a built-in function and cannot be overwritten." ) + + +def is_function(override_name: str): + """ + Whether the definition is a function or not. + """ + return override_name.startswith("%") + + +def to_function_name(function_key: str) -> str: + """ + Drop the % in %custom_function + """ + return function_key[1:] + + +def to_function_definition_name(function_key: str) -> str: + """ + Add % in %custom_function + """ + return f"%{function_key}" diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py index e8a8bc79..7598a526 100644 --- a/src/ytdl_sub/script/utils/type_checking.py +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -2,6 +2,7 @@ import inspect from dataclasses import dataclass from inspect import FullArgSpec +from types import NoneType from typing import Callable from typing import List from typing import Optional @@ -50,7 +51,7 @@ def get_optional_type(optional_type: Type) -> Type[NamedType]: ------- Type within the Optional[Type] """ - return [arg for arg in optional_type.__args__ if arg != type(None)][0] + return [arg for arg in optional_type.__args__ if not isinstance(arg, NoneType)][0] def _is_union_compatible( diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 970f142d..553eb74d 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -8,10 +8,15 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.variable_validation import ResolutionLevel +from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.variables.override_variables import SubscriptionVariables +from ytdl_sub.utils.exceptions import SubscriptionPermissionError +from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get("subscription") @@ -74,6 +79,14 @@ class BaseSubscription(ABC): } ) + # Validate after adding the subscription name + _ = VariableValidation( + overrides=self.overrides, + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + ).ensure_proper_usage() + self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = ( _initialize_download_archive( output_options=self.output_options, @@ -86,14 +99,20 @@ class BaseSubscription(ABC): # Add post-archive variables self.overrides.add( { - SubscriptionVariables.subscription_has_download_archive(): f"""{{ - %bool({self.download_archive.num_entries > 0}) - }}""", + SubscriptionVariables.subscription_has_download_archive(): ( + f"{{%bool({self.download_archive.num_entries > 0})}}" + ), } ) self._exception: Optional[Exception] = None + if not FileHandler.is_path_writable(self.output_directory): + raise SubscriptionPermissionError( + "ytdl-sub does not have write permissions to the output directory: " + f"{self.output_directory}" + ) + @property def download_archive(self) -> EnhancedDownloadArchive: """ @@ -236,4 +255,22 @@ class BaseSubscription(ABC): ------- Subscription in yaml format """ - return self._preset_options.yaml + return self._preset_options.yaml(subscription_only=False) + + def resolved_yaml(self, resolution_level: int = ResolutionLevel.RESOLVE) -> str: + """ + Returns + ------- + Human-readable, condensed YAML definition of the subscription. + """ + if resolution_level == ResolutionLevel.ORIGINAL: + return self._preset_options.yaml(subscription_only=True) + + out = VariableValidation( + overrides=self.overrides, + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + resolution_level=resolution_level, + ).ensure_proper_usage(partial_resolve_formatters=True) + return dump_yaml(out) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 43b811a4..082260cc 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -73,6 +73,7 @@ class SubscriptionDownload(BaseSubscription, ABC): file_metadata=entry_metadata, output_file_name=output_file_name, entry=entry, + preserve_mtime=self.output_options.preserve_mtime, ) # Always pretend to include the thumbnail in a dry-run @@ -87,6 +88,7 @@ class SubscriptionDownload(BaseSubscription, ABC): output_file_name=output_thumbnail_name, entry=entry, copy_file=True, + preserve_mtime=self.output_options.preserve_mtime, ) elif not entry.is_thumbnail_downloaded(): logger.warning( @@ -106,6 +108,7 @@ class SubscriptionDownload(BaseSubscription, ABC): file_name=entry.get_download_info_json_name(), output_file_name=output_info_json_name, entry=entry, + preserve_mtime=self.output_options.preserve_mtime, ) def _delete_working_directory(self, is_error: bool = False) -> None: @@ -152,8 +155,8 @@ class SubscriptionDownload(BaseSubscription, ABC): keep_max_files: Optional[int] = None if self.output_options.keep_max_files: # validated it can be cast to int within the validator - keep_max_files = int( - self.overrides.apply_formatter(self.output_options.keep_max_files) + keep_max_files = self.overrides.apply_formatter( + self.output_options.keep_max_files, expected_type=int ) if date_range_to_keep or self.output_options.keep_max_files is not None: diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index b5772197..c689c88b 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -94,8 +94,8 @@ class SubscriptionYTDLOptions: self._enhanced_download_archive.working_ytdl_file_path ) if self._preset.output_options.keep_max_files: - keep_max_files = int( - self._overrides.apply_formatter(self._preset.output_options.keep_max_files) + keep_max_files = self._overrides.apply_formatter( + self._preset.output_options.keep_max_files, expected_type=int ) if keep_max_files > 0: # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure diff --git a/src/ytdl_sub/utils/exceptions.py b/src/ytdl_sub/utils/exceptions.py index bfc222ec..76d80780 100644 --- a/src/ytdl_sub/utils/exceptions.py +++ b/src/ytdl_sub/utils/exceptions.py @@ -40,3 +40,7 @@ class FileNotDownloadedException(ValueError): class ExperimentalFeatureNotEnabled(ValidationException): """Feature is not enabled for usage""" + + +class SubscriptionPermissionError(ValidationException): + """Early-caught permission error""" diff --git a/src/ytdl_sub/utils/ffmpeg.py b/src/ytdl_sub/utils/ffmpeg.py index 6d16814d..52374464 100644 --- a/src/ytdl_sub/utils/ffmpeg.py +++ b/src/ytdl_sub/utils/ffmpeg.py @@ -201,7 +201,11 @@ def add_ffmpeg_metadata_key_values(file_path: str, key_values: Dict[str, str]) - "-dn", # ignore data streams ] for key, value in key_values.items(): - ffmpeg_args.extend(["-metadata", f"{key}={value}"]) + # Some special characters (utf-16 maybe?) have null-byte + # which results in ffmpeg error, remove them outright + value_null_byte_removed = value.replace("\0", "") + ffmpeg_args.extend(["-metadata", f"{key}={value_null_byte_removed}"]) + ffmpeg_args.extend(["-codec", "copy", "-bitexact", tmp_file_path]) FFMPEG.run(ffmpeg_args) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index f7c6e44f..1db4bb6c 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -379,6 +379,36 @@ class FileHandler: """ return self._file_handler_transaction_log + @classmethod + def is_path_writable(cls, src_file_path: Union[str, Path]) -> bool: + """ + Check whether a path is writable. If it does not exist, try to find the base directory + and check permissions on that. + """ + path = os.path.abspath(src_file_path) + + while not os.path.exists(path): + new_path = os.path.dirname(path) + if new_path == path: # reached root + break + path = new_path + + return os.access(path, os.W_OK) + + @classmethod + def is_file_existent(cls, file_path: Union[str, Path]) -> bool: + """ + Check whether a file exists. + """ + return os.path.isfile(file_path) + + @classmethod + def is_file_readable(cls, file_path: Union[str, Path]) -> bool: + """ + Check whether a file exists and is readable. + """ + return cls.is_file_existent(file_path) and os.access(file_path, os.R_OK) + @classmethod def copy(cls, src_file_path: Union[str, Path], dst_file_path: Union[str, Path]): """ @@ -430,6 +460,25 @@ class FileHandler: if os.path.isfile(file_path): os.remove(file_path) + @classmethod + def set_mtime(cls, file_path: Union[str, Path], mtime: float): + """ + Set the modification time of a file + + Parameters + ---------- + file_path + Path to the file to modify + mtime + Modification time as a Unix timestamp + """ + try: + # Set both access time and modification time + os.utime(file_path, (mtime, mtime)) + except OSError: + # If file operation fails, silently continue + pass + def move_file_to_output_directory( self, file_name: str, diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py index e3a17b2f..7252d7c4 100644 --- a/src/ytdl_sub/utils/script.py +++ b/src/ytdl_sub/utils/script.py @@ -2,12 +2,14 @@ import json import re from typing import Any from typing import Dict +from typing import Optional from ytdl_sub.script.parser import parse -from ytdl_sub.script.script import _is_function +from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.array import UnresolvedArray from ytdl_sub.script.types.function import BuiltInFunction from ytdl_sub.script.types.function import Function +from ytdl_sub.script.types.map import Map from ytdl_sub.script.types.map import UnresolvedMap from ytdl_sub.script.types.resolvable import Argument from ytdl_sub.script.types.resolvable import Boolean @@ -15,8 +17,10 @@ from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.name_validation import is_function # pylint: disable=too-many-return-statements @@ -30,10 +34,26 @@ class ScriptUtils: sanitized_variables = { f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys() - if not _is_function(name) + if not is_function(name) } return dict(variables, **sanitized_variables) + @classmethod + def add_sanitized_parsed_variables( + cls, variables: Dict[str, SyntaxTree] + ) -> Dict[str, SyntaxTree]: + """ + Helper to add sanitized variables to a Script + """ + sanitized_variables = { + f"{name}_sanitized": SyntaxTree( + ast=[BuiltInFunction(name="sanitize", args=[Variable(name)])] + ) + for name in variables.keys() + if not is_function(name) + } + return variables | sanitized_variables + @classmethod def to_script(cls, value: Any, sort_keys: bool = True) -> str: """ @@ -67,9 +87,8 @@ class ScriptUtils: ast = parse(text=value).ast if len(ast) == 1: return ast[0] - return BuiltInFunction( - name="concat", args=[BuiltInFunction(name="string", args=[arg]) for arg in ast] - ) + + return BuiltInFunction(name="concat", args=ast) if isinstance(value, bool): return Boolean(value) if isinstance(value, int): @@ -88,6 +107,19 @@ class ScriptUtils: raise UNREACHABLE + @classmethod + def _get_quote_char(cls, arg: str) -> str: + contains_single_quote = "'" in arg + contains_double_quote = '"' in arg + + if not contains_single_quote and not contains_double_quote: + return '"' + if not contains_single_quote and contains_double_quote: + return "'" + if contains_single_quote and not contains_double_quote: + return '"' + return "'''" + @classmethod def _to_script_code(cls, arg: Argument, top_level: bool = False) -> str: if not top_level and isinstance(arg, (Integer, Boolean, Float)): @@ -96,7 +128,10 @@ class ScriptUtils: if isinstance(arg, String): if arg.native == "": return "" if top_level else "''" - return arg.native if top_level else f"'''{arg.native}'''" + + quote = cls._get_quote_char(arg.native) + + return arg.native if top_level else f"{quote}{arg.native}{quote}" if isinstance(arg, Integer): out = f"%int({arg.native})" @@ -104,9 +139,9 @@ class ScriptUtils: out = f"%bool({arg.native})" elif isinstance(arg, Float): out = f"%float({arg.native})" - elif isinstance(arg, UnresolvedArray): + elif isinstance(arg, (Array, UnresolvedArray)): out = f"[ {', '.join(cls._to_script_code(val) for val in arg.value)} ]" - elif isinstance(arg, UnresolvedMap): + elif isinstance(arg, (Map, UnresolvedMap)): kv_list = ( f"{cls._to_script_code(key)}: {cls._to_script_code(val)}" for key, val in arg.value.items() @@ -122,11 +157,43 @@ class ScriptUtils: raise UNREACHABLE return f"{{ {out} }}" if top_level else out + @classmethod + def _is_top_level_string(cls, tree: SyntaxTree) -> Optional[str]: + if not ( + len(tree.ast) == 1 + and isinstance(tree.ast[0], BuiltInFunction) + and tree.ast[0].name == "concat" + ): + return None + + output = "" + for arg in tree.ast[0].args: + if isinstance(arg, BuiltInFunction) and arg.name == "string" and len(arg.args) == 1: + output += cls._to_script_code(arg.args[0], top_level=True) + else: + output += cls._to_script_code(arg, top_level=True) + + return output + + @classmethod + def _syntax_tree_to_native_script(cls, tree: SyntaxTree) -> str: + + if (output := cls._is_top_level_string(tree)) is not None: + return output + + output = "" + for arg in tree.ast: + output += cls._to_script_code(arg, top_level=True) + return output + @classmethod def to_native_script(cls, value: Any) -> str: """ Converts any JSON-compatible value into equivalent script syntax """ + if isinstance(value, SyntaxTree): + return cls._syntax_tree_to_native_script(value) + return cls._to_script_code(cls._to_script_argument(value), top_level=True) @classmethod diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 17436700..b9fdd3ea 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -1,10 +1,10 @@ from datetime import datetime +from typing import Any from typing import Dict from typing import Set from typing import Union from typing import final -from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script from ytdl_sub.script.types.syntax_tree import SyntaxTree @@ -19,6 +19,8 @@ from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator +# pylint: disable=protected-access + class StringFormatterValidator(StringValidator): """ @@ -51,7 +53,10 @@ class StringFormatterValidator(StringValidator): def __init__(self, name, value: str): super().__init__(name=name, value=value) try: - _ = parse(str(value)) + self._parsed = parse( + text=str(value), + name=self.leaf_name, + ) except UserException as exc: raise self._validation_exception(exc) from exc @@ -65,27 +70,37 @@ class StringFormatterValidator(StringValidator): """ return self._value - def post_process(self, resolved: str) -> str: + @property + @final + def parsed(self) -> SyntaxTree: """ Returns ------- - Apply any post processing to the resolved value + The parsed format string. """ - return resolved + return self._parsed + + def post_process(self, resolved: Any) -> Any: + """ + Returns + ------- + Apply any post processing to the resolved value. Defaults to casting it to string. + """ + return str(resolved) class FloatFormatterValidator(StringFormatterValidator): _expected_value_type_name = "float" - def post_process(self, resolved: str) -> str: + def post_process(self, resolved: str) -> float: try: - float(resolved) + out = float(resolved) except Exception as exc: raise self._validation_exception( f"Expected a float, but received '{resolved}'" ) from exc - return resolved + return out class StandardizedDateValidator(StringFormatterValidator): @@ -102,6 +117,13 @@ class StandardizedDateValidator(StringFormatterValidator): return resolved +class BooleanFormatterValidator(StringFormatterValidator): + _expected_value_type_name = "boolean" + + def post_process(self, resolved: Any) -> bool: + return ScriptUtils.bool_formatter_output(output=str(resolved)) + + # pylint: disable=line-too-long class OverridesStringFormatterValidator(StringFormatterValidator): """ @@ -121,15 +143,14 @@ class OverridesStringFormatterValidator(StringFormatterValidator): class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator): _expected_value_type_name = "integer" - def post_process(self, resolved: str) -> str: + def post_process(self, resolved: str) -> int: try: - int(resolved) + out = int(resolved) except Exception as exc: raise self._validation_exception( f"Expected an integer, but received '{resolved}'" ) from exc - - return resolved + return out class OverridesFloatFormatterValidator(FloatFormatterValidator, OverridesStringFormatterValidator): @@ -138,9 +159,14 @@ class OverridesFloatFormatterValidator(FloatFormatterValidator, OverridesStringF """ -class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator): +class OverridesBooleanFormatterValidator( + BooleanFormatterValidator, OverridesStringFormatterValidator +): _expected_value_type_name = "boolean" + def post_process(self, resolved: Any) -> bool: + return ScriptUtils.bool_formatter_output(output=str(resolved)) + class ListFormatterValidator(ListValidator[StringFormatterValidator]): _inner_list_type = StringFormatterValidator @@ -158,18 +184,24 @@ class DictFormatterValidator(LiteralDictValidator): super().__init__(name, value) for key in self._keys: - self._value[key] = self._validate_key(key=key, validator=self._key_validator) + # Gets stored in __validator_dict + _ = self._validate_key(key=key, validator=self._key_validator) @property def dict(self) -> Dict[str, StringFormatterValidator]: """Returns dict with string formatter values""" - return self._value + return self._validator_dict @property def dict_with_format_strings(self) -> Dict[str, str]: - """Returns dict with the format strings themselves""" + """Returns dict with the format strings themselves.""" return {key: string_formatter.format_string for key, string_formatter in self.dict.items()} + @property + def dict_with_parsed_format_strings(self) -> Dict[str, SyntaxTree]: + """Returns dict with the parsed format strings.""" + return {key: string_formatter.parsed for key, string_formatter in self.dict.items()} + class OverridesDictFormatterValidator(DictFormatterValidator): """ @@ -180,7 +212,22 @@ class OverridesDictFormatterValidator(DictFormatterValidator): _key_validator = OverridesStringFormatterValidator +class AnyFormatterValidator(StringFormatterValidator): + """ + Applies no post-processing. + """ + + def post_process(self, resolved: Any) -> Any: + return resolved + + +class AnyOverridesFormatterValidator(AnyFormatterValidator, OverridesStringFormatterValidator): + pass + + class UnstructuredDictFormatterValidator(DictFormatterValidator): + _key_validator = AnyFormatterValidator + def __init__(self, name, value): # Convert the unstructured-ness into a script if isinstance(value, dict): @@ -189,40 +236,30 @@ class UnstructuredDictFormatterValidator(DictFormatterValidator): class UnstructuredOverridesDictFormatterValidator(UnstructuredDictFormatterValidator): - _key_validator = OverridesStringFormatterValidator - - -def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str: - """ - Create a dummy format string that contains all variable deps as a string. - """ - dummy_format_string = "" - for var in parsed_format_string.variables: - dummy_format_string += f"{{ {var.name} }}" - # pylint: disable=protected-access - for variable_dependency in script._variables[var.name].variables: - dummy_format_string += f"{{ {variable_dependency.name} }}" - # pylint: enable=protected-access - return dummy_format_string + _key_validator = AnyOverridesFormatterValidator def _validate_formatter( mock_script: Script, unresolved_variables: Set[str], + unresolved_runtime_variables: Set[str], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], -) -> None: - is_static_formatter = False - unresolvable = unresolved_variables - if isinstance(formatter_validator, OverridesStringFormatterValidator): - is_static_formatter = True - unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + partial_resolve_entry_formatters: bool, +) -> Any: + parsed = formatter_validator.parsed + if resolved := parsed.maybe_resolvable: + return formatter_validator.post_process(resolved.native) + + is_static_formatter = isinstance(formatter_validator, OverridesStringFormatterValidator) - parsed = parse( - text=formatter_validator.format_string, - ) variable_names = {var.name for var in parsed.variables} custom_function_names = {f"%{func.name}" for func in parsed.custom_functions} + # Add lambda functions to custom function names, if it's custom + for lambda_func in parsed.lambdas: + if lambda_func in mock_script.function_names: + custom_function_names.add(lambda_func.value) + if not variable_names.issubset(mock_script.variable_names): raise StringFormattingVariableNotFoundException( "contains the following variables that do not exist: " @@ -233,20 +270,32 @@ def _validate_formatter( "contains the following custom functions that do not exist: " f"{', '.join(sorted(custom_function_names - mock_script.function_names))}" ) - if unresolved := variable_names.intersection(unresolvable): + if unresolved := variable_names.intersection(unresolved_runtime_variables): raise StringFormattingVariableNotFoundException( "contains the following variables that are unresolved when executing this " f"formatter: {', '.join(sorted(unresolved))}" ) + + if partial_resolve_entry_formatters and not is_static_formatter: + parsed = mock_script.resolve_partial_once( + variable_definitions={"tmp_var": formatter_validator.parsed}, + unresolvable=unresolved_variables, + )["tmp_var"] + try: - mock_script.resolve_once( - { - "tmp_var": to_variable_dependency_format_string( - script=mock_script, parsed_format_string=parsed - ) - }, - unresolvable=unresolvable, - ) + if is_static_formatter: + return formatter_validator.post_process( + mock_script.resolve_once_parsed( + {"tmp_var": formatter_validator.parsed}, + unresolvable=unresolved_variables, + update=True, + )["tmp_var"].native + ) + + if maybe_resolved := parsed.maybe_resolvable: + return formatter_validator.post_process(maybe_resolved) + + return ScriptUtils.to_native_script(parsed) except RuntimeException as exc: if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter: raise StringFormattingVariableNotFoundException( @@ -259,40 +308,59 @@ def _validate_formatter( def validate_formatters( script: Script, unresolved_variables: Set[str], + unresolved_runtime_variables: Set[str], validator: Validator, -) -> None: + partial_resolve_formatters: bool, +) -> Dict: """ Ensure all OverridesStringFormatterValidator's only contain variables from the overrides and resolve. """ + resolved_dict: Dict = {} + if isinstance(validator, DictValidator): - # pylint: disable=protected-access + resolved_dict[validator.leaf_name] = {} # Usage of protected variables in other validators is fine. The reason to keep # them protected is for readability when using them in subscriptions. for validator_value in validator._validator_dict.values(): - validate_formatters( + resolved_dict[validator.leaf_name] |= validate_formatters( script=script, unresolved_variables=unresolved_variables, + unresolved_runtime_variables=unresolved_runtime_variables, validator=validator_value, + partial_resolve_formatters=partial_resolve_formatters, ) - # pylint: enable=protected-access elif isinstance(validator, ListValidator): + resolved_dict[validator.leaf_name] = [] for list_value in validator.list: - validate_formatters( + list_output = validate_formatters( script=script, unresolved_variables=unresolved_variables, + unresolved_runtime_variables=unresolved_runtime_variables, validator=list_value, + partial_resolve_formatters=partial_resolve_formatters, ) + assert len(list_output) == 1 + resolved_dict[validator.leaf_name].append(list(list_output.values())[0]) elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)): - _validate_formatter( + resolved_dict[validator.leaf_name] = _validate_formatter( mock_script=script, unresolved_variables=unresolved_variables, + unresolved_runtime_variables=unresolved_runtime_variables, formatter_validator=validator, + partial_resolve_entry_formatters=partial_resolve_formatters, ) elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)): + resolved_dict[validator.leaf_name] = {} for validator_value in validator.dict.values(): - _validate_formatter( + resolved_dict[validator.leaf_name] |= _validate_formatter( mock_script=script, unresolved_variables=unresolved_variables, + unresolved_runtime_variables=unresolved_runtime_variables, formatter_validator=validator_value, + partial_resolve_entry_formatters=partial_resolve_formatters, ) + else: + resolved_dict[validator.leaf_name] = validator._value + + return resolved_dict diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index fbb13c95..6c1356ae 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -95,21 +95,11 @@ class Validator(ABC): @final @property - def _root_name(self) -> str: + def leaf_name(self) -> str: """ Returns ------- - "first" from the first.element.of.the.name - """ - return self._name.split(".")[0] - - @final - @property - def _leaf_name(self) -> str: - """ - Returns - ------- - "first" from the first.element.of.the.name + "name" from the first.element.of.the.name """ return self._name.split(".")[-1] @@ -274,7 +264,7 @@ class DictValidator(Validator): value=self._dict.get(key, default), ) - self.__validator_dict[validator_name] = validator_instance + self.__validator_dict[key] = validator_instance return validator_instance @final diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 7a565412..50b12f42 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -1,6 +1,7 @@ import copy import json import os.path +import time from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -642,6 +643,7 @@ class EnhancedDownloadArchive: output_file_name: Optional[str] = None, entry: Optional[Entry] = None, copy_file: bool = False, + preserve_mtime: bool = False, ): """ Saves a file from the working directory to the output directory and record it in the @@ -660,6 +662,8 @@ class EnhancedDownloadArchive: Optional. Entry that this file belongs to copy_file Optional. If True, copy the file. Move otherwise + preserve_mtime + Optional. If True and entry has upload_date, set file mtime to upload date """ if output_file_name is None: output_file_name = file_name @@ -674,6 +678,22 @@ class EnhancedDownloadArchive: copy_file=copy_file, ) + # Set mtime if preserve_mtime is enabled and we have an entry with upload_date + if preserve_mtime and entry and not self._file_handler.dry_run: + upload_date = entry.get(v.ytdl_sub_keep_files_date_eval, str) + if upload_date: + try: + # Convert YYYY-mm-dd to timestamp + upload_datetime = datetime.strptime(upload_date, "%Y-%m-%d") + upload_timestamp = time.mktime(upload_datetime.timetuple()) + + # Set mtime on the output file + output_file_path = Path(self._file_handler.output_directory) / output_file_name + FileHandler.set_mtime(output_file_path, upload_timestamp) + except (ValueError, OSError): + # If date parsing or file operation fails, silently continue + pass + # Determine if it's the entry file by seeing if the file_name to move matches the entry # download file name is_entry_file = entry and entry.get_download_file_name() == file_name diff --git a/tests/conftest.py b/tests/conftest.py index aabbba8d..42d82178 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ from typing import Dict from typing import List from typing import Optional from typing import Tuple +from typing import Union from unittest.mock import patch import pytest @@ -116,7 +117,8 @@ def assert_logs( yield for call_args in patched_debug.call_args_list: - occurrences += int(expected_message in call_args.args[0]) + full_print = call_args.args[0] % call_args.args[1:] + occurrences += int(expected_message in full_print) if expected_occurrences is not None: assert ( @@ -165,19 +167,18 @@ def preset_dict_to_dl_args(preset_dict: Dict) -> str: @pytest.fixture -def preset_dict_to_subscription_yaml_generator() -> Callable: +def subscription_yaml_file_generator() -> Callable: @contextlib.contextmanager - def _preset_dict_to_subscription_yaml_generator(subscription_name: str, preset_dict: Dict): - subscription_dict = {subscription_name: preset_dict} + def _subscription_yaml_file_generator(yaml_dict: Dict): with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file: - tmp_file.write(json.dumps(subscription_dict).encode("utf-8")) + tmp_file.write(json.dumps(yaml_dict).encode("utf-8")) try: yield tmp_file.name finally: FileHandler.delete(tmp_file.name) - return _preset_dict_to_subscription_yaml_generator + return _subscription_yaml_file_generator ################################################################################################### @@ -224,8 +225,14 @@ def _load_config(config_path: Path, working_directory: str) -> ConfigFile: @pytest.fixture() -def music_video_subscription_path() -> Path: - return Path("examples/music_video_subscriptions.yaml") +def music_video_subscription_path( + output_directory: str, subscription_yaml_file_generator: Callable +) -> Path: + yaml = load_yaml("examples/music_video_subscriptions.yaml") + yaml["__preset__"]["overrides"]["music_video_directory"] = output_directory + + with subscription_yaml_file_generator(yaml) as mock_filename: + yield mock_filename @pytest.fixture() @@ -239,13 +246,25 @@ def tv_show_config(working_directory, tv_show_config_path) -> ConfigFile: @pytest.fixture() -def tv_show_subscriptions_path() -> Path: - return Path("examples/tv_show_subscriptions.yaml") +def tv_show_subscriptions_path( + output_directory: str, subscription_yaml_file_generator: Callable +) -> Path: + yaml = load_yaml("examples/tv_show_subscriptions.yaml") + yaml["__preset__"]["overrides"]["tv_show_directory"] = output_directory + + with subscription_yaml_file_generator(yaml) as mock_filename: + yield mock_filename @pytest.fixture() -def advanced_tv_show_subscriptions_path() -> Path: - return Path("examples/advanced/tv_show_subscriptions.yaml") +def advanced_tv_show_subscriptions_path( + output_directory: str, subscription_yaml_file_generator: Callable +) -> Path: + yaml = load_yaml("examples/advanced/tv_show_subscriptions.yaml") + yaml["__preset__"] = {"overrides": {"tv_show_directory": output_directory}} + + with subscription_yaml_file_generator(yaml) as mock_filename: + yield mock_filename @pytest.fixture() @@ -265,8 +284,25 @@ def default_config_path(default_config) -> str: @pytest.fixture() -def music_subscriptions_path() -> Path: - return Path("examples/music_subscriptions.yaml") +def music_subscriptions_path( + output_directory: str, subscription_yaml_file_generator: Callable +) -> Path: + yaml = load_yaml("examples/music_subscriptions.yaml") + yaml["__preset__"]["overrides"]["music_directory"] = output_directory + + with subscription_yaml_file_generator(yaml) as mock_filename: + yield mock_filename + + +@pytest.fixture() +def docker_default_subscription_path( + output_directory: str, subscription_yaml_file_generator: Callable +) -> Path: + yaml = load_yaml("docker/root/defaults/subscriptions.yaml") + yaml["__preset__"]["overrides"]["tv_show_directory"] = output_directory + + with subscription_yaml_file_generator(yaml) as mock_filename: + yield mock_filename def mock_run_from_cli(args: str) -> List[Subscription]: diff --git a/tests/integration/cli/test_entrypoint.py b/tests/integration/cli/test_entrypoint.py index 084df414..90ea3579 100644 --- a/tests/integration/cli/test_entrypoint.py +++ b/tests/integration/cli/test_entrypoint.py @@ -9,6 +9,7 @@ import pytest from ytdl_sub.cli.entrypoint import _download_subscriptions_from_yaml_files from ytdl_sub.cli.entrypoint import main +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled @@ -59,6 +60,7 @@ def test_subscription_logs_write_to_file( subscription_override_dict={}, update_with_info_json=False, dry_run=dry_run, + shuffle=False, ) except ValueError: assert not mock_success_output @@ -120,3 +122,40 @@ def test_update_with_info_json_requires_experimental_flag( pytest.raises(ExperimentalFeatureNotEnabled), ): _ = main() + + +def test_subscription_shuffle( + default_config: ConfigFile, + mock_subscription_download_factory: Callable, + music_video_subscription_path: Path, +): + + subscription_paths = [str(music_video_subscription_path)] + + with ( + patch.object( + Subscription, + "download", + new=mock_subscription_download_factory(mock_success_output=True), + ), + ): + out1 = _download_subscriptions_from_yaml_files( + config=default_config, + subscription_paths=subscription_paths, + subscription_matches=[], + subscription_override_dict={}, + update_with_info_json=False, + dry_run=True, + shuffle=True, + ) + out2 = _download_subscriptions_from_yaml_files( + config=default_config, + subscription_paths=subscription_paths, + subscription_matches=[], + subscription_override_dict={}, + update_with_info_json=False, + dry_run=True, + shuffle=True, + ) + + assert [sub.name for sub in out1] != [sub.name for sub in out2] diff --git a/tests/integration/plugins/test_output_options.py b/tests/integration/plugins/test_output_options.py index 458416b4..b65692b5 100644 --- a/tests/integration/plugins/test_output_options.py +++ b/tests/integration/plugins/test_output_options.py @@ -236,24 +236,14 @@ class TestOutputOptions: ): output_options_subscription_dict["output_options"]["keep_files_date_eval"] = "nope" - subscription = Subscription.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=output_options_subscription_dict, - ) - expected_error_msg = ( "Validation error in subscription_test.output_options.keep_files_date_eval: " "Expected a standardized date in the form of YYYY-MM-DD, but received 'nope'" ) - with ( - mock_download_collection_entries( - is_youtube_channel=False, - num_urls=1, - is_extracted_audio=False, - is_dry_run=True, - ), - pytest.raises(ValidationException, match=re.escape(expected_error_msg)), - ): - subscription.download(dry_run=True) + with pytest.raises(ValidationException, match=re.escape(expected_error_msg)): + _ = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=output_options_subscription_dict, + ) diff --git a/tests/integration/plugins/test_throttle_protection.py b/tests/integration/plugins/test_throttle_protection.py index 6154f70a..bd56f652 100644 --- a/tests/integration/plugins/test_throttle_protection.py +++ b/tests/integration/plugins/test_throttle_protection.py @@ -71,7 +71,7 @@ class TestThrottleProtectionPlugin: ), assert_logs( logger=throttle_protection_logger, - expected_message="Sleeping between subscriptions for %0.2f seconds", + expected_message="Sleeping between subscriptions for 0.02 seconds", log_level="info", expected_occurrences=1, ), @@ -139,7 +139,7 @@ class TestThrottleProtectionPlugin: ), assert_logs( logger=throttle_protection_logger, - expected_message="Reached subscription max downloads of %d", + expected_message="Reached subscription max downloads of 0 for throttle protection", log_level="info", expected_occurrences=1, ), diff --git a/tests/resources.py b/tests/resources.py index c1aa18d4..a327be02 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -1,18 +1,37 @@ +import json import os import shutil from pathlib import Path +from typing import Dict DISABLE_YOUTUBE_TESTS: bool = True REGENERATE_FIXTURES: bool = False RESOURCE_PATH: Path = Path("tests") / "resources" _FILE_FIXTURE_PATH: Path = RESOURCE_PATH / "file_fixtures" +_EXPECTED_JSON_PATH: Path = RESOURCE_PATH / "expected_json" def file_fixture_path(fixture_name: str) -> Path: return _FILE_FIXTURE_PATH / fixture_name +def expected_json(input_json: Dict, json_name: str) -> Dict: + expected_json_path = _EXPECTED_JSON_PATH / json_name + os.makedirs(os.path.dirname(expected_json_path), exist_ok=True) + + if REGENERATE_FIXTURES: + with open(expected_json_path, "w", encoding="utf-8") as json_file: + json.dump(input_json, json_file, sort_keys=True, indent=2) + + return input_json + + with open(expected_json_path, "r", encoding="utf-8") as json_file: + expected_json = json.load(json_file) + + return expected_json + + def copy_file_fixture(fixture_name: str, output_file_path: Path) -> None: os.makedirs(os.path.dirname(output_file_path), exist_ok=True) shutil.copy(file_fixture_path(fixture_name), output_file_path) diff --git a/tests/resources/expected_downloads_summaries/plugins/file_convert/custom_ffmpeg_args.json b/tests/resources/expected_downloads_summaries/plugins/file_convert/custom_ffmpeg_args.json index 642ea323..13bbe1fa 100644 --- a/tests/resources/expected_downloads_summaries/plugins/file_convert/custom_ffmpeg_args.json +++ b/tests/resources/expected_downloads_summaries/plugins/file_convert/custom_ffmpeg_args.json @@ -2,18 +2,18 @@ ".ytdl-sub-subscription_test-download-archive.json": "19cf39d57914ba9cbd1e57ba6f1e0683", "JMC/Mock Entry 20-1.info.json": "INFO_JSON", "JMC/Mock Entry 20-1.jpg": "e80c508c4818454300133fe1dc1a9cd7", - "JMC/Mock Entry 20-1.mkv": "eb9a8ff61701ab673c2f06147fe29a4e", + "JMC/Mock Entry 20-1.mkv": "cbc8c05ea5cf4deefd735b93af9c0259", "JMC/Mock Entry 20-1.nfo": "fefcf0b3e4f4ff80ad636584d50dadec", "JMC/Mock Entry 20-2.info.json": "INFO_JSON", "JMC/Mock Entry 20-2.jpg": "e80c508c4818454300133fe1dc1a9cd7", - "JMC/Mock Entry 20-2.mkv": "65d047562c61a068e8a92bd8df8801fa", + "JMC/Mock Entry 20-2.mkv": "85cc274346abc0c71ab7702b0d61abde", "JMC/Mock Entry 20-2.nfo": "025c0b631da5ff5470382b38fce78d2d", "JMC/Mock Entry 20-3.info.json": "INFO_JSON", "JMC/Mock Entry 20-3.jpg": "e80c508c4818454300133fe1dc1a9cd7", - "JMC/Mock Entry 20-3.mkv": "98582526fcb3bea10ab7b3df7a5bc65d", + "JMC/Mock Entry 20-3.mkv": "c5a9ce268475c3b8fa9d4d0e3c0c25ae", "JMC/Mock Entry 20-3.nfo": "618b0ff948d9de2e10cf1da8c0dd6615", "JMC/Mock Entry 21-1.info.json": "INFO_JSON", "JMC/Mock Entry 21-1.jpg": "e80c508c4818454300133fe1dc1a9cd7", - "JMC/Mock Entry 21-1.mkv": "3fe368f6d4bd9662a24abca9215b4d26", + "JMC/Mock Entry 21-1.mkv": "ebdd8026e72625c56dd2aa8a2e19e814", "JMC/Mock Entry 21-1.nfo": "e5c715749efc1603a6e2f59244d87aba" } \ No newline at end of file diff --git a/tests/resources/expected_json/music/inspect_sub_fill.json b/tests/resources/expected_json/music/inspect_sub_fill.json new file mode 100644 index 00000000..b66bfe9b --- /dev/null +++ b/tests/resources/expected_json/music/inspect_sub_fill.json @@ -0,0 +1,242 @@ +{ + "audio_extract": { + "codec": "best", + "enable": true + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": true, + "playlist_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ], + "variables": {}, + "webpage_url": "{ modified_webpage_url }", + "ytdl_options": {} + } + ], + "format": "ba[ext=webm]/ba", + "music_tags": { + "album": [ + "{ track_album }" + ], + "albumartist": [ + "Lester Young" + ], + "albumartists": [ + "Lester Young" + ], + "artist": [ + "Lester Young" + ], + "artists": [ + "Lester Young" + ], + "date": [ + "{ track_date }" + ], + "genres": [ + "Jazz" + ], + "original_date": [ + "{ track_original_date }" + ], + "title": [ + "{ track_title }" + ], + "track": [ + "{ track_number }" + ], + "tracktotal": [ + "{ track_total }" + ], + "year": [ + "{ track_year }" + ] + }, + "output_options": { + "download_archive_name": ".ytdl-sub-Lester Young-download-archive.json", + "file_name": "{ track_full_path }", + "keep_files_date_eval": "{ upload_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpzald2h7x", + "preserve_mtime": false, + "thumbnail_name": "{ album_cover_path }" + }, + "overrides": { + "album_cover_path": "Lester Young/{ %concat( \"[\", playlist_max_upload_year, \"] \", %sanitize( playlist_title ) ) }/folder.{ thumbnail_ext }", + "album_dir": "[{ playlist_max_upload_year }] { %sanitize( playlist_title ) }", + "artist_dir": "Lester Young", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "include_sibling_metadata": true, + "modified_webpage_url": "{ webpage_url }", + "music_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpzald2h7x", + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "subscription_array": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ], + "subscription_indent_1": "Jazz", + "subscription_value": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "subscription_value_1": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "track_album": "{ playlist_title }", + "track_album_artist": "Lester Young", + "track_artist": "Lester Young", + "track_date": "{ upload_date_standardized }", + "track_file_name": "{ playlist_index_padded } - { %sanitize( title ) }.{ ext }", + "track_full_path": "Lester Young/{ %concat( \"[\", playlist_max_upload_year, \"] \", %sanitize( playlist_title ) ) }/{ %concat( playlist_index_padded, \" - \", %sanitize( title ), \".\", ext ) }", + "track_genre": "Jazz", + "track_genre_default": "Unset", + "track_number": "{ playlist_index }", + "track_number_padded": "{ playlist_index_padded }", + "track_original_date": "{ upload_date_standardized }", + "track_title": "{ title }", + "track_total": "{ playlist_count }", + "track_year": "{ playlist_max_upload_year }", + "url": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music/inspect_sub_internal.json b/tests/resources/expected_json/music/inspect_sub_internal.json new file mode 100644 index 00000000..2409b16d --- /dev/null +++ b/tests/resources/expected_json/music/inspect_sub_internal.json @@ -0,0 +1,242 @@ +{ + "audio_extract": { + "codec": "best", + "enable": true + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": true, + "playlist_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ], + "variables": {}, + "webpage_url": "{ %map_get( entry_metadata, \"webpage_url\" ) }", + "ytdl_options": {} + } + ], + "format": "ba[ext=webm]/ba", + "music_tags": { + "album": [ + "{ %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }" + ], + "albumartist": [ + "Lester Young" + ], + "albumartists": [ + "Lester Young" + ], + "artist": [ + "Lester Young" + ], + "artists": [ + "Lester Young" + ], + "date": [ + "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }" + ], + "genres": [ + "Jazz" + ], + "original_date": [ + "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }" + ], + "title": [ + "{ %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) }" + ], + "track": [ + "{ %map_get_non_empty( entry_metadata, \"playlist_index\", 1 ) }" + ], + "tracktotal": [ + "{ %map_get_non_empty( entry_metadata, \"playlist_count\", 1 ) }" + ], + "year": [ + "{ %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ) }" + ] + }, + "output_options": { + "download_archive_name": ".ytdl-sub-Lester Young-download-archive.json", + "file_name": "Lester Young/{ %concat( \"[\", %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ), \"] \", %sanitize( %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) }/{ %concat( %pad_zero( %map_get_non_empty( entry_metadata, \"playlist_index\", 1 ), 2 ), \" - \", %sanitize( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ), \".\", ext ) }", + "keep_files_date_eval": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpdu1vad67", + "preserve_mtime": false, + "thumbnail_name": "Lester Young/{ %concat( \"[\", %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ), \"] \", %sanitize( %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) }/folder.jpg" + }, + "overrides": { + "album_cover_path": "Lester Young/{ %concat( \"[\", %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ), \"] \", %sanitize( %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) }/folder.jpg", + "album_dir": "[{ %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ) }] { %sanitize( %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) }", + "artist_dir": "Lester Young", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "include_sibling_metadata": true, + "modified_webpage_url": "{ %map_get( entry_metadata, \"webpage_url\" ) }", + "music_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpdu1vad67", + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ), \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ), [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "subscription_array": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ], + "subscription_indent_1": "Jazz", + "subscription_value": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "subscription_value_1": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "track_album": "{ %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }", + "track_album_artist": "Lester Young", + "track_artist": "Lester Young", + "track_date": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "track_file_name": "{ %pad_zero( %map_get_non_empty( entry_metadata, \"playlist_index\", 1 ), 2 ) } - { %sanitize( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }.{ ext }", + "track_full_path": "Lester Young/{ %concat( \"[\", %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ), \"] \", %sanitize( %map_get_non_empty( entry_metadata, \"playlist_title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) }/{ %concat( %pad_zero( %map_get_non_empty( entry_metadata, \"playlist_index\", 1 ), 2 ), \" - \", %sanitize( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ), \".\", ext ) }", + "track_genre": "Jazz", + "track_genre_default": "Unset", + "track_number": "{ %map_get_non_empty( entry_metadata, \"playlist_index\", 1 ) }", + "track_number_padded": "{ %pad_zero( %map_get_non_empty( entry_metadata, \"playlist_index\", 1 ), 2 ) }", + "track_original_date": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "track_title": "{ %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) }", + "track_total": "{ %map_get_non_empty( entry_metadata, \"playlist_count\", 1 ) }", + "track_year": "{ %int( %map_get( %to_date_metadata( %array_reduce( %if_passthrough( %extract_field_from_siblings( \"upload_date\" ), [ upload_date ] ), %max ) ), \"year\" ) ) }", + "url": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music/inspect_sub_original.json b/tests/resources/expected_json/music/inspect_sub_original.json new file mode 100644 index 00000000..988e58a9 --- /dev/null +++ b/tests/resources/expected_json/music/inspect_sub_original.json @@ -0,0 +1,227 @@ +{ + "audio_extract": { + "codec": "best" + }, + "download": [ + { + "include_sibling_metadata": "{include_sibling_metadata}", + "playlist_thumbnails": [ + { + "name": "{avatar_uncropped_thumbnail_file_name}", + "uid": "avatar_uncropped" + }, + { + "name": "{banner_uncropped_thumbnail_file_name}", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "{avatar_uncropped_thumbnail_file_name}", + "uid": "avatar_uncropped" + }, + { + "name": "{banner_uncropped_thumbnail_file_name}", + "uid": "banner_uncropped" + } + ], + "url": "{ %array_at(urls, 0) }", + "webpage_url": "{modified_webpage_url}" + }, + { + "include_sibling_metadata": "{include_sibling_metadata}", + "url": "{ %array_slice(urls, 1) }", + "webpage_url": "{modified_webpage_url}" + } + ], + "format": "ba[ext=webm]/ba", + "music_tags": { + "album": "{track_album}", + "albumartist": "{track_album_artist}", + "albumartists": [ + "{track_album_artist}" + ], + "artist": "{track_artist}", + "artists": [ + "{track_artist}" + ], + "date": "{track_date}", + "genres": [ + "{track_genre}" + ], + "original_date": "{track_original_date}", + "title": "{track_title}", + "track": "{track_number}", + "tracktotal": "{track_total}", + "year": "{track_year}" + }, + "output_options": { + "file_name": "{track_full_path}", + "maintain_download_archive": true, + "output_directory": "{music_directory}", + "thumbnail_name": "{album_cover_path}" + }, + "overrides": { + "album_cover_path": "{artist_dir}/{album_dir}/folder.{thumbnail_ext}", + "album_dir": "[{track_year}] {track_album_sanitized}", + "artist_dir": "{track_artist_sanitized}", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "include_sibling_metadata": true, + "modified_webpage_url": "{webpage_url}", + "music_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpk6coazyn", + "resolution_assert": "{\n %if(\n %and(\n enable_resolution_assert,\n %ne( height, 0 ),\n %not(resolution_assert_is_ignored)\n ),\n %assert(\n %gte( height, resolution_assert_height_gte ),\n %concat(\n \"Entry \",\n title,\n \" downloaded at a low resolution (\",\n resolution_readable,\n \"), you've probably been throttled. \",\n \"Stopping further downloads, wait a few hours and try again. \",\n \"Disable using the override variable `enable_resolution_assert: False`.\"\n )\n ),\n \"false is no-op\"\n )\n}", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [] }", + "resolution_assert_is_ignored": "{\n %print_if_true(\n %concat(title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\"),\n %contains_any(title, resolution_assert_ignore_titles)\n )\n}", + "resolution_assert_print": "{\n %print(\n %if(\n enable_resolution_assert,\n \"Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`\",\n \"Resolution assert is disabled. Use at your own risk!\"\n ),\n enable_resolution_assert,\n -1\n )\n}", + "resolution_readable": "{width}x{height}", + "subscription_array": "{%from_json('''[\"https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists\"]''')}", + "subscription_indent_1": "Jazz", + "subscription_value": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "subscription_value_1": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "track_album": "{playlist_title}", + "track_album_artist": "{track_artist}", + "track_artist": "{subscription_name}", + "track_date": "{upload_date_standardized}", + "track_file_name": "{track_number_padded} - {track_title_sanitized}.{ext}", + "track_full_path": "{artist_dir}/{album_dir}/{track_file_name}", + "track_genre": "{subscription_indent_1}", + "track_genre_default": "Unset", + "track_number": "{playlist_index}", + "track_number_padded": "{playlist_index_padded}", + "track_original_date": "{track_date}", + "track_title": "{title}", + "track_total": "{playlist_count}", + "track_year": "{playlist_max_upload_year}", + "url": "{subscription_value}", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": "{subscription_array}" + }, + "preset": [ + "_music_base", + "_multi_url", + "_albums_from_playlists", + "_throttle_protection", + "YouTube Releases", + "__preset__" + ], + "throttle_protection": { + "enable": "{\n %print(\n %if(\n enable_throttle_protection,\n \"Throttle protection is enabled. Disable using the override variable `enable_throttle_protection: False`\",\n \"Throttle protection is disabled. Use at your own risk!\"\n ),\n enable_throttle_protection\n )\n}", + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "ytdl_options": { + "break_on_existing": true + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music/inspect_sub_resolve.json b/tests/resources/expected_json/music/inspect_sub_resolve.json new file mode 100644 index 00000000..347a61ac --- /dev/null +++ b/tests/resources/expected_json/music/inspect_sub_resolve.json @@ -0,0 +1,242 @@ +{ + "audio_extract": { + "codec": "best", + "enable": true + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": true, + "playlist_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ], + "variables": {}, + "webpage_url": "{ webpage_url }", + "ytdl_options": {} + } + ], + "format": "ba[ext=webm]/ba", + "music_tags": { + "album": [ + "{ playlist_title }" + ], + "albumartist": [ + "Lester Young" + ], + "albumartists": [ + "Lester Young" + ], + "artist": [ + "Lester Young" + ], + "artists": [ + "Lester Young" + ], + "date": [ + "{ upload_date_standardized }" + ], + "genres": [ + "Jazz" + ], + "original_date": [ + "{ upload_date_standardized }" + ], + "title": [ + "{ title }" + ], + "track": [ + "{ playlist_index }" + ], + "tracktotal": [ + "{ playlist_count }" + ], + "year": [ + "{ playlist_max_upload_year }" + ] + }, + "output_options": { + "download_archive_name": ".ytdl-sub-Lester Young-download-archive.json", + "file_name": "Lester Young/{ %concat( \"[\", playlist_max_upload_year, \"] \", %sanitize( playlist_title ) ) }/{ %concat( playlist_index_padded, \" - \", %sanitize( title ), \".\", ext ) }", + "keep_files_date_eval": "{ upload_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpd5oeacb3", + "preserve_mtime": false, + "thumbnail_name": "Lester Young/{ %concat( \"[\", playlist_max_upload_year, \"] \", %sanitize( playlist_title ) ) }/folder.{ thumbnail_ext }" + }, + "overrides": { + "album_cover_path": "Lester Young/{ %concat( \"[\", playlist_max_upload_year, \"] \", %sanitize( playlist_title ) ) }/folder.{ thumbnail_ext }", + "album_dir": "[{ playlist_max_upload_year }] { %sanitize( playlist_title ) }", + "artist_dir": "Lester Young", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "include_sibling_metadata": true, + "modified_webpage_url": "{ webpage_url }", + "music_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpd5oeacb3", + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "subscription_array": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ], + "subscription_indent_1": "Jazz", + "subscription_value": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "subscription_value_1": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "track_album": "{ playlist_title }", + "track_album_artist": "Lester Young", + "track_artist": "Lester Young", + "track_date": "{ upload_date_standardized }", + "track_file_name": "{ playlist_index_padded } - { %sanitize( title ) }.{ ext }", + "track_full_path": "Lester Young/{ %concat( \"[\", playlist_max_upload_year, \"] \", %sanitize( playlist_title ) ) }/{ %concat( playlist_index_padded, \" - \", %sanitize( title ), \".\", ext ) }", + "track_genre": "Jazz", + "track_genre_default": "Unset", + "track_number": "{ playlist_index }", + "track_number_padded": "{ playlist_index_padded }", + "track_original_date": "{ upload_date_standardized }", + "track_title": "{ title }", + "track_total": "{ playlist_count }", + "track_year": "{ playlist_max_upload_year }", + "url": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music_video/inspect_sub_fill.json b/tests/resources/expected_json/music_video/inspect_sub_fill.json new file mode 100644 index 00000000..5999a01f --- /dev/null +++ b/tests/resources/expected_json/music_video/inspect_sub_fill.json @@ -0,0 +1,314 @@ +{ + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + ], + "variables": {}, + "webpage_url": "{ modified_webpage_url }", + "ytdl_options": {} + } + ], + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-Rick Astley-download-archive.json", + "file_name": "{ music_video_file_name }.{ ext }", + "info_json_name": "{ music_video_file_name }.{ info_json_ext }", + "keep_files_date_eval": "{ upload_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpyukbh6ta", + "preserve_mtime": false, + "thumbnail_name": "{ music_video_file_name }.jpg" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%contains_url_field": "{ %not( %is_null( %get_url_field( $0, '' ) ) ) }", + "%flat_array__category_to_map_format": "{ %array_apply_fixed( %assert_then( %is_array( $1 ), $1, \"If using album categories, each category must map to an array\" ), $0, %flat_array__url_to_map_format ) }", + "%flat_array__url_keyed_map_format": "{ { %map_get( $0, \"url\" ): $0 } }", + "%flat_array__url_to_map_format": "{ %elif( %is_map( $0 ), %map_extend( $0, { \"category\": $1 } ), %is_string( $0 ), { \"url\": $0, \"category\": $1 }, %throw( \"If using album categories, each URL must be either a string or map with metadata fields\" ) ) }", + "%get_url_field": "{ %map_get( %map( %map_get( category_url_map, ytdl_sub_input_url, { } ) ), $0, $1 ) }", + "%get_url_i": "{ %map_get( %map( %array_at( category_url_array, %int( %sub( $0, 1 ) ), { } ) ), \"url\", %array_at( subscription_array, %int( %sub( $0, 1 ) ), '' ) ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "category_url_array": "{ [ ] }", + "category_url_map": "{ [ ] }", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "file_title": "{ title_sanitized_plex }", + "file_uid": "{ uid_sanitized_plex }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ webpage_url }", + "music_video_album": "{ %get_url_field( \"category\", \"Music Videos\" ) }", + "music_video_album_default": "Music Videos", + "music_video_artist": "Rick Astley", + "music_video_date": "{ %elif( %contains_url_field( \"date\" ), %get_url_field( \"date\", upload_date_standardized ), %contains_url_field( \"year\" ), %concat( %get_url_field( \"date\", upload_year ), \"-01-01\" ), upload_date_standardized ) }", + "music_video_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpyukbh6ta", + "music_video_file_name": "Rick Astley/{ %sanitize( %get_url_field( \"title\", title ) ) }", + "music_video_file_name_suffix": "", + "music_video_genre": "Pop", + "music_video_genre_default": "ytdl-sub", + "music_video_title": "{ %get_url_field( \"title\", title ) }", + "music_video_year": "{ %int( %elif( %contains_url_field( \"date\" ), %slice( %get_url_field( \"date\", upload_date_standardized ), 0, 4 ), %contains_url_field( \"year\" ), %get_url_field( \"date\", upload_year ), upload_year ) ) }", + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "subscription_array": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + ], + "subscription_indent_1": "Pop", + "subscription_map": {}, + "subscription_value": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "subscription_value_1": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "album": "{ music_video_album }", + "artist": "Rick Astley", + "genre": "Pop", + "premiered": "{ music_video_date }", + "title": "{ music_video_title }", + "year": "{ music_video_year }" + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music_video/inspect_sub_internal.json b/tests/resources/expected_json/music_video/inspect_sub_internal.json new file mode 100644 index 00000000..3cd8a4cd --- /dev/null +++ b/tests/resources/expected_json/music_video/inspect_sub_internal.json @@ -0,0 +1,314 @@ +{ + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + ], + "variables": {}, + "webpage_url": "{ %map_get( entry_metadata, \"webpage_url\" ) }", + "ytdl_options": {} + } + ], + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-Rick Astley-download-archive.json", + "file_name": "{ %concat( \"Rick Astley\", \"/\", %sanitize( %get_url_field( \"title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ), '' ) }.{ %map_get( entry_metadata, \"ext\" ) }", + "info_json_name": "{ %concat( \"Rick Astley\", \"/\", %sanitize( %get_url_field( \"title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ), '' ) }.info.json", + "keep_files_date_eval": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmput7fc_rs", + "preserve_mtime": false, + "thumbnail_name": "{ %concat( \"Rick Astley\", \"/\", %sanitize( %get_url_field( \"title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ), '' ) }.jpg" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%contains_url_field": "{ %not( %is_null( %get_url_field( $0, '' ) ) ) }", + "%flat_array__category_to_map_format": "{ %array_apply_fixed( %assert_then( %is_array( $1 ), $1, \"If using album categories, each category must map to an array\" ), $0, %flat_array__url_to_map_format ) }", + "%flat_array__url_keyed_map_format": "{ { %map_get( $0, \"url\" ): $0 } }", + "%flat_array__url_to_map_format": "{ %elif( %is_map( $0 ), %map_extend( $0, { \"category\": $1 } ), %is_string( $0 ), { \"url\": $0, \"category\": $1 }, %throw( \"If using album categories, each URL must be either a string or map with metadata fields\" ) ) }", + "%get_url_field": "{ %map_get( %map( %map_get( category_url_map, ytdl_sub_input_url, { } ) ), $0, $1 ) }", + "%get_url_i": "{ %map_get( %map( %array_at( category_url_array, %int( %sub( $0, 1 ) ), { } ) ), \"url\", %array_at( subscription_array, %int( %sub( $0, 1 ) ), '' ) ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "category_url_array": "{ [ ] }", + "category_url_map": "{ [ ] }", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "file_title": "{ %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }", + "file_uid": "{ %sanitize_plex_episode( %map_get( entry_metadata, \"id\" ) ) }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ %map_get( entry_metadata, \"webpage_url\" ) }", + "music_video_album": "{ %get_url_field( \"category\", \"Music Videos\" ) }", + "music_video_album_default": "Music Videos", + "music_video_artist": "Rick Astley", + "music_video_date": "{ %elif( %contains_url_field( \"date\" ), %get_url_field( \"date\", upload_date_standardized ), %contains_url_field( \"year\" ), %concat( %get_url_field( \"date\", upload_year ), \"-01-01\" ), upload_date_standardized ) }", + "music_video_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmput7fc_rs", + "music_video_file_name": "Rick Astley/{ %sanitize( %get_url_field( \"title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) }", + "music_video_file_name_suffix": "", + "music_video_genre": "Pop", + "music_video_genre_default": "ytdl-sub", + "music_video_title": "{ %get_url_field( \"title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }", + "music_video_year": "{ %int( %elif( %contains_url_field( \"date\" ), %slice( %get_url_field( \"date\", upload_date_standardized ), 0, 4 ), %contains_url_field( \"year\" ), %get_url_field( \"date\", upload_year ), upload_year ) ) }", + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ), \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ), [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "subscription_array": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + ], + "subscription_indent_1": "Pop", + "subscription_map": {}, + "subscription_value": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "subscription_value_1": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "album": "{ %get_url_field( \"category\", \"Music Videos\" ) }", + "artist": "Rick Astley", + "genre": "Pop", + "premiered": "{ %elif( %contains_url_field( \"date\" ), %get_url_field( \"date\", upload_date_standardized ), %contains_url_field( \"year\" ), %concat( %get_url_field( \"date\", upload_year ), \"-01-01\" ), upload_date_standardized ) }", + "title": "{ %get_url_field( \"title\", %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }", + "year": "{ %int( %elif( %contains_url_field( \"date\" ), %slice( %get_url_field( \"date\", upload_date_standardized ), 0, 4 ), %contains_url_field( \"year\" ), %get_url_field( \"date\", upload_year ), upload_year ) ) }" + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music_video/inspect_sub_original.json b/tests/resources/expected_json/music_video/inspect_sub_original.json new file mode 100644 index 00000000..cdd52c9f --- /dev/null +++ b/tests/resources/expected_json/music_video/inspect_sub_original.json @@ -0,0 +1,235 @@ +{ + "download": [ + { + "download_reverse": false, + "url": "{ %array_apply(urls, %bilateral_url) }", + "webpage_url": "{modified_webpage_url}", + "ytdl_options": { + "playlist_items": "-1:0:-1" + } + }, + { + "include_sibling_metadata": "{include_sibling_metadata}", + "playlist_thumbnails": [ + { + "name": "{avatar_uncropped_thumbnail_file_name}", + "uid": "avatar_uncropped" + }, + { + "name": "{banner_uncropped_thumbnail_file_name}", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "{avatar_uncropped_thumbnail_file_name}", + "uid": "avatar_uncropped" + }, + { + "name": "{banner_uncropped_thumbnail_file_name}", + "uid": "banner_uncropped" + } + ], + "url": "{ %array_at(urls, 0) }", + "webpage_url": "{modified_webpage_url}" + }, + { + "include_sibling_metadata": "{include_sibling_metadata}", + "url": "{ %array_slice(urls, 1) }", + "webpage_url": "{modified_webpage_url}" + } + ], + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "file_name": "{music_video_file_name}.{ext}", + "info_json_name": "{music_video_file_name}.{info_json_ext}", + "maintain_download_archive": true, + "output_directory": "{music_video_directory}", + "thumbnail_name": "{music_video_file_name}.jpg" + }, + "overrides": { + "%bilateral_url": "{ \n %if(\n %and(\n enable_bilateral_scraping,\n subscription_has_download_archive,\n %is_bilateral_url($0)\n ),\n $0,\n \"\"\n )\n}", + "%contains_url_field": "{ %not( %is_null( %get_url_field( $0, null ) ) ) }", + "%flat_array__category_to_map_format": "{ \n %array_apply_fixed(\n %assert_then(\n %is_array( $1 ),\n $1,\n \"If using album categories, each category must map to an array\"\n ),\n $0,\n %flat_array__url_to_map_format \n )\n}", + "%flat_array__url_keyed_map_format": "{ { %map_get($0, \"url\"): $0 } }", + "%flat_array__url_to_map_format": "{\n %elif(\n %is_map($0),\n %map_extend( $0, { \"category\": $1 } ),\n %is_string($0),\n { \"url\": $0, \"category\": $1 },\n %throw(\"If using album categories, each URL must be either a string or map with metadata fields\")\n )\n}", + "%get_url_field": "{ %map_get( %map( %map_get( category_url_map, ytdl_sub_input_url, {} ) ), $0, $1 ) }", + "%get_url_i": "{ \n %map_get(\n %map( %array_at( category_url_array, %int( %sub($0, 1) ), {} ) ),\n \"url\",\n %array_at(subscription_array, %int( %sub($0, 1) ), null)\n )\n}", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "category_url_array": "{ %array_flatten( %map_apply( subscription_map, %flat_array__category_to_map_format ) ) }", + "category_url_map": "{\n %array_reduce(\n %array_apply( category_url_array, %flat_array__url_keyed_map_format ),\n %map_extend\n )\n}", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "file_title": "{title_sanitized_plex}", + "file_uid": "{uid_sanitized_plex}", + "include_sibling_metadata": false, + "modified_webpage_url": "{webpage_url}", + "music_video_album": "{ %get_url_field(\"category\", music_video_album_default) }", + "music_video_album_default": "Music Videos", + "music_video_artist": "{subscription_name}", + "music_video_date": "{ \n %elif(\n %contains_url_field(\"date\"),\n %get_url_field(\"date\", upload_date_standardized),\n\n %contains_url_field(\"year\"),\n %concat( %get_url_field(\"date\", upload_year), \"-01-01\"),\n\n upload_date_standardized\n )\n}", + "music_video_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp3bmucwsg", + "music_video_file_name": "{music_video_artist_sanitized}/{music_video_title_sanitized}{music_video_file_name_suffix}", + "music_video_file_name_suffix": "", + "music_video_genre": "{subscription_indent_1}", + "music_video_genre_default": "ytdl-sub", + "music_video_title": "{ %get_url_field(\"title\", title) }", + "music_video_year": "{\n %int(%elif(\n %contains_url_field(\"date\"),\n %slice( %get_url_field(\"date\", upload_date_standardized), 0, 4 ),\n\n %contains_url_field(\"year\"),\n %get_url_field(\"date\", upload_year),\n\n upload_year\n ))\n}", + "resolution_assert": "{\n %if(\n %and(\n enable_resolution_assert,\n %ne( height, 0 ),\n %not(resolution_assert_is_ignored)\n ),\n %assert(\n %gte( height, resolution_assert_height_gte ),\n %concat(\n \"Entry \",\n title,\n \" downloaded at a low resolution (\",\n resolution_readable,\n \"), you've probably been throttled. \",\n \"Stopping further downloads, wait a few hours and try again. \",\n \"Disable using the override variable `enable_resolution_assert: False`.\"\n )\n ),\n \"false is no-op\"\n )\n}", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [] }", + "resolution_assert_is_ignored": "{\n %print_if_true(\n %concat(title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\"),\n %contains_any(title, resolution_assert_ignore_titles)\n )\n}", + "resolution_assert_print": "{\n %print(\n %if(\n enable_resolution_assert,\n \"Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`\",\n \"Resolution assert is disabled. Use at your own risk!\"\n ),\n enable_resolution_assert,\n -1\n )\n}", + "resolution_readable": "{width}x{height}", + "subscription_array": "{%from_json('''[\"https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc\"]''')}", + "subscription_indent_1": "Pop", + "subscription_map": "{ {} }", + "subscription_value": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "subscription_value_1": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url": "{subscription_value}", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": "{ %array_apply(%range(100, 1), %get_url_i) }" + }, + "preset": [ + "_base", + "_plex_base", + "_url_bilateral_overrides", + "_multi_url_bilateral_inner", + "_multi_url", + "_multi_url_bilateral", + "_throttle_protection", + "_url_categorized", + "_plex_video_base", + "_music_video_base", + "_music_video_tags", + "Plex Music Videos", + "__preset__" + ], + "throttle_protection": { + "enable": "{\n %print(\n %if(\n enable_throttle_protection,\n \"Throttle protection is enabled. Disable using the override variable `enable_throttle_protection: False`\",\n \"Throttle protection is disabled. Use at your own risk!\"\n ),\n enable_throttle_protection\n )\n}", + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "album": "{music_video_album}", + "artist": "{music_video_artist}", + "genre": "{music_video_genre}", + "premiered": "{music_video_date}", + "title": "{music_video_title}", + "year": "{music_video_year}" + }, + "ytdl_options": { + "break_on_existing": true + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/music_video/inspect_sub_resolve.json b/tests/resources/expected_json/music_video/inspect_sub_resolve.json new file mode 100644 index 00000000..f3594e9c --- /dev/null +++ b/tests/resources/expected_json/music_video/inspect_sub_resolve.json @@ -0,0 +1,314 @@ +{ + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "", + "uid": "avatar_uncropped" + }, + { + "name": "", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + ], + "variables": {}, + "webpage_url": "{ webpage_url }", + "ytdl_options": {} + } + ], + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-Rick Astley-download-archive.json", + "file_name": "{ %concat( \"Rick Astley\", \"/\", %sanitize( %get_url_field( \"title\", title ) ), '' ) }.{ ext }", + "info_json_name": "{ %concat( \"Rick Astley\", \"/\", %sanitize( %get_url_field( \"title\", title ) ), '' ) }.{ info_json_ext }", + "keep_files_date_eval": "{ upload_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpstvsa5ot", + "preserve_mtime": false, + "thumbnail_name": "{ %concat( \"Rick Astley\", \"/\", %sanitize( %get_url_field( \"title\", title ) ), '' ) }.jpg" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%contains_url_field": "{ %not( %is_null( %get_url_field( $0, '' ) ) ) }", + "%flat_array__category_to_map_format": "{ %array_apply_fixed( %assert_then( %is_array( $1 ), $1, \"If using album categories, each category must map to an array\" ), $0, %flat_array__url_to_map_format ) }", + "%flat_array__url_keyed_map_format": "{ { %map_get( $0, \"url\" ): $0 } }", + "%flat_array__url_to_map_format": "{ %elif( %is_map( $0 ), %map_extend( $0, { \"category\": $1 } ), %is_string( $0 ), { \"url\": $0, \"category\": $1 }, %throw( \"If using album categories, each URL must be either a string or map with metadata fields\" ) ) }", + "%get_url_field": "{ %map_get( %map( %map_get( category_url_map, ytdl_sub_input_url, { } ) ), $0, $1 ) }", + "%get_url_i": "{ %map_get( %map( %array_at( category_url_array, %int( %sub( $0, 1 ) ), { } ) ), \"url\", %array_at( subscription_array, %int( %sub( $0, 1 ) ), '' ) ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "avatar_uncropped_thumbnail_file_name": "", + "banner_uncropped_thumbnail_file_name": "", + "category_url_array": "{ [ ] }", + "category_url_map": "{ [ ] }", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "file_title": "{ title_sanitized_plex }", + "file_uid": "{ uid_sanitized_plex }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ webpage_url }", + "music_video_album": "{ %get_url_field( \"category\", \"Music Videos\" ) }", + "music_video_album_default": "Music Videos", + "music_video_artist": "Rick Astley", + "music_video_date": "{ %elif( %contains_url_field( \"date\" ), %get_url_field( \"date\", upload_date_standardized ), %contains_url_field( \"year\" ), %concat( %get_url_field( \"date\", upload_year ), \"-01-01\" ), upload_date_standardized ) }", + "music_video_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpstvsa5ot", + "music_video_file_name": "Rick Astley/{ %sanitize( %get_url_field( \"title\", title ) ) }", + "music_video_file_name_suffix": "", + "music_video_genre": "Pop", + "music_video_genre_default": "ytdl-sub", + "music_video_title": "{ %get_url_field( \"title\", title ) }", + "music_video_year": "{ %int( %elif( %contains_url_field( \"date\" ), %slice( %get_url_field( \"date\", upload_date_standardized ), 0, 4 ), %contains_url_field( \"year\" ), %get_url_field( \"date\", upload_year ), upload_year ) ) }", + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "subscription_array": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + ], + "subscription_indent_1": "Pop", + "subscription_map": {}, + "subscription_value": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "subscription_value_1": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "album": "{ %get_url_field( \"category\", \"Music Videos\" ) }", + "artist": "Rick Astley", + "genre": "Pop", + "premiered": "{ %elif( %contains_url_field( \"date\" ), %get_url_field( \"date\", upload_date_standardized ), %contains_url_field( \"year\" ), %concat( %get_url_field( \"date\", upload_year ), \"-01-01\" ), upload_date_standardized ) }", + "title": "{ %get_url_field( \"title\", title ) }", + "year": "{ %int( %elif( %contains_url_field( \"date\" ), %slice( %get_url_field( \"date\", upload_date_standardized ), 0, 4 ), %contains_url_field( \"year\" ), %get_url_field( \"date\", upload_year ), upload_year ) ) }" + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/tv_show/inspect_sub_fill.json b/tests/resources/expected_json/tv_show/inspect_sub_fill.json new file mode 100644 index 00000000..6d1a8662 --- /dev/null +++ b/tests/resources/expected_json/tv_show/inspect_sub_fill.json @@ -0,0 +1,250 @@ +{ + "chapters": { + "allow_chapters_from_comments": false, + "embed_chapters": true, + "enable": true, + "force_key_frames": false + }, + "date_range": { + "breaks": true, + "enable": true, + "type": "upload_date" + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/@novapbs" + ], + "variables": {}, + "webpage_url": "{ modified_webpage_url }", + "ytdl_options": {} + } + ], + "file_convert": { + "convert_to": "mp4", + "convert_with": "yt-dlp", + "enable": true + }, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-NOVA PBS-download-archive.json", + "file_name": "{ episode_file_path }.{ ext }", + "info_json_name": "{ episode_file_path }.{ info_json_ext }", + "keep_files_date_eval": "{ episode_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpc_p6gjjw/NOVA PBS", + "preserve_mtime": false, + "thumbnail_name": "{ thumbnail_file_name }" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%episode_ordering_": "{ %eq( %lower( tv_show_by_date_episode_ordering ), $0 ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "%ordering_pair_eq": "{ %eq( [ tv_show_by_date_season_ordering, tv_show_by_date_episode_ordering ], [ $0, $1 ] ) }", + "%season_ordering_": "{ %eq( %lower( tv_show_by_date_season_ordering ), $0 ) }", + "assert_not_collection": true, + "avatar_uncropped_thumbnail_file_name": "poster.jpg", + "banner_uncropped_thumbnail_file_name": "fanart.jpg", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "episode_content_rating": "TV-14", + "episode_date_standardized": "{ upload_date_standardized }", + "episode_file_name": "s{ upload_year }.e{ %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ) } - { title_sanitized_plex }", + "episode_file_path": "{ %sanitize( %concat( \"Season \", upload_year ) ) }/{ %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) }", + "episode_number": "{ upload_month }{ upload_day_padded }{ upload_date_index_padded }", + "episode_number_and_padded_": "{ [ %concat( upload_month, upload_day_padded, upload_date_index_padded ), 6 ] }", + "episode_number_padded": "{ %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ) }", + "episode_plot": "{ webpage_url }\n\n{ description }", + "episode_title": "{ upload_date_standardized } - { title }", + "episode_year": "{ %slice( upload_date_standardized, 0, 4 ) }", + "file_title": "{ title_sanitized_plex }", + "file_uid": "{ uid_sanitized_plex }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ webpage_url }", + "only_recent_date_range": "2months", + "only_recent_max_files": 30, + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "s01_name": "", + "s01_url": "", + "season_directory_name": "Season { upload_year }", + "season_number": "{ upload_year }", + "season_number_padded": "{ upload_year }", + "season_poster_file_name": "{ %sanitize( %concat( \"Season \", upload_year ) ) }/Season{ upload_year }.jpg", + "subscription_array": [ + "https://www.youtube.com/@novapbs" + ], + "subscription_indent_1": "Documentaries", + "subscription_indent_2": "TV-14", + "subscription_value": "https://www.youtube.com/@novapbs", + "subscription_value_1": "https://www.youtube.com/@novapbs", + "thumbnail_file_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }-thumb.jpg", + "tv_show_by_date_episode_ordering": "upload-month-day", + "tv_show_by_date_ordering_pair_validation_": "{ [ %concat( upload_month, upload_day_padded, upload_date_index_padded ), 6 ] }", + "tv_show_by_date_season_ordering": "upload-year", + "tv_show_content_rating": "TV-14", + "tv_show_content_rating_default": "TV-14", + "tv_show_date_range_type": "upload_date", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpc_p6gjjw", + "tv_show_fanart_file_name": "fanart.jpg", + "tv_show_genre": "Documentaries", + "tv_show_genre_default": "ytdl-sub", + "tv_show_name": "NOVA PBS", + "tv_show_poster_file_name": "poster.jpg", + "url": "https://www.youtube.com/@novapbs", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/@novapbs" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "contentRating": "TV-14", + "date": "{ episode_date_standardized }", + "episode_id": "{ episode_number }", + "genre": "Documentaries", + "show": "NOVA PBS", + "synopsis": "{ episode_plot }", + "title": "{ episode_title }", + "year": "{ episode_year }" + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/tv_show/inspect_sub_internal.json b/tests/resources/expected_json/tv_show/inspect_sub_internal.json new file mode 100644 index 00000000..3f1c1eaf --- /dev/null +++ b/tests/resources/expected_json/tv_show/inspect_sub_internal.json @@ -0,0 +1,250 @@ +{ + "chapters": { + "allow_chapters_from_comments": false, + "embed_chapters": true, + "enable": true, + "force_key_frames": false + }, + "date_range": { + "breaks": true, + "enable": true, + "type": "upload_date" + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/@novapbs" + ], + "variables": {}, + "webpage_url": "{ %map_get( entry_metadata, \"webpage_url\" ) }", + "ytdl_options": {} + } + ], + "file_convert": { + "convert_to": "mp4", + "convert_with": "yt-dlp", + "enable": true + }, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-NOVA PBS-download-archive.json", + "file_name": "{ %concat( %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ), \"/\", %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) ) }.{ ext }", + "info_json_name": "{ %concat( %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ), \"/\", %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) ) }.info.json", + "keep_files_date_eval": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpps6shcpt/NOVA PBS", + "preserve_mtime": false, + "thumbnail_name": "{ %concat( %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ), \"/\", %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) ) }-thumb.jpg" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%episode_ordering_": "{ %eq( %lower( tv_show_by_date_episode_ordering ), $0 ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "%ordering_pair_eq": "{ %eq( [ tv_show_by_date_season_ordering, tv_show_by_date_episode_ordering ], [ $0, $1 ] ) }", + "%season_ordering_": "{ %eq( %lower( tv_show_by_date_season_ordering ), $0 ) }", + "assert_not_collection": true, + "avatar_uncropped_thumbnail_file_name": "poster.jpg", + "banner_uncropped_thumbnail_file_name": "fanart.jpg", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "episode_content_rating": "TV-14", + "episode_date_standardized": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "episode_file_name": "s{ %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) }.e{ %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ) } - { %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }", + "episode_file_path": "{ %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ) }/{ %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) }", + "episode_number": "{ %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ) }{ %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) }{ %pad_zero( upload_date_index, 2 ) }", + "episode_number_and_padded_": "{ [ %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ), 6 ] }", + "episode_number_padded": "{ %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ) }", + "episode_plot": "{ %map_get( entry_metadata, \"webpage_url\" ) }\n\n{ %map_get_non_empty( entry_metadata, \"description\", '' ) }", + "episode_title": "{ %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) } - { %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) }", + "episode_year": "{ %slice( %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ), 0, 4 ) }", + "file_title": "{ %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) }", + "file_uid": "{ %sanitize_plex_episode( %map_get( entry_metadata, \"id\" ) ) }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ %map_get( entry_metadata, \"webpage_url\" ) }", + "only_recent_date_range": "2months", + "only_recent_max_files": 30, + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ), \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ), [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "s01_name": "", + "s01_url": "", + "season_directory_name": "Season { %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) }", + "season_number": "{ %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) }", + "season_number_padded": "{ %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) }", + "season_poster_file_name": "{ %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ) }/Season{ %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) }.jpg", + "subscription_array": [ + "https://www.youtube.com/@novapbs" + ], + "subscription_indent_1": "Documentaries", + "subscription_indent_2": "TV-14", + "subscription_value": "https://www.youtube.com/@novapbs", + "subscription_value_1": "https://www.youtube.com/@novapbs", + "thumbnail_file_name": "{ %concat( %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ), \"/\", %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) ) }-thumb.jpg", + "tv_show_by_date_episode_ordering": "upload-month-day", + "tv_show_by_date_ordering_pair_validation_": "{ [ %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ), 6 ] }", + "tv_show_by_date_season_ordering": "upload-year", + "tv_show_content_rating": "TV-14", + "tv_show_content_rating_default": "TV-14", + "tv_show_date_range_type": "upload_date", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpps6shcpt", + "tv_show_fanart_file_name": "fanart.jpg", + "tv_show_genre": "Documentaries", + "tv_show_genre_default": "ytdl-sub", + "tv_show_name": "NOVA PBS", + "tv_show_poster_file_name": "poster.jpg", + "url": "https://www.youtube.com/@novapbs", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/@novapbs" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "contentRating": "TV-14", + "date": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", + "episode_id": "{ %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ) }{ %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) }{ %pad_zero( upload_date_index, 2 ) }", + "genre": "Documentaries", + "show": "NOVA PBS", + "synopsis": "{ %map_get( entry_metadata, \"webpage_url\" ) }\n\n{ %map_get_non_empty( entry_metadata, \"description\", '' ) }", + "title": "{ %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) } - { %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) }", + "year": "{ %slice( %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ), 0, 4 ) }" + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/tv_show/inspect_sub_original.json b/tests/resources/expected_json/tv_show/inspect_sub_original.json new file mode 100644 index 00000000..0a7540d3 --- /dev/null +++ b/tests/resources/expected_json/tv_show/inspect_sub_original.json @@ -0,0 +1,255 @@ +{ + "chapters": { + "embed_chapters": true + }, + "date_range": { + "type": "{tv_show_date_range_type}" + }, + "download": [ + { + "download_reverse": false, + "url": "{ %array_apply(urls, %bilateral_url) }", + "webpage_url": "{modified_webpage_url}", + "ytdl_options": { + "playlist_items": "-1:0:-1" + } + }, + { + "include_sibling_metadata": "{include_sibling_metadata}", + "playlist_thumbnails": [ + { + "name": "{avatar_uncropped_thumbnail_file_name}", + "uid": "avatar_uncropped" + }, + { + "name": "{banner_uncropped_thumbnail_file_name}", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "{avatar_uncropped_thumbnail_file_name}", + "uid": "avatar_uncropped" + }, + { + "name": "{banner_uncropped_thumbnail_file_name}", + "uid": "banner_uncropped" + } + ], + "url": "{ %array_at(urls, 0) }", + "webpage_url": "{modified_webpage_url}" + }, + { + "include_sibling_metadata": "{include_sibling_metadata}", + "url": "{ %array_slice(urls, 1) }", + "webpage_url": "{modified_webpage_url}" + } + ], + "file_convert": { + "convert_to": "mp4" + }, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "file_name": "{episode_file_path}.{ext}", + "info_json_name": "{episode_file_path}.{info_json_ext}", + "keep_files_date_eval": "{episode_date_standardized}", + "maintain_download_archive": true, + "output_directory": "{tv_show_directory}/{tv_show_name_sanitized}", + "thumbnail_name": "{thumbnail_file_name}" + }, + "overrides": { + "%bilateral_url": "{ \n %if(\n %and(\n enable_bilateral_scraping,\n subscription_has_download_archive,\n %is_bilateral_url($0)\n ),\n $0,\n \"\"\n )\n}", + "%episode_ordering_": "{ %eq( %lower(tv_show_by_date_episode_ordering), $0 ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "%ordering_pair_eq": "{\n %eq([tv_show_by_date_season_ordering, tv_show_by_date_episode_ordering], [$0, $1])\n}", + "%season_ordering_": "{ %eq( %lower(tv_show_by_date_season_ordering), $0 ) }", + "assert_not_collection": "{\n %assert(\n %and(\n %not( %bool(s01_url) ),\n %not( %bool(s01_name) )\n ),\n \"Provided `s01_url` or `s01_name` variable to TV Show by Date preset when it expects `url`. Perhaps you meant to use the `TV Show Collection` preset?\"\n )\n}", + "avatar_uncropped_thumbnail_file_name": "{tv_show_poster_file_name}", + "banner_uncropped_thumbnail_file_name": "{tv_show_fanart_file_name}", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "episode_content_rating": "{tv_show_content_rating}", + "episode_date_standardized": "{\n %if(\n %contains(tv_show_by_date_season_ordering, \"release\"),\n release_date_standardized,\n upload_date_standardized\n )\n}", + "episode_file_name": "s{season_number_padded}.e{episode_number_padded} - {file_title}", + "episode_file_path": "{season_directory_name_sanitized}/{episode_file_name_sanitized}", + "episode_number": "{ %array_at(episode_number_and_padded_, 0) }", + "episode_number_and_padded_": "{\n %elif(\n %episode_ordering_( \"upload-day\" ), [ %concat(upload_day, upload_date_index_padded), 4 ],\n %episode_ordering_( \"upload-month-day\" ), [ %concat(upload_month, upload_day_padded, upload_date_index_padded), 6],\n %episode_ordering_( \"upload-month-day-reversed\" ), [ %concat(upload_day_of_year_reversed, upload_date_index_reversed_padded), 5],\n %episode_ordering_( \"release-day\" ), [ %concat(release_day, upload_date_index_padded), 4 ],\n %episode_ordering_( \"release-month-day\" ), [ %concat(release_month, release_day_padded, upload_date_index_padded), 6],\n %episode_ordering_( \"release-month-day-reversed\" ), [ %concat(release_day_of_year_reversed, upload_date_index_reversed_padded), 5],\n %episode_ordering_( \"download-index\" ), [ download_index, 6 ],\n %throw(\n 'tv_show_by_date_episode_ordering must be one of the following: \"upload-day\", \"upload-month-day\", \"upload-month-day-reversed\", \"release-day\", \"release-month-day\", \"release-month-day-reversed\", \"download-index\"'\n )\n )\n}", + "episode_number_padded": "{ %pad_zero( %int(episode_number), %int(%array_at(episode_number_and_padded_, 1))) }", + "episode_plot": "{webpage_url}\n\n{description}", + "episode_title": "{episode_date_standardized} - {title}", + "episode_year": "{%slice(episode_date_standardized, 0, 4)}", + "file_title": "{title_sanitized_plex}", + "file_uid": "{uid_sanitized_plex}", + "include_sibling_metadata": false, + "modified_webpage_url": "{webpage_url}", + "only_recent_date_range": "2months", + "only_recent_max_files": 30, + "resolution_assert": "{\n %if(\n %and(\n enable_resolution_assert,\n %ne( height, 0 ),\n %not(resolution_assert_is_ignored)\n ),\n %assert(\n %gte( height, resolution_assert_height_gte ),\n %concat(\n \"Entry \",\n title,\n \" downloaded at a low resolution (\",\n resolution_readable,\n \"), you've probably been throttled. \",\n \"Stopping further downloads, wait a few hours and try again. \",\n \"Disable using the override variable `enable_resolution_assert: False`.\"\n )\n ),\n \"false is no-op\"\n )\n}", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [] }", + "resolution_assert_is_ignored": "{\n %print_if_true(\n %concat(title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\"),\n %contains_any(title, resolution_assert_ignore_titles)\n )\n}", + "resolution_assert_print": "{\n %print(\n %if(\n enable_resolution_assert,\n \"Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`\",\n \"Resolution assert is disabled. Use at your own risk!\"\n ),\n enable_resolution_assert,\n -1\n )\n}", + "resolution_readable": "{width}x{height}", + "s01_name": "", + "s01_url": "", + "season_directory_name": "Season {season_number_padded}", + "season_number": "{\n %elif(\n %season_ordering_( \"upload-year\" ), upload_year,\n %season_ordering_( \"upload-year-month\" ), %concat(upload_year, upload_month_padded),\n %season_ordering_( \"release-year\" ), release_year,\n %season_ordering_( \"release-year-month\" ), %concat(release_year, release_month_padded),\n %throw(\n 'tv_show_by_date_season_ordering must be one of the following: \"upload-year\", \"upload-year-month\", \"release-year\", \"release-year-month\"'\n )\n )\n}", + "season_number_padded": "{season_number}", + "season_poster_file_name": "{season_directory_name_sanitized}/Season{season_number_padded}.jpg", + "subscription_array": "{%from_json('''[\"https://www.youtube.com/@novapbs\"]''')}", + "subscription_indent_1": "Documentaries", + "subscription_indent_2": "{tv_show_content_rating_default}", + "subscription_value": "https://www.youtube.com/@novapbs", + "subscription_value_1": "https://www.youtube.com/@novapbs", + "thumbnail_file_name": "{episode_file_path}-thumb.jpg", + "tv_show_by_date_episode_ordering": "upload-month-day", + "tv_show_by_date_ordering_pair_validation_": "{\n %assert_then(\n %or(\n %ordering_pair_eq(\"upload-year\", \"upload-month-day\"),\n %ordering_pair_eq(\"upload-year\", \"upload-month-day-reversed\"),\n %ordering_pair_eq(\"upload-year\", \"download-index\"),\n %ordering_pair_eq(\"upload-year-month\", \"upload-day\"),\n %ordering_pair_eq(\"release-year\", \"release-month-day\"),\n %ordering_pair_eq(\"release-year\", \"release-month-day-reversed\"),\n %ordering_pair_eq(\"release-year\", \"download-index\"),\n %ordering_pair_eq(\"release-year-month\", \"release-day\")\n ),\n episode_number_and_padded_,\n \"Detected incompatibility between tv_show_by_date_season_ordering and tv_show_by_date_episode_ordering. Ensure you are not using both upload and release date, and that the year/month/day are included in the combined season and episode.\"\n )\n}", + "tv_show_by_date_season_ordering": "upload-year", + "tv_show_content_rating": "{subscription_indent_2}", + "tv_show_content_rating_default": "TV-14", + "tv_show_date_range_type": "{\n %if(\n %contains(tv_show_by_date_season_ordering, \"release\"),\n \"release_date\",\n \"upload_date\"\n )\n}", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp5yx9y73k", + "tv_show_fanart_file_name": "fanart.jpg", + "tv_show_genre": "{subscription_indent_1}", + "tv_show_genre_default": "ytdl-sub", + "tv_show_name": "{subscription_name}", + "tv_show_poster_file_name": "poster.jpg", + "url": "{subscription_value}", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": "{subscription_array}" + }, + "preset": [ + "_season_by_year", + "plex_tv_show_by_date", + "season_by_year__episode_by_month_day", + "Plex TV Show by Date", + "__preset__" + ], + "throttle_protection": { + "enable": "{\n %print(\n %if(\n enable_throttle_protection,\n \"Throttle protection is enabled. Disable using the override variable `enable_throttle_protection: False`\",\n \"Throttle protection is disabled. Use at your own risk!\"\n ),\n enable_throttle_protection\n )\n}", + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "contentRating": "{episode_content_rating}", + "date": "{episode_date_standardized}", + "episode_id": "{episode_number}", + "genre": "{tv_show_genre}", + "show": "{tv_show_name}", + "synopsis": "{episode_plot}", + "title": "{episode_title}", + "year": "{episode_year}" + }, + "ytdl_options": { + "break_on_existing": true + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/tv_show/inspect_sub_resolve.json b/tests/resources/expected_json/tv_show/inspect_sub_resolve.json new file mode 100644 index 00000000..3c2898ea --- /dev/null +++ b/tests/resources/expected_json/tv_show/inspect_sub_resolve.json @@ -0,0 +1,250 @@ +{ + "chapters": { + "allow_chapters_from_comments": false, + "embed_chapters": true, + "enable": true, + "force_key_frames": false + }, + "date_range": { + "breaks": true, + "enable": true, + "type": "upload_date" + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/@novapbs" + ], + "variables": {}, + "webpage_url": "{ webpage_url }", + "ytdl_options": {} + } + ], + "file_convert": { + "convert_to": "mp4", + "convert_with": "yt-dlp", + "enable": true + }, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-NOVA PBS-download-archive.json", + "file_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }.{ ext }", + "info_json_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }.{ info_json_ext }", + "keep_files_date_eval": "{ upload_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp3j0cm_tw/NOVA PBS", + "preserve_mtime": false, + "thumbnail_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }-thumb.jpg" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%episode_ordering_": "{ %eq( %lower( tv_show_by_date_episode_ordering ), $0 ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "%ordering_pair_eq": "{ %eq( [ tv_show_by_date_season_ordering, tv_show_by_date_episode_ordering ], [ $0, $1 ] ) }", + "%season_ordering_": "{ %eq( %lower( tv_show_by_date_season_ordering ), $0 ) }", + "assert_not_collection": true, + "avatar_uncropped_thumbnail_file_name": "poster.jpg", + "banner_uncropped_thumbnail_file_name": "fanart.jpg", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "episode_content_rating": "TV-14", + "episode_date_standardized": "{ upload_date_standardized }", + "episode_file_name": "s{ upload_year }.e{ %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ) } - { title_sanitized_plex }", + "episode_file_path": "{ %sanitize( %concat( \"Season \", upload_year ) ) }/{ %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) }", + "episode_number": "{ upload_month }{ upload_day_padded }{ upload_date_index_padded }", + "episode_number_and_padded_": "{ [ %concat( upload_month, upload_day_padded, upload_date_index_padded ), 6 ] }", + "episode_number_padded": "{ %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ) }", + "episode_plot": "{ webpage_url }\n\n{ description }", + "episode_title": "{ upload_date_standardized } - { title }", + "episode_year": "{ %slice( upload_date_standardized, 0, 4 ) }", + "file_title": "{ title_sanitized_plex }", + "file_uid": "{ uid_sanitized_plex }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ webpage_url }", + "only_recent_date_range": "2months", + "only_recent_max_files": 30, + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "s01_name": "", + "s01_url": "", + "season_directory_name": "Season { upload_year }", + "season_number": "{ upload_year }", + "season_number_padded": "{ upload_year }", + "season_poster_file_name": "{ %sanitize( %concat( \"Season \", upload_year ) ) }/Season{ upload_year }.jpg", + "subscription_array": [ + "https://www.youtube.com/@novapbs" + ], + "subscription_indent_1": "Documentaries", + "subscription_indent_2": "TV-14", + "subscription_value": "https://www.youtube.com/@novapbs", + "subscription_value_1": "https://www.youtube.com/@novapbs", + "thumbnail_file_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }-thumb.jpg", + "tv_show_by_date_episode_ordering": "upload-month-day", + "tv_show_by_date_ordering_pair_validation_": "{ [ %concat( upload_month, upload_day_padded, upload_date_index_padded ), 6 ] }", + "tv_show_by_date_season_ordering": "upload-year", + "tv_show_content_rating": "TV-14", + "tv_show_content_rating_default": "TV-14", + "tv_show_date_range_type": "upload_date", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp3j0cm_tw", + "tv_show_fanart_file_name": "fanart.jpg", + "tv_show_genre": "Documentaries", + "tv_show_genre_default": "ytdl-sub", + "tv_show_name": "NOVA PBS", + "tv_show_poster_file_name": "poster.jpg", + "url": "https://www.youtube.com/@novapbs", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/@novapbs" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "contentRating": "TV-14", + "date": "{ upload_date_standardized }", + "episode_id": "{ upload_month }{ upload_day_padded }{ upload_date_index_padded }", + "genre": "Documentaries", + "show": "NOVA PBS", + "synopsis": "{ webpage_url }\n\n{ description }", + "title": "{ upload_date_standardized } - { title }", + "year": "{ %slice( upload_date_standardized, 0, 4 ) }" + } +} \ No newline at end of file diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index cb3c0b81..a42439be 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -6,7 +6,6 @@ import pytest from ytdl_sub.config.preset import Preset from ytdl_sub.plugins.nfo_tags import NfoTagsOptions -from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import ValidationException @@ -120,6 +119,26 @@ class TestPreset: == "today-2months" ) + def test_preset_preserve_mtime_option(self, config_file, youtube_video, output_options): + # Test preserve_mtime defaults to False + preset_default = Preset( + config=config_file, + name="test_default", + value={"download": youtube_video, "output_options": output_options}, + ) + assert preset_default.output_options.preserve_mtime is False + + # Test preserve_mtime can be set to True + preset_enabled = Preset( + config=config_file, + name="test_enabled", + value={ + "download": youtube_video, + "output_options": dict(output_options, **{"preserve_mtime": True}), + }, + ) + assert preset_enabled.output_options.preserve_mtime is True + @pytest.mark.parametrize( "parent_preset", ["preset_self_loop", "preset_loop_0", "preset_loop_1"] ) @@ -137,100 +156,6 @@ class TestPreset: }, ) - def test_preset_error__source_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, - }, - ) - - def test_preset_error__override_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": {"output_directory": "{dne_var}", "file_name": "file"}, - }, - ) - - def test_preset_error__dict_source_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": {"output_directory": "dir", "file_name": "file"}, - "nfo_tags": { - "nfo_name": "the nfo name", - "nfo_root": "the root", - "tags": {"tag_a": "{dne_var}"}, - }, - }, - ) - - def test_preset_error__dict_override_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": output_options, - "output_directory_nfo_tags": { - "nfo_name": "the nfo name", - "nfo_root": "the root", - "tags": {"tag_a": "{dne_var}"}, - }, - }, - ) - - def test_preset_error__dict_override_variable_not_static( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="static formatters must contain variables that " - "have no dependency to entry variables", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": { - "output_directory": "{title}", - "file_name": "{uid}", - }, - }, - ) - def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): _ = Preset( config=config_file, @@ -392,26 +317,3 @@ class TestPreset: "overrides": {name: "ack"}, }, ) - - def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options): - with pytest.raises( - ValidationException, - match=re.escape( - "variable the_bad_one cannot use the variables subtitles_ext because it " - "depends on other variables that are computed later in execution" - ), - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": { - "url": "youtube.com/watch?v=123abc", - "variables": {"the_bad_one": "{subtitles_ext}"}, - }, - "subtitles": { - "embed_subtitles": True, - }, - "output_options": {"output_directory": "dir", "file_name": "acjk"}, - }, - ) diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 044509bf..719ac9f4 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -1,4 +1,3 @@ -import json import re from contextlib import contextmanager from pathlib import Path @@ -6,11 +5,15 @@ from typing import Dict from unittest.mock import patch import pytest +import yaml from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.validators.variable_validation import ResolutionLevel +from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.plugins.nfo_tags import NfoTagsOptions from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.script import ScriptUtils @contextmanager @@ -472,23 +475,6 @@ def test_advanced_tv_show_subscriptions( overrides = subs[5].overrides assert overrides.script.get("subscription_name").native == "Gardening with Ciscoe" - assert ( - overrides.apply_formatter(overrides.dict["url"]) - == "https://www.youtube.com/@gardeningwithciscoe4430" - ) - assert ( - overrides.apply_formatter(overrides.dict["url2"]) - == "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_" - ) - - assert overrides.apply_formatter(overrides.dict["subscription_array"]) == json.dumps( - [ - "https://www.youtube.com/@gardeningwithciscoe4430", - "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_", - "https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn", - "https://www.youtube.com/watch?v=2vq-vPubS5I", - ] - ) def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_path: Path): @@ -524,7 +510,7 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc ) assert jackson.get("subscription_indent_1").native == "Pop" assert ( - jackson.get("url").native + jackson.get("urls").native[0] == "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" ) @@ -532,17 +518,7 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc gnr = subs[3].overrides.script assert gnr.get("subscription_name").native == "Guns N' Roses" - assert ( - gnr.get("url").native - == "https://www.youtube.com/playlist?list=PLOTK54q5K4INNXaHKtmXYr6J7CajWjqeJ" - ) + gnr_urls = gnr.get("urls").native + assert gnr_urls[0] == "https://www.youtube.com/playlist?list=PLOTK54q5K4INNXaHKtmXYr6J7CajWjqeJ" assert gnr.get("subscription_indent_1").native == "Rock" - assert gnr.get("url2").native == "https://www.youtube.com/watch?v=OldpIhHPsbs" - - -def test_default_docker_config_and_subscriptions(): - default_config = ConfigFile.from_file_path("docker/root/defaults/config.yaml") - default_subs = Subscription.from_file_path( - config=default_config, subscription_path=Path("docker/root/defaults/subscriptions.yaml") - ) - assert len(default_subs) == 1 + assert gnr_urls[1] == "https://www.youtube.com/watch?v=OldpIhHPsbs" diff --git a/tests/unit/config/test_subscription_resolution.py b/tests/unit/config/test_subscription_resolution.py new file mode 100644 index 00000000..bfa04d16 --- /dev/null +++ b/tests/unit/config/test_subscription_resolution.py @@ -0,0 +1,92 @@ +from pathlib import Path + +import pytest +import yaml +from resources import expected_json + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.validators.variable_validation import ResolutionLevel +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.file_path import FilePathTruncater + + +def _ensure_resolved_yaml( + sub: Subscription, output_directory: str, preset_type: str, resolution_level: int +) -> None: + output_yaml = sub.resolved_yaml(resolution_level=resolution_level) + out = yaml.safe_load(output_yaml) + + expected_out_filename = ( + f"{preset_type}/inspect_sub_{ResolutionLevel.name_of(resolution_level)}.json" + ) + expected_out = expected_json(out, expected_out_filename) + + if resolution_level > ResolutionLevel.ORIGINAL: + output_path = Path(output_directory) + if "tv_show_directory" in expected_out["overrides"]: + output_path = output_path / sub.name + + expected_out["output_options"]["output_directory"] = FilePathTruncater.to_native_filepath( + str(output_path) + ) + + if "tv_show_directory" in expected_out["overrides"]: + expected_out["overrides"]["tv_show_directory"] = output_directory + if "music_directory" in expected_out["overrides"]: + expected_out["overrides"]["music_directory"] = output_directory + if "music_video_directory" in expected_out["overrides"]: + expected_out["overrides"]["music_video_directory"] = output_directory + + assert out == expected_out + + +@pytest.mark.parametrize("resolution_level", ResolutionLevel.all()) +class TestResolution: + + def test_resolution_tv_show( + self, + resolution_level: int, + default_config: ConfigFile, + tv_show_subscriptions_path: Path, + output_directory: str, + ): + _ensure_resolved_yaml( + sub=Subscription.from_file_path( + config=default_config, subscription_path=tv_show_subscriptions_path + )[0], + output_directory=output_directory, + preset_type="tv_show", + resolution_level=resolution_level, + ) + + def test_resolution_music( + self, + resolution_level: int, + default_config: ConfigFile, + music_subscriptions_path: Path, + output_directory: str, + ): + _ensure_resolved_yaml( + sub=Subscription.from_file_path( + config=default_config, subscription_path=music_subscriptions_path + )[0], + output_directory=output_directory, + preset_type="music", + resolution_level=resolution_level, + ) + + def test_resolution_music_video( + self, + resolution_level: int, + default_config: ConfigFile, + music_video_subscription_path: Path, + output_directory: str, + ): + _ensure_resolved_yaml( + sub=Subscription.from_file_path( + config=default_config, subscription_path=music_video_subscription_path + )[0], + output_directory=output_directory, + preset_type="music_video", + resolution_level=resolution_level, + ) diff --git a/tests/unit/config/test_subscription_validation.py b/tests/unit/config/test_subscription_validation.py new file mode 100644 index 00000000..3a6fd13d --- /dev/null +++ b/tests/unit/config/test_subscription_validation.py @@ -0,0 +1,172 @@ +import re + +import pytest + +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException +from ytdl_sub.utils.exceptions import ValidationException + + +class TestSubscriptionValidation: + def test_preset_error__source_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, + }, + ), + config=config_file, + ) + + def test_preset_error__override_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "{dne_var}", "file_name": "file"}, + }, + ), + config=config_file, + ) + + def test_preset_error__dict_source_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "file"}, + "nfo_tags": { + "nfo_name": "the nfo name", + "nfo_root": "the root", + "tags": {"tag_a": "{dne_var}"}, + }, + }, + ), + config=config_file, + ) + + def test_preset_error__dict_override_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": output_options, + "output_directory_nfo_tags": { + "nfo_name": "the nfo name", + "nfo_root": "the root", + "tags": {"tag_a": "{dne_var}"}, + }, + }, + ), + config=config_file, + ) + + def test_preset_error__dict_override_variable_not_static( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="static formatters must contain variables that " + "have no dependency to entry variables", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": { + "output_directory": "{title}", + "file_name": "{uid}", + }, + }, + ), + config=config_file, + ) + + def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options): + with pytest.raises( + ValidationException, + match=re.escape( + "variable the_bad_one cannot use the variables subtitles_ext because it " + "depends on other variables that are computed later in execution" + ), + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"the_bad_one": "{subtitles_ext}"}, + }, + "subtitles": { + "embed_subtitles": True, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + }, + ), + config=config_file, + ) + + def test_preset_error_override_name_conflicts_with_plugin(self, config_file, output_options): + with pytest.raises( + ValidationException, + match=re.escape( + "Override variable with name throttle_protection cannot be used since it is the " + "name of a plugin. Perhaps you meant to define it as a plugin? If so, indent it " + "left to make it at the same level as overrides." + ), + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + }, + "subtitles": { + "embed_subtitles": True, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + "overrides": {"throttle_protection": "nope"}, + }, + ), + config=config_file, + ) diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py index 5d326839..5cfb8f92 100644 --- a/tests/unit/docgen/test_docgen.py +++ b/tests/unit/docgen/test_docgen.py @@ -1,5 +1,6 @@ from typing import Type +from tools.docgen.configuration import ConfigurationDocGen from tools.docgen.docgen import DocGen from tools.docgen.entry_variables import EntryVariablesDocGen from tools.docgen.plugins import PluginsDocGen @@ -28,3 +29,6 @@ class TestDocGen: def test_plugins_generated(self): _test_doc_gen(PluginsDocGen) + + def test_configuration_generated(self): + _test_doc_gen(ConfigurationDocGen) diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 9421d35e..e3302010 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -65,8 +65,8 @@ def mock_entry_to_dict( "epoch": 1596878400, "epoch_date": "20200808", "epoch_hour": "09", - "title": "entry ｛title｝", - "title_sanitized": "entry ｛title｝", + "title": "entry {title}", + "title_sanitized": "entry {title}", "ext": ext, "description": "", "creator": "abc123", @@ -113,8 +113,8 @@ def mock_entry_to_dict( "playlist_count": 1, "playlist_max_upload_year": 2021, "playlist_max_upload_year_truncated": 21, - "playlist_title": "entry ｛title｝", - "playlist_title_sanitized": "entry ｛title｝", + "playlist_title": "entry {title}", + "playlist_title_sanitized": "entry {title}", "playlist_description": "", "playlist_webpage_url": "https://yourname.here", "playlist_uid": "abc123", @@ -126,15 +126,15 @@ def mock_entry_to_dict( "source_description": "", "source_index": 1, "source_index_padded": "01", - "source_title": "entry ｛title｝", - "source_title_sanitized": "entry ｛title｝", + "source_title": "entry {title}", + "source_title_sanitized": "entry {title}", "source_webpage_url": "https://yourname.here", "source_uid": "abc123", "source_uploader": "abc123", "source_uploader_id": "abc123", "source_uploader_url": "https://yourname.here", "uid_sanitized_plex": "abc１２３", - "title_sanitized_plex": "entry ｛title｝", + "title_sanitized_plex": "entry {title}", "release_date": upload_date, "release_date_standardized": "2021-01-12", "release_year": 2021, diff --git a/tests/unit/main/test_main.py b/tests/unit/main/test_main.py index b9a6b07d..32ff6d57 100644 --- a/tests/unit/main/test_main.py +++ b/tests/unit/main/test_main.py @@ -244,7 +244,9 @@ def test_no_positional_arg_command(mock_sys_exit, tv_show_config_path): main() assert mock_error.call_count == 1 - assert mock_error.call_args.args[0] == "Must provide one of the commands: sub, dl, view" + assert mock_error.call_args.args[0] == ( + "Must provide one of the commands: sub, dl, view, cli-to-sub" + ) def test_bad_config_path(mock_sys_exit): @@ -264,3 +266,18 @@ def test_bad_config_path(mock_sys_exit): "The config file 'does_not_exist.yaml' could not be found. " "Did you set --config correctly?" ) + + +def test_cli_to_sub(mock_sys_exit, capsys): + with ( + mock_sys_exit(expected_exit_code=0), + patch.object( + sys, + "argv", + ["ytdl-sub", "cli-to-sub", "--mark-watched", "--mtime"], + ), + ): + main() + + captured = capsys.readouterr() + assert captured.out == ("ytdl_options:\n mark_watched: true\n updatetime: true\n\n") diff --git a/tests/unit/plugins/test_ytdl_options.py b/tests/unit/plugins/test_ytdl_options.py new file mode 100644 index 00000000..205e1750 --- /dev/null +++ b/tests/unit/plugins/test_ytdl_options.py @@ -0,0 +1,111 @@ +import re +from typing import Any +from typing import Dict + +import pytest +import yt_dlp +from conftest import assert_logs + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.downloaders.ytdlp import YTDLP +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.ffmpeg import FFMPEG +from ytdl_sub.utils.file_path import FilePathTruncater + + +@pytest.fixture +def preset_dict(output_directory) -> Dict[str, Any]: + return { + "download": "https://your.name.here", + "output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"}, + } + + +class TestYtdlOptions: + + def test_ytdl_options_are_strings( + self, + default_config: ConfigFile, + preset_dict: Dict[str, Any], + working_directory, + ): + expected_ytdl_options = { + "ignoreerrors": True, + "outtmpl": FilePathTruncater.to_native_filepath( + f"{working_directory}/test_ytdl_options/%(id)S.%(ext)s" + ), + "writethumbnail": False, + "ffmpeg_location": FFMPEG.ffmpeg_path(), + "match_filter": yt_dlp.utils.match_filter_func( + ["!is_live & !is_upcoming & !post_live"], [] + ), + "skip_download": True, + "writeinfojson": True, + "extract_flat": "discard", + } + + with ( + assert_logs( + logger=YTDLP.logger, + expected_message=f"ytdl_options: {str(expected_ytdl_options)}", + log_level="debug", + expected_occurrences=1, + ), + ): + _ = Subscription.from_dict( + config=default_config, + preset_name="test_ytdl_options", + preset_dict=preset_dict, + ).download(dry_run=True) + + def test_cookiefile_does_not_exist( + self, + default_config: ConfigFile, + preset_dict: Dict[str, Any], + ): + preset_dict["ytdl_options"] = { + "cookiefile": "/path/to/nowhere", + } + + error_msg = "Specified cookiefile /path/to/nowhere but it does not exist as a file." + + with pytest.raises(ValidationException, match=re.escape(error_msg)): + Subscription.from_dict( + config=default_config, + preset_name="test_ytdl_options", + preset_dict=preset_dict, + ).download(dry_run=False) + + def test_ytdl_option_types_preserved( + self, + default_config: ConfigFile, + output_directory: str, + ): + preset_dict = { + "download": "https://your.name.here", + "output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"}, + "ytdl_options": { + "break_on_existing": True, + "js_runtimes": {"deno": {"path": "/usr/local/bin/{dnope}"}}, + "string_path": "verify overrides: {test_string}", + "list_test": ["hmmm"], + }, + "overrides": {"test_string": "hi", "dnope": "deno"}, + } + + sub = Subscription.from_dict( + config=default_config, + preset_name="test_ytdl_options", + preset_dict=preset_dict, + ) + + out = sub.ytdl_options.to_native_dict(sub.overrides) + expected = { + "break_on_existing": True, + "js_runtimes": {"deno": {"path": "/usr/local/bin/deno"}}, + "string_path": "verify overrides: hi", + "list_test": ["hmmm"], + } + + assert out == expected diff --git a/tests/unit/prebuilt_presets/test_tv_show_by_date.py b/tests/unit/prebuilt_presets/test_tv_show_by_date.py index f8da99a7..2b21e050 100644 --- a/tests/unit/prebuilt_presets/test_tv_show_by_date.py +++ b/tests/unit/prebuilt_presets/test_tv_show_by_date.py @@ -1,6 +1,7 @@ import re import pytest +import yaml from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.subscriptions.subscription import Subscription @@ -24,3 +25,68 @@ class TestTvShowByDatePreset: "overrides": {"tv_show_directory": "abc", "s01_url": "test"}, }, ) + + def test_backward_compatibility_single_download(self, default_config): + a = yaml.safe_load( + Subscription.from_dict( + config=default_config, + preset_name="a", + preset_dict={ + "preset": "Jellyfin TV Show by Date", + "overrides": {"tv_show_directory": "abc", "url": "test_1"}, + }, + ).resolved_yaml() + ) + + b = yaml.safe_load( + Subscription.from_dict( + config=default_config, + preset_name="a", + preset_dict={ + "preset": "Jellyfin TV Show by Date", + "overrides": {"tv_show_directory": "abc", "subscription_value": "test_1"}, + }, + ).resolved_yaml() + ) + + assert a["download"] == b["download"] + + def test_backward_compatibility_multi_download(self, default_config): + a = yaml.safe_load( + Subscription.from_dict( + config=default_config, + preset_name="a", + preset_dict={ + "preset": "Jellyfin TV Show by Date", + "overrides": {"tv_show_directory": "abc", "url": "test_1", "url2": "test_2"}, + }, + ).resolved_yaml() + ) + + b = yaml.safe_load( + Subscription.from_dict( + config=default_config, + preset_name="a", + preset_dict={ + "preset": "Jellyfin TV Show by Date", + "overrides": { + "tv_show_directory": "abc", + "subscription_array": ["test_1", "test_2"], + }, + }, + ).resolved_yaml() + ) + + c = yaml.safe_load( + Subscription.from_dict( + config=default_config, + preset_name="a", + preset_dict={ + "preset": "Jellyfin TV Show by Date", + "overrides": {"tv_show_directory": "abc", "urls": ["test_1", "test_2"]}, + }, + ).resolved_yaml() + ) + + assert a["download"] == b["download"] + assert a["download"] == c["download"] diff --git a/tests/unit/prebuilt_presets/test_tv_show_collection.py b/tests/unit/prebuilt_presets/test_tv_show_collection.py index 3621f3cb..0f7e9c5e 100644 --- a/tests/unit/prebuilt_presets/test_tv_show_collection.py +++ b/tests/unit/prebuilt_presets/test_tv_show_collection.py @@ -24,3 +24,90 @@ class TestTvShowCollectionPreset: "overrides": {"tv_show_directory": "abc", "url": "test"}, }, ) + + def test_multi_url(self, default_config): + num_seasons = 40 # excluding season 0 + num_urls_per_season = 11 + + overrides = { + "tv_show_directory": "abc", + } + for season_num in range(0, num_seasons + 1): + overrides[f"s{season_num:02d}_name"] = f"The Season {season_num}" + overrides[f"s{season_num:02d}_url"] = [ + f"youtube.com/playlist?url_{season_num}_{i}" for i in range(num_urls_per_season) + ] + + sub = Subscription.from_dict( + config=default_config, + preset_name="test", + preset_dict={"preset": "Jellyfin TV Show Collection", "overrides": overrides}, + ) + + assert len(sub.downloader_options.urls.list) == (num_seasons + 1) * 3 + url_list = sub.downloader_options.urls.list + itr = 0 + + # loop twice for bilateral + for i in range(2): + for season_num in range(1, num_seasons + 2): + # Season 0 is placed last, adjust here + if season_num == num_seasons + 1: + season_num = 0 + + # is_bilateral + if i == 0: + url = sub.overrides.apply_formatter( + url_list[itr].url, + function_overrides={ + # mock so bilateral url gets enabled + "subscription_has_download_archive": "True" + }, + expected_type=list, + ) + assert url == [ + f"youtube.com/playlist?url_{season_num}_{i}" + for i in range(num_urls_per_season) + ] + variables = url_list[itr].variables.dict + assert ( + sub.overrides.apply_formatter(variables["collection_season_number"]) + == f"{season_num}" + ) + assert ( + sub.overrides.apply_formatter(variables["collection_season_name"]) + == f"The Season {season_num}" + ) + itr += 1 + # not bilateral + else: + for j in range(2): + url = sub.overrides.apply_formatter( + url_list[itr + j].url, + function_overrides={ + # mock so bilateral url gets enabled + "subscription_has_download_archive": "True" + }, + expected_type=list, + ) + + # First instance is the first url to get thumbnails + if j == 0: + assert url == [f"youtube.com/playlist?url_{season_num}_0"] + # Next one contains remaining urls + else: + assert url == [ + f"youtube.com/playlist?url_{season_num}_{i}" + for i in range(1, num_urls_per_season) + ] + + variables = url_list[itr].variables.dict + assert ( + sub.overrides.apply_formatter(variables["collection_season_number"]) + == f"{season_num}" + ) + assert ( + sub.overrides.apply_formatter(variables["collection_season_name"]) + == f"The Season {season_num}" + ) + itr += 2 diff --git a/tests/unit/script/functions/test_conditional_functions.py b/tests/unit/script/functions/test_conditional_functions.py index eff73214..d2078a80 100644 --- a/tests/unit/script/functions/test_conditional_functions.py +++ b/tests/unit/script/functions/test_conditional_functions.py @@ -3,6 +3,9 @@ import re import pytest from unit.script.conftest import single_variable_output +from ytdl_sub.script.script import Script +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.utils.exceptions import FunctionRuntimeException @@ -135,3 +138,29 @@ class TestConditionalFunction: ): output = single_variable_output(function_str) assert output == expected_output + + def test_if_partial_resolve(self): + assert ( + Script( + { + "aa": "a", + "bb": "unresolvable!", + "cc": "{%if( true, aa, bb )}", + } + ) + .resolve_partial(unresolvable={"bb"}) + .get("cc") + .native + == "a" + ) + + def test_if_partial_resolve_unresolved(self): + assert Script( + { + "aa": "a", + "bb": "unresolvable!", + "cc": "{%if( false, aa, bb )}", + } + ).resolve_partial(unresolvable={"bb"}).definition_of("cc") == SyntaxTree( + ast=[Variable("bb")] + ) diff --git a/tests/unit/script/functions/test_numeric_functions.py b/tests/unit/script/functions/test_numeric_functions.py index c41f4c59..b05dbb70 100644 --- a/tests/unit/script/functions/test_numeric_functions.py +++ b/tests/unit/script/functions/test_numeric_functions.py @@ -76,3 +76,15 @@ class TestNumericFunctions: def test_pow(self, values: str, expected_output: float): output = single_variable_output(f"{{ %pow({values}) }}") assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("5", [0, 1, 2, 3, 4]), + ("5, 1", [1, 2, 3, 4]), + ("5, 1, 2", [1, 3]), + ], + ) + def test_range(self, values: str, expected_output: float): + output = single_variable_output(f"{{ %range({values}) }}") + assert output == expected_output diff --git a/tests/unit/script/test_parser.py b/tests/unit/script/test_parser.py index b40e13ca..c6694562 100644 --- a/tests/unit/script/test_parser.py +++ b/tests/unit/script/test_parser.py @@ -16,6 +16,7 @@ from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import ResolvedSyntaxTree from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.utils.exceptions import InvalidSyntaxException @@ -24,7 +25,7 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException class TestParser: def test_simple(self): parsed = parse("hello world") - assert parsed == SyntaxTree([String(value="hello world")]) + assert parsed == ResolvedSyntaxTree([String(value="hello world")]) assert parsed.variables == set() def test_single_function_one_arg(self): diff --git a/tests/unit/script/types/test_array.py b/tests/unit/script/types/test_array.py index f816b1bd..312ec325 100644 --- a/tests/unit/script/types/test_array.py +++ b/tests/unit/script/types/test_array.py @@ -10,6 +10,8 @@ from ytdl_sub.script.script_output import ScriptOutput from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.utils.exceptions import InvalidSyntaxException @@ -117,3 +119,29 @@ class TestArray: ).resolve() == ScriptOutput( {"aa": String("a"), "bb": String("b"), "cc": String('return ["a", "b"]')} ) + + def test_partial_resolve(self): + assert ( + Script( + { + "aa": "a", + "bb": "unresolvable!", + "cc": "{%array_at( [aa, bb], 0 )}", + } + ) + .resolve_partial(unresolvable={"bb"}) + .get("cc") + .native + == "a" + ) + + def test_partial_resolve_unresolved(self): + assert Script( + { + "aa": "a", + "bb": "unresolvable!", + "cc": "{%array_at( [aa, bb], 1 )}", + } + ).resolve_partial(unresolvable={"bb"}).definition_of("cc") == SyntaxTree( + ast=[Variable("bb")] + ) diff --git a/tests/unit/script/types/test_custom_function.py b/tests/unit/script/types/test_custom_function.py index 61096cc3..17919cc0 100644 --- a/tests/unit/script/types/test_custom_function.py +++ b/tests/unit/script/types/test_custom_function.py @@ -5,7 +5,9 @@ import pytest from ytdl_sub.script.parser import CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS from ytdl_sub.script.script import Script from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.function import CustomFunction from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.utils.exceptions import CycleDetected from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName @@ -251,3 +253,30 @@ class TestCustomFunction: "output": "{%mul(%func1(1), 1)}", } ) + + def test_partial_resolve_custom_functions_any_order_via_init(self): + assert ( + Script( + { + "%custom_cubed": "{%mul(%custom_square($0),$0)}", + "%custom_square": "{%mul($0, $0)}", + "output": "{%custom_cubed(3)}", + } + ) + .resolve_partial() + .get("output") + .native + == 27 + ) + + def test_partial_resolve_unresolved(self): + assert Script( + { + "aa": "nope", + "%custom_cubed": "{%mul(%custom_square($0),$0)}", + "%custom_square": "{%mul($0, aa)}", + "output": "{%custom_cubed(3)}", + } + ).resolve_partial(unresolvable={"aa"}).definition_of("output") == SyntaxTree( + ast=[CustomFunction(name="custom_cubed", args=[Integer(3)])] + ) diff --git a/tests/unit/script/types/test_lambda_function.py b/tests/unit/script/types/test_lambda_function.py index 07105477..0474a1d3 100644 --- a/tests/unit/script/types/test_lambda_function.py +++ b/tests/unit/script/types/test_lambda_function.py @@ -6,6 +6,7 @@ from ytdl_sub.script.script import Script from ytdl_sub.script.script_output import ScriptOutput from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import CycleDetected from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments @@ -76,6 +77,28 @@ class TestLambdaFunction: assert script.resolve().get("category_url_map").native == {1: 1, 2: 2, 3: 3} + def test_array_apply_custom_function(self): + output = ( + Script( + { + "the_array": "{ ['a', 'B', 'c', 'D'] }", + "output": "{ %array_apply(the_array, %custom_cap) }", + "should_lower": "{%bool(True)}", + "%custom_cap": """{ + %if( + %bool(should_lower), + %lower($0), + %upper($0) + ) + }""", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == ["a", "b", "c", "d"] + class TestLambdaFunctionIncompatibleNumArguments: @pytest.mark.parametrize( @@ -147,3 +170,27 @@ class TestLambdaFunctionIncompatibleNumArguments: "%output": f"{{%array_enumerate(array1, {lambda_value})}}", } ) + + def test_lambda_with_custom_function_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape("Cycle detected within these variables: two -> %times_two -> two"), + ): + Script({"%times_two": "{%mul($0, two)}", "two": "{%times_two(2)}"}) + + def test_partial_resolve_nested_lambda_custom_functions_within_custom_functions(self): + assert ( + Script( + { + "%nest4": "{%mul($0, 2)}", + "%nest3": "{%array_at(%array_apply([$0], %nest4), 0)}", + "%nest2": "{%array_at(%array_apply([$0], %nest3), 0)}", + "%nest1": "{%array_at(%array_apply([$0], %nest2), 0)}", + "output": "{%array_at(%array_apply([2], %nest1), 0)}", + } + ) + .resolve_partial() + .get("output") + .native + == 4 + ) diff --git a/tests/unit/utils/test_file_handler.py b/tests/unit/utils/test_file_handler.py new file mode 100644 index 00000000..80505e24 --- /dev/null +++ b/tests/unit/utils/test_file_handler.py @@ -0,0 +1,20 @@ +import pytest + +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.system import IS_WINDOWS + + +class TestFileHandler: + + def test_directory_exists(self): + if IS_WINDOWS: + return + + assert FileHandler.is_path_writable("/tmp") + assert FileHandler.is_path_writable("/tmp/") + assert FileHandler.is_path_writable("/tmp/non-existent") + assert FileHandler.is_path_writable("/tmp/non-existent/") + assert FileHandler.is_path_writable("/tmp/non-existent/nested") + + assert not FileHandler.is_path_writable("/lol-in-root") + assert not FileHandler.is_path_writable("/") diff --git a/tests/unit/utils/test_script_utils.py b/tests/unit/utils/test_script_utils.py index f9d9b395..792020ba 100644 --- a/tests/unit/utils/test_script_utils.py +++ b/tests/unit/utils/test_script_utils.py @@ -18,6 +18,9 @@ class TestScriptUtils: "string": "value", "quotes": "has '' and \"\"", "triple-single-quote": "right here! '''''''''''''''''''''''''''''' ack '''''''", + "has-double-quotes": 'i got "some double quotes" in here', + "has-single-quotes": "i got 'some single quotes' in here", + "has-both-quotes": "i got 'both quotes\" in here", "int": 1, "bool": True, "list": [1, 2, 3], @@ -60,7 +63,15 @@ class TestScriptUtils: def test_to_syntax_tree(self): out = ScriptUtils.to_native_script( - {"{var_a}": "{var_b}", "static_a": "string with {var_c} in it"} + { + "{var_a}": "{var_b}", + "static_a": "string with {var_c} in it", + "quotes": "has '' and \"\"", + "triple-single-quote": "right here! '''''''''''''''''''''''''''''' ack '''''''", + "has-double-quotes": 'i got "some double quotes" in here', + "has-single-quotes": "i got 'some single quotes' in here", + "has-both-quotes": "i got 'both quotes\" in here", + } ) assert parse(out) == SyntaxTree( ast=[ @@ -70,11 +81,24 @@ class TestScriptUtils: String(value="static_a"): BuiltInFunction( name="concat", args=[ - BuiltInFunction(name="string", args=[String(value="string with ")]), - BuiltInFunction(name="string", args=[Variable(name="var_c")]), - BuiltInFunction(name="string", args=[String(value=" in it")]), + String(value="string with "), + Variable(name="var_c"), + String(value=" in it"), ], ), + String(value="quotes"): String(value="has '' and \"\""), + String(value="triple-single-quote"): String( + value="right here! '''''''''''''''''''''''''''''' ack '''''''" + ), + String(value="has-double-quotes"): String( + value='i got "some double quotes" in here' + ), + String(value="has-single-quotes"): String( + value="i got 'some single quotes' in here" + ), + String(value="has-both-quotes"): String( + value="i got 'both quotes\" in here" + ), } ) ] diff --git a/tests/unit/validators/test_string_formatter_validator.py b/tests/unit/validators/test_string_formatter_validator.py index 4b2baef6..888dc58d 100644 --- a/tests/unit/validators/test_string_formatter_validator.py +++ b/tests/unit/validators/test_string_formatter_validator.py @@ -115,12 +115,12 @@ class TestUnstructuredDictFormatterValidator(object): assert len(validator.dict) == 8 assert all(isinstance(val, expected_formatter_class) for val in validator.dict.values()) assert validator.dict_with_format_strings == { - "key1": "{ %concat( %string( '''string with ''' ), %string( variable ) ) }", + "key1": '{ %concat( "string with ", variable ) }', "key2": "no variables", "key3": "{ %int(3) }", "key4": "{ %float(4.132) }", "key5": "{ %bool(True) }", - "key6": "{ { %concat( %string( variable ), %string( '''_key''' ) ): '''value''', '''static_key''': %concat( %string( variable ), %string( '''_value''' ) ) } }", - "key7": "{ [ '''list_1''', %concat( %string( '''list_''' ), %string( variable_2 ) ) ] }", - "key8": "{ %concat( %string( '''string ''' ), %string( variable1 ), %string( ''' with multiple ''' ), %string( variable2 ) ) }", + "key6": '{ { %concat( variable, "_key" ): "value", "static_key": %concat( variable, "_value" ) } }', + "key7": '{ [ "list_1", %concat( "list_", variable_2 ) ] }', + "key8": '{ %concat( "string ", variable1, " with multiple ", variable2 ) }', } diff --git a/tools/docgen/configuration.py b/tools/docgen/configuration.py new file mode 100644 index 00000000..958c09a1 --- /dev/null +++ b/tools/docgen/configuration.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import generate_options_validator_docs +from ytdl_sub.config.config_validator import ConfigOptions +from ytdl_sub.config.preset import Preset + + +class ConfigurationDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/config_yaml.rst") + DOCSTRING_LOCATION = ( + "The respective function docstrings within ytdl_sub/config/config_validator.py" + ) + + @classmethod + def generate(cls): + docs = generate_options_validator_docs( + name="Configuration File", + options=ConfigOptions, + offset=0, + skip_properties=False, + recurse_property_options=True, + property_sections=True, + ) + + docs += generate_options_validator_docs( + name="Presets", + options=Preset, + offset=0, + skip_properties=True, + recurse_property_options=False, + property_sections=False, + ) + + return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py index a7f7bed7..f0331acc 100644 --- a/tools/docgen/plugins.py +++ b/tools/docgen/plugins.py @@ -1,12 +1,11 @@ -import inspect from pathlib import Path -from typing import Any from typing import Dict +from typing import Set from typing import Type from tools.docgen.docgen import DocGen +from tools.docgen.utils import generate_options_validator_docs from tools.docgen.utils import line_section -from tools.docgen.utils import properties from tools.docgen.utils import section from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_mapping import PluginMapping @@ -15,57 +14,17 @@ from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator - -def should_filter_all_properties(plugin_name: str) -> bool: - return plugin_name in ( - "format", - "match_filters", - "music_tags", - "filter_include", - "filter_exclude", - "embed_thumbnail", - "square_thumbnail", - "video_tags", - "download", - ) - - -def should_filter_property(property_name: str) -> bool: - return property_name.startswith("_") or property_name in ( - "value", - "source_variable_capture_dict", - "dict", - "keys", - "dict_with_format_strings", - "subscription_name", - "list", - "script", - "unresolvable", - ) - - -def get_function_docs(function_name: str, obj: Any, level: int) -> str: - docs = f"\n``{function_name}``\n\n" - docs += inspect.cleandoc(getattr(obj, function_name).__doc__) - docs += "\n\n" - return docs - - -def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: - docs = "" - docs += section(name, level=offset + 0) - - docs += inspect.cleandoc(options.__doc__) - docs += "\n" - - if should_filter_all_properties(name): - return docs - - property_names = [prop for prop in properties(options) if not should_filter_property(prop)] - for property_name in sorted(property_names): - docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) - - return docs +PLUGIN_NAMES_TO_SKIP_PROPERTIES: Set[str] = { + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "square_thumbnail", + "video_tags", + "download", +} class PluginsDocGen(DocGen): @@ -89,6 +48,11 @@ class PluginsDocGen(DocGen): docs = section("Plugins", level=0) for idx, name in enumerate(sorted(options_dict.keys())): docs += line_section(section_idx=idx) - docs += generate_plugin_docs(name, options_dict[name], offset=1) + docs += generate_options_validator_docs( + name=name, + options=options_dict[name], + offset=1, + skip_properties=name in PLUGIN_NAMES_TO_SKIP_PROPERTIES, + ) return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py index 81ddc3fa..5cd6f357 100644 --- a/tools/docgen/utils.py +++ b/tools/docgen/utils.py @@ -6,12 +6,51 @@ from typing import List from typing import Optional from typing import Type +from ytdl_sub.script.utils.type_checking import get_optional_type +from ytdl_sub.script.utils.type_checking import is_optional +from ytdl_sub.validators.validators import Validator + LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} -def section(name: str, level: int, as_code: bool = False) -> str: - if as_code: - name = f"``{name}``" +def _should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + "script", + "unresolvable", + "dict_with_parsed_format_strings", + "leaf_name", + ) + + +def _is_validator_property( + options: Type[Validator], property_name: str +) -> Optional[Type[Validator]]: + property_return_type = inspect.getfullargspec(getattr(options, property_name).fget).annotations[ + "return" + ] + if is_optional(property_return_type): + property_return_type = get_optional_type(property_return_type) + + try: + if issubclass(property_return_type, Validator): + return property_return_type + except TypeError: + return None + + return None + + +def section(name: str, level: Optional[int]) -> str: + if level is None: + return f"\n{name}\n\n" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" @@ -40,10 +79,20 @@ def camel_case_to_human(string: str) -> str: return output_str +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return "" + + def get_function_docs( function_name: str, obj: Any, - level: int, + level: Optional[int], display_function_name: Optional[str] = None, pre_docstring: Optional[str] = None, ) -> str: @@ -56,11 +105,42 @@ def get_function_docs( return docs -def line() -> str: - return "\n" + ("-" * 100) + "\n" +def generate_options_validator_docs( + name: str, + options: Type[Validator], + offset: int, + skip_properties: bool, + recurse_property_options: bool = False, + property_sections: bool = False, +) -> str: + docs = "" + docs += section(name, level=offset + 0) + docs += inspect.cleandoc(options.__doc__) + docs += "\n" -def line_section(section_idx: int) -> str: - if section_idx > 0: - return line() - return "" + if skip_properties: + return docs + + property_names = [prop for prop in properties(options) if not _should_filter_property(prop)] + for property_name in sorted(property_names): + maybe_validator_property = ( + _is_validator_property(options, property_name) if recurse_property_options else None + ) + if maybe_validator_property: + docs += generate_options_validator_docs( + name=property_name, + options=maybe_validator_property, + offset=offset + 1, + skip_properties=False, + recurse_property_options=False, + ) + else: + docs += get_function_docs( + function_name=property_name, + obj=options, + level=(offset + 1) if property_sections else None, + display_function_name=f"``{property_name}``" if not property_sections else None, + ) + + return docs