diff --git a/.gitignore b/.gitignore index 20d1b188..e18f90d3 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,5 @@ docker/testing/volumes ffmpeg.exe ffprobe.exe + +tools/docgen/out \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 72264f8b..07eb81c9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,16 +1,17 @@ version: 2 build: - os: ubuntu-20.04 + os: "ubuntu-22.04" tools: python: "3.10" sphinx: - configuration: docs/conf.py + configuration: docs/source/conf.py python: install: + - requirements: docs/source/requirements.txt - method: pip path: . extra_requirements: - - docs + - docs \ No newline at end of file diff --git a/Makefile b/Makefile index 30ca26b3..dd52f307 100644 --- a/Makefile +++ b/Makefile @@ -17,12 +17,10 @@ lint: @-isort . @-black . @-pylint src/ - @-pydocstyle src/* check_lint: isort . --check-only --diff \ && black . --check \ - && pylint src/ \ - && pydocstyle src/* + && pylint src/ wheel: clean $(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py) cat src/ytdl_sub/__init__.py @@ -41,14 +39,14 @@ executable: clean pyinstaller ytdl-sub.spec mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX} docs: - sphinx-build -a -b html docs docs/_html + sphinx-build -M html docs/source/ docs/build/ clean: rm -rf \ .pytest_cache/ \ build/ \ dist/ \ src/ytdl_sub.egg-info/ \ - docs/_html/ \ + docs/build/ \ .coverage \ docker/root/*.whl \ docker/root/defaults/examples \ diff --git a/README.md b/README.md index 8f28f16e..f2ba4fd5 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,10 @@ __preset__: # Root folder of all ytdl-sub Music Videos music_video_directory: "/music_videos" - # For 'Only Recent' preset, only keep vids uploaded in this range - date_range: "2months" - + # For 'Only Recent' preset, only keep vids within this range and limit + only_recent_date_range: "2months" + only_recent_max_files: 30 + # Pass any arg directly to yt-dlp's Python API ytdl_options: cookiefile: "/config/cookie.txt" @@ -185,24 +186,24 @@ Any part of this process is modifiable by using custom configs. See our on how to build your first config from scratch. Ready-to-use [example configurations](https://github.com/jmbannon/ytdl-sub/tree/master/examples) can be found here alongside our -[readthedocs](https://ytdl-sub.readthedocs.io/en/latest/config.html#) +[readthedocs](https://ytdl-sub.readthedocs.io/en/latest/index.html) for detailed information on all config fields. ## Installation `ytdl-sub` can be installed on the following platforms. -- [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker-compose) - - [Web-GUI](https://ytdl-sub.readthedocs.io/en/latest/install.html#gui) - - [Headless](https://ytdl-sub.readthedocs.io/en/latest/install.html#headless) - - [CPU / GPU Passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#passthrough) -- [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker) -- [Windows](https://ytdl-sub.readthedocs.io/en/latest/install.html#windows) -- [Unraid](https://ytdl-sub.readthedocs.io/en/latest/install.html#unraid) -- [Linux](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux) -- [Linux ARM](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux-arm) -- [PIP](https://ytdl-sub.readthedocs.io/en/latest/install.html#pip) -- [Local Install](https://ytdl-sub.readthedocs.io/en/latest/install.html#local-install) -- [Local Docker Build](https://ytdl-sub.readthedocs.io/en/latest/install.html#local-docker-build) +- [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose) + - [Web-GUI](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose) + - [Headless](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose) + - [CPU / GPU Passthrough](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#device-passthrough) +- [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#docker-cli) +- [Windows](https://ytdl-sub.readthedocs.io/en/latest/guides/install/windows.html) +- [Unraid](https://ytdl-sub.readthedocs.io/en/latest/guides/install/unraid.html) +- [Linux](https://ytdl-sub.readthedocs.io/en/latest/guides/install/linux.html) +- [Linux ARM](https://ytdl-sub.readthedocs.io/en/latest/guides/install/linux.html) +- [PIP](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#pip-install) +- [Local Install](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#local-install) +- [Local Docker Build](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#local-docker-build) ### Docker Installation Docker installs can be either headless or use the Web-GUI image, which comprises @@ -222,5 +223,5 @@ pick up a bug. We are pretty active in our [Discord channel](https://discord.gg/v8j9RAHb4k) if you have any questions. Also see our -[FAQ](https://github.com/jmbannon/ytdl-sub/wiki/FAQ) +[FAQ](https://ytdl-sub.readthedocs.io/en/latest/faq/index.html) for commonly asked questions. diff --git a/docker/Dockerfile b/docker/Dockerfile index d928c0fd..e9015270 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,7 @@ RUN mkdir -p /config && \ g++ \ nano \ make \ + libffi-dev \ "python3>=3.10" \ py3-pip \ fontconfig \ @@ -46,6 +47,7 @@ RUN mkdir -p /config && \ apk del \ g++ \ make \ + libffi-dev \ py3-pip \ py3-setuptools diff --git a/docker/root/defaults/subscriptions.yaml b/docker/root/defaults/subscriptions.yaml index a63c7994..a7e129dc 100644 --- a/docker/root/defaults/subscriptions.yaml +++ b/docker/root/defaults/subscriptions.yaml @@ -14,8 +14,9 @@ __preset__: # Root folder of all ytdl-sub Music Videos music_video_directory: "/music_videos" - # For 'Only Recent' preset, only keep vids uploaded in this range - date_range: "2months" + # For 'Only Recent' preset, only keep vids within this range and limit + only_recent_date_range: "2months" + only_recent_max_files: 30 # Pass any arg directly to yt-dlp's Python API ytdl_options: diff --git a/docs/Makefile b/docs/Makefile index d4bb2cbb..d0c3cbf1 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,8 +5,8 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build +SOURCEDIR = source +BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 47a478be..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,67 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys - -sys.path.insert(0, os.path.abspath("../src")) - - -# -- Project information ----------------------------------------------------- - -project = "ytdl-sub" -copyright = "2022, Jesse Bannon" -author = "Jesse Bannon" - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "sphinx_rtd_theme" - -html_theme_options = {"navigation_depth": 10} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ["_static"] - -# Do not show full module path in api docs -add_module_names = False -python_use_unqualified_type_names = False -napoleon_numpy_docstrings = True -napoleon_use_rtype = False - -# -- Options for autodocs ------------------------------------------------- - -autodoc_default_options = {"autodoc_typehints_format": "short"} diff --git a/docs/config.rst b/docs/config.rst deleted file mode 100644 index b1f05b7f..00000000 --- a/docs/config.rst +++ /dev/null @@ -1,454 +0,0 @@ -Config -====== -ytdl-sub is configured using a ``config.yaml`` file. - -.. _config: - -config.yaml ------------ - -The ``config.yaml`` is made up of two sections: - -.. code-block:: yaml - - configuration: - presets: - -You can jump to any section and subsection of the config using the navigation -section to the left. - -Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. -If you wish to represent paths like Windows, you will need to ``C:\\double\\bashslash\\paths`` -in order to escape the backslash character. - -configuration -^^^^^^^^^^^^^ -The ``configuration`` section contains app-wide configs applied to all presets -and subscriptions. - -.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions() - :members: - :member-order: bysource - :exclude-members: subscription_value, persist_logs, experimental - -persist_logs -"""""""""""" -Within ``configuration``, define whether logs from subscription downloads -should be persisted. - -.. code-block:: yaml - - configuration: - persist_logs: - logs_directory: "/path/to/log/directory" - -Log files are stored as -``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. - -.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator() - :members: - :member-order: bysource - -presets -^^^^^^^ -``presets`` define a `formula` for how to format downloaded media and metadata. - -download_strategy -""""""""""""""""" -Download strategies dictate what is getting downloaded from a source. Each -download strategy has its own set of parameters. - -.. _url: - -url -''' -.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions() - :members: url, playlist_thumbnails, source_thumbnails, download_reverse - :member-order: bysource - -multi_url -''''''''' -.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions() - :members: urls, variables - -------------------------------------------------------------------------------- - -output_options -"""""""""""""" - -.. autoclass:: ytdl_sub.config.preset_options.OutputOptions() - :members: - :member-order: bysource - :exclude-members: get_upload_date_range_to_keep, partial_validate - -------------------------------------------------------------------------------- - -.. _ytdl_options: - -ytdl_options -"""""""""""" -.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions() - -------------------------------------------------------------------------------- - -.. _overrides: - -overrides -""""""""" -.. autoclass:: ytdl_sub.config.preset_options.Overrides() - -.. _parent preset: - -preset -"""""" -Presets support inheritance by defining a parent preset: - -.. code-block:: yaml - - presets: - custom_preset: - ... - parent_preset: - ... - child_preset: - preset: "parent_preset" - -In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``. -It is advantageous to use parent presets where possible to reduce duplicate yaml -definitions. - -Presets also support inheritance from multiple presets: - -.. code-block:: yaml - - child_preset: - preset: - - "custom_preset" - - "parent_preset" - -In this example, ``child_preset`` will inherit all fields from ``custom_preset`` -and ``parent_preset`` in that order. The bottom-most preset has the highest -priority. - -If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is -valid YAML. Inheriting from multiple presets require use of a list. - -------------------------------------------------------------------------------- - - -Plugins -""""""" -Plugins are used to perform any type of post-processing to the already downloaded files. - -audio_extract -''''''''''''' -.. autoclass:: ytdl_sub.plugins.audio_extract.AudioExtractOptions() - :members: - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -chapters -'''''''' -.. autoclass:: ytdl_sub.plugins.chapters.ChaptersOptions() - :members: - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -date_range -'''''''''' -.. autoclass:: ytdl_sub.plugins.date_range.DateRangeOptions() - :members: - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -embed_thumbnail -'''''''''''''''' - -.. autoclass:: ytdl_sub.plugins.embed_thumbnail.EmbedThumbnailOptions() - -------------------------------------------------------------------------------- - -file_convert -'''''''''''' -.. autoclass:: ytdl_sub.plugins.file_convert.FileConvertOptions() - :members: - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -format -'''''' -.. autoclass:: ytdl_sub.plugins.format.FormatOptions() - -------------------------------------------------------------------------------- - -match_filters -''''''''''''' -.. autoclass:: ytdl_sub.plugins.match_filters.MatchFiltersOptions() - :members: - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -music_tags -'''''''''' -.. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions() - -------------------------------------------------------------------------------- - -nfo_tags -'''''''' -.. autoclass:: ytdl_sub.plugins.nfo_tags.NfoTagsOptions() - :members: nfo_name, nfo_root, tags, kodi_safe - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -output_directory_nfo_tags -''''''''''''''''''''''''' -.. autoclass:: ytdl_sub.plugins.output_directory_nfo_tags.OutputDirectoryNfoTagsOptions() - :members: nfo_name, nfo_root, tags, kodi_safe - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -regex -''''' -.. autoclass:: ytdl_sub.plugins.regex.RegexOptions() - :members: skip_if_match_fails - -.. autoclass:: ytdl_sub.plugins.regex.VariableRegex() - :members: match, capture_group_names, capture_group_defaults, exclude - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -split_by_chapters -''''''''''''''''' -.. autoclass:: ytdl_sub.plugins.split_by_chapters.SplitByChaptersOptions() - :members: when_no_chapters - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -subtitles -''''''''' -.. autoclass:: ytdl_sub.plugins.subtitles.SubtitleOptions() - :members: subtitles_name, subtitles_type, embed_subtitles, languages, allow_auto_generated_subtitles - :member-order: bysource - :exclude-members: partial_validate - -------------------------------------------------------------------------------- - -throttle_protection -''''''''''''''''''' -.. autoclass:: ytdl_sub.plugins.throttle_protection.ThrottleProtectionOptions() - :members: - :member-order: bysource - -------------------------------------------------------------------------------- - -video_tags -'''''''''' -.. autoclass:: ytdl_sub.plugins.video_tags.VideoTagsOptions() - -------------------------------------------------------------------------------- - -.. _subscription_yaml: - -subscription.yaml ------------------ -The ``subscription.yaml`` file is where we use our `presets`_ in the `config.yaml`_ -to define a `subscription`: something we want to recurrently download such as a specific -channel or playlist. - -The only difference between a ``subscription`` and ``preset`` is that the subscription -must have all required fields and ``{variables}`` defined so it can perform a download. - -Below is an example that downloads a YouTube playlist: - -.. code-block:: yaml - :caption: config.yaml - - presets: - playlist_preset_ex: - download: - download_strategy: "url" - url: "{url}" - output_options: - output_directory: "{output_directory}/{playlist_name}" - file_name: "{playlist_name}.{title}.{ext}" - overrides: - output_directory: "/path/to/ytdl-sub-videos" - -.. code-block:: yaml - :caption: subscription.yaml - - my_subscription_name: - preset: "playlist_preset_ex" - overrides: - playlist_name: "diy-playlist" - url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - -Our preset ``playlist_preset_ex`` defines three -custom variables: ``{output_directory}``, ``{playlist_name}``, and ``{url}``. The subscription sets -the `parent preset`_ to ``playlist_preset_ex``, and must define the variables ``{playlist_name}`` -and ``{url}`` since the preset did not. - -.. _beautifying subscriptions: - -Beautifying Subscriptions -^^^^^^^^^^^^^^^^^^^^^^^^^ -Subscriptions support using presets as keys, and using keys to set override variables as values. -For example: - -.. code-block:: yaml - :caption: subscription.yaml - - TV Show Full Archive: - = News: - "Breaking News": "https://www.youtube.com/@SomeBreakingNews" - - TV Show Only Recent: - = Tech | TV-Y: - "Two Minute Papers": "https://www.youtube.com/@TwoMinutePapers" - -Will create two subscriptions named "Breaking News" and "Two Minute Papers", equivalent to: - -.. code-block:: yaml - - "Breaking News": - preset: - - "TV Show Full Archive" - - overrides: - subscription_indent_1: "News" - subscription_name: "Breaking News" - subscription_value: "https://www.youtube.com/@SomeBreakingNews" - - "Two Minute Papers": - preset: - - "TV Show Only Recent" - - overrides: - subscription_indent_1: "Tech" - subscription_indent_2: "TV-Y" - subscription_name: "Two Minute Papers" - subscription_value: "https://www.youtube.com/@TwoMinutePapers" - -You can provide as many parent presets in the form of keys, and subscription indents as ``=keys``. -This can drastically simplify subscription definitions by setting things like so in your -parent preset: - -.. code-block:: yaml - - presets: - "TV Show Preset": - overrides: - subscription_indent_1: "default-genre" - subscription_indent_2: "default-content-rating" - - tv_show_name: "{subscription_name}" - url: "{subscription_value}" - genre: "{subscription_indent_1}" - content_rating: "{subscription_indent_2}" - -.. _subscription value: - -File Preset -^^^^^^^^^^^ -NOTE: This is deprecated in favor of using the method in :ref:`beautifying subscriptions`. - -You can apply a preset to all subscriptions in the ``subscription.yaml`` file -by using the file-wide ``__preset__``: - -.. code-block:: yaml - :caption: subscription.yaml - - __preset__: - preset: "playlist_preset_ex" - - my_subscription_name: - overrides: - url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - playlist_name: "diy-playlist" - -This ``subscription.yaml`` is equivalent to the one above it because all -subscriptions automatically set ``__preset__`` as a `parent preset`_. - - -Subscription Value -^^^^^^^^^^^^^^^^^^^ -NOTE: This is deprecated in favor of using the method in :ref:`beautifying subscriptions`. - -With a clever config and use of ``__preset__``, your subscriptions can typically boil -down to a name and url. You can set ``__value__`` to the name of an override variable, -and use the override variable ``subscription_name`` to achieve one-liner subscriptions. -Using the example above, we can do: - -.. code-block:: yaml - :caption: subscription.yaml - - __preset__: - preset: - - "tv_show" - overrides: - tv_show_name: "{subscription_name}" - - __value__: "url" - - # single-line subscription, sets "Brandon Acker" and the subscription value - # to the override variables tv_show_name and url - "Brandon Acker": "https://www.youtube.com/@brandonacker" - -Traditional subscriptions that can override presets will still work when using ``__value__``. -``__value__`` can also be set within a :ref:`config`. - -------------------------------------------------------------------------------- - -.. _source-variables: - -Source Variables ----------------- - -.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables - :members: - :inherited-members: - :undoc-members: - -Override Variables ------------------- - -.. autoclass:: ytdl_sub.entries.variables.override_variables.OverrideVariables() - :members: - :member-order: bysource - -------------------------------------------------------------------------------- - -Config Types ------------- -The `config.yaml`_ uses various types for its configurable fields. Below is a definition for each type. - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator() - -.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator() diff --git a/docs/getting_started.rst b/docs/getting_started.rst deleted file mode 100644 index 8e1d449f..00000000 --- a/docs/getting_started.rst +++ /dev/null @@ -1,29 +0,0 @@ -Getting Started -=============== - -Walk-through Guide -------------------- -If you haven't read it yet, it's highly recommended to go through our -`walk-through guide `_ -to get familiar with how ``ytdl-sub`` works. - -Example Configs ---------------- -If you are ready to start downloading, see our -`examples directory `_ -for ready-to-use configs and subscriptions. Read through them carefully before use. - -Using Example Configs -^^^^^^^^^^^^^^^^^^^^^^ -Copy and paste the examples into local yaml files, modify the -``working_directory`` and ``output_directory`` with your desired paths, -and perform a dry-run using - -.. code-block:: bash - - ytdl-sub \ - --dry-run \ - --config path/to/config.yaml \ - sub path/to/subscriptions.yaml - -This will simulate what a download will look like. diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 246ff447..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -ytdl-sub readthedocs -==================== - -Our readthedocs page is dedicated towards ytdl-sub config documentation. - -If you are new to ytdl-sub, head over to the -`GitHub Wiki `_ -to see our -`walkthrough `_ and -`FAQ `_. For full examples of -ytdl-sub configs, check out the -`examples directory `_. - - -For navigating config docs, use the left-side bar on the -:ref:`config` page to find every available ytdl-sub field. - -Contents -======== - -.. toctree:: - :maxdepth: 2 - - install - usage - getting_started - presets - config - deprecation_notices diff --git a/docs/install.rst b/docs/install.rst deleted file mode 100644 index bea0667b..00000000 --- a/docs/install.rst +++ /dev/null @@ -1,256 +0,0 @@ -Install -======= -``ytdl-sub`` can be installed on the following platforms. - -.. contents:: - :depth: 3 - -All installations require a 64-bit CPU. 32-bit is not supported. - -Docker Compose --------------- -The ytdl-sub Docker images use -`LSIO-based images `_ -and installs ytdl-sub on top. There are a few flavors to choose from. - -For automating ``subscriptions.yaml`` downloads to pull new media, see -`this guide `_ -on how set up a cron job in any of the docker containers. - -GUI -^^^^ - -The GUI image uses LSIO's -`code-server `_ -for its base image. More info on other code-server environment variables -can be found within its documentation. This is the recommended way to use ``ytdl-sub``. - -After starting, code-server will be running at http://localhost:8443/ - -.. code-block:: yaml - - services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub-gui:latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - ports: - - 8443:8443 - restart: unless-stopped - -Headless -^^^^^^^^^^ - -The headless image uses LSIO's -`baseimage-alpine `_ -for its base image. With this image, ``ytdl-sub`` is meant to be ran from console -via exec'ing into the image using the command: - -.. code-block:: bash - - docker exec -u abc -it ytdl-sub /bin/bash - - -.. code-block:: yaml - - services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub:latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - - DOCKER_MODS=linuxserver/mods:universal-cron - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - restart: unless-stopped - -Passthrough -^^^^^^^^^^^ -For CPU or GPU passthrough, you must use either the GUI image or the headless Ubuntu image -``ghcr.io/jmbannon/ytdl-sub:ubuntu-latest``. - -The docker-compose examples use the GUI image. - -CPU -____ - -.. code-block:: yaml - - services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub-gui:latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - ports: - - 8443:8443 - devices: - - /dev/dri:/dev/dri # CPU passthrough - restart: unless-stopped - -GPU -____ - -.. code-block:: yaml - - services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub-gui:latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - - NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args - - NVIDIA_VISIBLE_DEVICES=all - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - ports: - - 8443:8443 - deploy: - resources: - reservations: - devices: - - capabilities: [gpu] # GPU passthrough - restart: unless-stopped - -Docker --------------- -.. code-block:: bash - - docker run -d \ - --name=ytdl-sub \ - -e PUID=1000 \ - -e PGID=1000 \ - -e TZ=America/Los_Angeles \ - -p 8443:8443 \ - -v :/config \ - -v :/tv_shows \ - -v :/movies \ - -v :/music_videos \ - -v :/music \ - --restart unless-stopped \ - ghcr.io/jmbannon/ytdl-sub-gui:latest - -Windows --------------- -From powershell, run: - -.. code-block:: powershell - - # Download ffmpeg/ffprobe dependencies from yt-dlp - curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip - tar -xf ffmpeg.zip - move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe" - move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe" - - # Download ytdl-sub - curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe - ytdl-sub.exe -h - -Unraid --------------- -See the -`community app `_ -``ytdl-sub``. Uses Docker under the hood. - -Linux --------------- -Requires ffmpeg as a dependency. Can typically be installed with any Linux package manager. - -.. code-block:: bash - - curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub - chmod +x ytdl-sub - ytdl-sub -h - -You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date: - -.. code-block:: bash - - curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz - tar -xf ffmpeg.tar.gz - chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg - chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe - - # May need sudo / root permissions to perform - mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg - mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe - -Linux ARM --------------- -Requires ffmpeg as a dependency. Can typically be installed with any Linux package manager. - -.. code-block:: bash - - curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub_aarch64 - chmod +x ytdl-sub - ytdl-sub -h - -You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date: - -.. code-block:: bash - - curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz - tar -xf ffmpeg.tar.gz - chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg - chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe - - # May need sudo / root permissions to perform - mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg - mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe - - -PIP --------------- -You can install our -`PyPI package `_. -Both ffmpeg and Python 3.10 or greater are required. - -.. code-block:: bash - - python3 -m pip install -U ytdl-sub - -Local Install --------------- -With a Python 3.10 virtual environment, you can clone and install the repo using - -.. code-block:: bash - - git clone https://github.com/jmbannon/ytdl-sub.git - cd ytdl-sub - - pip install -e . - -Local Docker Build -------------------- -Run ``make docker`` in the root directory of this repo to build the image. This -will build the python wheel and install it in the Dockerfile. - diff --git a/docs/make.bat b/docs/make.bat index 32bb2452..747ffb7b 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -7,8 +7,8 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) -set SOURCEDIR=. -set BUILDDIR=_build +set SOURCEDIR=source +set BUILDDIR=build %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( diff --git a/docs/presets.rst b/docs/presets.rst deleted file mode 100644 index b69a019a..00000000 --- a/docs/presets.rst +++ /dev/null @@ -1,183 +0,0 @@ -Presets -======= -``ytdl-sub`` offers a number of built-in presets using best practices for formatting -media in various players. For advanced users, you can find the prebuilt preset -definitions -`here `_. - -TV Shows --------- - -There are two main methods for downloading and formatting videos as a TV show. - -TV Show by Date -^^^^^^^^^^^^^^^ - -TV Show by Date will organize something like a YouTube channel or playlist -into a tv show, where seasons and episodes are organized using upload date. - -Player Presets -"""""""""""""" - -* ``kodi_tv_show_by_date`` -* ``jellyfin_tv_show_by_date`` -* ``plex_tv_show_by_date`` - -Episode Formatting Presets -"""""""""""""""""""""""""" - -* ``season_by_year__episode_by_month_day`` -* ``season_by_year_month__episode_by_day`` -* ``season_by_year__episode_by_month_day_reversed`` - * Episode numbers are reversed, meaning more recent episodes appear at the - top of a season by having a lower value. -* ``season_by_year__episode_by_download_index`` - * Episodes are numbered by the download order. NOTE that this fetched using - the length of the download archive. Do not use if you intend to remove - old videos. - -Usage -""""" - -A preset/subscription requires specifying a player and episode formatting preset -and overriding the following variables: - -.. code-block:: yaml - - rick_a_tv_show_by_date: - preset: - - "jellyfin_tv_show_by_date" - - "season_by_year__episode_by_month_day" - overrides: - # required - tv_show_name: "Rick A" - tv_show_directory: "/path/to/youtube_shows" - url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" - # can be modified from their default value - # tv_show_genre: "ytdl-sub" - # tv_show_content_rating: "TV-14" - # episode_title: "{upload_date_standardized} - {title}" - # episode_description: "{webpage_url}" - -In addition, you can add additional URLs to create a single TV by using the override variables -``url2``, ``url3``, ..., ``url20``: - -.. code-block:: yaml - - overrides: - tv_show_name: "Rick A" - tv_show_directory: "/path/to/youtube_shows" - url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" - url2: "https://www.youtube.com/@just.rick_6" - - -TV Show Collection -^^^^^^^^^^^^^^^^^^ - -TV Show Collections are made up from multiple URLs, where each URL is a season. -If a video belongs to multiple URLs (i.e. a channel and a channel's playlist), -it will resolve to the bottom-most season. - -Two main use cases of a collection are: - 1. Organize a YouTube channel TV show where Season 1 contains any video - not in a 'season playlist', Season 2 for 'Playlist A', Season 3 for - 'Playlist B', etc. - 2. Organize one or more YouTube channels/playlists, where each season - represents a separate channel/playlist. - -Player Presets -"""""""""""""" - -* ``kodi_tv_show_collection`` -* ``jellyfin_tv_show_collection`` -* ``plex_tv_show_collection`` - -Episode Formatting Presets -"""""""""""""""""""""""""" - -* ``season_by_collection__episode_by_year_month_day`` -* ``season_by_collection__episode_by_year_month_day_reversed`` -* ``season_by_collection__episode_by_playlist_index`` - * Only use playlist_index episode formatting for playlists that - will be fully downloaded once and never again. Otherwise, - indices can change. -* ``season_by_collection__episode_by_playlist_index_reversed`` - -Season Presets -"""""""""""""" - -* ``collection_season_1`` -* ``collection_season_2`` -* ``collection_season_3`` -* ``collection_season_4`` -* ``...`` -* ``collection_season_40`` - -Example -""""""" - -A preset/subscription requires specifying a player, episode formatting, and -one or more season presets, with the following override variables: - -.. code-block:: yaml - - rick_a_tv_show_collection: - preset: - - "jellyfin_tv_show_collection" - - "season_by_collection__episode_by_year_month_day_reversed" - - "collection_season_1" - - "collection_season_2" - overrides: - # required - tv_show_name: "Rick A" - tv_show_directory: "/path/to/youtube_shows" - collection_season_1_url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" - collection_season_1_name: "All Videos" - collection_season_2_url: "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" - collection_season_2_name: "Official Music Videos" - # can be modified from their default value - # tv_show_genre: "ytdl-sub" - # episode_title: "{upload_date_standardized} - {title}" - # episode_description: "{webpage_url}" - -Common ------- - -Common presets are applicable to any config. - -Best Video Quality -^^^^^^^^^^^^^^^^^^ - -Add the following preset to download the best available video and audio quality, and remux -it into an MP4 container: - -* ``best_video_quality`` - - -Max 1080p -^^^^^^^^^^^^^^^^^^ - -Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container: - -* ``max_1080p`` - -Chunk Initial Download -^^^^^^^^^^^^^^^^^^^^^^ - -If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from -newest to oldest before starting any downloads. It is a long process and not ideal. A better method -is to chunk the process by using the following preset: - -* ``chunk_initial_download`` - -It will download videos starting from the oldest one, and only download 20 at a time. You can -change this number by setting: - -.. code-block:: yaml - - ytdl_options: - max_downloads: 30 # Desired number to download per invocation - -Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to -oldest again, and stop pulling additional metadata once it reaches a video that has already been -downloaded. diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..7e551b67 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,88 @@ +# Configuration file for the Sphinx documentation builder. + +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "ytdl-sub" +copyright = "2023, Jesse Bannon" +author = "Jesse Bannon" +release = "2023.12.15" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosectionlabel", + "sphinx.ext.extlinks", + "sphinx.ext.napoleon", + "sphinx_copybutton", + "sphinx_design", +] + +templates_path = ["_templates"] +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_book_theme" + +html_theme_options = { + "icon_links": [ + { + "name": "GitHub", + "url": "https://github.com/jmbannon/ytdl-sub", + "icon": "fa-brands fa-square-github", + "type": "fontawesome", + }, + { + "name": "Discord", + "url": "https://discord.gg/v8j9RAHb4k", + "icon": "https://img.shields.io/discord/994270357957648404?logo=Discord", + "type": "url", + }, + ], + "announcement": ("Please excuse our mess as we update these documents"), + "navigation_depth": 10, + "show_toc_level": 10, +} + +html_static_path = ["_static"] +html_css_files = ["custom.css"] + + +# Make sure the all autosectionlabel targets are unique +autosectionlabel_prefix_document = True + +suppress_warnings = [ + "autosectionlabel.*", +] + +extlinks = { + "yt-dlp": ("https://github.com/yt-dlp/yt-dlp/%s", "yt-dlp%s"), + "unraid": ("https://unraid.net/%s", "unraid%s"), + "lsio": ("https://www.linuxserver.io/%s", "lsio%s"), + "lsio-gh": ("https://github.com/linuxserver/%s", "%s image"), + "ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"), +} + +# -- Options for autodoc ---------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration + +# Automatically extract typehints when specified and place them in +# descriptions of the relevant function/method. +autodoc_default_options = { + "autodoc_typehints_format": "short", + "autodoc_class_signature": "separated", + "add_module_names": False, + # "add_class_names": False, +} + +python_use_unqualified_type_names = True +napoleon_numpy_docstring = True +napoleon_use_rtype = False diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst new file mode 100644 index 00000000..8e4097c3 --- /dev/null +++ b/docs/source/config_reference/config_yaml.rst @@ -0,0 +1,90 @@ +================== +Configuration File +================== +----------- +config.yaml +----------- + +ytdl-sub is configured using a ``config.yaml`` file. + +The ``config.yaml`` is made up of two sections: + +.. code-block:: yaml + + configuration: + presets: + +You can jump to any section and subsection of the config using the navigation +section to the left. + +Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. +If you wish to represent paths like Windows, you will need to ``C:\\double\\bashslash\\paths`` +in order to escape the backslash character. + +configuration +~~~~~~~~~~~~~ +The ``configuration`` section contains app-wide configs applied to all presets +and subscriptions. + +.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions() + :members: + :member-order: bysource + :exclude-members: subscription_value, persist_logs, experimental + +persist_logs +"""""""""""" +Within ``configuration``, define whether logs from subscription downloads +should be persisted. + +.. code-block:: yaml + + configuration: + persist_logs: + logs_directory: "/path/to/log/directory" + +Log files are stored as +``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. + +.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator() + :members: + :member-order: bysource + +presets +~~~~~~~ +``presets`` define a `formula` for how to format downloaded media and metadata. + +This section is work-in-progress! + +preset +"""""" +Presets support inheritance by defining a parent preset: + +.. code-block:: yaml + + presets: + custom_preset: + ... + parent_preset: + ... + child_preset: + preset: "parent_preset" + +In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``. +It is advantageous to use parent presets where possible to reduce duplicate yaml +definitions. + +Presets also support inheritance from multiple presets: + +.. code-block:: yaml + + child_preset: + preset: + - "custom_preset" + - "parent_preset" + +In this example, ``child_preset`` will inherit all fields from ``custom_preset`` +and ``parent_preset`` in that order. The bottom-most preset has the highest +priority. + +If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is +valid YAML. Inheriting from multiple presets require use of a list. \ No newline at end of file diff --git a/docs/source/config_reference/index.rst b/docs/source/config_reference/index.rst new file mode 100644 index 00000000..daea6857 --- /dev/null +++ b/docs/source/config_reference/index.rst @@ -0,0 +1,12 @@ +========= +Reference +========= + +This section contains direct references to the code of ``ytdl-sub`` and information on how it functions. + +.. toctree:: + config_yaml + subscriptions_yaml + plugins + scripting/index + prebuilt_presets/index \ No newline at end of file diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst new file mode 100644 index 00000000..69512e8f --- /dev/null +++ b/docs/source/config_reference/plugins.rst @@ -0,0 +1,1015 @@ + +Plugins +======= + +audio_extract +------------- +Extracts audio from a video file. + +:Usage: + +.. code-block:: yaml + + audio_extract: + codec: "mp3" + quality: 128 + +``codec`` + +:expected type: String +:description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. + + +``quality`` + +:expected type: Float +:description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + + +---------------------------------------------------------------------------------------------------- + +chapters +-------- +Embeds chapters to video files if they are present. Additional options to add SponsorBlock +chapters and remove specific ones. Can also remove chapters using regex. + +:Usage: + +.. code-block:: yaml + + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" + + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False + +``allow_chapters_from_comments`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. + + +``embed_chapters`` + +:expected type: Optional[Boolean] +:description: + Defaults to True. Embed chapters into the file. + + +``force_key_frames`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. + + +``remove_chapters_regex`` + +:expected type: Optional[List[RegexString] +:description: + List of regex patterns to match chapter titles against and remove them from the + entry. + + +``remove_sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. + + +``sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. + + +---------------------------------------------------------------------------------------------------- + +date_range +---------- +Only download files uploaded within the specified date range. +Dates must adhere to a yt-dlp datetime. From their docs: + +.. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + +Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. +Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest +granularity possible. + +:Usage: + +.. code-block:: yaml + + date_range: + before: "now" + after: "today-2weeks" + +``after`` + +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +``before`` + +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +---------------------------------------------------------------------------------------------------- + +download +-------- +Sets the URL(s) to download from. Can be used in many forms, including + +:Single URL: + +.. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + +:Multi URL: + +.. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + +:Thumbnails + Variables: + +All variables must be defined for the top-most url. All subsequent URL variables can be either +overwritten or default to the top-most value. + +If an entry is returned from more than one URL, it will use the variables in the bottom-most +URL. + +.. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + +---------------------------------------------------------------------------------------------------- + +embed_thumbnail +--------------- +Whether to embed thumbnails to the audio/video file or not. + +:Usage: + +.. code-block:: yaml + + embed_thumbnail: True + +---------------------------------------------------------------------------------------------------- + +file_convert +------------ +Converts video files from one extension to another. + +:Usage: + +.. code-block:: yaml + + file_convert: + convert_to: "mp4" + +Also supports custom ffmpeg conversions: + +:Usage: + +.. code-block:: yaml + + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text + +``convert_to`` + +:expected type: String +:description: + Convert to a desired file type. Supports + + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav + + +``convert_with`` + +:expected type: Optional[String] +:description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. + + +``ffmpeg_post_process_args`` + +:expected type: Optional[OverridesFormatter] +:description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of + + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. + + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. + + +---------------------------------------------------------------------------------------------------- + +filter_exclude +-------------- +Applies a conditional OR on any number of filters comprised of either variables or scripts. +If any filter evaluates to True, the entry will be excluded. + +:Usage: + +.. code-block:: yaml + + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } + +---------------------------------------------------------------------------------------------------- + +filter_include +-------------- +Applies a conditional AND on any number of filters comprised of either variables or scripts. +If all filters evaluate to True, the entry will be included. + +:Usage: + +.. code-block:: yaml + + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + +---------------------------------------------------------------------------------------------------- + +format +------ +Set ``--format`` to pass into yt-dlp to download a specific format quality. +Uses the same syntax as yt-dlp. + +Usage: + +.. code-block:: yaml + + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + +---------------------------------------------------------------------------------------------------- + +match_filters +------------- +Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. +Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. +For logical AND's between match filters, use the ``&`` operator in a single match filter. + +:Usage: + +.. code-block:: yaml + + match_filters: + filters: + - "age_limit?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" + +---------------------------------------------------------------------------------------------------- + +music_tags +---------- +Adds tags to every download audio file using +`MediaFile `_, +the same audio file tagging package used by +`beets `_. +It supports basic tags like ``title``, ``album``, ``artist`` and ``albumartist``. You can find +a full list of tags for various file types in MediaFile's +`source code `_. + +:Usage: + +.. code-block:: yaml + + presets: + my_example_preset: + music_tags: + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" + +---------------------------------------------------------------------------------------------------- + +nfo_tags +-------- +Adds an NFO file for every download file. An NFO file is simply an XML file +with a ``.nfo`` extension. You can add any values into the NFO. + +:Usage: + +.. code-block:: yaml + + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False + +``kodi_safe`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. + + +``nfo_name`` + +:expected type: EntryFormatter +:description: + The NFO file name. + + +``nfo_root`` + +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like + + .. code-block:: xml + + + + + + +``tags`` + +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like + + .. code-block:: xml + + + + Awesome Youtube Video + 2022 + 502 + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + 2022 + Comedy + Drama + + +---------------------------------------------------------------------------------------------------- + +output_directory_nfo_tags +------------------------- +Adds a single NFO file in the output directory. An NFO file is simply an XML file with a +``.nfo`` extension. It uses the last entry's source variables which can change per download +invocation. Be cautious of which variables you use. + +Usage: + +.. code-block:: yaml + + presets: + my_example_preset: + output_directory_nfo_tags: + # required + nfo_name: "tvshow.nfo" + nfo_root: "tvshow" + tags: + title: "Sweet youtube TV show" + # optional + kodi_safe: False + +``kodi_safe`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. + + +``nfo_name`` + +:expected type: EntryFormatter +:description: + The NFO file name. + + +``nfo_root`` + +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like + + .. code-block:: xml + + + + + + +``tags`` + +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like + + .. code-block:: xml + + + + Sweet youtube TV show + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> + + +---------------------------------------------------------------------------------------------------- + +output_options +-------------- +Defines where to output files and thumbnails after all post-processing has completed. + +:Usage: + +.. code-block:: yaml + + presets: + my_example_preset: + output_options: + # required + output_directory: "/path/to/videos_or_music" + file_name: "{title_sanitized}.{ext}" + # optional + thumbnail_name: "{title_sanitized}.{thumbnail_ext}" + info_json_name: "{title_sanitized}.{info_json_ext}" + download_archive_name: ".ytdl-sub-{subscription_name}-download-archive.json" + migrated_download_archive_name: ".ytdl-sub-{subscription_name_sanitized}-download-archive.json" + maintain_download_archive: True + keep_files_before: now + keep_files_after: 19000101 + +``download_archive_name`` + +:expected type: Optional[OverridesFormatter] +:description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` + + +``file_name`` + +:expected type: EntryFormatter +:description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. + + +``info_json_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. + + +``keep_files_after`` + +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. + + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. + + +``keep_files_before`` + +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. + + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. + + +``keep_max_files`` + +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + + +``maintain_download_archive`` + +:expected type: Optional[Boolean] +:description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. + + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. + + Defaults to False. + + +``migrated_download_archive_name`` + +:expected type: Optional[OverridesFormatter] +:description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. + + +``output_directory`` + +:expected type: OverridesFormatter +:description: + The output directory to store all media files downloaded. + + +``thumbnail_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. + + +---------------------------------------------------------------------------------------------------- + +overrides +--------- +Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + +:Usage: + +.. code-block:: yaml + + presets: + my_example_preset: + overrides: + output_directory: "/path/to/media" + custom_file_name: "{upload_date_standardized}.{title_sanitized}" + + # Then use the override variables in the output options + output_options: + output_directory: "{output_directory}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + +Override variables can contain explicit values and other variables, including both override +and source variables. + +In addition, any override variable defined will automatically create a ``sanitized`` variable +for use. In the example above, ``output_directory_sanitized`` will exist and perform +sanitization on the value when used. + +---------------------------------------------------------------------------------------------------- + +regex +----- +.. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + +Performs regex matching on an entry's source or override variables. Regex can be used to filter +entries from proceeding with download or capture groups to create new source variables. + +NOTE that YAML differentiates between single-quote (``'``) and double-quote (``"``), which can +affect regex. Double-quote implies string literals, i.e. ``"\n"`` is the literal chars ``\n``, +whereas single-quote, ``'\n'`` gets evaluated to a new line. To escape ``\`` when using +single-quote, use ``\\``. This is necessary if you want your regex to be something like +``\d\n`` to match a number and adjacent new-line. It must be written as ``\\d\n``. + +If you want to regex-search multiple source variables to create a logical OR effect, you can +create an override variable that contains the concatenation of them, and search that with regex. +For example, creating the override variable ``"title_and_description": "{title} {description}"`` +and using ``title_and_description`` can regex match/exclude from either ``title`` or +``description``. + +:Usage: + +.. code-block:: yaml + + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True + + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' + + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' + + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" + + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" + +``skip_if_match_fails`` + +:expected type: Optional[Boolean] +:description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. + + +---------------------------------------------------------------------------------------------------- + +split_by_chapters +----------------- +Splits a file by chapters into multiple files. Each file becomes its own entry with the +new variables + + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` + +Note that when using this plugin and performing dry-run, it assumes embedded chapters are being +used with no modifications. + +:Usage: + +.. code-block:: yaml + + split_by_chapters: + when_no_chapters: "pass" + +``when_no_chapters`` + +:expected type: String +:description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + + +---------------------------------------------------------------------------------------------------- + +subtitles +--------- +Defines how to download and store subtitles. Using this plugin creates two new variables: +``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. +It will set the respective language to the correct subtitle file. + +:Usage: + +.. code-block:: yaml + + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False + +``allow_auto_generated_subtitles`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to allow auto generated subtitles. + + +``embed_subtitles`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. + + +``languages`` + +:expected type: Optional[List[String]] +:description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". + + +``subtitles_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. + + +``subtitles_type`` + +:expected type: Optional[String] +:description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". + + +---------------------------------------------------------------------------------------------------- + +throttle_protection +------------------- +Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For +range-based values, a random number will be chosen within the range to avoid sleeps looking +scripted. + +:Usage: + +.. code-block:: yaml + + presets: + my_example_preset: + throttle_protection: + sleep_per_download_s: + min: 2.2 + max: 10.8 + sleep_per_subscription_s: + min: 9.0 + max: 14.1 + max_downloads_per_subscription: + min: 10 + max: 36 + subscription_download_probability: 1.0 + +``max_downloads_per_subscription`` + +:expected type: Optional[Range] +:description: + Number of downloads to perform per subscription. + + +``sleep_per_download_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. + + +``sleep_per_subscription_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each subscription. + + +``subscription_download_probability`` + +:expected type: Optional[Float] +:description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. + + +---------------------------------------------------------------------------------------------------- + +video_tags +---------- +Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. + +:Usage: + +.. code-block:: yaml + + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" + +---------------------------------------------------------------------------------------------------- + +ytdl_options +------------ +Allows you to add any ytdl argument to ytdl-sub's downloader. +The argument names can differ slightly from the command-line argument names. See +`this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ +for more details. + +:Usage: + +.. code-block:: yaml + + presets: + my_example_preset: + ytdl_options: + # Ignore any download related errors and continue + ignoreerrors: True + # Stop downloading additional metadata/videos if it + # exists in your download archive + break_on_existing: True + # Stop downloading additional metadata/videos if it + # is out of your date range + break_on_reject: True + # Path to your YouTube cookies file to download 18+ restricted content + cookiefile: "/path/to/cookies/file.txt" + # Only download this number of videos/audio + max_downloads: 10 + # Download and use English title/description/etc YouTube metadata + extractor_args: + youtube: + lang: + - "en" + + +where each key is a ytdl argument. Include in the example are some popular ytdl_options. diff --git a/docs/source/config_reference/prebuilt_presets/helpers_common.rst b/docs/source/config_reference/prebuilt_presets/helpers_common.rst new file mode 100644 index 00000000..3dc5e105 --- /dev/null +++ b/docs/source/config_reference/prebuilt_presets/helpers_common.rst @@ -0,0 +1,7 @@ +======================= +Common Preset Reference +======================= + +.. highlight:: yaml + +.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/common.yaml diff --git a/docs/source/config_reference/prebuilt_presets/helpers_players.rst b/docs/source/config_reference/prebuilt_presets/helpers_players.rst new file mode 100644 index 00000000..404c38ab --- /dev/null +++ b/docs/source/config_reference/prebuilt_presets/helpers_players.rst @@ -0,0 +1,7 @@ +======================== +Players Preset Reference +======================== + +.. highlight:: yaml + +.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/players.yaml diff --git a/docs/source/config_reference/prebuilt_presets/helpers_url.rst b/docs/source/config_reference/prebuilt_presets/helpers_url.rst new file mode 100644 index 00000000..b6928288 --- /dev/null +++ b/docs/source/config_reference/prebuilt_presets/helpers_url.rst @@ -0,0 +1,7 @@ +==================== +URL Preset Reference +==================== + +.. highlight:: yaml + +.. literalinclude:: /../../src/ytdl_sub/prebuilt_presets/helpers/url.yaml diff --git a/docs/source/config_reference/prebuilt_presets/index.rst b/docs/source/config_reference/prebuilt_presets/index.rst new file mode 100644 index 00000000..9af49b74 --- /dev/null +++ b/docs/source/config_reference/prebuilt_presets/index.rst @@ -0,0 +1,12 @@ +========================= +Prebuilt Preset Reference +========================= + +This section contains the code for the prebuilt presets. If you just want to understand how to use the presets, check :doc:`this section instead</prebuilt_presets/index>`. + + + +.. toctree:: + helpers_common + helpers_players + helpers_url \ No newline at end of file diff --git a/docs/source/config_reference/scripting/entry_variables.rst b/docs/source/config_reference/scripting/entry_variables.rst new file mode 100644 index 00000000..8aa85ca6 --- /dev/null +++ b/docs/source/config_reference/scripting/entry_variables.rst @@ -0,0 +1,649 @@ + +Entry Variables +=============== + +Entry Variables +--------------- + +channel +~~~~~~~ +:type: ``String`` +:description: + The channel name if it exists, otherwise returns the uploader. + +channel_id +~~~~~~~~~~ +:type: ``String`` +:description: + The channel id if it exists, otherwise returns the entry uploader ID. + +chapters +~~~~~~~~ +:type: ``Array`` +:description: + Chapters if they exist + +comments +~~~~~~~~ +:type: ``Array`` +:description: + Comments if they are requested + +creator +~~~~~~~ +:type: ``String`` +:description: + The creator name if it exists, otherwise returns the channel. + +description +~~~~~~~~~~~ +:type: ``String`` +:description: + The description if it exists. Otherwise, returns an emtpy string. + +duration +~~~~~~~~ +:type: ``Integer`` +:description: + The duration of the entry in seconds if it exists. Defaults to zero otherwise. + +epoch +~~~~~ +:type: ``Integer`` +:description: + The unix epoch of when the metadata was scraped by yt-dlp. + +epoch_date +~~~~~~~~~~ +:type: ``String`` +:description: + The epoch's date, in YYYYMMDD format. + +epoch_hour +~~~~~~~~~~ +:type: ``String`` +:description: + The epoch's hour + +ext +~~~ +:type: ``String`` +:description: + The downloaded entry's file extension + +extractor +~~~~~~~~~ +:type: ``String`` +:description: + The yt-dlp extractor name + +extractor_key +~~~~~~~~~~~~~ +:type: ``String`` +:description: + The yt-dlp extractor key + +ie_key +~~~~~~ +:type: ``String`` +:description: + The ie_key, used in legacy yt-dlp things as the 'info-extractor key'. + If it does not exist, return ``extractor_key`` + +info_json_ext +~~~~~~~~~~~~~ +:type: ``String`` +:description: + The "info.json" extension + +requested_subtitles +~~~~~~~~~~~~~~~~~~~ +:type: ``Map`` +:description: + Subtitles if they are requested and exist + +sponsorblock_chapters +~~~~~~~~~~~~~~~~~~~~~ +:type: ``Array`` +:description: + Sponsorblock Chapters if they are requested and exist + +thumbnail_ext +~~~~~~~~~~~~~ +:type: ``String`` +:description: + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. + +title +~~~~~ +:type: ``String`` +:description: + The title of the entry. If a title does not exist, returns its unique ID. + +title_sanitized_plex +~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + +uid +~~~ +:type: ``String`` +:description: + The entry's unique ID + +uid_sanitized_plex +~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + +uploader +~~~~~~~~ +:type: ``String`` +:description: + The uploader if it exists, otherwise return the uploader ID. + +uploader_id +~~~~~~~~~~~ +:type: ``String`` +:description: + The uploader id if it exists, otherwise return the unique ID. + +uploader_url +~~~~~~~~~~~~ +:type: ``String`` +:description: + The uploader url if it exists, otherwise returns the webpage_url. + +webpage_url +~~~~~~~~~~~ +:type: ``String`` +:description: + The url to the webpage. + +---------------------------------------------------------------------------------------------------- + +Metadata Variables +------------------ + +entry_metadata +~~~~~~~~~~~~~~ +:type: ``Map`` +:description: + The entry's info.json + +playlist_metadata +~~~~~~~~~~~~~~~~~ +:type: ``Map`` +:description: + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) + +sibling_metadata +~~~~~~~~~~~~~~~~ +:type: ``Array`` +:description: + Metadata from any sibling entries that reside in the same playlist as this entry. + +source_metadata +~~~~~~~~~~~~~~~ +:type: ``Map`` +:description: + Metadata from the source + (i.e. the grandparent metadata, like channel -> playlist -> entry) + +---------------------------------------------------------------------------------------------------- + +Playlist Variables +------------------ + +playlist_count +~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + Playlist count if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + +playlist_description +~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The playlist description if it exists, otherwise returns the entry's description. + +playlist_index +~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + +playlist_index_padded +~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + playlist_index padded two digits + +playlist_index_padded6 +~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + playlist_index padded six digits. + +playlist_index_reversed +~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + Playlist index reversed via ``playlist_count - playlist_index + 1`` + +playlist_index_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + playlist_index_reversed padded two digits + +playlist_index_reversed_padded6 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + playlist_index_reversed padded six digits. + +playlist_max_upload_date +~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` + +playlist_max_upload_year +~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + +playlist_max_upload_year_truncated +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + +playlist_title +~~~~~~~~~~~~~~ +:type: ``String`` +:description: + Name of its parent playlist/channel if it exists, otherwise returns its title. + +playlist_uid +~~~~~~~~~~~~ +:type: ``String`` +:description: + The playlist unique ID if it exists, otherwise return the entry unique ID. + +playlist_uploader +~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The playlist uploader if it exists, otherwise return the entry uploader. + +playlist_uploader_id +~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The playlist uploader id if it exists, otherwise returns the entry uploader ID. + +playlist_uploader_url +~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + +playlist_webpage_url +~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + +---------------------------------------------------------------------------------------------------- + +Release Date Variables +---------------------- + +release_date +~~~~~~~~~~~~ +:type: ``String`` +:description: + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + +release_date_standardized +~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The uploaded date formatted as YYYY-MM-DD + +release_day +~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload day as an integer (no padding). + +release_day_of_year +~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The day of the year, i.e. February 1st returns ``32`` + +release_day_of_year_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The upload day of year, but padded i.e. February 1st returns "032" + +release_day_of_year_reversed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + +release_day_of_year_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The reversed upload day of year, but padded i.e. December 31st returns "001" + +release_day_padded +~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" + +release_day_reversed +~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + +release_day_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The reversed upload day, but padded. i.e. August 30th returns "02". + +release_month +~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload month as an integer (no padding). + +release_month_padded +~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The entry's upload month padded to two digits, i.e. March returns "03" + +release_month_reversed +~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10`` + +release_month_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The reversed upload month, but padded. i.e. November returns "02" + +release_year +~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The entry's upload year + +release_year_truncated +~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The last two digits of the upload year, i.e. 22 in 2022 + +release_year_truncated_reversed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + +---------------------------------------------------------------------------------------------------- + +Source Variables +---------------- + +source_count +~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The source count if it exists, otherwise returns ``1``. + +source_description +~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The source description if it exists, otherwise returns the playlist description. + +source_index +~~~~~~~~~~~~ +:type: ``Integer`` +:description: + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + +source_index_padded +~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The source index, padded two digits. + +source_title +~~~~~~~~~~~~ +:type: ``String`` +:description: + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + +source_uid +~~~~~~~~~~ +:type: ``String`` +:description: + The source unique id if it exists, otherwise returns the playlist unique ID. + +source_uploader +~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The source uploader if it exists, otherwise return the playlist_uploader + +source_uploader_id +~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The source uploader id if it exists, otherwise returns the playlist_uploader_id + +source_uploader_url +~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The source uploader url if it exists, otherwise returns the source webpage_url. + +source_webpage_url +~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The source webpage url if it exists, otherwise returns the playlist webpage url. + +---------------------------------------------------------------------------------------------------- + +Upload Date Variables +--------------------- + +upload_date +~~~~~~~~~~~ +:type: ``String`` +:description: + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + +upload_date_standardized +~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The uploaded date formatted as YYYY-MM-DD + +upload_day +~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload day as an integer (no padding). + +upload_day_of_year +~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The day of the year, i.e. February 1st returns ``32`` + +upload_day_of_year_padded +~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The upload day of year, but padded i.e. February 1st returns "032" + +upload_day_of_year_reversed +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + +upload_day_of_year_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The reversed upload day of year, but padded i.e. December 31st returns "001" + +upload_day_padded +~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" + +upload_day_reversed +~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + +upload_day_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The reversed upload day, but padded. i.e. August 30th returns "02". + +upload_month +~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload month as an integer (no padding). + +upload_month_padded +~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The entry's upload month padded to two digits, i.e. March returns "03" + +upload_month_reversed +~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + +upload_month_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The reversed upload month, but padded. i.e. November returns "02" + +upload_year +~~~~~~~~~~~ +:type: ``Integer`` +:description: + The entry's upload year + +upload_year_truncated +~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The last two digits of the upload year, i.e. 22 in 2022 + +upload_year_truncated_reversed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + +---------------------------------------------------------------------------------------------------- + +Ytdl-Sub Variables +------------------ + +download_index +~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. + +download_index_padded6 +~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The download_index padded six digits + +upload_date_index +~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + The i'th entry downloaded with this upload date. + +upload_date_index_padded +~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The upload_date_index padded two digits + +upload_date_index_reversed +~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``Integer`` +:description: + 100 - upload_date_index + +upload_date_index_reversed_padded +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The upload_date_index padded two digits + +ytdl_sub_input_url +~~~~~~~~~~~~~~~~~~ +:type: ``String`` +:description: + The input URL used in ytdl-sub to create this entry. diff --git a/docs/source/config_reference/scripting/index.rst b/docs/source/config_reference/scripting/index.rst new file mode 100644 index 00000000..bee9a4e8 --- /dev/null +++ b/docs/source/config_reference/scripting/index.rst @@ -0,0 +1,172 @@ +========= +Scripting +========= + +``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The links below +contain reference documentation for each built-in variable and scripting function. + +.. toctree:: + :maxdepth: 1 + + entry_variables + override_variables + scripting_functions + scripting_types + +How it Works +------------ + +Fields in the config that support ``formatters`` mean they support scripting, and will +*format* the field using its defined script. + +In its most basic form, a script is a string comprised of variables and/or functions. + +Static String +~~~~~~~~~~~~~ + +The following example sets ``ytdl-sub``'s output directory. It is +considered *static* because it does not depend on anything from an entry. + +.. code-block:: yaml + + output_options: + output_directory: "Custom YTDL-SUB TV Show" + +Static Variables +~~~~~~~~~~~~~~~~ + +``ytdl-sub`` offers a few built-in static variables, including ``subscription_name``. +We can use this instead of hard-coding it above: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + +The syntax for variable usage is curly-braces with the variable name within it. Assuming +our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub`` +will actually write to that directory. + +Entry Variables +~~~~~~~~~~~~~~~ + +For context, an *entry* is a video or audio file downloaded from ``yt-dlp``. +*Entry variables* are variables that are derived from an entry's ``info.json`` file. This file +comes from ``yt-dlp`` and contains every piece of metadata that it scraped. + +These variables are not considered static since they change per entry download. There are a +few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For others, +we are free to use values that derive from an entry. + +Suppose we want to customize the name of an entry's output file and thumbnail to include its +title in its name. We can do that using entry variables: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + file_name: "{title}.{ext}" + thumbnail_name: "{title}.{thumbnail_ext}" + +Creating Custom Variables +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Suppose we want to include the date in our file names. This means we'd need to update +both the ``file_name`` and ``thumbnail_name`` fields to include it. + +Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method +for creating and overriding custom variables. + +These are created in the ``overrides`` section. Let's take our above example and create +a ``custom_file_name`` variable to use for the entry file and thumbnail fields: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + custom_file_name: "{upload_date_standardized} {title}" + +Sanitizing Variables +~~~~~~~~~~~~~~~~~~~~ + +For experienced ``yt-dlp`` scrapers, you may be thinking: + +- What if the title has characters that do not play nice with my operating system? + +``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic characters +with safe alternatives that can be used in file names. We can ensure our file names and directories +are safe by using: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name_sanitized}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + custom_file_name: "{upload_date_standardized} {title_sanitized}" + +Simply add a ``_sanitized`` suffix to any variable name to make it sanitized. + +.. note:: + + Make sure you do not sanitize custom variables that intentionally create directories, otherwise + they will... be sanitized and not resolve to directories! + + +Using Scripting Functions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's suppose you are an avid command-line user, and like all of your file names to be +``snake_cased_with_no_spaces``. We can use the +`replace <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_ +*scripting function* to create and use a snake-cased title. + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name_sanitized}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + snake_cased_title: >- + { + %replace( title, ' ', '_' ) + } + custom_file_name: "{upload_date_standardized}_{snake_cased_title_sanitized}" + +Scripting functions are similar to variables - they must be used within curly-braces. +It is good practice to use ``>-`` when defining variables that use functions. It is YAML's way of +saying: + +- Allow a string to be multi-lined, and do not include newlines before or after it. + +See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=canonical_yaml>`_. +Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make +scripting function usage readable. + +.. important:: + + It is important to use ``>-`` over other YAML new-line directives like ``>`` because they + add newlines before or after curly-braces, and will be included in your variable's output string. + +Advanced Scripting +------------------ + +Accessing ``info.json`` Fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +WIP + +Creating Custom Functions +~~~~~~~~~~~~~~~~~~~~~~~~~ +WIP + +Parsing Maps and Arrays +~~~~~~~~~~~~~~~~~~~~~~~ +WIP diff --git a/docs/source/config_reference/scripting/override_variables.rst b/docs/source/config_reference/scripting/override_variables.rst new file mode 100644 index 00000000..c7a9991b --- /dev/null +++ b/docs/source/config_reference/scripting/override_variables.rst @@ -0,0 +1,70 @@ + +Override Variables +================== + +subscription_indent_i +--------------------- +For subscriptions in the form of + +.. code-block:: yaml + + Preset | = Indent Value 1: + = Indent Value 2: + "Subscription Name": "https://..." + +``subscription_indent_1`` and ``subscription_indent_2`` get set to +``Indent Value 1`` and ``Indent Value 2``. + +subscription_map +---------------- +For subscriptions in the form of + +.. code-block:: yaml + + + Subscription Name: + Music Videos: + - "https://url1.com/..." + Concerts: + - "https://url2.com/..." + +Stores all the contents under the subscription name into the override variable +``subscription_map`` as a Map value. The above example is stored as: + +.. code-block:: python + + { + "Music Videos": [ + "https://url1.com/..." + ], + "Concerts: [ + "https://url2.com/..." + ] + } + +subscription_name +----------------- +Name of the subscription + +subscription_value +------------------ +For subscriptions in the form of + +.. code-block:: yaml + + "Subscription Name": "https://..." + +``subscription_value`` gets set to ``https://...``. + +subscription_value_i +-------------------- +For subscriptions in the form of + +.. code-block:: yaml + + "Subscription Name": + - "https://url1.com/..." + - "https://url2.com/..." + +``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...`` +and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to +``subscription_value``. diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst new file mode 100644 index 00000000..c271669e --- /dev/null +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -0,0 +1,631 @@ + +Scripting Functions +=================== + +Array Functions +--------------- + +array +~~~~~ +:spec: ``array(maybe_array: AnyArgument) -> Array`` + +:description: + Tries to cast an unknown variable type to an Array. + +array_apply +~~~~~~~~~~~ +:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array`` + +:description: + Apply a lambda function on every element in the Array. +:usage: + +.. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] + +array_apply_fixed +~~~~~~~~~~~~~~~~~ +:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array`` + +:description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. + +array_at +~~~~~~~~ +:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` + +:description: + Return the element in the Array at index ``idx``. + +array_contains +~~~~~~~~~~~~~~ +:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean`` + +:description: + Return True if the value exists in the Array. False otherwise. + +array_enumerate +~~~~~~~~~~~~~~~ +:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` + +:description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. + +array_extend +~~~~~~~~~~~~ +:spec: ``array_extend(arrays: Array, ...) -> Array`` + +:description: + Combine multiple Arrays into a single Array. + +array_first +~~~~~~~~~~~ +:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument`` + +:description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. + +array_flatten +~~~~~~~~~~~~~ +:spec: ``array_flatten(array: Array) -> Array`` + +:description: + Flatten any nested Arrays into a single-dimensional Array. + +array_index +~~~~~~~~~~~ +:spec: ``array_index(array: Array, value: AnyArgument) -> Integer`` + +:description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. + +array_overlay +~~~~~~~~~~~~~ +:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array`` + +:description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. + +array_product +~~~~~~~~~~~~~ +:spec: ``array_product(arrays: Array, ...) -> Array`` + +:description: + Returns the Cartesian product of elements from different arrays + +array_reduce +~~~~~~~~~~~~ +:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` + +:description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. + +array_reverse +~~~~~~~~~~~~~ +:spec: ``array_reverse(array: Array) -> Array`` + +:description: + Reverse an Array. + +array_size +~~~~~~~~~~ +:spec: ``array_size(array: Array) -> Integer`` + +:description: + Returns the size of an Array. + +array_slice +~~~~~~~~~~~ +:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` + +:description: + Returns the slice of the Array. + +---------------------------------------------------------------------------------------------------- + +Boolean Functions +----------------- + +and +~~~ +:spec: ``and(values: AnyArgument, ...) -> Boolean`` + +:description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. + +bool +~~~~ +:spec: ``bool(value: AnyArgument) -> Boolean`` + +:description: + Cast any type to a Boolean. + +eq +~~ +:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` + +:description: + ``==`` operator. Returns True if left == right. False otherwise. + +gt +~~ +:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` + +:description: + ``>`` operator. Returns True if left > right. False otherwise. + +gte +~~~ +:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` + +:description: + ``>=`` operator. Returns True if left >= right. False otherwise. + +is_null +~~~~~~~ +:spec: ``is_null(value: AnyArgument) -> Boolean`` + +:description: + Returns True if a value is null (i.e. an empty string). False otherwise. + +lt +~~ +:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` + +:description: + ``<`` operator. Returns True if left < right. False otherwise. + +lte +~~~ +:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` + +:description: + ``<=`` operator. Returns True if left <= right. False otherwise. + +ne +~~ +:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` + +:description: + ``!=`` operator. Returns True if left != right. False otherwise. + +not +~~~ +:spec: ``not(value: Boolean) -> Boolean`` + +:description: + ``not`` operator. Returns the opposite of value. + +or +~~ +:spec: ``or(values: AnyArgument, ...) -> Boolean`` + +:description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. + +xor +~~~ +:spec: ``xor(values: AnyArgument, ...) -> Boolean`` + +:description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + +---------------------------------------------------------------------------------------------------- + +Conditional Functions +--------------------- + +if +~~ +:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` + +:description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. + +if_passthrough +~~~~~~~~~~~~~~ +:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` + +:description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. + +---------------------------------------------------------------------------------------------------- + +Date Functions +-------------- + +datetime_strftime +~~~~~~~~~~~~~~~~~ +:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` + +:description: + Converts a posix timestamp to a date using strftime formatting. + +---------------------------------------------------------------------------------------------------- + +Error Functions +--------------- + +assert +~~~~~~ +:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. + +assert_eq +~~~~~~~~~ +:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. + +assert_ne +~~~~~~~~~ +:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. + +assert_then +~~~~~~~~~~~ +:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. + +throw +~~~~~ +:spec: ``throw(error_message: String) -> AnyArgument`` + +:description: + Explicitly throw an error with the provided error message. + +---------------------------------------------------------------------------------------------------- + +Json Functions +-------------- + +from_json +~~~~~~~~~ +:spec: ``from_json(argument: String) -> AnyArgument`` + +:description: + Converts a JSON string into an actual type. + +---------------------------------------------------------------------------------------------------- + +Map Functions +------------- + +map +~~~ +:spec: ``map(maybe_mapping: AnyArgument) -> Map`` + +:description: + Tries to cast an unknown variable type to a Map. + +map_apply +~~~~~~~~~ +:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` + +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. + +map_contains +~~~~~~~~~~~~ +:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` + +:description: + Returns True if the key is in the Map. False otherwise. + +map_enumerate +~~~~~~~~~~~~~ +:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` + +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. + +map_get +~~~~~~~ +:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` + +:description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. + +map_get_non_empty +~~~~~~~~~~~~~~~~~ +:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` + +:description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. + +map_size +~~~~~~~~ +:spec: ``map_size(mapping: Map) -> Integer`` + +:description: + Returns the size of a Map. + +---------------------------------------------------------------------------------------------------- + +Numeric Functions +----------------- + +add +~~~ +:spec: ``add(values: Numeric, ...) -> Numeric`` + +:description: + ``+`` operator. Returns the sum of all values. + +div +~~~ +:spec: ``div(left: Numeric, right: Numeric) -> Numeric`` + +:description: + ``/`` operator. Returns ``left / right``. + +float +~~~~~ +:spec: ``float(value: AnyArgument) -> Float`` + +:description: + Cast to Float. + +int +~~~ +:spec: ``int(value: AnyArgument) -> Integer`` + +:description: + Cast to Integer. + +max +~~~ +:spec: ``max(values: Numeric, ...) -> Numeric`` + +:description: + Returns max of all values. + +min +~~~ +:spec: ``min(values: Numeric, ...) -> Numeric`` + +:description: + Returns min of all values. + +mod +~~~ +:spec: ``mod(left: Numeric, right: Numeric) -> Numeric`` + +:description: + ``%`` operator. Returns ``left % right``. + +mul +~~~ +:spec: ``mul(values: Numeric, ...) -> Numeric`` + +:description: + ``*`` operator. Returns the product of all values. + +pow +~~~ +:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric`` + +:description: + ``**`` operator. Returns the exponential of the base and exponent value. + +sub +~~~ +:spec: ``sub(values: Numeric, ...) -> Numeric`` + +:description: + ``-`` operator. Subtracts all values from left to right. + +---------------------------------------------------------------------------------------------------- + +Regex Functions +--------------- + +regex_capture_groups +~~~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_groups(regex: String) -> Integer`` + +:description: + Returns number of capture groups in regex + +regex_fullmatch +~~~~~~~~~~~~~~~ +:spec: ``regex_fullmatch(regex: String, string: String) -> Array`` + +:description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + +regex_match +~~~~~~~~~~~ +:spec: ``regex_match(regex: String, string: String) -> Array`` + +:description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + +regex_search +~~~~~~~~~~~~ +:spec: ``regex_search(regex: String, string: String) -> Array`` + +:description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + +---------------------------------------------------------------------------------------------------- + +String Functions +---------------- + +capitalize +~~~~~~~~~~ +:spec: ``capitalize(string: String) -> String`` + +:description: + Capitalize the first character in the string. + +concat +~~~~~~ +:spec: ``concat(values: String, ...) -> String`` + +:description: + Concatenate multiple Strings into a single String. + +contains +~~~~~~~~ +:spec: ``contains(string: String, contains: String) -> Boolean`` + +:description: + Returns True if ``contains`` is in ``string``. False otherwise. + +lower +~~~~~ +:spec: ``lower(string: String) -> String`` + +:description: + Lower-case the entire String. + +pad +~~~ +:spec: ``pad(string: String, length: Integer, char: String) -> String`` + +:description: + Pads the string to the given length + +pad_zero +~~~~~~~~ +:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String`` + +:description: + Pads a numeric with zeros to the given length + +replace +~~~~~~~ +:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` + +:description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. + +slice +~~~~~ +:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` + +:description: + Returns the slice of the Array. + +string +~~~~~~ +:spec: ``string(value: AnyArgument) -> String`` + +:description: + Cast to String. + +titlecase +~~~~~~~~~ +:spec: ``titlecase(string: String) -> String`` + +:description: + Capitalize each word in the string. + +upper +~~~~~ +:spec: ``upper(string: String) -> String`` + +:description: + Upper-case the entire String. + +---------------------------------------------------------------------------------------------------- + +Ytdl-Sub Functions +------------------ + +legacy_bracket_safety +~~~~~~~~~~~~~~~~~~~~~ +:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` + +ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not +interfere with its legacy variable scripting system. This function replicates that +behavior. + +sanitize +~~~~~~~~ +:spec: ``sanitize(value: AnyArgument) -> String`` + +Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use +for file/directory names on any OS. + +sanitize_plex_episode +~~~~~~~~~~~~~~~~~~~~~ +:spec: ``sanitize_plex_episode(string: String) -> String`` + +Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width +numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the +season and/or episode. + +to_date_metadata +~~~~~~~~~~~~~~~~ +:spec: ``to_date_metadata(yyyymmdd: String) -> Map`` + +Takes a date in the form of YYYYMMDD and returns a Map containing: + +- date (String, YYYYMMDD) +- date_standardized (String, YYYY-MM-DD) +- year (Integer) +- month (Integer) +- day (Integer) +- year_truncated (Integer, YY from YY[YY]) +- month_padded (String) +- day_padded (String) +- year_truncated_reversed (Integer, 100 - year_truncated) +- month_reversed (Integer, 13 - month) +- month_reversed_padded (String) +- day_reversed (Integer, total_days_in_month + 1 - day) +- day_reversed_padded (String) +- day_of_year (Integer) +- day_of_year_padded (String, padded 3) +- day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year) +- day_of_year_reversed_padded (String, padded 3) + +to_native_filepath +~~~~~~~~~~~~~~~~~~ +:spec: ``to_native_filepath(filepath: String) -> String`` + +Convert any unix-based path separators ('/') with the OS's native +separator. + +truncate_filepath_if_too_long +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:spec: ``truncate_filepath_if_too_long(filepath: String) -> String`` + +If a file-path is too long for the OS, this function will truncate it while preserving +the extension. diff --git a/docs/source/config_reference/scripting/scripting_types.rst b/docs/source/config_reference/scripting/scripting_types.rst new file mode 100644 index 00000000..d389c2a9 --- /dev/null +++ b/docs/source/config_reference/scripting/scripting_types.rst @@ -0,0 +1,319 @@ + +Scripting Types +=============== + +Types +----- + +String +~~~~~~ + +Strings are a series of characters surrounded by quotes and can be defined in a few ways, including: + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + string_variable: "This is a String variable" + + .. tab-item:: In-Line + + .. code-block:: yaml + + string_variable: "{ %string('This is a String variable') }" + + .. tab-item:: Single Quote + + .. code-block:: yaml + + string_variable: >- + { + %string('This is a String variable') + } + + .. tab-item:: Double Quote + + .. code-block:: yaml + + string_variable: >- + { + %string("This is a String variable") + } + + .. tab-item:: Triple Quote + + .. code-block:: yaml + + string_variable: >- + { + %string('''This is a String variable''') + } + + .. tab-item:: Triple-Double Quote + + .. code-block:: yaml + + string_variable: >- + { + %string("""This is a String variable""") + } + +.. note:: + + For non-String types, they must be defined as parameters to scripting functions. This is because + anything in a variable definition that is not within curly-braces gets evaluated as a String. + +Integer +~~~~~~~ + +Integers are whole numbers with no decimal. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + int_variable: >- + { + %int(2022) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + int_variable: "{ %int(2022) }" + +Float +~~~~~ + +Floats are floating-point decimals numbers. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + float_variable: >- + { + %float(3.14) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + float_variable: "{ %float(3.14) }" + +Boolean +~~~~~~~ + +A type is considered boolean if it spells out ``True`` or ``False``, case-insensitive. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + bool_variable: >- + { + %bool(True) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + bool_variable: "{ %bool(FALSE) }" + +Array +~~~~~ + +An Array contains multiple types of any kind, including nested Arrays and Maps. +Arrays are defined using brackets (``[ ]``), and are accessed using zero-based indexing. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + array_variable: >- + { + [ + "element with index 0", + 1, + 2.0, + [ "Nested Array 3" ] + ] + } + element_0: >- + { + %array_at(array_variable, 0) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + array_variable: "{ ['element with index 0', 1, 2.0, ['Nested Array 3' ]] }" + element_0: "{ %array_at(array_variable, 0) }" + +Map +~~~ + +A Map is a key-value store, containing mappings between keys and values. +Maps are defined using curley-braces (``{ }``), and are accessed using their keys. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + map_variable: >- + { + { + "string_key": "string_value", + 1: "int_key", + "list_value": [ "elem0", 1, 2.0 ] + } + } + string_value: >- + { + %map_get(map_variable, "string_key") + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + map_variable: "{ {'string_key': 'string_value', 1: 'int_key', 'list_value': [ 'elem0', 1, 2.0 ]} }" + string_value: "{ %map_get(map_variable, 'string_key') }" + +Null +~~~~ +Null is represented by an empty String, and can be conveyed by spelling out ``null``, +case-insensitive. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + null_variable: "" + + .. tab-item:: In-Line + + .. code-block:: yaml + + null_variable: "{ %string(null) }" + + +Function Type-Hints +------------------- + +AnyArgument +~~~~~~~~~~~ +AnyArgument means any of the above Types are valid as input or output to a scripting function. + +.. note:: + + Strict typing is enforced. For functions that return ``AnyArgument`` need to be casted before + passing into functions that expect a particular type. + +Numeric +~~~~~~~ +Numeric refers to either an Integer or Float. + +Optional +~~~~~~~~ +Optional means a particular scripting function argument can be either provided or not included. +For example, the function +`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_ +has an optional default value. Both of these usages are valid: + +.. tab-set:: + + .. tab-item:: Map Get + + .. code-block:: yaml + + will_throw_key_does_not_exist_error: "{ %map_get( {}, 'key' ) }" + + .. tab-item:: Map Get with Optional Default Value + + .. code-block:: yaml + + will_return_default: "{ %map_get( {}, 'key', 'default value' ) }" + +Lambda +~~~~~~ +Lambda parameters are a reference to a function, and will call that lambda function +on the input. In this example, + +.. code-block:: yaml + + lambda_array_numeric_to_string: >- + { + %array_apply( [ 1, 2, 3, 4], %string ) + } + +We apply ``%string`` as a lambda function to +`array_apply <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_, +which is called on every element in the input array. The output becomes ``["1", "2", "3", "4"]``. + +This example has one input-argument being passed into the lambda. For other lambda-based functions +like `array_enumerate <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_, +it expects the lambda function to have two input arguments. These are denoted using +``LambdaTwo``, ``LambdaThree``, etc within the function spec. + +LambdaReduce +~~~~~~~~~~~~ +LambdaReduce is special type of lambda that reduces an Array to a single value by calling the +LabmdaReduce function repeatedly on two elements in the Array until it is reduced to a single value. + +In this example, + +.. code-block:: yaml + + lambda_reduce_sum: >- + { + %array_reduce( [ 1, 2, 3, 4], %add ) + } + +We call +`array_reduce <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_ +on the input array, using +`add <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_ +as the LambdaReduce function. This will reduce the Array to a single value by internally calling + +.. code-block:: + + - %add(1, 2) = 3 + - %add(3, 3) = 6 + - %add(6, 4) = 10 + +And evaluate to ``10``. + +ReturnableArguments +~~~~~~~~~~~~~~~~~~~ + +Returnable arguments are used in conditional functions like +`if <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_, +which implies the argument passed into the function is the function's output. For example, + +.. code-block:: yaml + + conditional_function: >- + { + %if( True, "Return this if True", "Return this if False" ) + } + +is going to return ``"Return this if True"`` since the condition parameter is ``True``. \ No newline at end of file diff --git a/docs/source/config_reference/subscriptions_yaml.rst b/docs/source/config_reference/subscriptions_yaml.rst new file mode 100644 index 00000000..5c976149 --- /dev/null +++ b/docs/source/config_reference/subscriptions_yaml.rst @@ -0,0 +1,149 @@ +================== +Subscriptions File +================== +------------------ +subscriptions.yaml +------------------ + +The ``subscriptions.yaml`` file is where we use our :ref:`config_reference/config_yaml:presets` in the :ref:`config_reference/config_yaml:config.yaml` +to define a ``subscription``: something we want to recurrently download such as a specific +channel or playlist. + +The only difference between a ``subscription`` and ``preset`` is that the subscription +must have all required fields and ``{override_variables}`` defined so it can perform a download. + +Below is an example that downloads a YouTube playlist: + +.. code-block:: yaml + :caption: config.yaml + + presets: + playlist_preset_ex: + download: + download_strategy: "url" + url: "{url}" + output_options: + output_directory: "{output_directory}/{playlist_name}" + file_name: "{playlist_name}.{title}.{ext}" + overrides: + output_directory: "/path/to/ytdl-sub-videos" + +.. code-block:: yaml + :caption: subscription.yaml + + my_subscription_name: + preset: "playlist_preset_ex" + overrides: + playlist_name: "diy-playlist" + url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + +Our preset ``playlist_preset_ex`` defines three +custom variables: ``{output_directory}``, ``{playlist_name}``, and ``{url}``. The subscription sets +the ``parent preset`` to ``playlist_preset_ex``, and must define the variables ``{playlist_name}`` +and ``{url}`` since the preset did not. + +Beautifying Subscriptions +~~~~~~~~~~~~~~~~~~~~~~~~~ +Subscriptions support using presets as keys, and using keys to set override variables as values. +For example: + +.. code-block:: yaml + :caption: subscription.yaml + + TV Show Full Archive: + = News: + "Breaking News": "https://www.youtube.com/@SomeBreakingNews" + + TV Show Only Recent: + = Tech | TV-Y: + "Two Minute Papers": "https://www.youtube.com/@TwoMinutePapers" + +Will create two subscriptions named "Breaking News" and "Two Minute Papers", equivalent to: + +.. code-block:: yaml + + "Breaking News": + preset: + - "TV Show Full Archive" + + overrides: + subscription_indent_1: "News" + subscription_name: "Breaking News" + subscription_value: "https://www.youtube.com/@SomeBreakingNews" + + "Two Minute Papers": + preset: + - "TV Show Only Recent" + + overrides: + subscription_indent_1: "Tech" + subscription_indent_2: "TV-Y" + subscription_name: "Two Minute Papers" + subscription_value: "https://www.youtube.com/@TwoMinutePapers" + +You can provide as many parent presets in the form of ``keys``, and subscription indents as ``= keys``. +This can drastically simplify subscription definitions by setting things like so in your +parent preset: + +.. code-block:: yaml + + presets: + "TV Show Preset": + overrides: + subscription_indent_1: "default-genre" + subscription_indent_2: "default-content-rating" + + tv_show_name: "{subscription_name}" + url: "{subscription_value}" + genre: "{subscription_indent_1}" + content_rating: "{subscription_indent_2}" + +.. _subscription value: + +File Preset +~~~~~~~~~~~ + +You can apply a preset to all subscriptions in the ``subscription.yaml`` file +by using the file-wide ``__preset__``: + +.. code-block:: yaml + :caption: subscription.yaml + + __preset__: + preset: "playlist_preset_ex" + + my_subscription_name: + overrides: + url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + playlist_name: "diy-playlist" + +This ``subscription.yaml`` is equivalent to the one above it because all +subscriptions automatically set ``__preset__`` as a ``parent preset``. + + +Subscription Value +~~~~~~~~~~~~~~~~~~~ +NOTE: This is deprecated in favor of using the method in :ref:`config_reference/subscriptions_yaml:beautifying subscriptions`. + +With a clever config and use of ``__preset__``, your subscriptions can typically boil +down to a name and url. You can set ``__value__`` to the name of an override variable, +and use the override variable ``subscription_name`` to achieve one-liner subscriptions. +Using the example above, we can do: + +.. code-block:: yaml + :caption: subscription.yaml + + __preset__: + preset: + - "tv_show" + overrides: + tv_show_name: "{subscription_name}" + + __value__: "url" + + # single-line subscription, sets "Brandon Acker" and the subscription value + # to the override variables tv_show_name and url + "Brandon Acker": "https://www.youtube.com/@brandonacker" + +Traditional subscriptions that can override presets will still work when using ``__value__``. +``__value__`` can also be set within a :ref:`config_reference/config_yaml:config.yaml`. \ No newline at end of file diff --git a/docs/deprecation_notices.rst b/docs/source/deprecation_notices.rst similarity index 55% rename from docs/deprecation_notices.rst rename to docs/source/deprecation_notices.rst index 56c42d0a..29e3c5b0 100644 --- a/docs/deprecation_notices.rst +++ b/docs/source/deprecation_notices.rst @@ -5,55 +5,55 @@ Oct 2023 -------- subscription preset and value -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The use of ``__value__`` will go away in Dec 2023 in favor of the method found in -:ref:`beautifying subscriptions`. ``__preset__`` will still be supported for the time being. +:ref:`config_reference/subscriptions_yaml:beautifying subscriptions`. ``__preset__`` will still be supported for the time being. July 2023 --------- music_tags -^^^^^^^^^^ +~~~~~~~~~~ Music tags are getting simplified. ``tags`` will now reside directly under music_tags, and ``embed_thumbnail`` is getting moved to its own plugin (supports video files as well). Convert from: .. code-block:: yaml - my_example_preset: - music_tags: - embed_thumbnail: True - tags: - artist: "Elvis Presley" + my_example_preset: + music_tags: + embed_thumbnail: True + tags: + artist: "Elvis Presley" To the following: .. code-block:: yaml - my_example_preset: - embed_thumbnail: True - music_tags: - artist: "Elvis Presley" + my_example_preset: + embed_thumbnail: True + music_tags: + artist: "Elvis Presley" The old format will be removed in October 2023. video_tags -^^^^^^^^^^ +~~~~~~~~~~ Video tags are getting simplified as well. ``tags`` will now reside directly under video_tags. Convert from: .. code-block:: yaml - my_example_preset: - video_tags: - tags: - title: "Elvis Presley Documentary" + my_example_preset: + video_tags: + tags: + title: "Elvis Presley Documentary" To the following: .. code-block:: yaml - my_example_preset: - video_tags: - title: "Elvis Presley Documentary" + my_example_preset: + video_tags: + title: "Elvis Presley Documentary" diff --git a/docs/source/faq/index.rst b/docs/source/faq/index.rst new file mode 100644 index 00000000..c728b7ca --- /dev/null +++ b/docs/source/faq/index.rst @@ -0,0 +1,59 @@ +FAQ +=== + +Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as +more questions get asked. + +.. contents:: Frequently Asked Questions + :depth: 3 + +How do I... +----------- + +...download age-restricted YouTube videos? +'''''''''''''''''''''''''''''''''''''''''' +See +`ytdls recommended way <https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_ +to download your YouTube cookie, then add it to your +`ytdl options <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl-options>`_ section of your config: + +.. code-block:: yaml + + ytdl_options: + cookiefile: "/path/to/cookies/file.txt" + +...automate my downloads? +''''''''''''''''''''''''' +`This part of the wiki <https://github.com/jmbannon/ytdl-sub/wiki/7.-Automate-Downloading-New-Content-Using-Your-Configs>`_ shows how to set up ``ytdl-sub`` to run in a cron job within Docker. + +There is a bug where... +----------------------- + +...date_range is not downloading older videos after I changed the range +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether. + +.. code-block:: yaml + + ytdl_options: + break_on_existing: False + +After your download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads. + +...it is downloading non-English title and description metadata +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using + +.. code-block:: yaml + + ytdl_options: + extractor_args: + youtube: + lang: + - "en" + +...Plex is not showing my TV shows correctly +'''''''''''''''''''''''''''''''''''''''''''' +Set the following +`Scanner and Agent <https://i.imgur.com/zdZhCLZ.png>`_ +for your library. \ No newline at end of file diff --git a/docs/source/guides/development/index.rst b/docs/source/guides/development/index.rst new file mode 100644 index 00000000..e606005d --- /dev/null +++ b/docs/source/guides/development/index.rst @@ -0,0 +1,5 @@ +Development and Contributing +============================ + +.. toctree:: + \ No newline at end of file diff --git a/docs/source/guides/getting_started/advanced_configuration.rst b/docs/source/guides/getting_started/advanced_configuration.rst new file mode 100644 index 00000000..385ed3b7 --- /dev/null +++ b/docs/source/guides/getting_started/advanced_configuration.rst @@ -0,0 +1,21 @@ +Advanced Configuration +====================== + +If the :doc:`prebuilt presets </prebuilt_presets/index>` aren't suitable for your needs, you may want to set up an advanced configuration. + +Layout of a Config file +----------------------- + +The layout of the ``config.yaml`` file is relatively straightforward: + +.. code-block:: yaml + + presets: + + preset_name: + plugin1: + plugin1_option1: value1 + + +Preset Inheritance +------------------ diff --git a/docs/source/guides/getting_started/automating_downloads.rst b/docs/source/guides/getting_started/automating_downloads.rst new file mode 100644 index 00000000..53791dde --- /dev/null +++ b/docs/source/guides/getting_started/automating_downloads.rst @@ -0,0 +1,140 @@ +Automating Downloads +==================== + +One of the key capabilities of ``ytdl-sub`` is how well it runs without user input, but to take advantage of this you must set up scheduling to execute the commands at some interval. How you set up this scheduling depends on which version of ``ytdl-sub`` you downloaded. + + +:ref:`Guide for Docker and Unraid Containers <guides/getting_started/automating_downloads:docker and unraid>` + +:ref:`Guide for Linux <guides/getting_started/automating_downloads:linux>` + +:ref:`Guide for Windows <guides/getting_started/automating_downloads:windows>` + +.. _cron tab manpage: https://man7.org/linux/man-pages/man5/crontab.5.html#EXAMPLE_CRON_FILE + +.. _docker-unraid-setup: + +Docker and Unraid +----------------- + +.. tab-set:: + + .. tab-item:: GUI Image + + The script that will execute automatically is located at ``/config/ytdl-sub-configs/run-cron``. + + Access your container at http://localhost:8443/, then in the GUI terminal run these commands: + + .. code-block:: shell + + echo '#!/bin/bash' > /config/ytdl-sub-configs/run_cron + echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/ytdl-sub-configs/run_cron + echo "echo 'Cron started, running ytdl-sub...'" >> /config/ytdl-sub-configs/run_cron + echo "cd /config/ytdl-sub-configs" >> /config/ytdl-sub-configs/run_cron + echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/ytdl-sub-configs/run_cron + chmod +x /config/ytdl-sub-configs/run_cron + chown abc:abc /config/ytdl-sub-configs/run_cron + + You can test the newly created script by running: + + .. code-block:: shell + + /config/ytdl-sub-configs/run_cron + + To create the cron definition, run the following command: + + .. code-block:: shell + + echo "# min hour day month weekday command" > /config/crontabs/abc + echo " 0 */6 * * * /config/ytdl-sub-configs/run_cron" >> /config/crontabs/abc + + This will run the script every 6 hours. To run every hour, change ``*/6`` to ``*/1``, or to run once a day, change the same value to the hour (in 24hr format) that you want it to run at. See the `cron tab manpage`_ for more options. + + .. tab-item:: Headless Image + + .. _LinuxServer's Universal Cron mod: https://github.com/linuxserver/docker-mods/tree/universal-cron + + The first step is to ensure you have `LinuxServer's Universal Cron mod`_ enabled via the environment variable. For the GUI image, this is already included (no need to add it). + + .. code-block:: yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub:latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + - DOCKER_MODS=linuxserver/mods:universal-cron # <-- Make sure you have this! + volumes: + # ensure directories have user permissions + - </path/to/ytdl-sub/config>:/config + - </path/to/ytdl-sub/tv_shows>:/tv_shows + restart: unless-stopped + + This line will tell your container to install and enable cron on start. + + If you had to add this line, you will need to restart your container. + + .. code-block:: shell + + docker compose restart + + The script that will execute automatically is located at ``/config/run-cron``. + + Access your container from the terminal by running: + + .. code-block:: shell + + docker exec -itu abc ytdl-sub /bin/bash + + then in the terminal run these commands: + + .. code-block:: shell + + echo '#!/bin/bash' > /config/ytdl-sub-configs/run_cron + echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/ytdl-sub-configs/run_cron + echo "echo 'Cron started, running ytdl-sub...'" >> /config/ytdl-sub-configs/run_cron + echo "cd /config/ytdl-sub-configs" >> /config/ytdl-sub-configs/run_cron + echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/ytdl-sub-configs/run_cron + chmod +x /config/ytdl-sub-configs/run_cron + chown abc:abc /config/ytdl-sub-configs/run_cron + + You can test the newly created script by running: + + .. code-block:: + + /config/run_cron + + To create the cron definition, run the following command: + + .. code-block:: shell + + echo "# min hour day month weekday command" > /config/crontabs/abc + echo " 0 */6 * * * /config/run_cron" >> /config/crontabs/abc + + This will run the script every 6 hours. To run every hour, change ``*/6`` to ``*/1``, or to run once a day, change the same value to the hour (in 24hr format) that you want it to run at. See the `cron tab manpage`_ for more options. + +.. _linux-setup: + +Linux +----- + +.. code-block:: shell + + crontab -e + 0 */6 * * * /config/run_cron + + + + +.. _windows-setup: + +Windows +------- +To be tested (please contact code owner or join the discord server if you can test this out for us) + +.. code-block:: powershell + + ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml \ No newline at end of file diff --git a/docs/source/guides/getting_started/examples.rst b/docs/source/guides/getting_started/examples.rst new file mode 100644 index 00000000..7045ef7b --- /dev/null +++ b/docs/source/guides/getting_started/examples.rst @@ -0,0 +1,15 @@ +===================== +Using Example Configs +===================== +Copy and paste the examples into local yaml files, modify the +``working_directory`` and ``output_directory`` with your desired paths, +and perform a dry-run using + +.. code-block:: bash + + ytdl-sub \ + --dry-run \ + --config path/to/config.yaml \ + sub path/to/subscriptions.yaml + +This will simulate what a download will look like. \ No newline at end of file diff --git a/docs/source/guides/getting_started/first_config.rst b/docs/source/guides/getting_started/first_config.rst new file mode 100644 index 00000000..3deac497 --- /dev/null +++ b/docs/source/guides/getting_started/first_config.rst @@ -0,0 +1,25 @@ +Basic Configuration +=================== + +Your first configuration will look pretty simple: + +.. code-block:: yaml + :linenos: + + configuration: + working_directory: '.ytdl-sub-downloads' + + + +The first two lines in this ``config.yaml`` file are the ``configuration``, and define the ``working_directory``, which is described near the bottom of :ref:`this section <guides/getting_started/index:quick overview of \`\`ytdl-sub\`\`>` + + +Line 4 begins the definition of your custom ``presets``, with line 5 being the name of your first custom ``preset``. + +Lines 7 and 8 tell ``ytdl-sub`` which :doc:`/prebuilt_presets/index` to expand on; these ``presets`` already indicate that the downloaded files should be: + +- in a format usable by, and with metadata accessible to, Jellyfin +- sorted by upload date, and +- only uploaded in the last 2 months (and will also delete any files in the media library which were uploaded over 2 months ago) + +Line 11 is an override variable, ``tv_show_directory``, that tells ``ytdl-sub`` where to save your downloaded files once they've been processed, also known as the ``output_directory``. In this case, the downloaded files will be saved to the ``youtube`` folder in the root ``tv_shows`` directory. \ No newline at end of file diff --git a/docs/source/guides/getting_started/first_download.rst b/docs/source/guides/getting_started/first_download.rst new file mode 100644 index 00000000..8babc19e --- /dev/null +++ b/docs/source/guides/getting_started/first_download.rst @@ -0,0 +1,30 @@ +Initial Download +================ + +Once you have the ``config.yaml`` and ``subscriptions.yaml`` files created and filled out, you can perform your first download. Access ``ytdl-sub``, navigate to the directory containing your ``config.yaml`` and ``subscriptions.yaml`` files, then run the below command: + +.. tab-set:: + + .. tab-item:: Dry run + + A dry run lets you check that your configuration doesn't throw any errors and what the expected output files of actually doing the download are, without actually downloading the full media. + + .. code-block:: shell + + ytdl-sub --dry-run sub + + .. tab-item:: Normal run + + A normal run will download all files as determined by your ``presets`` and, once processing is finished, move the downloaded and processed files to your ``output_directory``. + + .. code-block:: shell + + ytdl-sub sub + + .. tab-item:: One-time download + + Sometimes you may only want to download media once, in which case adding them to your ``subscriptions.yaml`` file is unneccessary. As an example, the below code will download the same videos as our subscription file: + + .. code-block:: shell + + ytdl-sub dl --preset "My Favorite YouTube Channels" --overrides.subscription_name "Rick Astley" --overrides.subscription_value "https://www.youtube.com/@RickAstleyYT/videos" \ No newline at end of file diff --git a/docs/source/guides/getting_started/first_sub.rst b/docs/source/guides/getting_started/first_sub.rst new file mode 100644 index 00000000..b2a83a88 --- /dev/null +++ b/docs/source/guides/getting_started/first_sub.rst @@ -0,0 +1,21 @@ +Initial Subscription +==================== + +Your first subscription should look similar to the below: + +.. code-block:: yaml + :linenos: + + __preset__: + overrides: + tv_show_directory: "/tv_shows" + + "My Favorite YouTube Channels": + "Rick Astley": "https://www.youtube.com/@RickAstleyYT/videos" + + +The first three lines in this subscription file define where to save the downloaded files associated with all subscriptions in this file. + +The fifth line in this subscription file is the ``preset``, which provides the "definitions" for the subscription as listed in :doc:`/guides/getting_started/first_config`. + +The sixth line is the actual ``subscription``, named ``Rick Astley``, with a link to a :yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`, in this case a YouTube channel. \ No newline at end of file diff --git a/docs/source/guides/getting_started/index.rst b/docs/source/guides/getting_started/index.rst new file mode 100644 index 00000000..c503a5ad --- /dev/null +++ b/docs/source/guides/getting_started/index.rst @@ -0,0 +1,88 @@ +Getting Started +=============== + +Now that you've completed your install of ``ytdl-sub``, it's time to get started. This is a 3-step process: + +- Create your configuration file (if the :doc:`/prebuilt_presets/index` don't fit your needs) +- Create your subscription file +- Automate starting YTDL-Sub + +Prerequisite Knowledge +---------------------- + +.. _navigate directories: https://en.wikipedia.org/wiki/Cd_(command) +.. _YAML syntax: https://yaml.org/spec/1.2.2/#chapter-2-language-overview + + +In order to use ``ytdl-sub`` in any of the forms listed in these docs, you will need some basic knowledge. + +Be sure that you: + ☑ Can `navigate directories`_ in a command line interface (or CLI) + + ☑ Have a basic understanding of `YAML syntax`_ + +If you plan on using the headless image of ``ytdl-sub``, you: + ☑ Can use ``nano`` or ``vim`` to edit OR + + ☑ Can mount the config directory somewhere you can open it using gui text editors + +Additional useful (but not required) knowledge: + ☑ Understanding how :yt-dlp:`\ ` works + + +Quick Overview of ``ytdl-sub`` +------------------------------ + +``ytdl-sub`` uses two types of YAML files: + +- ``config.yaml`` defines ``presets``, which are the "definitions" of your media. ``presets`` "define" how you want your media downloaded, which formats, naming conventions to follow when saving them, etc. These ``presets`` can also inherit other ``presets``, so that you can easily modify an existing ``preset``. +- ``subscriptions.yaml`` defines ``subscriptions``, which specify the media we want to recurrently download, like YouTube channels and playlists, SoundCloud artists, or any :yt-dlp:`yt-dlp supported site <blob/master/supportedsites.md>`. ``subscriptions`` use ``presets`` to define how ``ytdl-sub`` should handle downloading, processing, and saving them. + +When ``ytdl-sub`` is run, in its most basic form: + +.. tab-set-code:: + + .. code-block:: shell + + ytdl-sub sub + + .. code-block:: powershell + + ytdl-sub.exe sub + +``ytdl-sub`` initially downloads all files to a defined ``working_directory``. This is a temporary storage spot for metadata and media files so that errors during processing- if they occur- don't affect your existing media library. Once all file processing is complete, your media files are moved to the ``output_directory``. + +Ready to Start? +--------------- + +Now that you have installed ``ytdl-sub``, checked your skills, and gotten a bit of background on how ``ytdl-sub`` functions, read through the articles below to get started: + +:doc:`Step 1: Initial Subscriptions <first_sub>` + +:doc:`Step 2: Your First Download <first_download>` + +:doc:`Step 3: Automating Downloads <automating_downloads>` + +Want to go a step further? + +If you want to use atypical paths or specific configuration options, check out :doc:`Basic Configuration <first_config>` + +For tips on creating your own presets when the prebuilt presets aren't cutting it, check out :doc:`Advanced Configuration <advanced_configuration>` + +Other docs that may be of use: + +:doc:`/prebuilt_presets/index` + +:doc:`examples` + +.. toctree:: + :hidden: + :caption: Getting Started Guide + :maxdepth: 1 + + first_sub + first_download + automating_downloads + first_config + advanced_configuration + examples \ No newline at end of file diff --git a/docs/source/guides/index.rst b/docs/source/guides/index.rst new file mode 100644 index 00000000..114f5acc --- /dev/null +++ b/docs/source/guides/index.rst @@ -0,0 +1,7 @@ +Guides +====== + +.. toctree:: + install/index + getting_started/index + development/index \ No newline at end of file diff --git a/docs/source/guides/install/agnostic.rst b/docs/source/guides/install/agnostic.rst new file mode 100644 index 00000000..658dd5ed --- /dev/null +++ b/docs/source/guides/install/agnostic.rst @@ -0,0 +1,43 @@ +==================== +Environment Agnostic +==================== + +The PIP install method is not recommended; use of this method may cause unintended requirement conflicts if you have other locally installed apps that depend on ffmpeg. + +PIP Install +-------------- +You can install our +`PyPI package <https://pypi.org/project/ytdl-sub/>`_. +Both ffmpeg and Python 3.10 or greater are required. + +.. code-block:: bash + + python3 -m pip install -U ytdl-sub + +Install for Development +======================= + +These environment-agnostic methods of installing ``ytdl-sub`` are meant for local development of ``ytdl-sub``. If you want to contribute your changes, please read :doc:`/guides/development/index`. + +Local Install +-------------- +With a Python 3.10 virtual environment, you can clone and install the repo. + +.. code-block:: bash + + git clone https://github.com/jmbannon/ytdl-sub.git + cd ytdl-sub + + pip install -e . + +Local Docker Build +------------------- +Run ``make docker`` in the root directory of this repo to build the image. This +will build the python wheel and install it in the Dockerfile. + +.. code-block:: bash + + git clone https://github.com/jmbannon/ytdl-sub.git + cd ytdl-sub + + make docker \ No newline at end of file diff --git a/docs/source/guides/install/docker.rst b/docs/source/guides/install/docker.rst new file mode 100644 index 00000000..9c89e9be --- /dev/null +++ b/docs/source/guides/install/docker.rst @@ -0,0 +1,155 @@ +====== +Docker +====== + +For automating ``subscriptions.yaml`` downloads to pull new media, see :ref:`this page <guides/getting_started/automating_downloads:docker and unraid>` on how to set up a cron job in any of the docker containers. + +The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install ytdl-sub on top. There are two flavors to choose from. + +.. margin:: + + .. tip:: + + The recommended docker image is the GUI image. + + :ref:`Docker Compose <guides/install/docker:install with docker compose>` is the recommended way of setting up a ``ytdl-sub`` docker container. + +GUI Image +--------- + +The GUI image uses LSIO's :lsio-gh:`docker-code-server image` for its base image. More info on other code-server environment variables can be found within its documentation. + +After starting, code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``. + +Headless Image +-------------- + +The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image` for its base image. Execute the following command to access and interact with ``ytdl-sub``: + +.. code-block:: bash + + docker exec -u abc -it ytdl-sub /bin/bash + +Install with Docker Compose +--------------------------- + +Docker Compose is an easy "set it and forget it" install method. Follow the instructions below to create a ``compose.yaml`` file for your chosen ``ytdl-sub`` image. + +.. margin:: + + .. important:: + + Set the PUID and PGID to the UID and GID associated with the user you want to own the downloaded files. Setting these values to root UID and GID may create issues with your media managers. + +.. tab-set:: + + .. tab-item:: GUI Image + + .. code-block:: yaml + :caption: compose.yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub-gui:latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + volumes: + - <path/to/ytdl-sub/config>:/config + - <path/to/tv_shows>:/tv_shows # optional + - <path/to/movies>:/movies # optional + - <path/to/music_videos>:/music_videos # optional + - <path/to/music>:/music # optional + ports: + - 8443:8443 + restart: unless-stopped + + .. tab-item:: Headless Image + + .. code-block:: yaml + :caption: compose.yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub:latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + - DOCKER_MODS=linuxserver/mods:universal-cron + volumes: + - <path/to/ytdl-sub/config>:/config + - <path/to/tv_shows>:/tv_shows # optional + - <path/to/movies>:/movies # optional + - <path/to/music_videos>:/music_videos # optional + - <path/to/music>:/music # optional + restart: unless-stopped + +Device Passthrough +~~~~~~~~~~~~~~~~~~~ +For CPU or GPU passthrough, you must use either the GUI image or the headless Ubuntu image +``ghcr.io/jmbannon/ytdl-sub:ubuntu-latest``. + +The docker-compose examples use the GUI image. + +CPU Passthrough +^^^^^^^^^^^^^^^ + +.. code-block:: yaml + :emphasize-lines: 5-6 + :caption: compose.yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub-gui:latest + container_name: ytdl-sub + devices: + - /dev/dri:/dev/dri # CPU passthrough + restart: unless-stopped + +GPU Passthrough +^^^^^^^^^^^^^^^ + +.. Awe + +.. code-block:: yaml + :caption: compose.yaml + :emphasize-lines: 5-13 + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub-gui:latest + container_name: ytdl-sub + environment: + - .. + - NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args + - NVIDIA_VISIBLE_DEVICES=all + deploy: + resources: + reservations: + devices: + - capabilities: ["gpu"] # GPU passthrough + restart: unless-stopped + +Docker CLI +---------- + +If you prefer to only run the container once, you can use the CLI command instead. The following command is for the gui image, and will not restart if it comes down for any reason. See `the Docker reference <https://docs.docker.com/engine/reference/run/>`_ for further information on the parameters and other options you can use. + +.. code-block:: bash + + docker run -d \ + --name=ytdl-sub \ + -e PUID=1000 \ + -e PGID=1000 \ + -e TZ=America/Los_Angeles \ + -p 8443:8443 \ + -v <path/to/ytdl-sub/config>:/config \ + -v <OPTIONAL/path/to/tv_shows>:/tv_shows \ + -v <OPTIONAL/path/to/movies>:/movies \ + -v <OPTIONAL/path/to/music_videos>:/music_videos \ + -v <OPTIONAL/path/to/music>:/music \ + ghcr.io/jmbannon/ytdl-sub-gui:latest \ No newline at end of file diff --git a/docs/source/guides/install/index.rst b/docs/source/guides/install/index.rst new file mode 100644 index 00000000..ff1d293f --- /dev/null +++ b/docs/source/guides/install/index.rst @@ -0,0 +1,34 @@ +Install by Platform +=================== +``ytdl-sub`` can be installed on the following platforms. + +All installations require a 64-bit CPU. 32-bit is not supported. + +.. margin:: + + .. tip:: + + The recommended install method of ``ytdl-sub`` is one of our :doc:`docker containers </guides/install/docker>`. For install on Unraid, check out our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>`. + +:doc:`/guides/install/docker` + +:doc:`/guides/install/unraid` + +:doc:`/guides/install/linux` + +:doc:`/guides/install/windows` + +:doc:`/guides/install/agnostic` + + + +Once you've completed your installation, please refer to the :doc:`../getting_started/index` guide for next steps + +.. toctree:: + :hidden: + + docker + linux + unraid + windows + agnostic diff --git a/docs/source/guides/install/linux.rst b/docs/source/guides/install/linux.rst new file mode 100644 index 00000000..1eaf6982 --- /dev/null +++ b/docs/source/guides/install/linux.rst @@ -0,0 +1,50 @@ +===== +Linux +===== + +``ytdl-sub`` should be installable using any Linux package manager, and requires ffmpeg to be installed. + + +.. tab-set:: + + .. tab-item:: Linux + + .. code-block:: bash + + curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub + chmod +x ytdl-sub + ytdl-sub -h + + You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date: + + .. code-block:: bash + + curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz + tar -xf ffmpeg.tar.gz + chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg + chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe + + # May need sudo / root permissions to perform + mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg + mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe + + .. tab-item:: Linux ARM + + .. code-block:: bash + + curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub_aarch64 + chmod +x ytdl-sub + ytdl-sub -h + + You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date: + + .. code-block:: bash + + curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz + tar -xf ffmpeg.tar.gz + chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg + chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe + + # May need sudo / root permissions to perform + mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg + mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe \ No newline at end of file diff --git a/docs/source/guides/install/unraid.rst b/docs/source/guides/install/unraid.rst new file mode 100644 index 00000000..ac979a99 --- /dev/null +++ b/docs/source/guides/install/unraid.rst @@ -0,0 +1,3 @@ +Unraid +-------------- +You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>` through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_. Uses Docker under the hood. \ No newline at end of file diff --git a/docs/source/guides/install/windows.rst b/docs/source/guides/install/windows.rst new file mode 100644 index 00000000..7c4050c2 --- /dev/null +++ b/docs/source/guides/install/windows.rst @@ -0,0 +1,15 @@ +Windows +-------------- +From powershell, run: + +.. code-block:: powershell + + # Download ffmpeg/ffprobe dependencies from yt-dlp + curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip + tar -xf ffmpeg.zip + move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe" + move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe" + + # Download ytdl-sub + curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe + ytdl-sub.exe -h \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..2fda8ee7 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,16 @@ +ytdl-sub User Guide +=================== + +.. toctree:: + :maxdepth: 2 + :titlesonly: + + introduction + guides/index + prebuilt_presets/index + usage + config_reference/index + faq/index + deprecation_notices + +.. note:: The docs are heavily work-in-progress. Please bear with us while we're under construction! \ No newline at end of file diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst new file mode 100644 index 00000000..0126202a --- /dev/null +++ b/docs/source/introduction.rst @@ -0,0 +1,43 @@ +================= +What is ytdl-sub? +================= + +.. _yt-dlp: https://github.com/yt-dlp/yt-dlp +.. _kodi: https://github.com/xbmc/xbmc +.. _jellyfin: https://github.com/jellyfin/jellyfin +.. _plex: https://github.com/plexinc/pms-docker +.. _emby: https://github.com/plexinc/pms-docker + +``ytdl-sub`` is a command-line tool that downloads media via `yt-dlp`_ and prepares it for your favorite media player (`Kodi`_, `Jellyfin`_, `Plex`_, `Emby`_, modern music players). + +Visual examples +=============== + +.. figure:: https://user-images.githubusercontent.com/10107080/182677243-b4184e51-9780-4094-bd40-ea4ff58555d0.PNG + :alt: The Jellyfin web interface, showing the thumbnails of various YouTube shows. + + Youtube channels as TV shows in Jellyfin + +.. figure:: https://user-images.githubusercontent.com/10107080/182677256-43aeb029-0c3f-4648-9fd2-352b9666b262.PNG + :alt: The Jellyfin web interace, showing the thumbnails of various music videos starring the Red Hot Chili Peppers + + Music videos and concerts in Jellyfin + +.. figure:: https://user-images.githubusercontent.com/10107080/182677268-d1bf2ff0-9b9c-4a04-98ec-443a67ada734.png + :alt: The Kodi app interface, showing a list of artists available to watch under the "Music videos" heading + + Music videos and concerts in Kodi + +.. figure:: https://user-images.githubusercontent.com/10107080/182685415-06adf477-3dd3-475d-bbcd-53b0152b9f0a.PNG + :alt: The MusicBee app interface, showing a list of album artists and the thumbnails of all downloaded songs produced by the currently selected artist + + SoundCloud albums and singles in MusicBee + + +Why ytdl-sub? +------------- +There is a lack of open-source tools to download media and generate metadata to play it in these players. Most solutions involve using multiple tools or bash scripts to achieve this. ``ytdl-sub`` aims to consolidate all of this logic into a single easy-to-use application that can run automatically once configured. + +Why download instead of stream? +------------------------------- +We believe it is important to download what you like because there is no guarantee it will stay online forever. We also believe it is important to download it in such a way that it is easy to consume. Most solutions today force you to watch/listen to your downloaded content via file system or web browser. ``ytdl-sub`` aims to format downloaded content for any media player. \ No newline at end of file diff --git a/docs/source/prebuilt_presets/helpers.rst b/docs/source/prebuilt_presets/helpers.rst new file mode 100644 index 00000000..f0392aa7 --- /dev/null +++ b/docs/source/prebuilt_presets/helpers.rst @@ -0,0 +1,37 @@ +============== +Helper Presets +============== + +Common presets are not usable by themselves- setting one of these as the sole preset of your subscription and attempting to download will not work. But you can add these presets to quickly modify an existing preset to better suit your needs. + +Best A/V Quality +---------------- + +Add the following preset to download the best available video and audio quality, and remux it into an MP4 container: + +``best_video_quality`` + + +Max 1080p Video +--------------- + +Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container: + +``max_1080p`` + +Chunk Initial Download +---------------------- + +If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the following preset: + +``chunk_initial_download`` + +It will download videos starting from the oldest one, and only download 20 at a time. You can +change this number by setting: + +.. code-block:: yaml + + ytdl_options: + max_downloads: 30 # Desired number to download per invocation + +Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to oldest again, and stop pulling additional metadata once it reaches a video that has already been downloaded. \ No newline at end of file diff --git a/docs/source/prebuilt_presets/index.rst b/docs/source/prebuilt_presets/index.rst new file mode 100644 index 00000000..00efc8ba --- /dev/null +++ b/docs/source/prebuilt_presets/index.rst @@ -0,0 +1,14 @@ +================ +Prebuilt Presets +================ + +``ytdl-sub`` offers a number of built-in presets using best practices for formatting +media in various players. For advanced users, you can review the prebuilt preset +definitions :doc:`here </config_reference/prebuilt_presets/index>`. + +.. toctree:: + :titlesonly: + + helpers + tv_shows + music \ No newline at end of file diff --git a/docs/source/prebuilt_presets/music.rst b/docs/source/prebuilt_presets/music.rst new file mode 100644 index 00000000..60552eea --- /dev/null +++ b/docs/source/prebuilt_presets/music.rst @@ -0,0 +1,3 @@ +============= +Music Presets +============= \ No newline at end of file diff --git a/docs/source/prebuilt_presets/tv_shows.rst b/docs/source/prebuilt_presets/tv_shows.rst new file mode 100644 index 00000000..f2b1da1d --- /dev/null +++ b/docs/source/prebuilt_presets/tv_shows.rst @@ -0,0 +1,171 @@ +=============== +TV Show Presets +=============== + +Player-Specific Presets +======================= + +``ytdl-sub`` provides player-specific versions of certain presets, which apply settings to optimize the downloads for that player. + +The following actions are taken based on the indicated player: + + +Jellyfin +~~~~~~~~ +* Places any season-specific poster art in the main show folder +* Generates NFO tags + +Kodi +~~~~ +* Everything that the Jellyfin version does +* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□`` + +Plex +~~~~ +* :ref:`Special sanitization <config_reference/scripting/entry_variables:title_sanitized_plex>` of numbers so Plex doesn't recognize numbers that are part of the title as the episode number +* Converts all downloaded videos to the mp4 format +* Places any season-specific poster art into the season folder + +---------------------------------------------- + +Generic Presets +=============== + +There are two main methods for downloading and formatting videos as a TV show. + +TV Show by Date +--------------- + +TV Show by Date will organize something like a YouTube channel or playlist into a tv show, where seasons and episodes are organized using upload date. + +Plug and Play Presets +~~~~~~~~~~~~~~~~~~~~~ + +You can use any of these presets in your ``subscriptions.yaml`` as a "One size fits all" solution- they should set all appropriate values. These will organize seasons by year and episodes by month, then day. + +Must define ``tv_show_directory`` + +* ``"Kodi TV Show by Date"`` +* ``"Jellyfin TV Show by Date"`` +* ``"Plex TV Show by Date"`` + +Advanced Usage +~~~~~~~~~~~~~~ + +If you prefer a different organization method, you can instead apply multiple presets to your subscriptions. + +You will need a base of one of the below: + +* ``kodi_tv_show_by_date`` +* ``jellyfin_tv_show_by_date`` +* ``plex_tv_show_by_date`` + +And then add one of these: + +* ``season_by_year__episode_by_month_day`` +* ``season_by_year_month__episode_by_day`` +* ``season_by_year__episode_by_month_day_reversed`` + + * Episode numbers are reversed, meaning more recent episodes appear at the top of a season by having a lower value. +* ``season_by_year__episode_by_download_index`` + + * Episodes are numbered by the download order. NOTE that this is fetched using the length of the download archive. Do not use if you intend to remove old videos. + +An example of a subscription that will be played on Kodi, organized by year with the most recent episode at the top (having a lower episode number), with a genre of "Pop": + +.. code-block:: yaml + :caption: subscriptions.yaml + + __preset__: + overrides: + tv_show_directory: "/tv_shows" + + kodi_tv_show_by_date: + season_by_year_episode_by_month_day_reversed: + = Pop: + "Rick A": "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" + +You can also choose to combine multiple URLs into one show. This will result in your videos being downloaded to the same folder, and the episode numbers being shared between them (so you won't have two episode 10's, for example). Note that you may :ytdl-sub-gh:`experience issues <issues/833>` if you use more than 20 URLs at this time. + +.. code-block:: yaml + :caption: subscriptions.yaml + + __preset__: + overrides: + tv_show_directory: "/tv_shows" + + kodi_tv_show_by_date: + season_by_year_episode_by_month_day_reversed: + = Pop: + "~Rick A": + url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" + url2: "https://www.youtube.com/@just.rick_6" + + +TV Show Collection +------------------ + +TV Show Collections are made up of multiple URLs, where each URL is a season. +If a video belongs to multiple URLs (i.e. a channel and a channel's playlist), +it will resolve to the bottom-most season, as defined in the subscription. + +Two main use cases of a collection are: + 1. Organize a YouTube channel TV show where Season 1 contains any video + not in a 'season playlist', Season 2 for 'Playlist A', Season 3 for + 'Playlist B', etc. + 2. Organize one or more YouTube channels/playlists, where each season + represents a separate channel/playlist. + +Player Presets +~~~~~~~~~~~~~~ + +* ``kodi_tv_show_collection`` +* ``jellyfin_tv_show_collection`` +* ``plex_tv_show_collection`` + +Episode Formatting Presets +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``season_by_collection__episode_by_year_month_day`` +* ``season_by_collection__episode_by_year_month_day_reversed`` +* ``season_by_collection__episode_by_playlist_index`` + + * Only use playlist_index episode formatting for playlists that will be fully downloaded once and never again. Otherwise, indices can change. +* ``season_by_collection__episode_by_playlist_index_reversed`` + +Season Presets +~~~~~~~~~~~~~~ + +* ``collection_season_1`` +* ``collection_season_2`` +* ``collection_season_3`` +* ``collection_season_4`` +* ``...`` +* ``collection_season_40`` + +Example +~~~~~~~ + +A preset/subscription requires specifying a player, episode formatting, and +one or more season presets, with the following override variables: + +.. code-block:: yaml + + rick_a_tv_show_collection: + preset: + - "jellyfin_tv_show_collection" + - "season_by_collection__episode_by_year_month_day_reversed" + - "collection_season_1" + - "collection_season_2" + overrides: + # required + tv_show_name: "Rick A" + tv_show_directory: "/path/to/youtube_shows" + collection_season_1_url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" + collection_season_1_name: "All Videos" + collection_season_2_url: "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" + collection_season_2_name: "Official Music Videos" + # can be modified from their default value + # tv_show_genre: "ytdl-sub" + # episode_title: "{upload_date_standardized} - {title}" + # episode_description: "{webpage_url}" \ No newline at end of file diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt new file mode 100644 index 00000000..c8203d0f --- /dev/null +++ b/docs/source/requirements.txt @@ -0,0 +1,3 @@ +sphinx-book-theme==1.0.1 +sphinx-copybutton==0.5.2 +sphinx-design==0.5.0 \ No newline at end of file diff --git a/docs/usage.rst b/docs/source/usage.rst similarity index 82% rename from docs/usage.rst rename to docs/source/usage.rst index b0d01cb9..0a9f1d3e 100644 --- a/docs/usage.rst +++ b/docs/source/usage.rst @@ -3,7 +3,7 @@ Usage .. code-block:: - ytdl-sub [GENERAL OPTIONS] {sub,dl,view} [COMMAND OPTIONS] + ytdl-sub [GENERAL OPTIONS] {sub,dl,view} [COMMAND OPTIONS] For Windows users, it would be ``ytdl-sub.exe`` @@ -43,28 +43,28 @@ Download a single subscription in the form of CLI arguments. .. code-block:: - ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS] + ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS] ``SUBSCRIPTION ARGUMENTS`` are exactly the same as YAML arguments, but use periods (``.``) instead of indents for specifying YAML from the CLI. For example, you can represent this subscription: .. code-block:: yaml - rick_a: - preset: - - "tv_show" - overrides: - tv_show_name: "Rick A" - url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" + rick_a: + preset: + - "tv_show" + overrides: + tv_show_name: "Rick A" + url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" Using the command: .. code-block:: bash - ytdl-sub dl \ - --preset "tv_show" \ - --overrides.tv_show_name "Rick A" \ - --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" + ytdl-sub dl \ + --preset "tv_show" \ + --overrides.tv_show_name "Rick A" \ + --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" See how to shorten commands using `download aliases <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_. diff --git a/examples/advanced/tv_show_config.yaml b/examples/advanced/tv_show_config.yaml index b2d0c694..9cfa4ca7 100644 --- a/examples/advanced/tv_show_config.yaml +++ b/examples/advanced/tv_show_config.yaml @@ -97,4 +97,5 @@ presets: - "Only Recent" overrides: - date_range: "2months" + only_recent_date_range: "2months" + only_recent_max_files: 30 \ No newline at end of file diff --git a/examples/tv_show_subscriptions.yaml b/examples/tv_show_subscriptions.yaml index e8fea998..d8b914cb 100644 --- a/examples/tv_show_subscriptions.yaml +++ b/examples/tv_show_subscriptions.yaml @@ -23,7 +23,10 @@ __preset__: overrides: tv_show_directory: "/tv_shows" # Root folder of all ytdl-sub TV Shows - date_range: "2months" # For 'Only Recent' preset, only keep vids uploaded in this range + + # For 'Only Recent' preset, only keep vids within this range and limit + only_recent_date_range: "2months" + only_recent_max_files: 30 # Choose the player you intend to use by setting the top-level key to be either: # - Plex TV Show by Date: diff --git a/pyproject.toml b/pyproject.toml index f04e9334..d78c5633 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ force_single_line = true [tool.black] line_length = 100 +target-version = ["py310"] [tool.pylint.MASTER] disable = [ @@ -43,3 +44,8 @@ ignore = [ include = [ "src/*" ] + +[tool.coverage.report] +exclude_also = [ + "raise UNREACHABLE.*", +] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 64241293..db9e581d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ package_dir = packages=find: install_requires = - yt-dlp==2023.10.13 + yt-dlp==2023.11.16 argparse==1.4.0 colorama==0.4.6 mergedeep==1.3.4 @@ -49,7 +49,6 @@ lint = black==22.3.0 isort==5.10.1 pylint==2.13.5 - pydocstyle[toml]==6.1.1 docs = sphinx==4.5.0 sphinx-rtd-theme==1.0.0 diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index 9332df66..bc7fc04e 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -221,6 +221,7 @@ def main() -> List[Subscription]: "full backup before usage. You have been warned!", ) + logger.info("Validating subscriptions...") subscriptions = _download_subscriptions_from_yaml_files( config=config, subscription_paths=args.subscription_paths, @@ -230,6 +231,7 @@ def main() -> List[Subscription]: # One-off download elif args.subparser == "dl": + logger.info("Validating presets...") subscriptions.append( _download_subscription_from_cli( config=config, dry_run=args.dry_run, extra_args=extra_args diff --git a/src/ytdl_sub/config/config_file.py b/src/ytdl_sub/config/config_file.py index 67a906cc..d0c117ef 100644 --- a/src/ytdl_sub/config/config_file.py +++ b/src/ytdl_sub/config/config_file.py @@ -6,8 +6,8 @@ from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.preset import Preset from ytdl_sub.utils.exceptions import FileNotFoundException from ytdl_sub.utils.ffmpeg import FFMPEG +from ytdl_sub.utils.file_path import FilePathTruncater from ytdl_sub.utils.yaml import load_yaml -from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin class ConfigFile(ConfigValidator): @@ -36,7 +36,7 @@ class ConfigFile(ConfigValidator): ffprobe_path=self.config_options.ffprobe_path, ) - FilePathValidatorMixin.set_max_file_name_bytes( + FilePathTruncater.set_max_file_name_bytes( max_file_name_bytes=self.config_options.file_name_max_bytes ) diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py index 897684ea..ff18b3fc 100644 --- a/src/ytdl_sub/config/config_validator.py +++ b/src/ytdl_sub/config/config_validator.py @@ -10,7 +10,6 @@ from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS -from ytdl_sub.subscriptions.utils import SUBSCRIPTION_VALUE_CONFIG_KEY from ytdl_sub.validators.file_path_validators import FFmpegFileValidator from ytdl_sub.validators.file_path_validators import FFprobeFileValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -107,7 +106,6 @@ class ConfigOptions(StrictDictValidator): "ffprobe_path", "file_name_max_bytes", "experimental", - SUBSCRIPTION_VALUE_CONFIG_KEY, } def __init__(self, name: str, value: Any): @@ -142,9 +140,6 @@ class ConfigOptions(StrictDictValidator): self._file_name_max_bytes = self._validate_key( key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES ) - self._subscription_value = self._validate_key_if_present( - key=SUBSCRIPTION_VALUE_CONFIG_KEY, validator=StringValidator - ) @property def working_directory(self) -> str: @@ -237,14 +232,6 @@ class ConfigOptions(StrictDictValidator): """ return self._ffprobe_path.value - @property - def subscription_value(self) -> Optional[str]: - """ - Sets the :ref:`subscription value` for subscription - files that use this config. - """ - return self._subscription_value.value if self._subscription_value else None - class ConfigValidator(StrictDictValidator): _optional_keys = {"configuration", "presets"} diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py new file mode 100644 index 00000000..59b787b7 --- /dev/null +++ b/src/ytdl_sub/config/overrides.py @@ -0,0 +1,199 @@ +from typing import Any +from typing import Dict +from typing import Optional +from typing import Set + +import mergedeep + +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.variables.override_variables import OverrideHelpers +from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.script.parser import parse +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.utils.exceptions import InvalidVariableNameException +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.script import ScriptUtils +from ytdl_sub.utils.scriptable import Scriptable +from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator + + +class Overrides(DictFormatterValidator, Scriptable): + """ + Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + + :Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + overrides: + output_directory: "/path/to/media" + custom_file_name: "{upload_date_standardized}.{title_sanitized}" + + # Then use the override variables in the output options + output_options: + output_directory: "{output_directory}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + Override variables can contain explicit values and other variables, including both override + and source variables. + + In addition, any override variable defined will automatically create a ``sanitized`` variable + for use. In the example above, ``output_directory_sanitized`` will exist and perform + sanitization on the value when used. + """ + + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + dict_formatter = DictFormatterValidator(name=name, value=value) + _ = [parse(format_string) for format_string in dict_formatter.dict_with_format_strings] + + def __init__(self, name, value): + DictFormatterValidator.__init__(self, name, value) + Scriptable.__init__(self) + + for key in self._keys: + self.ensure_variable_name_valid(key) + + self.unresolvable.add(VARIABLES.entry_metadata.variable_name) + + def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool: + """ + Returns False if the variable exists as a non-override. + + Raises + ------ + ValidationException + If the variable is already added as an override variable. + """ + try: + self.ensure_variable_name_valid(added_variable) + except ValidationException: + return False + + if added_variable in self.keys: + raise self._validation_exception( + f"Override variable with name {added_variable} cannot be used since it is" + " added by a plugin." + ) + + return True + + def ensure_variable_name_valid(self, name: str) -> None: + """ + Ensures the variable name does not collide with any entry variables or built-in functions. + """ + if not OverrideHelpers.is_valid_name(name): + override_type = "function" if name.startswith("%") else "variable" + raise self._validation_exception( + f"Override {override_type} with name {name} is invalid. Names must be" + " lower_snake_cased and begin with a letter.", + exception_class=InvalidVariableNameException, + ) + + if OverrideHelpers.is_entry_variable_name(name): + raise self._validation_exception( + f"Override variable with name {name} cannot be used since it is a" + " built-in ytdl-sub entry variable name.", + exception_class=InvalidVariableNameException, + ) + + if OverrideHelpers.is_function_name(name): + raise self._validation_exception( + f"Override function definition with name {name} cannot be used since it is" + " a built-in ytdl-sub function name.", + exception_class=InvalidVariableNameException, + ) + + def initial_variables( + self, unresolved_variables: Optional[Dict[str, str]] = None + ) -> Dict[str, str]: + """ + Returns + ------- + Variables and format strings for all Override variables + additional variables (Optional) + """ + initial_variables: Dict[str, str] = {} + mergedeep.merge( + initial_variables, + self.dict_with_format_strings, + unresolved_variables if unresolved_variables else {}, + ) + return ScriptUtils.add_sanitized_variables(initial_variables) + + def initialize_script( + self, subscription_name: str, unresolved_variables: Set[str] + ) -> "Overrides": + """ + Initialize the override script with override variables + any unresolved variables + """ + self.script.add( + ScriptUtils.add_sanitized_variables( + {OverrideVariables.subscription_name(): subscription_name} + ) + ) + self.script.add( + self.initial_variables( + unresolved_variables={ + var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" + for var_name in unresolved_variables + } + ) + ) + self.unresolvable.update(unresolved_variables) + self.update_script() + return self + + def apply_formatter( + self, + formatter: StringFormatterValidator, + entry: Optional[Entry] = None, + function_overrides: Dict[str, str] = None, + ) -> str: + """ + Parameters + ---------- + formatter + Formatter to apply + entry + Optional. Entry to add source variables to the formatter + function_overrides + Optional. Explicit values to override the overrides themselves and source variables + + Returns + ------- + The format_string after .format has been called + + Raises + ------ + StringFormattingException + If the formatter that is trying to be resolved cannot + """ + script: Script = self.script + unresolvable: Set[str] = self.unresolvable + if entry: + script = entry.script + unresolvable = entry.unresolvable + + try: + return formatter.post_process( + str( + script.resolve_once( + dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), + unresolvable=unresolvable, + )["tmp_var"] + ) + ) + except ScriptVariableNotResolved as exc: + raise StringFormattingException( + "Tried to resolve the following script, but could not due to unresolved " + f"variables:\n {formatter.format_string}\n" + "This is most likely due to circular dependencies in variables. " + "If you think otherwise, please file a bug on GitHub and post your config. Thanks!" + ) from exc diff --git a/src/ytdl_sub/config/plugin/__init__.py b/src/ytdl_sub/config/plugin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin/plugin.py similarity index 77% rename from src/ytdl_sub/config/plugin.py rename to src/ytdl_sub/config/plugin/plugin.py index 9ddc083c..24d616f6 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin/plugin.py @@ -7,44 +7,13 @@ from typing import Optional from typing import Tuple from typing import Type -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive - -class PluginPriority: - """ - Defines priority for plugins, 0 is highest priority - """ - - # If modify_entry priority is >= to this value, run after split - MODIFY_ENTRY_AFTER_SPLIT = 10 - - # if post_process is >= to this value, run after file_convert - POST_PROCESS_AFTER_FILE_CONVERT = 10 - - MODIFY_ENTRY_FIRST = 0 - - def __init__( - self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5 - ): - self.modify_entry_metadata = modify_entry_metadata - self.modify_entry = modify_entry - self.post_process = post_process - - @property - def modify_entry_after_split(self) -> bool: - """ - Returns - ------- - True if the plugin should modify an entry after a potential split. False otherwise. - """ - return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT - - # pylint: disable=no-self-use,unused-argument @@ -53,7 +22,6 @@ class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC): Shared code amongst all SourcePlugins (downloaders) and Plugins (post-download modification) """ - priority: PluginPriority = PluginPriority() plugin_options_type: Type[TOptionsValidator] def __init__( diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py new file mode 100644 index 00000000..dadc4a20 --- /dev/null +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -0,0 +1,208 @@ +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type + +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin +from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin +from ytdl_sub.plugins.audio_extract import AudioExtractPlugin +from ytdl_sub.plugins.chapters import ChaptersPlugin +from ytdl_sub.plugins.date_range import DateRangePlugin +from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin +from ytdl_sub.plugins.file_convert import FileConvertPlugin +from ytdl_sub.plugins.filter_exclude import FilterExcludePlugin +from ytdl_sub.plugins.filter_include import FilterIncludePlugin +from ytdl_sub.plugins.format import FormatPlugin +from ytdl_sub.plugins.internal.view import ViewPlugin +from ytdl_sub.plugins.match_filters import MatchFiltersPlugin +from ytdl_sub.plugins.music_tags import MusicTagsPlugin +from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin +from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin +from ytdl_sub.plugins.regex import RegexPlugin +from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin +from ytdl_sub.plugins.subtitles import SubtitlesPlugin +from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin +from ytdl_sub.plugins.video_tags import VideoTagsPlugin + + +class PluginMapping: + """ + Maps plugins defined in the preset to its respective plugin class + """ + + _MAPPING: Dict[str, Type[Plugin]] = { + "_view": ViewPlugin, + "audio_extract": AudioExtractPlugin, + "date_range": DateRangePlugin, + "embed_thumbnail": EmbedThumbnailPlugin, + "file_convert": FileConvertPlugin, + "format": FormatPlugin, + "match_filters": MatchFiltersPlugin, + "music_tags": MusicTagsPlugin, + "video_tags": VideoTagsPlugin, + "nfo_tags": NfoTagsPlugin, + "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, + "regex": RegexPlugin, + "subtitles": SubtitlesPlugin, + "chapters": ChaptersPlugin, + "split_by_chapters": SplitByChaptersPlugin, + "throttle_protection": ThrottleProtectionPlugin, + "filter_include": FilterIncludePlugin, + "filter_exclude": FilterExcludePlugin, + } + + # All other plugins are added after the defined ordered ones + _ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [ + ThrottleProtectionPlugin, + UrlDownloaderCollectionVariablePlugin, + SubtitlesPlugin, + FilterExcludePlugin, + FilterIncludePlugin, + # add all others + ] + + _ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [ + UrlDownloaderThumbnailPlugin, + AudioExtractPlugin, + FileConvertPlugin, + ChaptersPlugin, + SplitByChaptersPlugin, + FilterExcludePlugin, + FilterIncludePlugin, + RegexPlugin, + # add all others + ] + + _ORDER_POST_PROCESS: List[Type[Plugin]] = [ + AudioExtractPlugin, + FileConvertPlugin, + ChaptersPlugin, + SubtitlesPlugin, + MusicTagsPlugin, + VideoTagsPlugin, + NfoTagsPlugin, + EmbedThumbnailPlugin, + ] + + @classmethod + def _order_by( + cls, plugin_types: List[Type[Plugin]], operation: PluginOperation + ) -> List[Type[Plugin]]: + if operation == PluginOperation.MODIFY_ENTRY_METADATA: + ordering = cls._ORDER_MODIFY_ENTRY_METADATA + elif operation == PluginOperation.MODIFY_ENTRY: + ordering = cls._ORDER_MODIFY_ENTRY + elif operation == PluginOperation.POST_PROCESS: + ordering = cls._ORDER_POST_PROCESS + else: + raise ValueError("PluginOperation does not support ordering") + + ordered_plugin_operations: List[Type[Plugin]] = [] + for pl_type in reversed(ordering): + for plugin_type in plugin_types: + if plugin_type == pl_type: + ordered_plugin_operations.insert(0, plugin_type) + else: + ordered_plugin_operations.append(plugin_type) + + return ordered_plugin_operations + + @classmethod + def order_options_by( + cls, zipped: List[Tuple[Type[Plugin], OptionsValidator]], operation: PluginOperation + ) -> List[OptionsValidator]: + """ + Returns + ------- + Ordered plugin options with respect to the PluginOperation. + """ + ordered_types: List[Type[Plugin]] = cls._order_by( + plugin_types=[val[0] for val in zipped], operation=operation + ) + + ordered_options: List[OptionsValidator] = [] + for plugin_type, plugin_options in zipped: + sorted_idx = ordered_types.index(plugin_type) + ordered_options.insert(sorted_idx, plugin_options) + + return ordered_options + + @classmethod + def _is_modified_after_split(cls, plugin: Plugin) -> bool: + if type(plugin) not in cls._ORDER_MODIFY_ENTRY: + return True + return cls._ORDER_MODIFY_ENTRY.index(type(plugin)) > cls._ORDER_MODIFY_ENTRY.index( + SplitByChaptersPlugin + ) + + @classmethod + def order_plugins_by( + cls, plugins: List[Plugin], operation: PluginOperation, before_split: Optional[bool] = None + ) -> List[Plugin]: + """ + Returns + ------- + Ordered plugins with respect to the PluginOperation. Optionally only return plugins + before/after a split plugin. + """ + ordered_types: List[Type[Plugin]] = cls._order_by( + plugin_types=[type(plugin) for plugin in plugins], operation=operation + ) + + ordered_plugins: List[Plugin] = [] + for plugin in plugins: + sorted_idx = ordered_types.index(type(plugin)) + ordered_plugins.insert(sorted_idx, plugin) + + if before_split is None: + return ordered_plugins + + # Remove the split plugin if differentiating + ordered_plugins = [ + plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin) + ] + if before_split: + return [ + plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin) + ] + + # before_split is False + return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)] + + @classmethod + def plugins(cls) -> List[str]: + """ + Returns + ------- + Available download sources + """ + return sorted(list(cls._MAPPING.keys())) + + @classmethod + def get(cls, plugin: str) -> Type[Plugin]: + """ + Parameters + ---------- + plugin + Name of the plugin + + Returns + ------- + The plugin class + + Raises + ------ + ValueError + Raised if the plugin does not exist + """ + if plugin not in cls.plugins(): + raise ValueError( + f"Tried to use plugin '{plugin}' that does not exist. Available plugins: " + f"{', '.join(cls.plugins())}" + ) + return cls._MAPPING[plugin] diff --git a/src/ytdl_sub/config/plugin/plugin_operation.py b/src/ytdl_sub/config/plugin/plugin_operation.py new file mode 100644 index 00000000..d3323aca --- /dev/null +++ b/src/ytdl_sub/config/plugin/plugin_operation.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class PluginOperation(Enum): + ANY = -2 + DOWNLOADER = -1 + MODIFY_ENTRY_METADATA = 0 + MODIFY_ENTRY = 1 + POST_PROCESS = 2 diff --git a/src/ytdl_sub/config/plugin/preset_plugins.py b/src/ytdl_sub/config/plugin/preset_plugins.py new file mode 100644 index 00000000..2a86af6a --- /dev/null +++ b/src/ytdl_sub/config/plugin/preset_plugins.py @@ -0,0 +1,46 @@ +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type + +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.config.validators.options import TOptionsValidator + + +class PresetPlugins: + def __init__(self): + self.plugin_types: List[Type[Plugin]] = [] + self.plugin_options: List[OptionsValidator] = [] + + def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins": + """ + Add a pair of plugin type and options to the list + """ + self.plugin_types.append(plugin_type) + self.plugin_options.append(plugin_options) + return self + + def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]: + """ + Returns + ------- + Plugin and PluginOptions zipped + """ + return list(zip(self.plugin_types, self.plugin_options)) + + def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]: + """ + Parameters + ---------- + plugin_type + Fetch the plugin options for this type + + Returns + ------- + Options of this plugin if they exit. Otherwise, return None. + """ + plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] + if plugin_type in plugin_option_types: + return self.plugin_options[plugin_option_types.index(plugin_type)] + return None diff --git a/src/ytdl_sub/config/plugin_mapping.py b/src/ytdl_sub/config/plugin_mapping.py deleted file mode 100644 index 5a446a26..00000000 --- a/src/ytdl_sub/config/plugin_mapping.py +++ /dev/null @@ -1,79 +0,0 @@ -from typing import Dict -from typing import List -from typing import Type - -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.plugins.audio_extract import AudioExtractPlugin -from ytdl_sub.plugins.chapters import ChaptersPlugin -from ytdl_sub.plugins.date_range import DateRangePlugin -from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin -from ytdl_sub.plugins.file_convert import FileConvertPlugin -from ytdl_sub.plugins.format import FormatPlugin -from ytdl_sub.plugins.internal.view import ViewPlugin -from ytdl_sub.plugins.match_filters import MatchFiltersPlugin -from ytdl_sub.plugins.music_tags import MusicTagsPlugin -from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin -from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin -from ytdl_sub.plugins.regex import RegexPlugin -from ytdl_sub.plugins.split_by_chapters import SplitByChaptersPlugin -from ytdl_sub.plugins.subtitles import SubtitlesPlugin -from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin -from ytdl_sub.plugins.video_tags import VideoTagsPlugin - - -class PluginMapping: - """ - Maps plugins defined in the preset to its respective plugin class - """ - - _MAPPING: Dict[str, Type[Plugin]] = { - "_view": ViewPlugin, - "audio_extract": AudioExtractPlugin, - "date_range": DateRangePlugin, - "embed_thumbnail": EmbedThumbnailPlugin, - "file_convert": FileConvertPlugin, - "format": FormatPlugin, - "match_filters": MatchFiltersPlugin, - "music_tags": MusicTagsPlugin, - "video_tags": VideoTagsPlugin, - "nfo_tags": NfoTagsPlugin, - "output_directory_nfo_tags": OutputDirectoryNfoTagsPlugin, - "regex": RegexPlugin, - "subtitles": SubtitlesPlugin, - "chapters": ChaptersPlugin, - "split_by_chapters": SplitByChaptersPlugin, - "throttle_protection": ThrottleProtectionPlugin, - } - - @classmethod - def plugins(cls) -> List[str]: - """ - Returns - ------- - Available download sources - """ - return sorted(list(cls._MAPPING.keys())) - - @classmethod - def get(cls, plugin: str) -> Type[Plugin]: - """ - Parameters - ---------- - plugin - Name of the plugin - - Returns - ------- - The plugin class - - Raises - ------ - ValueError - Raised if the plugin does not exist - """ - if plugin not in cls.plugins(): - raise ValueError( - f"Tried to use plugin '{plugin}' that does not exist. Available plugins: " - f"{', '.join(cls.plugins())}" - ) - return cls._MAPPING[plugin] diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index e88e4fc0..4fec587d 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -1,39 +1,25 @@ import copy from typing import Any from typing import Dict -from typing import Iterable from typing import List -from typing import Optional -from typing import Tuple -from typing import Type -from typing import Union from mergedeep import mergedeep from ytdl_sub.config.config_validator import ConfigValidator -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_mapping import PluginMapping -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.entry import Entry from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator -from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator -from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -from ytdl_sub.validators.validators import DictValidator -from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringListValidator -from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import validation_exception PRESET_KEYS = { @@ -61,44 +47,6 @@ def _parent_preset_error_message( ) -class PresetPlugins: - def __init__(self): - self.plugin_types: List[Type[Plugin]] = [] - self.plugin_options: List[OptionsValidator] = [] - - def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins": - """ - Add a pair of plugin type and options to the list - """ - self.plugin_types.append(plugin_type) - self.plugin_options.append(plugin_options) - return self - - def zipped(self) -> Iterable[Tuple[Type[Plugin], OptionsValidator]]: - """ - Returns - ------- - Plugin and PluginOptions zipped - """ - return zip(self.plugin_types, self.plugin_options) - - def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]: - """ - Parameters - ---------- - plugin_type - Fetch the plugin options for this type - - Returns - ------- - Options of this plugin if they exit. Otherwise, return None. - """ - plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] - if plugin_type in plugin_option_types: - return self.plugin_options[plugin_option_types.index(plugin_type)] - return None - - class _PresetShell(StrictDictValidator): # Have all present keys optional since parent presets could not have all the # required keys. They will get validated in the init after the mergedeep of dicts @@ -154,11 +102,7 @@ class Preset(_PresetShell): validator=PluginMapping.get(plugin_name).plugin_options_type, ) - @property - def _source_variables(self) -> List[str]: - return Entry.source_variables() - - def __validate_and_get_plugins(self) -> PresetPlugins: + def _validate_and_get_plugins(self) -> PresetPlugins: preset_plugins = PresetPlugins() for key in self._keys: @@ -172,88 +116,6 @@ class Preset(_PresetShell): return preset_plugins - def __validate_added_variables(self): - source_variables = copy.deepcopy(self._source_variables) - - # Validate added download option variables here since plugins could subsequently use them - self.downloader_options.validate_with_variables( - source_variables=source_variables, - override_variables=self.overrides.dict_with_format_strings, - ) - source_variables.extend(self.downloader_options.added_source_variables()) - - for _, plugin_options in sorted( - self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry - ): - # Validate current plugin using source + added plugin variables - plugin_options.validate_with_variables( - source_variables=source_variables, - override_variables=self.overrides.dict_with_format_strings, - ) - - # Extend existing source variables with ones created from this plugin - source_variables.extend(plugin_options.added_source_variables()) - - def __validate_override_string_formatter_validator( - self, - formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], - ): - # Set the formatter variables to be the overrides - variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings) - - # If the formatter supports source variables, set the formatter variables to include - # both source and override variables - if not isinstance(formatter_validator, OverridesStringFormatterValidator): - source_variables = { - source_var: "dummy_string" - for source_var in self._source_variables - + self.downloader_options.added_source_variables() - } - variable_dict = dict(source_variables, **variable_dict) - - # For all plugins, add in any extra added source variables - for plugin_options in self.plugins.plugin_options: - added_plugin_variables = { - source_var: "dummy_string" - for source_var in plugin_options.added_source_variables() - } - # sanity check plugin variables do not override source variables - expected_len = len(variable_dict) + len(added_plugin_variables) - variable_dict = dict(variable_dict, **added_plugin_variables) - - assert ( - len(variable_dict) == expected_len - ), "plugin variables overwrote source variables" - - _ = formatter_validator.apply_formatter(variable_dict=variable_dict) - - def __recursive_preset_validate( - self, - validator: Optional[Validator] = None, - ) -> None: - """ - Ensure all OverridesStringFormatterValidator's only contain variables from the overrides - and resolve. - """ - if validator is None: - validator = self - - if isinstance(validator, DictValidator): - # pylint: disable=protected-access - # Usage of protected variables in other validators is fine. The reason to keep - # them protected is for readability when using them in subscriptions. - for validator_value in validator._validator_dict.values(): - self.__recursive_preset_validate(validator_value) - # pylint: enable=protected-access - elif isinstance(validator, ListValidator): - for list_value in validator.list: - self.__recursive_preset_validate(list_value) - elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)): - self.__validate_override_string_formatter_validator(validator) - elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)): - for validator_value in validator.dict.values(): - self.__validate_override_string_formatter_validator(validator_value) - def _get_presets_to_merge( self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator ) -> List[Dict]: @@ -291,7 +153,7 @@ class Preset(_PresetShell): return presets_to_merge - def __merge_parent_preset_dicts_if_present(self, config: ConfigValidator): + def _merge_parent_preset_dicts_if_present(self, config: ConfigValidator): parent_preset_validator = self._validate_key_if_present( key="preset", validator=StringListValidator ) @@ -314,7 +176,7 @@ class Preset(_PresetShell): super().__init__(name=name, value=value) # Perform the merge of parent presets before validating any keys - self.__merge_parent_preset_dicts_if_present(config=config) + self._merge_parent_preset_dicts_if_present(config=config) self.downloader_options: MultiUrlValidator = self._validate_key( key="download", validator=MultiUrlValidator @@ -329,13 +191,16 @@ class Preset(_PresetShell): key="ytdl_options", validator=YTDLOptions, default={} ) + self.plugins: PresetPlugins = self._validate_and_get_plugins() self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) - self.plugins: PresetPlugins = self.__validate_and_get_plugins() - self.__validate_added_variables() - # After all options are initialized, perform a recursive post-validate that requires - # values from multiple validators - self.__recursive_preset_validate() + VariableValidation( + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + ).initialize_overrides( + subscription_name=self.name, overrides=self.overrides + ).ensure_proper_usage() @property def name(self) -> str: diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index ff07ce92..8cc4697c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,97 +1,26 @@ -from abc import ABC from typing import Any -from typing import Dict -from typing import List from typing import Optional -from typing import TypeVar - -from yt_dlp.utils import sanitize_filename from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME -from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator -from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import LiteralDictValidator -from ytdl_sub.validators.validators import Validator - - -# pylint: disable=no-self-use -# pylint: disable=unused-argument -class OptionsValidator(Validator, ABC): - """ - Abstract class that validates options for preset sections (plugins, downloaders) - """ - - def validation_exception( - self, - error_message: str | Exception, - ) -> ValidationException: - """ - Parameters - ---------- - error_message - Error message to include in the validation exception - - Returns - ------- - Validation exception that points to the location in the config. To be used to throw good - validation exceptions at runtime from code outside this class. - """ - return self._validation_exception(error_message=error_message) - - def added_source_variables(self) -> List[str]: - """ - If the plugin adds source variables, list them here. - - Returns - ------- - List of added source variables this plugin creates - """ - return [] - - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Optional validation after init with the session's source and override variables. - - Parameters - ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin - """ - return None - - -TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) - - -class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): - pass - - -# pylint: enable=no-self-use -# pylint: enable=unused-argument class YTDLOptions(LiteralDictValidator): """ - Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. + Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. - ytdl_options should be formatted like: + :Usage: .. code-block:: yaml @@ -123,108 +52,13 @@ class YTDLOptions(LiteralDictValidator): # Disable for proper docstring formatting # pylint: disable=line-too-long -class Overrides(DictFormatterValidator): - """ - Optional. This section allows you to define variables that can be used in any string formatter. - For example, if you want your file and thumbnail files to match without copy-pasting a large - format string, you can define something like: - - .. code-block:: yaml - - presets: - my_example_preset: - overrides: - output_directory: "/path/to/media" - custom_file_name: "{upload_year}.{upload_month_padded}.{upload_day_padded}.{title_sanitized}" - - # Then use the override variables in the output options - output_options: - output_directory: "{output_directory}" - file_name: "{custom_file_name}.{ext}" - thumbnail_name: "{custom_file_name}.{thumbnail_ext}" - - Override variables can contain explicit values and other variables, including both override - and source variables. - - In addition, any override variable defined will automatically create a ``sanitized`` variable - for use. In the example above, ``output_directory_sanitized`` will exist and perform - sanitization on the value when used. - """ - - # pylint: enable=line-too-long - - def _add_override_variable(self, key_name: str, format_string: str, sanitize: bool = False): - if sanitize: - key_name = f"{key_name}_sanitized" - format_string = sanitize_filename(format_string) - - self._value[key_name] = StringFormatterValidator( - name="__should_never_fail__", - value=format_string, - ) - - def __init__(self, name, value): - super().__init__(name, value) - - # Add sanitized overrides - for key in self._keys: - self._add_override_variable( - key_name=key, - format_string=self._value[key].format_string, - sanitize=True, - ) - - if SUBSCRIPTION_NAME not in self._value: - for sanitized in [True, False]: - self._add_override_variable( - key_name=SUBSCRIPTION_NAME, - format_string=self.subscription_name, - sanitize=sanitized, - ) - - @property - def subscription_name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - return self._root_name - - def apply_formatter( - self, - formatter: StringFormatterValidator, - entry: Optional[Entry] = None, - function_overrides: Dict[str, str] = None, - ) -> str: - """ - Parameters - ---------- - formatter - Formatter to apply - entry - Optional. Entry to add source variables to the formatter - function_overrides - Optional. Explicit values to override the overrides themselves and source variables - - Returns - ------- - The format_string after .format has been called - """ - variable_dict = self.dict_with_format_strings - if entry: - variable_dict = dict(entry.to_dict(), **variable_dict) - if function_overrides: - variable_dict = dict(variable_dict, **function_overrides) - - return formatter.apply_formatter(variable_dict) class OutputOptions(StrictDictValidator): """ Defines where to output files and thumbnails after all post-processing has completed. - Usage: + :Usage: .. code-block:: yaml @@ -253,6 +87,7 @@ class OutputOptions(StrictDictValidator): "maintain_download_archive", "keep_files_before", "keep_files_after", + "keep_max_files", } @classmethod @@ -307,96 +142,133 @@ class OutputOptions(StrictDictValidator): self._keep_files_after = self._validate_key_if_present( "keep_files_after", StringDatetimeValidator ) + self._keep_max_files = self._validate_key_if_present( + "keep_max_files", OverridesIntegerFormatterValidator + ) if ( - self._keep_files_before or self._keep_files_after + self._keep_files_before or self._keep_files_after or self._keep_max_files ) and not self.maintain_download_archive: raise self._validation_exception( - "keep_files requires maintain_download_archive set to True" + "keep_files/keep_max requires maintain_download_archive set to True" ) @property def output_directory(self) -> OverridesStringFormatterValidator: """ - Required. The output directory to store all media files downloaded. + :expected type: OverridesFormatter + :description: + The output directory to store all media files downloaded. """ return self._output_directory @property def file_name(self) -> StringFormatterValidator: """ - Required. The file name for the media file. This can include directories such as - ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. + :expected type: EntryFormatter + :description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. """ return self._file_name @property def thumbnail_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's thumbnail image. This can include directories such - as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable thumbnail writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. """ return self._thumbnail_name @property def info_json_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's info json file. This can include directories such - as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable info json writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. """ return self._info_json_name @property def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. The file name to store a subscriptions download archive placed relative to - the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` + :expected type: Optional[OverridesFormatter] + :description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` """ return self._download_archive_name @property def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. Intended to be used if you are migrating a subscription with either a new - subscription name or output directory. It will try to load the archive file using this name - first, and fallback to ``download_archive_name``. It will always save to this file - and remove the original ``download_archive_name``. + :expected type: Optional[OverridesFormatter] + :description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. """ return self._migrated_download_archive_name @property def maintain_download_archive(self) -> bool: """ - Optional. Maintains a download archive file in the output directory for a subscription. - It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the - output directory. + :expected type: Optional[Boolean] + :description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. - The download archive contains a mapping of ytdl IDs to downloaded files. This is used to - create a ytdl download-archive file when invoking a download on a subscription. This will - prevent ytdl from redownloading media already downloaded. + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. - Defaults to False. + Defaults to False. """ return self._maintain_download_archive.value @property def keep_files_before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep - files before ``now``, which implies all files. + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_before @property def keep_files_after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep - files after ``19000101``, which implies all files. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_after + + @property + def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: + """ + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + """ + return self._keep_max_files diff --git a/src/ytdl_sub/config/validators/__init__.py b/src/ytdl_sub/config/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py new file mode 100644 index 00000000..c35defa2 --- /dev/null +++ b/src/ytdl_sub/config/validators/options.py @@ -0,0 +1,59 @@ +from abc import ABC +from typing import Dict +from typing import Set +from typing import TypeVar + +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.validators import Validator + +# pylint: disable=no-self-use +# pylint: disable=unused-argument + + +class OptionsValidator(Validator, ABC): + """ + Abstract class that validates options for preset sections (plugins, downloaders) + """ + + def validation_exception( + self, + error_message: str | Exception, + ) -> ValidationException: + """ + Parameters + ---------- + error_message + Error message to include in the validation exception + + Returns + ------- + Validation exception that points to the location in the config. To be used to throw good + validation exceptions at runtime from code outside this class. + """ + return self._validation_exception(error_message=error_message) + + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + """ + If the plugin modifies existing variables, define them here + """ + return {} + + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: + """ + If the plugin adds source variables, list them here. + """ + return {} + + +TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) + + +class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): + pass diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py new file mode 100644 index 00000000..1b2ea1e4 --- /dev/null +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -0,0 +1,199 @@ +import copy +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.script.script import Script +from ytdl_sub.validators.string_formatter_validators import validate_formatters + + +def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: + dummy_variables: Dict[str, str] = {} + for var in variables: + dummy_variables[var] = "" + dummy_variables[f"{var}_sanitized"] = "" + + return dummy_variables + + +def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: + # Have the dummy override variable contain all variable deps that it uses in the string + dummy_overrides: Dict[str, str] = {} + for override_name in _override_variables(overrides): + dummy_overrides[override_name] = "" + # pylint: disable=protected-access + for variable_dependency in overrides.script._variables[override_name].variables: + dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" + # pylint: enable=protected-access + return dummy_overrides + + +def _get_added_and_modified_variables( + plugins: PresetPlugins, downloader_options: MultiUrlValidator +) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: + """ + Iterates and returns the plugin options, added variables, modified variables + """ + options: List[OptionsValidator] = plugins.plugin_options + options.append(downloader_options) + + for plugin_options in options: + added_variables: Set[str] = set() + modified_variables: Set[str] = set() + + for plugin_added_variables in plugin_options.added_variables( + resolved_variables=set(), + unresolved_variables=set(), + plugin_op=PluginOperation.ANY, + ).values(): + added_variables |= set(plugin_added_variables) + + for plugin_modified_variables in plugin_options.modified_variables().values(): + modified_variables = plugin_modified_variables + + yield plugin_options, added_variables, modified_variables + + +def _override_variables(overrides: Overrides) -> Set[str]: + return set(list(overrides.initial_variables().keys())) | {OverrideVariables.subscription_name()} + + +def _entry_variables() -> Set[str]: + return set(list(VARIABLE_SCRIPTS.keys())) + + +class VariableValidation: + def __init__( + self, + downloader_options: MultiUrlValidator, + output_options: OutputOptions, + plugins: PresetPlugins, + ): + self.downloader_options = downloader_options + self.output_options = output_options + self.plugins = plugins + + self.script: Optional[Script] = None + self.resolved_variables: Set[str] = set() + self.unresolved_variables: Set[str] = set() + + def initialize_overrides( + self, subscription_name: str, overrides: Overrides + ) -> "VariableValidation": + """ + Do some gymnastics to initialize the Overrides script. + """ + entry_variables = _entry_variables() + override_variables = _override_variables(overrides) + + # Set resolved variables as all entry + override variables + # at this point to generate every possible added/modified variable + self.resolved_variables = entry_variables | override_variables + + for ( + plugin_options, + added_variables, + modified_variables, + ) in _get_added_and_modified_variables( + plugins=self.plugins, + downloader_options=self.downloader_options, + ): + + for added_variable in added_variables: + if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable): + # pylint: disable=protected-access + raise plugin_options._validation_exception( + f"Cannot use the variable name {added_variable} because it exists as a" + " built-in ytdl-sub variable name." + ) + # pylint: enable=protected-access + + # Set unresolved as variables that are added but do not exist as + # entry/override variables since they are created at run-time + self.unresolved_variables |= added_variables | modified_variables + + # Then update resolved variables to reflect that + self.resolved_variables -= self.unresolved_variables + + # Initialize overrides with unresolved variables + modified variables to throw an error. + # For modified variables, this is to prevent a resolve(update=True) to setting any + # dependencies until it has been explicitly added + overrides = overrides.initialize_script( + subscription_name=subscription_name, unresolved_variables=self.unresolved_variables + ) + + # copy the script and mock entry variables + self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) + self.script.add( + variables=_add_dummy_overrides(overrides=overrides), + unresolvable=self.unresolved_variables, + ) + + return self + + def _update_script(self) -> None: + _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + + def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]: + added_variables = options.added_variables( + resolved_variables=self.resolved_variables, + unresolved_variables=self.unresolved_variables, + plugin_op=plugin_op, + ).get(plugin_op, set()) + modified_variables = options.modified_variables().get(plugin_op, set()) + + resolved_variables = added_variables | modified_variables + + self.script.add(_add_dummy_variables(resolved_variables)) + self.resolved_variables |= resolved_variables + self.unresolved_variables -= resolved_variables + + return added_variables + + def ensure_proper_usage(self) -> None: + """ + Validate variables resolve as plugins are executed, and return + a mock script which contains actualized added variables from the plugins + """ + self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) + + # Metadata variables to be added + for plugin_options in PluginMapping.order_options_by( + self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA + ): + self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) + + self._update_script() + for plugin_options in PluginMapping.order_options_by( + self.plugins.zipped(), PluginOperation.MODIFY_ENTRY + ): + added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) + if added: + self._update_script() + + # Validate that any formatter in the plugin options can resolve + validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=plugin_options, + ) + + validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=self.output_options, + ) + + assert not self.unresolved_variables diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index a7fac294..509a4f8d 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -6,17 +6,22 @@ from typing import Iterable from typing import List from typing import Optional -from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import get_file_extension from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive +v: VariableDefinitions = VARIABLES + class InfoJsonDownloaderOptions(OptionsDictValidator): _optional_keys = {"no-op"} @@ -97,14 +102,26 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): for download_mapping in self._enhanced_download_archive.mapping.entry_mappings.values(): entry = self._get_entry_from_download_mapping(download_mapping) + + # See if prior variables exist. If so, delete them from metadata + # to avoid saving them recursively on multiple updates + prior_variables = entry.maybe_get_prior_variables() + + entry.initialize_script(self.overrides).add( + { + inj: prior_variables.get( + inj.variable_name, + VARIABLE_SCRIPTS[inj.variable_name], + ) + for inj in v.injected_variables() + } + ) entries.append(entry) - # Remove each entry from the live download archive since it will get re-added - # unless it is filtered - for entry in entries: + for entry in sorted(entries, key=lambda ent: ent.get(v.download_index, int)): + # Remove each entry from the live download archive since it will get re-added + # unless it is filtered self._enhanced_download_archive.mapping.remove_entry(entry.uid) - - for entry in sorted(entries, key=lambda ent: ent.download_index): yield entry # If the original entry file_path is no longer maintained in the new mapping, then diff --git a/src/ytdl_sub/downloaders/source_plugin.py b/src/ytdl_sub/downloaders/source_plugin.py index b11b1940..167f67fd 100644 --- a/src/ytdl_sub/downloaders/source_plugin.py +++ b/src/ytdl_sub/downloaders/source_plugin.py @@ -8,10 +8,10 @@ from typing import Optional from typing import Type from typing import final -from ytdl_sub.config.plugin import BasePlugin -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import BasePlugin +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 4fb8f273..67feabd0 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -11,8 +11,7 @@ from typing import Tuple from yt_dlp.utils import RejectedVideoReached -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.source_plugin import SourcePluginExtension from ytdl_sub.downloaders.url.validators import MultiUrlValidator @@ -22,14 +21,8 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent -from ytdl_sub.entries.variables.kwargs import COLLECTION_URL -from ytdl_sub.entries.variables.kwargs import COMMENTS -from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES -from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY -from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes @@ -37,6 +30,8 @@ from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive +v: VariableDefinitions = VARIABLES + download_logger = Logger.get(name="downloader") @@ -47,8 +42,6 @@ class URLDownloadState: class UrlDownloaderThumbnailPlugin(SourcePluginExtension): - priority = PluginPriority(modify_entry=0) - def __init__( self, options: MultiUrlValidator, @@ -119,22 +112,18 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. """ - if entry.kwargs_contains(PLAYLIST_ENTRY): + if playlist_metadata := entry.get(v.playlist_metadata, dict): self._download_parent_thumbnails( thumbnail_list_info=collection_url.playlist_thumbnails, entry=entry, - parent=EntryParent( - entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory - ), + parent=EntryParent(playlist_metadata, working_directory=self.working_directory), ) - if entry.kwargs_contains(SOURCE_ENTRY): + if source_metadata := entry.get(v.source_metadata, dict): self._download_parent_thumbnails( thumbnail_list_info=collection_url.source_thumbnails, entry=entry, - parent=EntryParent( - entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory - ), + parent=EntryParent(source_metadata, working_directory=self.working_directory), ) def modify_entry(self, entry: Entry) -> Optional[Entry]: @@ -147,17 +136,15 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): if not self.is_dry_run: try_convert_download_thumbnail(entry=entry) - if entry.kwargs_get(COLLECTION_URL) in self._collection_url_mapping: + if (input_url := entry.get(v.ytdl_sub_input_url, str)) in self._collection_url_mapping: self._download_url_thumbnails( - collection_url=self._collection_url_mapping[entry.kwargs(COLLECTION_URL)], + collection_url=self._collection_url_mapping[input_url], entry=entry, ) return entry class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): - priority = PluginPriority(modify_entry_metadata=0) - def __init__( self, options: MultiUrlValidator, @@ -181,7 +168,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): """ # COLLECTION_URL is a recent variable that may not exist for old entries when updating. # Try to use source_webpage_url if it does not exist - entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.source_webpage_url) + entry_collection_url = entry.get(v.ytdl_sub_input_url, str) # If the collection URL cannot find its mapping, use the last URL collection_url = ( @@ -189,7 +176,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): or list(self._collection_url_mapping.values())[-1] ) - entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings) + entry.add(collection_url.variables.dict_with_format_strings) return entry @@ -356,7 +343,10 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): else entry.is_thumbnail_downloaded_via_ytdlp, url=entry.webpage_url, ) - return Entry(download_entry_dict, working_directory=self.working_directory) + return Entry( + download_entry_dict, + working_directory=self.working_directory, + ) def _iterate_child_entries( self, url_validator: UrlValidator, entries: List[Entry] @@ -395,7 +385,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ): yield entry_child - def _download_url_metadata(self, url: str) -> Tuple[List[EntryParent], List[Entry]]: + def _download_url_metadata( + self, url: str, include_sibling_metadata: bool + ) -> Tuple[List[EntryParent], List[Entry]]: """ Downloads only info.json files and forms EntryParent trees """ @@ -411,9 +403,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): url=url, entry_dicts=entry_dicts, working_directory=self.working_directory, + include_sibling_metadata=include_sibling_metadata, ) orphans = EntryParent.from_entry_dicts_with_no_parents( - parents=parents, entry_dicts=entry_dicts, working_directory=self.working_directory + parents=parents, + entry_dicts=entry_dicts, + working_directory=self.working_directory, ) return parents, orphans @@ -446,7 +441,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): if not (url := self.overrides.apply_formatter(collection_url.url)): continue - parents, orphan_entries = self._download_url_metadata(url=url) + parents, orphan_entries = self._download_url_metadata( + url=url, include_sibling_metadata=collection_url.include_sibling_metadata + ) # TODO: Encapsulate this logic into its own class self._url_state = URLDownloadState( @@ -459,9 +456,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): for entry in self._iterate_entries( url_validator=collection_url, parents=parents, orphans=orphan_entries ): - # Add the collection URL to the info_dict to trace where it came from - entry.add_kwargs( - {COLLECTION_URL: self.overrides.apply_formatter(collection_url.url)} + entry.initialize_script(self.overrides).add( + {v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)} ) yield entry @@ -497,22 +493,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): return None upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( - upload_date_standardized=entry.upload_date_standardized + upload_date_standardized=entry.get(v.upload_date_standardized, str) ) download_idx = self._enhanced_download_archive.num_entries - entry.add_kwargs( - { - # Subtitles are not downloaded in metadata run, only here, so move over - REQUESTED_SUBTITLES: download_entry.kwargs_get(REQUESTED_SUBTITLES), - # Same with sponsorblock chapters - SPONSORBLOCK_CHAPTERS: download_entry.kwargs_get(SPONSORBLOCK_CHAPTERS), - COMMENTS: download_entry.kwargs_get(COMMENTS), - # Tracks number of entries downloaded - DOWNLOAD_INDEX: download_idx, - # Tracks number of entries with the same upload date to make them unique - UPLOAD_DATE_INDEX: upload_date_idx, - } + return entry.add_injected_variables( + download_entry=download_entry, + download_idx=download_idx, + upload_date_idx=upload_date_idx, ) - - return entry diff --git a/src/ytdl_sub/downloaders/url/multi_url.py b/src/ytdl_sub/downloaders/url/multi_url.py deleted file mode 100644 index b55b8eed..00000000 --- a/src/ytdl_sub/downloaders/url/multi_url.py +++ /dev/null @@ -1,37 +0,0 @@ -from ytdl_sub.downloaders.url.validators import MultiUrlValidator - - -# TODO: Remove later - keep for docstring -class MultiUrlDownloadOptions(MultiUrlValidator): - """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - urls: - - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "1" - season_name: "Uploads" - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "2" - season_name: "Playlist as Season" - playlist_thumbnails: - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - """ diff --git a/src/ytdl_sub/downloaders/url/url.py b/src/ytdl_sub/downloaders/url/url.py deleted file mode 100644 index e5589645..00000000 --- a/src/ytdl_sub/downloaders/url/url.py +++ /dev/null @@ -1,25 +0,0 @@ -from ytdl_sub.downloaders.url.validators import UrlValidator - - -# TODO: Remove later - keep for docstring -class UrlDownloadOptions(UrlValidator): - """ - Downloads from a single URL supported by yt-dlp. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - # optional - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - download_reverse: True - """ diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 2f799873..631ee979 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -1,10 +1,12 @@ import copy from typing import Any from typing import Dict -from typing import List from typing import Optional +from typing import Set -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.script.parser import parse from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -43,7 +45,13 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]): class UrlValidator(StrictDictValidator): _required_keys = {"url"} - _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails", "download_reverse"} + _optional_keys = { + "variables", + "source_thumbnails", + "playlist_thumbnails", + "download_reverse", + "include_sibling_metadata", + } @classmethod def partial_validate(cls, name: str, value: Any) -> None: @@ -72,6 +80,9 @@ class UrlValidator(StrictDictValidator): self._download_reverse = self._validate_key( key="download_reverse", validator=BoolValidator, default=True ) + self._include_sibling_metadata = self._validate_key( + key="include_sibling_metadata", validator=BoolValidator, default=False + ) @property def url(self) -> OverridesStringFormatterValidator: @@ -145,6 +156,16 @@ class UrlValidator(StrictDictValidator): """ return self._download_reverse.value + @property + def include_sibling_metadata(self) -> bool: + """ + Optional. Whether to include sibling metadata as an entry variable, which comprises basic + metadata from all other entries (including itself) that belong to the same playlist. For + channels or large playlists, this becomes memory-intensive since you are storing + ``n^2`` metadata. Defaults to False. + """ + return self._include_sibling_metadata.value + class UrlStringOrDictValidator(UrlValidator): """ @@ -194,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]): class MultiUrlValidator(OptionsValidator): """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. + Sets the URL(s) to download from. Can be used in many forms, including + + :Single URL: + + .. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + + :Multi URL: + + .. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + + :Thumbnails + Variables: + + All variables must be defined for the top-most url. All subsequent URL variables can be either + overwritten or default to the top-most value. + + If an entry is returned from more than one URL, it will use the variables in the bottom-most + URL. + + .. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" """ @classmethod @@ -243,45 +309,26 @@ class MultiUrlValidator(OptionsValidator): # keep for readthedocs documentation return self._urls.list[0].variables - def added_source_variables(self) -> List[str]: + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of variables added. The first collection url always contains all the variables. """ - return list(self._urls.list[0].variables.keys) + if plugin_op != PluginOperation.ANY: + for url in self._urls.list: + for variable_name, definition in url.variables.dict_with_format_strings.items(): + used_variables = set(var.name for var in parse(definition).variables) + if unresolved := used_variables & unresolved_variables: + raise self._validation_exception( + f"variable {variable_name} cannot use the variables " + f"{', '.join(sorted(list(unresolved)))} because it depends on other" + " variables that are computed later in execution" + ) - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Ensures new variables added are not existing variables - """ - for source_var_name in self.added_source_variables(): - if source_var_name in source_variables: - raise self._validation_exception( - f"'{source_var_name}' cannot be used as a variable name because it " - f"is an existing source variable" - ) - - base_variables = dict( - override_variables, **{source_var: "dummy_string" for source_var in source_variables} - ) - - # Apply formatting to each new source variable, ensure it resolves - for collection_url in self.urls.list: - for ( - source_var_name, - source_var_formatter_str, - ) in collection_url.variables.dict_with_format_strings.items(): - _ = StringFormatterValidator( - name=f"{self._name}.{source_var_name}", value=source_var_formatter_str - ).apply_formatter(base_variables) - - # Ensure at least URL is non-empty - has_non_empty_url = False - for url_validator in self.urls.list: - has_non_empty_url |= bool(url_validator.url.apply_formatter(override_variables)) - - if not has_non_empty_url: - raise self._validation_exception("Must contain at least one url that is non-empty") + return {PluginOperation.DOWNLOADER: set(self._urls.list[0].variables.keys)} diff --git a/src/ytdl_sub/downloaders/ytdl_options_builder.py b/src/ytdl_sub/downloaders/ytdl_options_builder.py index b159da42..9f389b51 100644 --- a/src/ytdl_sub/downloaders/ytdl_options_builder.py +++ b/src/ytdl_sub/downloaders/ytdl_options_builder.py @@ -17,7 +17,7 @@ class YTDLOptionsBuilder: self, *ytdl_option_dicts: Optional[Dict], before: bool = False, - strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE + strategy: mergedeep.Strategy = mergedeep.Strategy.TYPESAFE_ADDITIVE, ) -> "YTDLOptionsBuilder": """ Parameters diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 697bb1f6..c4243d63 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -1,241 +1,22 @@ +# pylint: disable=protected-access from abc import ABC -from datetime import datetime from pathlib import Path from typing import Any from typing import Dict -from typing import List from typing import Optional from typing import Type from typing import TypeVar from typing import final -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.entries.variables.kwargs import DESCRIPTION -from ytdl_sub.entries.variables.kwargs import EPOCH -from ytdl_sub.entries.variables.kwargs import EXTRACTOR -from ytdl_sub.entries.variables.kwargs import IE_KEY -from ytdl_sub.entries.variables.kwargs import TITLE -from ytdl_sub.entries.variables.kwargs import UID -from ytdl_sub.entries.variables.kwargs import UPLOADER -from ytdl_sub.entries.variables.kwargs import UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL - -# pylint: disable=no-member - - -def _sanitize_plex(string: str) -> str: - out = "" - for char in string: - match char: - case "0": - out += "０" - case "1": - out += "１" - case "2": - out += "２" - case "3": - out += "３" - case "4": - out += "４" - case "5": - out += "５" - case "6": - out += "６" - case "7": - out += "７" - case "8": - out += "８" - case "9": - out += "９" - case _: - out += char - return out - - -class BaseEntryVariables: - """ - Source variables are ``{variables}`` that contain metadata from downloaded media. - These variables can be used with fields that expect - :class:`~ytdl_sub.validators.string_formatter_validators.StringFormatterValidator`, - but not - :class:`~ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator`. - """ - - @property - def uid(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The entry's unique ID - """ - return str(self.kwargs(UID)) - - @property - def uid_sanitized(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The sanitized uid of the entry, which is safe to use for Unix and Windows file names. - """ - return sanitize_filename(self.uid) - - @property - def uid_sanitized_plex(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The sanitized uid with additional sanitizing for Plex. Replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. - """ - return _sanitize_plex(self.uid_sanitized) - - @property - def extractor(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The ytdl extractor name - """ - # pylint: disable=line-too-long - # Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514 - # pylint: enable=line-too-long - return self.kwargs_get(EXTRACTOR) or self.kwargs(IE_KEY) - - @property - def epoch(self: "BaseEntry") -> int: - """ - Returns - ------- - int - The unix epoch of when the metadata was scraped by yt-dlp. - """ - return self.kwargs(EPOCH) - - @property - def epoch_date(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The epoch's date, in YYYYMMDD format. - """ - return datetime.utcfromtimestamp(self.epoch).strftime("%Y%m%d") - - @property - def epoch_hour(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The epoch's hour, padded - """ - return datetime.utcfromtimestamp(self.epoch).strftime("%H") - - @property - def title(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The title of the entry. If a title does not exist, returns its unique ID. - """ - return self.kwargs_get(TITLE, self.uid) - - @property - def title_sanitized(self) -> str: - """ - Returns - ------- - str - The sanitized title of the entry, which is safe to use for Unix and Windows file names. - """ - return sanitize_filename(self.title) - - @property - def title_sanitized_plex(self) -> str: - """ - Returns - ------- - str - The sanitized title with additional sanitizing for Plex. Replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. - """ - return _sanitize_plex(self.title_sanitized) - - @property - def webpage_url(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The url to the webpage. - """ - return self.kwargs(WEBPAGE_URL) - - @property - def info_json_ext(self) -> str: - """ - Returns - ------- - str - The "info.json" extension - """ - return "info.json" - - @property - def description(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The description if it exists. Otherwise, returns an emtpy string. - """ - return self.kwargs_get(DESCRIPTION, "") - - @property - def uploader_id(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The uploader id if it exists, otherwise return the unique ID. - """ - return self.kwargs_get(UPLOADER_ID, self.uid) - - @property - def uploader(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The uploader if it exists, otherwise return the uploader ID. - """ - return self.kwargs_get(UPLOADER, self.uploader_id) - - @property - def uploader_url(self: "BaseEntry") -> str: - """ - Returns - ------- - str - The uploader url if it exists, otherwise returns the webpage_url. - """ - return self.kwargs_get(UPLOADER_URL, self.webpage_url) - - -# pylint: enable=no-member +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +v: VariableDefinitions = VARIABLES TBaseEntry = TypeVar("TBaseEntry", bound="BaseEntry") -class BaseEntry(BaseEntryVariables, ABC): +class BaseEntry(ABC): """ Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc). """ @@ -254,30 +35,63 @@ class BaseEntry(BaseEntryVariables, ABC): self._working_directory = working_directory self._kwargs = entry_dict - self._additional_variables: Dict[str, str | int] = {} + @property + def uid(self) -> str: + """ + Returns + ------- + str + The entry's unique ID + """ + return str(self._kwargs[v.uid.metadata_key]) - def kwargs_contains(self, key: str) -> bool: - """Returns whether internal kwargs contains the specified key""" - return key in self._kwargs + @property + def download_archive_extractor(self) -> str: + """ + The extractor name used in yt-dlp download archives + """ + # pylint: disable=line-too-long + # Taken from https://github.com/yt-dlp/yt-dlp/blob/e6ab678e36c40ded0aae305bbb866cdab554d417/yt_dlp/YoutubeDL.py#L3514 + # pylint: enable=line-too-long + return str( + self._kwargs_get(v.extractor_key.metadata_key) + or self._kwargs_get(v.ie_key.metadata_key) + or "NO_EXTRACTOR" + ).lower() - def kwargs(self, key) -> Any: - """Returns an internal kwarg value supplied from ytdl""" - if not self.kwargs_contains(key): - raise KeyError(f"Expected '{key}' in {self.__class__.__name__} but does not exist.") - output = self._kwargs[key] + @property + def title(self) -> str: + """ + The title of the entry. If a title does not exist, returns its unique ID. + """ + return self._kwargs_get(v.title.metadata_key, self.uid) - # Replace curly braces with unicode version to avoid variable shenanigans - if isinstance(output, str): - return output.replace("{", "｛").replace("}", "｝") - return output + @property + def webpage_url(self) -> str: + """ + The url to the webpage. + """ + return self._kwargs[v.webpage_url.metadata_key] - def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any: + @property + def info_json_ext(self) -> str: + """The "info.json" extension""" + return "info.json" + + @property + def uploader_id(self) -> str: + """ + The uploader id if it exists, otherwise return the unique ID. + """ + return self._kwargs_get(v.uploader_id.metadata_key, self.uid) + + def _kwargs_get(self, key: str, default: Optional[Any] = None) -> Any: """ Dict get on kwargs """ - if not self.kwargs_contains(key) or self.kwargs(key) is None: + if (out := self._kwargs.get(key)) is None: return default - return self.kwargs(key) + return out def working_directory(self) -> str: """ @@ -304,31 +118,6 @@ class BaseEntry(BaseEntryVariables, ABC): self._kwargs = dict(self._kwargs, **variables_to_add) return self - def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry": - """ - Parameters - ---------- - variables_to_add - Variables to add to this entry - - Returns - ------- - self - - Raises - ------ - ValueError - If a variable trying to be added already exists as a source variable - """ - for variable_name in variables_to_add.keys(): - if self.kwargs_contains(variable_name): - raise ValueError( - f"Cannot add variable '{variable_name}': already exists in the kwargs" - ) - - self._additional_variables = dict(self._additional_variables, **variables_to_add) - return self - def get_download_info_json_name(self) -> str: """ Returns @@ -345,36 +134,6 @@ class BaseEntry(BaseEntryVariables, ABC): """ return str(Path(self.working_directory()) / self.get_download_info_json_name()) - def _added_variables(self) -> Dict[str, str]: - """ - Returns - ------- - Dict of variables added to this entry - """ - return self._additional_variables - - @classmethod - def source_variables(cls) -> List[str]: - """ - Returns - ------- - List of all source variables - """ - property_names = [prop for prop in dir(cls) if isinstance(getattr(cls, prop), property)] - return property_names - - @final - def to_dict(self) -> Dict[str, str]: - """ - Returns - ------- - Dictionary containing all variables - """ - source_variable_dict = { - source_var: getattr(self, source_var) for source_var in self.source_variables() - } - return dict(source_variable_dict, **self._added_variables()) - @final def to_type(self, entry_type: Type[TBaseEntry]) -> TBaseEntry: """ @@ -393,7 +152,7 @@ class BaseEntry(BaseEntryVariables, ABC): """ entry_type: Optional[str] = None if isinstance(entry_dict, cls): - entry_type = entry_dict.kwargs_get("_type") + entry_type = entry_dict._kwargs_get("_type") if isinstance(entry_dict, dict): entry_type = entry_dict.get("_type") @@ -408,7 +167,7 @@ class BaseEntry(BaseEntryVariables, ABC): """ entry_ext: Optional[str] = None if isinstance(entry_dict, cls): - entry_ext = entry_dict.kwargs_get("ext") + entry_ext = entry_dict._kwargs_get("ext") if isinstance(entry_dict, dict): entry_ext = entry_dict.get("ext") @@ -420,4 +179,4 @@ class BaseEntry(BaseEntryVariables, ABC): ------- extractor + uid, making this a unique hash for any entry """ - return self.extractor + self.uid + return self.download_archive_extractor + self.uid diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index 0538c66c..c7442992 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -1,21 +1,109 @@ +# pylint: disable=protected-access import copy import json import os from pathlib import Path +from typing import Any +from typing import Dict from typing import Optional +from typing import Type +from typing import TypeVar from typing import final from ytdl_sub.entries.base_entry import BaseEntry -from ytdl_sub.entries.variables.entry_variables import EntryVariables +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.entries.script.variable_types import ArrayVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.utils.script import ScriptUtils +from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS +v: VariableDefinitions = VARIABLES -class Entry(EntryVariables, BaseEntry): +_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables" +ytdl_sub_chapters_from_comments = ArrayVariable( + "ytdl_sub_chapters_from_comments", definition="{ [] }" +) +ytdl_sub_split_by_chapters_parent_uid = StringVariable( + "ytdl_sub_split_by_chapters_parent_uid", definition="{ %string('') }" +) + +TypeT = TypeVar("TypeT") + + +class Entry(BaseEntry, Scriptable): """ Entry object to represent a single media object returned from yt-dlp. """ + def __init__(self, entry_dict: Dict, working_directory: str): + BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory) + Scriptable.__init__(self) + + def _add_entry_kwargs_to_script(self) -> None: + # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized + self.unresolvable.remove(v.entry_metadata.variable_name) + self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) + self.update_script() + + def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry": + """ + Initializes the entry script using the Overrides script, then adding + its kwargs to the entry metadata variable + """ + # Overrides contains added variables that are unresolvable, add them here + if other: + self.script = copy.deepcopy(other.script) + self.unresolvable = copy.deepcopy(other.unresolvable) + + self._add_entry_kwargs_to_script() + return self + + def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT: + """ + Gets a variable of an expected type. Will error if it does not exist or is not resolved. + """ + out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name) + return expected_type(out) + + def try_get(self, variable: Variable, expected_type: Type[TypeT]) -> Optional[TypeT]: + """ + Gets a variable of an expected type. Returns None if it does not exist or is not resolved. + """ + try: + return self.get(variable=variable, expected_type=expected_type) + except ScriptVariableNotResolved: + return None + + def add_injected_variables( + self, download_entry: "Entry", download_idx: int, upload_date_idx: int + ) -> "Entry": + """ + Adds variables that get injected into the Entry script that aren't available at + metadata scrape time (only after the actual download). + """ + self.add( + { + # Tracks number of entries downloaded + v.download_index: download_idx + 1, + # Tracks number of entries with the same upload date to make them unique + v.upload_date_index: upload_date_idx + 1, + v.requested_subtitles: download_entry._kwargs_get( + v.requested_subtitles.metadata_key, [] + ), + v.chapters: download_entry._kwargs_get(v.chapters.metadata_key, []), + v.sponsorblock_chapters: download_entry._kwargs_get( + v.sponsorblock_chapters.metadata_key, [] + ), + v.comments: download_entry._kwargs_get(v.comments.metadata_key, []), + } + ) + return self + @property def ext(self) -> str: """ @@ -23,12 +111,13 @@ class Entry(EntryVariables, BaseEntry): This is not reflected in the entry. See if the mkv file exists and return "mkv" if so, otherwise, return the original extension. """ - for possible_ext in [super().ext, "mkv"]: + ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key] + for possible_ext in [ext, "mkv"]: file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}") if os.path.isfile(file_path): return possible_ext - return super().ext + return ext def get_download_file_name(self) -> str: """ @@ -48,7 +137,7 @@ class Entry(EntryVariables, BaseEntry): ------- The download thumbnail's file name """ - return f"{self.uid}.{self.thumbnail_ext}" + return f"{self.uid}.{self.get(v.thumbnail_ext, str)}" def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" @@ -59,7 +148,7 @@ class Entry(EntryVariables, BaseEntry): The source `thumbnail` value and the actual downloaded thumbnail extension sometimes do not match. Return the actual downloaded thumbnail path. """ - thumbnails = self.kwargs_get("thumbnails", []) + thumbnails = self._kwargs_get("thumbnails", []) possible_thumbnail_exts = {"jpg", "webp"} # Always check for jpg and webp thumbs for thumbnail in thumbnails: @@ -77,7 +166,7 @@ class Entry(EntryVariables, BaseEntry): Write the entry's _kwargs back into the info.json file as well as its source variables """ kwargs_dict = copy.deepcopy(self._kwargs) - kwargs_dict["ytdl_sub_entry_variables"] = self.to_dict() + kwargs_dict[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] = self.to_dict() kwargs_json = json.dumps(kwargs_dict, ensure_ascii=False, sort_keys=True, indent=2) with open(self.get_download_info_json_path(), "w", encoding="utf-8") as file: @@ -119,3 +208,39 @@ class Entry(EntryVariables, BaseEntry): break return file_exists + + def maybe_get_prior_variables(self) -> Dict[str, Any]: + """ + If variables exist in the .info.json from a prior run, delete them + from kwargs (to prevent nested writes) and return them + """ + maybe_prior_variables: Dict[str, Any] = {} + if _YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY in self._kwargs: + maybe_prior_variables = self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] + del self._kwargs[_YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] + + return maybe_prior_variables + + @final + def to_dict(self) -> Dict[str, Any]: + """ + Returns + ------- + Dictionary containing all variables + """ + return self.script.resolve().as_native() + + @classmethod + def create_split_entry(cls, entry: "Entry", new_uid: str) -> "Entry": + """ + Creates a copy of an entry with a new uid to use as the starting point for a split entry + """ + new_entry = copy.deepcopy(entry) + new_entry._kwargs[v.uid.metadata_key] = new_uid + new_entry.add( + { + v.uid.variable_name: new_uid, + ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid, + } + ) + return new_entry diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 24e184af..6cf7dcb1 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -1,55 +1,32 @@ import math +from typing import Any from typing import Dict from typing import List from typing import Optional - -import mergedeep +from typing import Set from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import TBaseEntry from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import DESCRIPTION -from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED -from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT -from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY -from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX -from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE -from ytdl_sub.entries.variables.kwargs import SOURCE_UID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import TITLE -from ytdl_sub.entries.variables.kwargs import UID -from ytdl_sub.entries.variables.kwargs import UPLOADER -from ytdl_sub.entries.variables.kwargs import UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.entries.script.variable_types import MetadataVariable + +v: VariableDefinitions = VARIABLES -class ParentType: - PLAYLIST = "playlist" - SOURCE = "source" - - -def _sort_entries(entries: List[TBaseEntry]) -> List[TBaseEntry]: - """Try sorting by playlist_id first, then fall back to uid""" - return sorted(entries, key=lambda ent: (ent.kwargs_get(PLAYLIST_INDEX, math.inf), ent.uid)) +# pylint: disable=protected-access class EntryParent(BaseEntry): + @classmethod + def _sort_entries(cls, entries: List[TBaseEntry]) -> List[TBaseEntry]: + """Try sorting by playlist_id first, then fall back to uid""" + return sorted( + entries, + key=lambda ent: (ent._kwargs_get(v.playlist_index.metadata_key, math.inf), ent.uid), + ) + def __init__(self, entry_dict: Dict, working_directory: str): super().__init__(entry_dict=entry_dict, working_directory=working_directory) self._parent_children: List["EntryParent"] = [] @@ -73,87 +50,44 @@ class EntryParent(BaseEntry): self.entry_children() ) - def _playlist_variables(self, idx: int, children: List[TBaseEntry], parent_type: str) -> Dict: - _count = self.kwargs_get(PLAYLIST_COUNT, len(children)) - _index = children[idx].kwargs_get(PLAYLIST_INDEX, idx + 1) + def _sibling_entry_metadata(self) -> List[Dict[str, Any]]: + sibling_entry_metadata: List[Dict[str, Any]] = [] + variable_filter: Set[MetadataVariable] = ( + v.required_entry_variables() | v.default_entry_variables() + ) + for entry in self.entry_children(): + sibling_entry_metadata.append( + {var.metadata_key: entry._kwargs_get(var.metadata_key) for var in variable_filter} + ) + return sibling_entry_metadata - if parent_type == ParentType.SOURCE: - return {SOURCE_INDEX: _index, SOURCE_COUNT: _count} - return { - SOURCE_INDEX: self.kwargs_get(SOURCE_INDEX, 1), - SOURCE_COUNT: self.kwargs_get(SOURCE_INDEX, 1), - PLAYLIST_INDEX: _index, - PLAYLIST_COUNT: _count, - } - - def _parent_variables(self, parent_type: str) -> Dict: - def _(source_key: str, playlist_key: str) -> str: - return playlist_key if parent_type == ParentType.PLAYLIST else source_key - - def __(key: str) -> Optional[str]: - return self.kwargs_get(key=key) - - return { - _(SOURCE_ENTRY, PLAYLIST_ENTRY): self._kwargs, - _(SOURCE_TITLE, PLAYLIST_TITLE): __(TITLE), - _(SOURCE_WEBPAGE_URL, PLAYLIST_WEBPAGE_URL): __(WEBPAGE_URL), - _(SOURCE_UID, PLAYLIST_UID): __(UID), - _(SOURCE_DESCRIPTION, PLAYLIST_DESCRIPTION): __(DESCRIPTION), - _(SOURCE_UPLOADER, PLAYLIST_UPLOADER): __(UPLOADER), - _(SOURCE_UPLOADER_ID, PLAYLIST_UPLOADER_ID): __(UPLOADER_ID), - _(SOURCE_UPLOADER_URL, PLAYLIST_UPLOADER_URL): __(UPLOADER_URL), - } - - def _get_entry_children_variable_list(self, variable_name: str) -> List[str | int]: - return [getattr(entry_child, variable_name) for entry_child in self.entry_children()] - - def _entry_aggregate_variables(self) -> Dict: - if not self.entry_children(): - return {} - - return { - PLAYLIST_MAX_UPLOAD_YEAR: max(self._get_entry_children_variable_list("upload_year")), - PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED: max( - self._get_entry_children_variable_list("upload_year_truncated") - ), - } - - # pylint: disable=protected-access - - def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent": + def _set_child_variables( + self, include_sibling_metadata: bool, parents: Optional[List["EntryParent"]] = None + ) -> "EntryParent": if parents is None: parents = [self] - self.add_kwargs( - self._playlist_variables(idx=0, children=parents, parent_type=ParentType.SOURCE) - ) - kwargs_to_add: Dict = {} + kwargs_to_add: Dict[str, Any] = {} + if include_sibling_metadata: + kwargs_to_add[v.sibling_metadata.metadata_key] = self._sibling_entry_metadata() + if len(parents) >= 1: - mergedeep.merge(kwargs_to_add, parents[-1]._parent_variables(ParentType.PLAYLIST)) + kwargs_to_add[v.playlist_metadata.metadata_key] = parents[-1]._kwargs if len(parents) >= 2: - mergedeep.merge(kwargs_to_add, parents[-2]._parent_variables(ParentType.SOURCE)) + kwargs_to_add[v.source_metadata.metadata_key] = parents[-2]._kwargs if len(parents) >= 3: raise ValueError( "ytdl-sub currently does support more than 3 layers of playlists/entries. " "If you encounter this error, please file a ticket with the URLs used." ) - mergedeep.merge(kwargs_to_add, self._entry_aggregate_variables()) - for idx, entry_child in enumerate(self.entry_children()): - entry_child.add_kwargs( - self._playlist_variables( - idx=idx, children=self.entry_children(), parent_type=ParentType.PLAYLIST - ) - ) - entry_child.add_kwargs(kwargs_to_add) + for entry_child in self.entry_children(): + entry_child._kwargs = dict(entry_child._kwargs, **kwargs_to_add) - for idx, parent_child in enumerate(self.parent_children()): - parent_child.add_kwargs( - self._playlist_variables( - idx=idx, children=self.parent_children(), parent_type=ParentType.SOURCE - ) + for parent_child in self.parent_children(): + parent_child._set_child_variables( + include_sibling_metadata=include_sibling_metadata, parents=parents + [parent_child] ) - parent_child._set_child_variables(parents=parents + [parent_child]) return self @@ -170,8 +104,10 @@ class EntryParent(BaseEntry): if entry_dict in self ] - self._parent_children = _sort_entries([ent for ent in entries if self.is_entry_parent(ent)]) - self._entry_children = _sort_entries( + self._parent_children = self._sort_entries( + [ent for ent in entries if self.is_entry_parent(ent)] + ) + self._entry_children = self._sort_entries( [ent.to_type(Entry) for ent in entries if self.is_entry(ent)] ) @@ -190,7 +126,7 @@ class EntryParent(BaseEntry): ------- Desired thumbnail url if it exists. None if it does not. """ - for thumbnail in self.kwargs_get("thumbnails", []): + for thumbnail in self._kwargs_get("thumbnails", []): if thumbnail["id"] == thumbnail_id: return thumbnail["url"] return None @@ -200,7 +136,7 @@ class EntryParent(BaseEntry): if isinstance(item, dict): playlist_id = item.get("playlist_id") elif isinstance(item, BaseEntry): - playlist_id = item.kwargs_get("playlist_id") + playlist_id = item._kwargs_get("playlist_id") if not playlist_id: return False @@ -248,7 +184,11 @@ class EntryParent(BaseEntry): @classmethod def from_entry_dicts( - cls, url: str, entry_dicts: List[Dict], working_directory: str + cls, + url: str, + entry_dicts: List[Dict], + working_directory: str, + include_sibling_metadata: bool, ) -> List["EntryParent"]: """ Reads all entry dicts and builds a tree of EntryParents @@ -271,15 +211,16 @@ class EntryParent(BaseEntry): parents = [root_parent] for parent in parents: - parent._set_child_variables() + parent._set_child_variables(include_sibling_metadata=include_sibling_metadata) return parents - # pylint: enable=protected-access - @classmethod def from_entry_dicts_with_no_parents( - cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str + cls, + parents: List["EntryParent"], + entry_dicts: List[Dict], + working_directory: str, ) -> List[Entry]: """ Reads all entries that do not have any parents @@ -289,7 +230,10 @@ class EntryParent(BaseEntry): return any(entry_dict in parent for parent in parents) return [ - Entry(entry_dict=entry_dict, working_directory=working_directory) + Entry( + entry_dict=entry_dict, + working_directory=working_directory, + ) for entry_dict in entry_dicts if cls.is_entry(entry_dict) and not _in_any_parents(entry_dict) ] diff --git a/src/ytdl_sub/entries/script/__init__.py b/src/ytdl_sub/entries/script/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/entries/script/custom_functions.py b/src/ytdl_sub/entries/script/custom_functions.py new file mode 100644 index 00000000..690095de --- /dev/null +++ b/src/ytdl_sub/entries/script/custom_functions.py @@ -0,0 +1,178 @@ +import os +import posixpath + +from yt_dlp.utils import sanitize_filename + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import ReturnableArgument +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.utils.file_path import FilePathTruncater + + +def _pad(num: int, width: int): + return str(num).zfill(width) + + +_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + + +class CustomFunctions: + @staticmethod + def legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument: + """ + ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not + interfere with its legacy variable scripting system. This function replicates that + behavior. + """ + if isinstance(value, String): + value = String(value.value.replace("{", "｛").replace("}", "｝")) + return value + + @staticmethod + def to_native_filepath(filepath: String) -> String: + """ + Convert any unix-based path separators ('/') with the OS's native + separator. + """ + return String(filepath.value.replace(posixpath.sep, os.sep)) + + @staticmethod + def truncate_filepath_if_too_long(filepath: String) -> String: + """ + If a file-path is too long for the OS, this function will truncate it while preserving + the extension. + """ + return String(FilePathTruncater.maybe_truncate_file_path(filepath.value)) + + @staticmethod + def sanitize(value: AnyArgument) -> String: + """ + Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use + for file/directory names on any OS. + """ + return String(sanitize_filename(str(value))) + + @staticmethod + def sanitize_plex_episode(string: String) -> String: + """ + Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width + numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the + season and/or episode. + """ + sanitized_string = CustomFunctions.sanitize(string).value + out = "" + for char in sanitized_string: + match char: + case "0": + out += "０" + case "1": + out += "１" + case "2": + out += "２" + case "3": + out += "３" + case "4": + out += "４" + case "5": + out += "５" + case "6": + out += "６" + case "7": + out += "７" + case "8": + out += "８" + case "9": + out += "９" + case _: + out += char + return String(out) + + @staticmethod + def to_date_metadata(yyyymmdd: String) -> Map: + """ + Takes a date in the form of YYYYMMDD and returns a Map containing: + + - date (String, YYYYMMDD) + - date_standardized (String, YYYY-MM-DD) + - year (Integer) + - month (Integer) + - day (Integer) + - year_truncated (Integer, YY from YY[YY]) + - month_padded (String) + - day_padded (String) + - year_truncated_reversed (Integer, 100 - year_truncated) + - month_reversed (Integer, 13 - month) + - month_reversed_padded (String) + - day_reversed (Integer, total_days_in_month + 1 - day) + - day_reversed_padded (String) + - day_of_year (Integer) + - day_of_year_padded (String, padded 3) + - day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year) + - day_of_year_reversed_padded (String, padded 3) + """ + date_str = yyyymmdd.value + if not (date_str.isnumeric() and len(date_str) == 8): + raise RuntimeException( + f"Expected input of date_metadata to be YYYYMMDD, but received {date_str}" + ) + + year: int = int(date_str[:4]) + month_padded: str = date_str[4:6] + day_padded: str = date_str[6:8] + + month: int = int(month_padded) + day: int = int(day_padded) + year_truncated: int = int(str(year)[-2:]) + + day_of_year: int = sum(_days_in_month[:month]) + day + total_days_in_month: int = _days_in_month[month] + total_days_in_year: int = 365 + if year % 4 == 0: + total_days_in_year += 1 + if month == 2: + total_days_in_month += 1 + if month > 2: + day_of_year += 1 + + day_of_year_reversed: int = total_days_in_year + 1 - day_of_year + month_reversed: int = 13 - month + day_reversed: int = total_days_in_month + 1 - day + + return Map( + { + String("date"): yyyymmdd, + String("date_standardized"): String(f"{year}-{month_padded}-{day_padded}"), + String("year"): Integer(year), + String("month"): Integer(month), + String("day"): Integer(day), + String("year_truncated"): Integer(year_truncated), + String("month_padded"): String(month_padded), + String("day_padded"): String(day_padded), + String("year_truncated_reversed"): Integer(100 - year_truncated), + String("month_reversed"): Integer(month_reversed), + String("month_reversed_padded"): String(_pad(month_reversed, width=2)), + String("day_reversed"): Integer(day_reversed), + String("day_reversed_padded"): String(_pad(day_reversed, width=2)), + String("day_of_year"): Integer(day_of_year), + String("day_of_year_padded"): String(_pad(day_of_year, width=3)), + String("day_of_year_reversed"): Integer(day_of_year_reversed), + String("day_of_year_reversed_padded"): String(_pad(day_of_year_reversed, width=3)), + } + ) + + @staticmethod + def register(): + """ + Register Custom functions once and only once + """ + if not Functions.is_built_in("sanitize"): + Functions.register_function(CustomFunctions.legacy_bracket_safety) + Functions.register_function(CustomFunctions.truncate_filepath_if_too_long) + Functions.register_function(CustomFunctions.to_native_filepath) + Functions.register_function(CustomFunctions.sanitize) + Functions.register_function(CustomFunctions.sanitize_plex_episode) + Functions.register_function(CustomFunctions.to_date_metadata) diff --git a/src/ytdl_sub/entries/script/function_scripts.py b/src/ytdl_sub/entries/script/function_scripts.py new file mode 100644 index 00000000..68019a00 --- /dev/null +++ b/src/ytdl_sub/entries/script/function_scripts.py @@ -0,0 +1,86 @@ +from typing import Dict + +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +v: VariableDefinitions = VARIABLES + +CUSTOM_FUNCTION_SCRIPTS: Dict[str, str] = { + ############################################################################################# + # SIBLING GETTER + "%extract_field_from_siblings": f"""{{ + %if( + %bool({v.sibling_metadata.variable_name}), + %array_apply_fixed( + {v.sibling_metadata.variable_name}, + %string($0), + %map_get + ) + [] + ) + }}""", + ############################################################################################# + # REGEX PLUGIN + # $0 - input variable + # $1 - regex array + # $2 - defaults + # $3 - error message + "%regex_capture_inner": """{ + %assert_then( + %array_reduce( + %array_apply_fixed( + %array_apply( + $1, + %regex_capture_groups + ), + %array_size($2), + %lte + ), + %and + ), + %array_overlay( + %array( + %array_first( + %array_apply_fixed( + %array($1), + %string($0), + %regex_search + ), + [] + ) + ), + %array_extend( ['using all defaults'], $2 ), + True + ), + $3 + ) + }""", + "%regex_capture_many_required": """{ + %assert_ne( + %regex_capture_inner( + $0, $1, ['', '', '', '', '', '', '', '', '', ''], + 'When using %regex_capture_many, number of regex capture groups must be less than or equal to the number of defaults' + ), + ['using all defaults', '', '', '', '', '', '', '', '', '', ''], + 'When running %regex_capture_many_required, no regex strings were captured' + ) + }""", + "%regex_capture_many_with_defaults": """{ + %regex_capture_inner( + $0, $1, $2, + 'When using %regex_capture_with_defaults, number of regex capture groups must be less than or equal to the number of defaults' + ) + }""", + "%regex_search_any": """{ + %ne( + %array_at( + %regex_capture_inner( + $0, $1, [], + 'When using %regex_search_many, all regex strings must contain no capture groups' + ), + 0 + ), + 'using all defaults' + ) + }""", +} diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py new file mode 100644 index 00000000..d06b015f --- /dev/null +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -0,0 +1,1160 @@ +from abc import ABC +from functools import cache +from functools import cached_property +from typing import Dict +from typing import Set + +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.entries.script.variable_types import ArrayMetadataVariable +from ytdl_sub.entries.script.variable_types import IntegerMetadataVariable +from ytdl_sub.entries.script.variable_types import IntegerVariable +from ytdl_sub.entries.script.variable_types import MapMetadataVariable +from ytdl_sub.entries.script.variable_types import MapVariable +from ytdl_sub.entries.script.variable_types import MetadataVariable +from ytdl_sub.entries.script.variable_types import StringDateMetadataVariable +from ytdl_sub.entries.script.variable_types import StringDateVariable +from ytdl_sub.entries.script.variable_types import StringMetadataVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable + +# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion +# pylint: disable=no-member +# pylint: disable=too-many-public-methods +# pylint: disable=too-many-lines + + +class MetadataVariableDefinitions(ABC): + @cached_property + def entry_metadata(self: "VariableDefinitions") -> MapVariable: + """ + :description: + The entry's info.json + """ + return MapVariable(variable_name="entry_metadata", definition="{ {} }") + + @cached_property + def playlist_metadata(self: "VariableDefinitions") -> MapMetadataVariable: + """ + :description: + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) + """ + return MapMetadataVariable.from_entry( + metadata_key="playlist_metadata", + default={}, + ) + + @cached_property + def source_metadata(self: "VariableDefinitions") -> MapMetadataVariable: + """ + :description: + Metadata from the source + (i.e. the grandparent metadata, like channel -> playlist -> entry) + """ + return MapMetadataVariable.from_entry( + metadata_key="source_metadata", + default={}, + ) + + @cached_property + def sibling_metadata(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Metadata from any sibling entries that reside in the same playlist as this entry. + """ + return ArrayMetadataVariable.from_entry( + metadata_key="sibling_metadata", + default=[], + ) + + +class PlaylistVariableDefinitions(ABC): + @cached_property + def playlist_uid(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist unique ID if it exists, otherwise return the entry unique ID. + """ + return StringMetadataVariable.from_playlist( + metadata_key="playlist_id", variable_name="playlist_uid", default=self.uid + ) + + @cached_property + def playlist_title(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + Name of its parent playlist/channel if it exists, otherwise returns its title. + """ + return StringMetadataVariable.from_entry(metadata_key="playlist_title", default=self.title) + + @cached_property + def playlist_index(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return IntegerMetadataVariable.from_entry(metadata_key="playlist_index", default=1) + + @cached_property + def playlist_index_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + Playlist index reversed via ``playlist_count - playlist_index + 1`` + """ + return IntegerVariable( + variable_name="playlist_index_reversed", + definition=f"""{{ + %sub( + {self.playlist_count.variable_name}, + {self.playlist_index.variable_name}, + -1 + ) + }}""", + ) + + @cached_property + def playlist_index_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + playlist_index padded two digits + """ + return self.playlist_index.to_padded_int(variable_name="playlist_index_padded", pad=2) + + @cached_property + def playlist_index_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + playlist_index_reversed padded two digits + """ + return self.playlist_index_reversed.to_padded_int( + variable_name="playlist_index_reversed_padded", + pad=2, + ) + + @cached_property + def playlist_index_padded6(self: "VariableDefinitions") -> StringVariable: + """ + :description: + playlist_index padded six digits. + """ + return self.playlist_index.to_padded_int(variable_name="playlist_index_padded6", pad=6) + + @cached_property + def playlist_index_reversed_padded6(self: "VariableDefinitions") -> StringVariable: + """ + :description: + playlist_index_reversed padded six digits. + """ + return self.playlist_index_reversed.to_padded_int( + variable_name="playlist_index_reversed_padded6", pad=6 + ) + + @cached_property + def playlist_count(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + Playlist count if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return IntegerMetadataVariable.from_entry(metadata_key="playlist_count", default=1) + + @cached_property + def playlist_description(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist description if it exists, otherwise returns the entry's description. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.description.metadata_key, + variable_name="playlist_description", + default=self.description, + ) + + @cached_property + def playlist_webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.webpage_url.metadata_key, + variable_name="playlist_webpage_url", + default=self.webpage_url, + ) + + @cached_property + def playlist_max_upload_date(self: "VariableDefinitions") -> StringDateVariable: + """ + :description: + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` + """ + return StringVariable( + variable_name="playlist_max_upload_date", + definition=f"""{{ + %array_reduce( + %if_passthrough( + %extract_field_from_siblings('{self.upload_date.variable_name}'), + [{self.upload_date.variable_name}] + ), + %max + ) + }}""", + ).as_date_variable() + + @cached_property + def playlist_max_upload_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + """ + return self.playlist_max_upload_date.get_integer_date_metadata( + date_metadata_key="year", + variable_name="playlist_max_upload_year", + ) + + @cached_property + def playlist_max_upload_year_truncated(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + """ + return self.playlist_max_upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="playlist_max_upload_year_truncated" + ) + + @cached_property + def playlist_uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist uploader id if it exists, otherwise returns the entry uploader ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="playlist_uploader_id", + default=self.uploader_id, + ) + + @cached_property + def playlist_uploader(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist uploader if it exists, otherwise return the entry uploader. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.uploader.metadata_key, + variable_name="playlist_uploader", + default=self.uploader, + ) + + @cached_property + def playlist_uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.uploader_url.metadata_key, + variable_name="playlist_uploader_url", + default=self.webpage_url, + ) + + +class SourceVariableDefinitions(ABC): + @cached_property + def source_title(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + """ + return StringMetadataVariable.from_source( + metadata_key=self.title.metadata_key, + variable_name="source_title", + default=self.playlist_title, + ) + + @cached_property + def source_uid(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source unique id if it exists, otherwise returns the playlist unique ID. + """ + return StringMetadataVariable.from_source( + metadata_key=self.uid.metadata_key, + variable_name="source_uid", + default=self.playlist_uid, + ) + + @cached_property + def source_index(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + """ + return IntegerMetadataVariable.from_playlist( + metadata_key=self.playlist_index.metadata_key, + variable_name="source_index", + default=1, + ) + + @cached_property + def source_index_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The source index, padded two digits. + """ + return self.source_index.to_padded_int(variable_name="source_index_padded", pad=2) + + @cached_property + def source_count(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + The source count if it exists, otherwise returns ``1``. + """ + return IntegerMetadataVariable.from_playlist( + metadata_key=self.playlist_count.metadata_key, + variable_name="source_count", + default=1, + ) + + @cached_property + def source_webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source webpage url if it exists, otherwise returns the playlist webpage url. + """ + return StringMetadataVariable.from_source( + metadata_key=self.webpage_url.metadata_key, + variable_name="source_webpage_url", + default=self.playlist_webpage_url, + ) + + @cached_property + def source_description(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source description if it exists, otherwise returns the playlist description. + """ + return StringMetadataVariable.from_source( + metadata_key=self.description.metadata_key, + variable_name="source_description", + default=self.playlist_description, + ) + + @cached_property + def source_uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source uploader id if it exists, otherwise returns the playlist_uploader_id + """ + return StringMetadataVariable.from_source( + metadata_key=self.uploader_id.metadata_key, + variable_name="source_uploader_id", + default=self.playlist_uploader_id, + ) + + @cached_property + def source_uploader(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source uploader if it exists, otherwise return the playlist_uploader + """ + return StringMetadataVariable.from_source( + metadata_key=self.uploader.metadata_key, + variable_name="source_uploader", + default=self.playlist_uploader, + ) + + @cached_property + def source_uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source uploader url if it exists, otherwise returns the source webpage_url. + """ + return StringMetadataVariable.from_source( + metadata_key=self.uploader_url.metadata_key, + variable_name="source_uploader_url", + default=self.source_webpage_url, + ) + + +class UploadDateVariableDefinitions(ABC): + @cached_property + def upload_date(self: "VariableDefinitions") -> StringDateMetadataVariable: + """ + :description: + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + """ + return StringDateMetadataVariable.from_entry(metadata_key="upload_date").as_date_variable() + + @cached_property + def upload_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The entry's upload year + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year", variable_name="upload_year" + ) + + @cached_property + def upload_year_truncated(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The last two digits of the upload year, i.e. 22 in 2022 + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="upload_year_truncated" + ) + + @cached_property + def upload_year_truncated_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated_reversed", + variable_name="upload_year_truncated_reversed", + ) + + @cached_property + def upload_month_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="month_reversed", variable_name="upload_month_reversed" + ) + + @cached_property + def upload_month_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload month, but padded. i.e. November returns "02" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="month_reversed_padded", variable_name="upload_month_reversed_padded" + ) + + @cached_property + def upload_month_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="month_padded", variable_name="upload_month_padded" + ) + + @cached_property + def upload_day_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_padded", variable_name="upload_day_padded" + ) + + @cached_property + def upload_month(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month as an integer (no padding). + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="month", variable_name="upload_month" + ) + + @cached_property + def upload_day(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day as an integer (no padding). + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day", variable_name="upload_day" + ) + + @cached_property + def upload_day_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_reversed", variable_name="upload_day_reversed" + ) + + @cached_property + def upload_day_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_reversed_padded", variable_name="upload_day_reversed_padded" + ) + + @cached_property + def upload_day_of_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The day of the year, i.e. February 1st returns ``32`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_of_year", variable_name="upload_day_of_year" + ) + + @cached_property + def upload_day_of_year_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload day of year, but padded i.e. February 1st returns "032" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_of_year_padded", variable_name="upload_day_of_year_padded" + ) + + @cached_property + def upload_day_of_year_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_of_year_reversed", variable_name="upload_day_of_year_reversed" + ) + + @cached_property + def upload_day_of_year_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_of_year_reversed_padded", + variable_name="upload_day_of_year_reversed_padded", + ) + + @cached_property + def upload_date_standardized(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The uploaded date formatted as YYYY-MM-DD + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="date_standardized", variable_name="upload_date_standardized" + ) + + +class ReleaseDateVariableDefinitions(ABC): + @cached_property + def release_date(self: "VariableDefinitions") -> StringDateMetadataVariable: + """ + :description: + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + """ + return StringMetadataVariable.from_entry( + metadata_key="release_date", default=self.upload_date + ).as_date_variable() + + @cached_property + def release_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The entry's upload year + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="year", variable_name="release_year" + ) + + @cached_property + def release_year_truncated(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The last two digits of the upload year, i.e. 22 in 2022 + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="release_year_truncated" + ) + + @cached_property + def release_year_truncated_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="year_truncated_reversed", + variable_name="release_year_truncated_reversed", + ) + + @cached_property + def release_month_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="month_reversed", variable_name="release_month_reversed" + ) + + @cached_property + def release_month_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload month, but padded. i.e. November returns "02" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="month_reversed_padded", variable_name="release_month_reversed_padded" + ) + + @cached_property + def release_month_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="month_padded", variable_name="release_month_padded" + ) + + @cached_property + def release_day_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_padded", variable_name="release_day_padded" + ) + + @cached_property + def release_month(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month as an integer (no padding). + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="month", variable_name="release_month" + ) + + @cached_property + def release_day(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day as an integer (no padding). + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day", variable_name="release_day" + ) + + @cached_property + def release_day_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_reversed", variable_name="release_day_reversed" + ) + + @cached_property + def release_day_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_reversed_padded", variable_name="release_day_reversed_padded" + ) + + @cached_property + def release_day_of_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The day of the year, i.e. February 1st returns ``32`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_of_year", variable_name="release_day_of_year" + ) + + @cached_property + def release_day_of_year_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload day of year, but padded i.e. February 1st returns "032" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_of_year_padded", variable_name="release_day_of_year_padded" + ) + + @cached_property + def release_day_of_year_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_of_year_reversed", variable_name="release_day_of_year_reversed" + ) + + @cached_property + def release_day_of_year_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_of_year_reversed_padded", + variable_name="release_day_of_year_reversed_padded", + ) + + @cached_property + def release_date_standardized(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The uploaded date formatted as YYYY-MM-DD + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="date_standardized", variable_name="release_date_standardized" + ) + + +class YtdlSubVariableDefinitions(ABC): + @cached_property + def ytdl_sub_input_url(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The input URL used in ytdl-sub to create this entry. + """ + return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }") + + @cached_property + def download_index(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. + """ + return IntegerVariable(variable_name="download_index", definition="{ %int(1) }") + + @cached_property + def download_index_padded6(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The download_index padded six digits + """ + return self.download_index.to_padded_int(variable_name="download_index_padded6", pad=6) + + @cached_property + def upload_date_index(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The i'th entry downloaded with this upload date. + """ + return IntegerVariable(variable_name="upload_date_index", definition="{ %int(1) }") + + @cached_property + def upload_date_index_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload_date_index padded two digits + """ + return self.upload_date_index.to_padded_int(variable_name="upload_date_index_padded", pad=2) + + @cached_property + def upload_date_index_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + 100 - upload_date_index + """ + return IntegerVariable( + variable_name="upload_date_index_reversed", + definition=f"{{%sub(100, {self.upload_date_index.variable_name})}}", + ) + + @cached_property + def upload_date_index_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload_date_index padded two digits + """ + return self.upload_date_index_reversed.to_padded_int( + variable_name="upload_date_index_reversed_padded", pad=2 + ) + + +class EntryVariableDefinitions(ABC): + @cached_property + def uid(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The entry's unique ID + """ + return StringMetadataVariable.from_entry(metadata_key="id", variable_name="uid") + + @cached_property + def duration(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + The duration of the entry in seconds if it exists. Defaults to zero otherwise. + """ + return IntegerMetadataVariable.from_entry(metadata_key="duration", default=0) + + @cached_property + def uid_sanitized_plex(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + """ + return self.uid.to_sanitized_plex(variable_name="uid_sanitized_plex") + + @cached_property + def ie_key(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The ie_key, used in legacy yt-dlp things as the 'info-extractor key'. + If it does not exist, return ``extractor_key`` + """ + return StringMetadataVariable.from_entry(metadata_key="ie_key", default=self.extractor_key) + + @cached_property + def extractor_key(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The yt-dlp extractor key + """ + return StringMetadataVariable.from_entry(metadata_key="extractor_key") + + @cached_property + def extractor(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The yt-dlp extractor name + """ + return StringMetadataVariable.from_entry(metadata_key="extractor") + + @cached_property + def epoch(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + The unix epoch of when the metadata was scraped by yt-dlp. + """ + return IntegerMetadataVariable.from_entry(metadata_key="epoch") + + @cached_property + def epoch_date(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The epoch's date, in YYYYMMDD format. + """ + return StringVariable( + variable_name="epoch_date", + definition=f"{{%datetime_strftime({self.epoch.variable_name}, '%Y%m%d')}}", + ) + + @cached_property + def epoch_hour(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The epoch's hour + """ + return StringVariable( + variable_name="epoch_hour", + definition=f"{{%datetime_strftime({self.epoch.variable_name}, '%H')}}", + ) + + @cached_property + def title(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The title of the entry. If a title does not exist, returns its unique ID. + """ + return StringMetadataVariable.from_entry(metadata_key="title", default=self.uid) + + @cached_property + def title_sanitized_plex(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + """ + return self.title.to_sanitized_plex(variable_name="title_sanitized_plex") + + @cached_property + def webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The url to the webpage. + """ + return StringMetadataVariable.from_entry(metadata_key="webpage_url") + + @cached_property + def info_json_ext(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The "info.json" extension + """ + return StringVariable( + variable_name="info_json_ext", + definition="info.json", + ) + + @cached_property + def description(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The description if it exists. Otherwise, returns an emtpy string. + """ + return StringMetadataVariable.from_entry( + metadata_key="description", + default="", + ) + + @cached_property + def uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The uploader id if it exists, otherwise return the unique ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="uploader_id", + default=self.uid, + ) + + @cached_property + def uploader(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The uploader if it exists, otherwise return the uploader ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="uploader", + default=self.uploader_id, + ) + + @cached_property + def uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The uploader url if it exists, otherwise returns the webpage_url. + """ + return StringMetadataVariable.from_entry( + metadata_key="uploader_url", + default=self.webpage_url, + ) + + @cached_property + def creator(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The creator name if it exists, otherwise returns the channel. + """ + return StringMetadataVariable.from_entry( + metadata_key="creator", + default=self.channel, + ) + + @cached_property + def channel(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The channel name if it exists, otherwise returns the uploader. + """ + return StringMetadataVariable.from_entry( + metadata_key="channel", + default=self.uploader, + ) + + @cached_property + def channel_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The channel id if it exists, otherwise returns the entry uploader ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="channel_id", + default=self.uploader_id, + ) + + @cached_property + def ext(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The downloaded entry's file extension + """ + return StringMetadataVariable.from_entry(metadata_key="ext") + + @cached_property + def thumbnail_ext(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. + """ + return StringVariable( + variable_name="thumbnail_ext", + definition="jpg", + ) + + @cached_property + def comments(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Comments if they are requested + """ + return ArrayMetadataVariable( + metadata_key="comments", variable_name="comments", definition="{ [] }" + ) + + @cached_property + def chapters(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Chapters if they exist + """ + return ArrayMetadataVariable( + metadata_key="chapters", variable_name="chapters", definition="{ [] }" + ) + + @cached_property + def sponsorblock_chapters(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Sponsorblock Chapters if they are requested and exist + """ + return ArrayMetadataVariable( + metadata_key="sponsorblock_chapters", + variable_name="sponsorblock_chapters", + definition="{ [] }", + ) + + @cached_property + def requested_subtitles(self: "VariableDefinitions") -> MapMetadataVariable: + """ + :description: + Subtitles if they are requested and exist + """ + return MapMetadataVariable( + metadata_key="requested_subtitles", + variable_name="requested_subtitles", + definition="{ {} }", + ) + + +class VariableDefinitions( + EntryVariableDefinitions, + MetadataVariableDefinitions, + PlaylistVariableDefinitions, + SourceVariableDefinitions, + UploadDateVariableDefinitions, + ReleaseDateVariableDefinitions, + YtdlSubVariableDefinitions, +): + @cache + def scripts(self) -> Dict[str, str]: + """ + Returns all variables and their scripts in dict form + """ + return { + var.variable_name: var.definition + for var in [ + getattr(self, attr) + for attr in dir(self) + if isinstance(getattr(self, attr), Variable) + ] + } + + @cache + def injected_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that get injected in the download-stage + """ + return { + self.download_index, + self.upload_date_index, + self.comments, + self.requested_subtitles, + self.chapters, + self.sponsorblock_chapters, + self.ytdl_sub_input_url, + } + + @cache + def required_entry_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that the entry requires to exist + """ + return { + self.uid, + self.extractor_key, + self.epoch, + self.webpage_url, + self.ext, + } + + @cache + def default_entry_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that reside on the entry that may or may not exist, + but have defaults + """ + return { + self.title, + self.extractor, + self.description, + self.ie_key, + self.uploader_id, + self.uploader, + self.uploader_url, + self.upload_date, + self.release_date, + self.channel, + self.creator, + self.channel_id, + self.duration, + self.playlist_index, + self.playlist_count, + self.playlist_uid, + self.playlist_title, + self.playlist_uploader_id, + } + + @cache + def unresolvable_static_variables(self) -> Set[Variable]: + """ + Returns variables that are not static (i.e. depend on runtime) + """ + return { + VARIABLES.entry_metadata, + } | self.injected_variables() + + +# Singletons to use externally +VARIABLES: VariableDefinitions = VariableDefinitions() +VARIABLE_SCRIPTS: Dict[str, str] = VARIABLES.scripts() +UNRESOLVED_VARIABLES: Set[str] = { + var.variable_name for var in VARIABLES.unresolvable_static_variables() +} + +CustomFunctions.register() diff --git a/src/ytdl_sub/entries/script/variable_types.py b/src/ytdl_sub/entries/script/variable_types.py new file mode 100644 index 00000000..7cca58cd --- /dev/null +++ b/src/ytdl_sub/entries/script/variable_types.py @@ -0,0 +1,321 @@ +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Dict +from typing import List +from typing import Optional +from typing import Type +from typing import TypeVar + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String + +ENTRY_METADATA_VARIABLE_NAME = "entry_metadata" +PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata" +SOURCE_METADATA_VARIABLE_NAME = "source_metadata" + +TMetadataVariable = TypeVar("TMetadataVariable", bound="MetadataVariable") +TVariable = TypeVar("TVariable", bound="Variable") + + +def _get( + cast: str, + metadata_variable_name: str, + metadata_key: str, + variable_name: Optional[str], + default: Optional[TVariable | str | int | Dict | List], + as_type: Type[TMetadataVariable], +) -> TMetadataVariable: + if default is None: + # TODO: assert with good error message if key DNE + out = f"%map_get({metadata_variable_name}, '{metadata_key}')" + elif isinstance(default, Variable): + args = f"{metadata_variable_name}, '{metadata_key}', {default.variable_name}" + out = f"%map_get_non_empty({args})" + elif isinstance(default, str): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', '{default}')" + elif isinstance(default, dict): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {{}})" + elif isinstance(default, list): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', [])" + else: + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {default})" + + return as_type( + variable_name=variable_name or metadata_key, + metadata_key=metadata_key, + definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}", + ) + + +@dataclass(frozen=True) +class Variable(ABC): + variable_name: str + definition: str + + @classmethod + @abstractmethod + def human_readable_type(cls) -> str: + """ + Script type of the variable, for documentation + """ + + +@dataclass(frozen=True) +class StringVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return String.__name__ + + def to_sanitized_plex(self, variable_name: str) -> "StringVariable": + """ + Converts a String variable to be plex sanitized + """ + return StringVariable( + variable_name=variable_name, + definition=f"{{%sanitize_plex_episode({self.variable_name})}}", + ) + + def as_date_variable(self) -> "StringDateVariable": + """ + Converts a String variable to a date variable (which has metadata helpers) + """ + return StringDateVariable( + variable_name=self.variable_name, + definition=self.definition, + ) + + +@dataclass(frozen=True) +class StringDateVariable(StringVariable): + def get_string_date_metadata( + self, date_metadata_key: str, variable_name: Optional[str] = None + ) -> StringVariable: + """ + Gets a string-based date metadata variable + """ + return StringVariable( + variable_name=variable_name or date_metadata_key, + definition=f"""{{ + %string( + %map_get( + %to_date_metadata({self.variable_name}), + '{date_metadata_key}' + ) + ) + }}""", + ) + + def get_integer_date_metadata( + self, date_metadata_key: str, variable_name: str + ) -> "IntegerVariable": + """ + Gets an int-based date metadata variable + """ + return IntegerVariable( + variable_name=variable_name, + definition=f"""{{ + %int( + %map_get( + %to_date_metadata({self.variable_name}), + '{date_metadata_key}' + ) + ) + }}""", + ) + + +@dataclass(frozen=True) +class IntegerVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Integer.__name__ + + def to_padded_int(self, variable_name: str, pad: int) -> StringVariable: + """ + Pads an integer + """ + return StringVariable( + variable_name=variable_name, definition=f"{{%pad_zero({self.variable_name}, {pad})}}" + ) + + +@dataclass(frozen=True) +class ArrayVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Array.__name__ + + +@dataclass(frozen=True) +class MapVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Map.__name__ + + +@dataclass(frozen=True) +class MetadataVariable(Variable, ABC): + metadata_key: str + + +@dataclass(frozen=True) +class MapMetadataVariable(MetadataVariable, MapVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional["MapMetadataVariable" | Dict] = None, + ) -> "MapMetadataVariable": + """ + Creates a map variable from entry metadata + """ + return _get( + "map", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=MapMetadataVariable, + ) + + +@dataclass(frozen=True) +class ArrayMetadataVariable(MetadataVariable, ArrayVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional["ArrayMetadataVariable" | List] = None, + ) -> "ArrayMetadataVariable": + """ + Creates an array variable from entry metadata + """ + return _get( + "array", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=ArrayMetadataVariable, + ) + + +@dataclass(frozen=True) +class StringMetadataVariable(MetadataVariable, StringVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from entry metadata + """ + return _get( + "string", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + @classmethod + def from_playlist( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from playlist metadata + """ + return _get( + "string", + metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + @classmethod + def from_source( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from source metadata + """ + return _get( + "string", + metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + def as_date_variable(self) -> "StringDateMetadataVariable": + """ + Converts a String variable to a date variable (which has metadata helpers) + """ + return StringDateMetadataVariable( + metadata_key=self.metadata_key, + variable_name=self.variable_name, + definition=self.definition, + ) + + +@dataclass(frozen=True) +class StringDateMetadataVariable(StringMetadataVariable, StringDateVariable): + pass + + +@dataclass(frozen=True) +class IntegerMetadataVariable(MetadataVariable, IntegerVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[IntegerVariable | int] = None, + ) -> "IntegerMetadataVariable": + """ + Creates an int variable from entry metadata + """ + return _get( + "int", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=IntegerMetadataVariable, + ) + + @classmethod + def from_playlist( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[IntegerVariable | int] = None, + ) -> "IntegerMetadataVariable": + """ + Creates an int variable from playlist metadata + """ + return _get( + "int", + metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=IntegerMetadataVariable, + ) diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py deleted file mode 100644 index 8405b060..00000000 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ /dev/null @@ -1,857 +0,0 @@ -from datetime import datetime -from typing import Union - -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.entries.base_entry import BaseEntry -from ytdl_sub.entries.base_entry import BaseEntryVariables -from ytdl_sub.entries.variables.kwargs import CHANNEL -from ytdl_sub.entries.variables.kwargs import CHANNEL_ID -from ytdl_sub.entries.variables.kwargs import CREATOR -from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX -from ytdl_sub.entries.variables.kwargs import EXT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR -from ytdl_sub.entries.variables.kwargs import PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED -from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import PLAYLIST_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import PLAYLIST_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import RELEASE_DATE -from ytdl_sub.entries.variables.kwargs import SOURCE_COUNT -from ytdl_sub.entries.variables.kwargs import SOURCE_DESCRIPTION -from ytdl_sub.entries.variables.kwargs import SOURCE_INDEX -from ytdl_sub.entries.variables.kwargs import SOURCE_TITLE -from ytdl_sub.entries.variables.kwargs import SOURCE_UID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_ID -from ytdl_sub.entries.variables.kwargs import SOURCE_UPLOADER_URL -from ytdl_sub.entries.variables.kwargs import SOURCE_WEBPAGE_URL -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX - -# This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion -# pylint: disable=no-member -# pylint: disable=too-many-public-methods - - -def pad(num: int, width: int = 2): - """Pad integers""" - return str(num).zfill(width) - - -_days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - -Self = Union[BaseEntry, "EntryVariables"] - - -class EntryVariables(BaseEntryVariables): - @property - def source_title(self: Self) -> str: - """ - Returns - ------- - str - Name of the source (i.e. channel with multiple playlists) if it exists, otherwise - returns its playlist_title. - """ - return self.kwargs_get(SOURCE_TITLE, self.playlist_title) - - @property - def source_title_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The source title, sanitized - """ - return sanitize_filename(self.source_title) - - @property - def source_uid(self: Self) -> str: - """ - Returns - ------- - str - The source unique id if it exists, otherwise returns the playlist unique ID. - """ - return self.kwargs_get(SOURCE_UID, self.playlist_uid) - - @property - def source_index(self: Self) -> int: - """ - Returns - ------- - int - Source index if it exists, otherwise returns ``1``. - - It is recommended to not use this unless you know the source will never add new content - (it is easy for this value to change). - """ - return self.kwargs_get(SOURCE_INDEX, self.playlist_index) - - @property - def source_index_padded(self: Self) -> str: - """ - Returns - ------- - int - The source index, padded. - """ - return pad(self.source_index, 2) - - @property - def source_count(self: Self) -> int: - """ - Returns - ------- - int - The source count if it exists, otherwise returns the playlist count. - """ - return self.kwargs_get(SOURCE_COUNT, self.playlist_count) - - @property - def source_webpage_url(self: Self) -> str: - """ - Returns - ------- - str - The source webpage url if it exists, otherwise returns the playlist webpage url. - """ - return self.kwargs_get(SOURCE_WEBPAGE_URL, self.playlist_webpage_url) - - @property - def source_description(self: Self) -> str: - """ - Returns - ------- - str - The source description if it exists, otherwise returns the playlist description. - """ - return self.kwargs_get(SOURCE_DESCRIPTION, self.playlist_description) - - @property - def playlist_uid(self: Self) -> str: - """ - Returns - ------- - str - The playlist unique ID if it exists, otherwise return the entry unique ID. - """ - return self.kwargs_get(PLAYLIST_UID, self.uid) - - @property - def playlist_title(self: Self) -> str: - """ - Returns - ------- - str - Name of its parent playlist/channel if it exists, otherwise returns its title. - """ - return self.kwargs_get(PLAYLIST_TITLE, self.title) - - @property - def playlist_title_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The playlist name, sanitized - """ - return sanitize_filename(self.playlist_title) - - @property - def playlist_index(self: Self) -> int: - """ - Returns - ------- - int - Playlist index if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return self.kwargs_get(PLAYLIST_INDEX, 1) - - @property - def playlist_index_reversed(self: Self) -> int: - """ - Returns - ------- - int - Playlist index reversed via ``playlist_count - playlist_index + 1`` - """ - return self.playlist_count - self.playlist_index + 1 - - @property - def playlist_index_padded(self: Self) -> str: - """ - Returns - ------- - str - playlist_index padded two digits - """ - return pad(self.playlist_index, width=2) - - @property - def playlist_index_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - playlist_index_reversed padded two digits - """ - return pad(self.playlist_index_reversed, width=2) - - @property - def playlist_index_padded6(self: Self) -> str: - """ - Returns - ------- - str - playlist_index padded six digits. - """ - return pad(self.playlist_index, width=6) - - @property - def playlist_index_reversed_padded6(self: Self) -> str: - """ - Returns - ------- - str - playlist_index_reversed padded six digits. - """ - return pad(self.playlist_index_reversed, width=6) - - @property - def playlist_count(self: Self) -> int: - """ - Returns - ------- - int - Playlist count if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return self.kwargs_get(PLAYLIST_COUNT, 1) - - @property - def playlist_description(self: Self) -> str: - """ - Returns - ------- - str - The playlist description if it exists, otherwise returns the entry's description. - """ - return self.kwargs_get(PLAYLIST_DESCRIPTION, self.description) - - @property - def playlist_webpage_url(self: Self) -> str: - """ - Returns - ------- - str - The playlist webpage url if it exists. Otherwise, returns the entry webpage url. - """ - return self.kwargs_get(PLAYLIST_WEBPAGE_URL, self.webpage_url) - - @property - def playlist_max_upload_year(self: Self) -> int: - """ - Returns - ------- - int - Max upload_year for all entries in this entry's playlist if it exists, otherwise returns - ``upload_year`` - """ - # override in EntryParent - return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR, self.upload_year) - - @property - def playlist_max_upload_year_truncated(self: Self) -> int: - """ - Returns - ------- - int - The max playlist truncated upload year for all entries in this entry's playlist if it - exists, otherwise returns ``upload_year_truncated``. - """ - return self.kwargs_get(PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED, self.upload_year_truncated) - - @property - def playlist_uploader_id(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader id if it exists, otherwise returns the entry uploader ID. - """ - return self.kwargs_get(PLAYLIST_UPLOADER_ID, self.uploader_id) - - @property - def playlist_uploader(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader if it exists, otherwise return the entry uploader. - """ - return self.kwargs_get(PLAYLIST_UPLOADER, self.uploader) - - @property - def playlist_uploader_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader, sanitized. - """ - return sanitize_filename(self.playlist_uploader) - - @property - def playlist_uploader_url(self: Self) -> str: - """ - Returns - ------- - str - The playlist uploader url if it exists, otherwise returns the playlist webpage_url. - """ - return self.kwargs_get(PLAYLIST_UPLOADER_URL, self.playlist_webpage_url) - - @property - def source_uploader_id(self: Self) -> str: - """ - Returns - ------- - str - The source uploader id if it exists, otherwise returns the playlist_uploader_id - """ - return self.kwargs_get(SOURCE_UPLOADER_ID, self.playlist_uploader_id) - - @property - def source_uploader(self: Self) -> str: - """ - Returns - ------- - str - The source uploader if it exists, otherwise return the playlist_uploader - """ - return self.kwargs_get(SOURCE_UPLOADER, self.playlist_uploader) - - @property - def source_uploader_url(self: Self) -> str: - """ - Returns - ------- - str - The source uploader url if it exists, otherwise returns the source webpage_url. - """ - return self.kwargs_get(SOURCE_UPLOADER_URL, self.source_webpage_url) - - @property - def creator(self: Self) -> str: - """ - Returns - ------- - str - The creator name if it exists, otherwise returns the channel. - """ - return self.kwargs_get(CREATOR, self.channel) - - @property - def creator_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The creator name, sanitized - """ - return sanitize_filename(self.creator) - - @property - def channel(self: Self) -> str: - """ - Returns - ------- - str - The channel name if it exists, otherwise returns the uploader. - """ - return self.kwargs_get(CHANNEL, self.uploader) - - @property - def channel_sanitized(self: Self) -> str: - """ - Returns - ------- - str - The channel name, sanitized. - """ - return sanitize_filename(self.channel) - - @property - def channel_id(self: Self) -> str: - """ - Returns - ------- - str - The channel id if it exists, otherwise returns the entry uploader ID. - """ - return self.kwargs_get(CHANNEL_ID, self.uploader_id) - - @property - def ext(self: Self) -> str: - """ - Returns - ------- - str - The downloaded entry's file extension - """ - return self.kwargs(EXT) - - @property - def thumbnail_ext(self: Self) -> str: - """ - Returns - ------- - str - The download entry's thumbnail extension. Will always return 'jpg'. Until there is a - need to support other image types, we always convert to jpg. - """ - return "jpg" - - @property - def download_index(self: Self) -> int: - """ - Returns - ------- - int - The i'th entry downloaded. NOTE that this is fetched dynamically from the download - archive. - """ - return self.kwargs_get(DOWNLOAD_INDEX, 0) + 1 - - @property - def download_index_padded6(self: Self) -> str: - """ - Returns - ------- - str - The download_index padded six digits - """ - return pad(self.download_index, 6) - - @property - def upload_date_index(self: Self) -> int: - """ - Returns - ------- - int - The i'th entry downloaded with this upload date. - """ - return self.kwargs_get(UPLOAD_DATE_INDEX, 0) + 1 - - @property - def upload_date_index_padded(self: Self) -> str: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return pad(self.upload_date_index, 2) - - @property - def upload_date_index_reversed(self: Self) -> int: - """ - Returns - ------- - int - 100 - upload_date_index - """ - return 100 - self.upload_date_index - - @property - def upload_date_index_reversed_padded(self: Self) -> str: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return pad(self.upload_date_index_reversed, 2) - - @property - def upload_date(self: Self) -> str: - """ - Returns - ------- - str - The entry's uploaded date, in YYYYMMDD format. If not present, return today's date. - """ - return self.kwargs_get(UPLOAD_DATE, datetime.now().strftime("%Y%m%d")) - - @property - def upload_year(self: Self) -> int: - """ - Returns - ------- - int - The entry's upload year - """ - return int(self.upload_date[:4]) - - @property - def upload_year_truncated(self: Self) -> int: - """ - Returns - ------- - int - The last two digits of the upload year, i.e. 22 in 2022 - """ - return int(str(self.upload_year)[-2:]) - - @property - def upload_year_truncated_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return 100 - self.upload_year_truncated - - @property - def upload_month_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` - """ - return 13 - self.upload_month - - @property - def upload_month_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed upload month, but padded. i.e. November returns "02" - """ - return pad(self.upload_month_reversed) - - @property - def upload_month_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's upload month padded to two digits, i.e. March returns "03" - """ - return self.upload_date[4:6] - - @property - def upload_day_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's upload day padded to two digits, i.e. the fifth returns "05" - """ - return self.upload_date[6:8] - - @property - def upload_month(self: Self) -> int: - """ - Returns - ------- - int - The upload month as an integer (no padding). - """ - return int(self.upload_month_padded.lstrip("0")) - - @property - def upload_day(self: Self) -> int: - """ - Returns - ------- - int - The upload day as an integer (no padding). - """ - return int(self.upload_day_padded.lstrip("0")) - - @property - def upload_day_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, - i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - total_days_in_month = _days_in_month[self.upload_month] - if self.upload_month == 2 and self.upload_year % 4 == 0: # leap year - total_days_in_month += 1 - - return total_days_in_month + 1 - self.upload_day - - @property - def upload_day_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed upload day, but padded. i.e. August 30th returns "02". - """ - return pad(self.upload_day_reversed) - - @property - def upload_day_of_year(self: Self) -> int: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - output = sum(_days_in_month[: self.upload_month]) + self.upload_day - if self.upload_month > 2 and self.upload_year % 4 == 0: - output += 1 - - return output - - @property - def upload_day_of_year_padded(self: Self) -> str: - """ - Returns - ------- - str - The upload day of year, but padded i.e. February 1st returns "032" - """ - return pad(self.upload_day_of_year, width=3) - - @property - def upload_day_of_year_reversed(self: Self) -> int: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, - i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - total_days_in_year = 365 - if self.upload_year % 4 == 0: - total_days_in_year += 1 - - return total_days_in_year + 1 - self.upload_day_of_year - - @property - def upload_day_of_year_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed upload day of year, but padded i.e. December 31st returns "001" - """ - return pad(self.upload_day_of_year_reversed, width=3) - - @property - def upload_date_standardized(self: Self) -> str: - """ - Returns - ------- - str - The uploaded date formatted as YYYY-MM-DD - """ - return f"{self.upload_year}-{self.upload_month_padded}-{self.upload_day_padded}" - - @property - def release_date(self: Self) -> str: - """ - Returns - ------- - str - The entry's release date, in YYYYMMDD format. If not present, return the upload date. - """ - return self.kwargs_get(RELEASE_DATE, self.upload_date) - - @property - def release_year(self: Self) -> int: - """ - Returns - ------- - int - The entry's release year - """ - return int(self.release_date[:4]) - - @property - def release_year_truncated(self: Self) -> int: - """ - Returns - ------- - int - The last two digits of the release year, i.e. 22 in 2022 - """ - return int(str(self.release_year)[-2:]) - - @property - def release_year_truncated_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return 100 - self.release_year_truncated - - @property - def release_month_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release month, but reversed - using ``13 - {release_month}``, i.e. March returns ``10`` - """ - return 13 - self.release_month - - @property - def release_month_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed release month, but padded. i.e. November returns "02" - """ - return pad(self.release_month_reversed) - - @property - def release_month_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's release month padded to two digits, i.e. March returns "03" - """ - return self.release_date[4:6] - - @property - def release_day_padded(self: Self) -> str: - """ - Returns - ------- - str - The entry's release day padded to two digits, i.e. the fifth returns "05" - """ - return self.release_date[6:8] - - @property - def release_month(self: Self) -> int: - """ - Returns - ------- - int - The release month as an integer (no padding). - """ - return int(self.release_month_padded.lstrip("0")) - - @property - def release_day(self: Self) -> int: - """ - Returns - ------- - int - The release day as an integer (no padding). - """ - return int(self.release_day_padded.lstrip("0")) - - @property - def release_day_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, - i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - total_days_in_month = _days_in_month[self.release_month] - if self.release_month == 2 and self.release_year % 4 == 0: # leap year - total_days_in_month += 1 - - return total_days_in_month + 1 - self.release_day - - @property - def release_day_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed release day, but padded. i.e. August 30th returns "02". - """ - return pad(self.release_day_reversed) - - @property - def release_day_of_year(self: Self) -> int: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - output = sum(_days_in_month[: self.release_month]) + self.release_day - if self.release_month > 2 and self.release_year % 4 == 0: - output += 1 - - return output - - @property - def release_day_of_year_padded(self: Self) -> str: - """ - Returns - ------- - str - The release day of year, but padded i.e. February 1st returns "032" - """ - return pad(self.release_day_of_year, width=3) - - @property - def release_day_of_year_reversed(self: Self) -> int: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, - i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - total_days_in_year = 365 - if self.release_year % 4 == 0: - total_days_in_year += 1 - - return total_days_in_year + 1 - self.release_day_of_year - - @property - def release_day_of_year_reversed_padded(self: Self) -> str: - """ - Returns - ------- - str - The reversed release day of year, but padded i.e. December 31st returns "001" - """ - return pad(self.release_day_of_year_reversed, width=3) - - @property - def release_date_standardized(self: Self) -> str: - """ - Returns - ------- - str - The release date formatted as YYYY-MM-DD - """ - return f"{self.release_year}-{self.release_month_padded}-{self.release_day_padded}" diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py deleted file mode 100644 index 84f729f9..00000000 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import List - - -class KwargKeys: - keys: List[str] = [] - backend_keys: List[str] = [] - - -def _(key: str, backend: bool = False) -> str: - if backend: - assert key not in KwargKeys.backend_keys - KwargKeys.backend_keys.append(key) - else: - assert key not in KwargKeys.keys - KwargKeys.keys.append(key) - return key - - -SOURCE_ENTRY = _("source_entry", backend=True) -SOURCE_INDEX = _("source_index") -SOURCE_COUNT = _("source_count") -SOURCE_TITLE = _("source_title") -SOURCE_UID = _("source_uid") -SOURCE_DESCRIPTION = _("source_description") -SOURCE_WEBPAGE_URL = _("source_webpage_url") -SOURCE_UPLOADER = _("source_uploader") -SOURCE_UPLOADER_ID = _("source_uploader_id") -SOURCE_UPLOADER_URL = _("source_uploader_url") - -PLAYLIST_ENTRY = _("playlist_entry", backend=True) -PLAYLIST_WEBPAGE_URL = _("playlist_webpage_url") -PLAYLIST_INDEX = _("playlist_index") -PLAYLIST_COUNT = _("playlist_count") -PLAYLIST_MAX_UPLOAD_YEAR = _("playlist_max_upload_year") -PLAYLIST_MAX_UPLOAD_YEAR_TRUNCATED = _("playlist_max_upload_year_truncated") -PLAYLIST_TITLE = _("playlist_title") -PLAYLIST_DESCRIPTION = _("playlist_description") -PLAYLIST_UID = _("playlist_uid") -PLAYLIST_UPLOADER = _("playlist_uploader") -PLAYLIST_UPLOADER_ID = _("playlist_uploader_id") -PLAYLIST_UPLOADER_URL = _("playlist_uploader_url") - -COLLECTION_URL = _("collection_url", backend=True) -DOWNLOAD_INDEX = _("download_index", backend=True) -UPLOAD_DATE_INDEX = _("upload_date_index", backend=True) -REQUESTED_SUBTITLES = _("requested_subtitles", backend=True) -CHAPTERS = _("chapters", backend=True) -YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) -YTDL_SUB_REGEX_SOURCE_VARS = _("ytdl_sub_regex_source_vars", backend=True) -SPONSORBLOCK_CHAPTERS = _("sponsorblock_chapters", backend=True) -SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True) -COMMENTS = _("comments", backend=True) -UID = _("id") -EXTRACTOR = _("extractor") -IE_KEY = _("ie_key") -EPOCH = _("epoch") -CHANNEL = _("channel") -CHANNEL_ID = _("channel_id") -CREATOR = _("creator") -EXT = _("ext") -TITLE = _("title") -DESCRIPTION = _("description") -WEBPAGE_URL = _("webpage_url") -RELEASE_DATE = _("release_date") -UPLOAD_DATE = _("upload_date") -UPLOADER = _("uploader") -UPLOADER_ID = _("uploader_id") -UPLOADER_URL = _("uploader_url") diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 8a885f83..38f7160c 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -1,17 +1,22 @@ -SUBSCRIPTION_NAME = "subscription_name" -SUBSCRIPTION_VALUE = "subscription_value" +from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.name_validation import is_valid_name + +# TODO: use this +SUBSCRIPTION_ARRAY = "subscription_array" class OverrideVariables: - @classmethod - def subscription_name(cls) -> str: + @staticmethod + def subscription_name() -> str: """ Name of the subscription """ - return SUBSCRIPTION_NAME + return "subscription_name" - @classmethod - def subscription_value(cls) -> str: + @staticmethod + def subscription_value() -> str: """ For subscriptions in the form of @@ -21,10 +26,10 @@ class OverrideVariables: ``subscription_value`` gets set to ``https://...``. """ - return SUBSCRIPTION_VALUE + return "subscription_value" - @classmethod - def subscription_indent_i(cls, index: int) -> str: + @staticmethod + def subscription_indent_i(index: int) -> str: """ For subscriptions in the form of @@ -39,8 +44,8 @@ class OverrideVariables: """ return f"subscription_indent_{index + 1}" - @classmethod - def subscription_value_i(cls, index: int) -> str: + @staticmethod + def subscription_value_i(index: int) -> str: """ For subscriptions in the form of @@ -55,3 +60,66 @@ class OverrideVariables: ``subscription_value``. """ return f"subscription_value_{index + 1}" + + @staticmethod + def subscription_map() -> str: + """ + For subscriptions in the form of + + .. code-block:: yaml + + + Subscription Name: + Music Videos: + - "https://url1.com/..." + Concerts: + - "https://url2.com/..." + + Stores all the contents under the subscription name into the override variable + ``subscription_map`` as a Map value. The above example is stored as: + + .. code-block:: python + + { + "Music Videos": [ + "https://url1.com/..." + ], + "Concerts: [ + "https://url2.com/..." + ] + } + """ + return "subscription_map" + + +class OverrideHelpers: + @classmethod + def is_entry_variable_name(cls, name: str) -> bool: + """ + Returns + ------- + True if the name is an entry variable name. False otherwise. + """ + return name in VARIABLE_SCRIPTS + + @classmethod + def is_function_name(cls, name: str) -> bool: + """ + Returns + ------- + True if the name is a function name (either built-in or script). False otherwise. + """ + if name.startswith("%"): + return name in CUSTOM_FUNCTION_SCRIPTS or Functions.is_built_in(name[1:]) + return False + + @classmethod + def is_valid_name(cls, name: str) -> bool: + """ + Returns + ------- + True if the override name itself is valid. False otherwise. + """ + if name.startswith("%"): + return is_valid_name(name=name[1:]) + + return is_valid_name(name=name) diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 750429de..644fa1c1 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -2,11 +2,15 @@ import os.path from typing import Any from typing import Dict from typing import Optional +from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -14,20 +18,20 @@ from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_TYPES_EXTENSION from ytdl_sub.validators.audo_codec_validator import AudioTypeValidator from ytdl_sub.validators.validators import FloatValidator +v: VariableDefinitions = VARIABLES + class AudioExtractOptions(OptionsDictValidator): """ Extracts audio from a video file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 """ _required_keys = {"codec"} @@ -50,21 +54,31 @@ class AudioExtractOptions(OptionsDictValidator): @property def codec(self) -> str: """ - The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, - opus, vorbis, wav, and best to grab the best possible format at runtime. + :expected type: String + :description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. """ return self._codec @property def quality(self) -> Optional[float]: """ - Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` - (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + :expected type: Float + :description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. """ if self._quality is not None: return self._quality.value return None + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + """ + Possibly changes ``ext``, so do not resolve until this has run + """ + return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} + class AudioExtractPlugin(Plugin[AudioExtractOptions]): plugin_options_type = AudioExtractOptions @@ -125,7 +139,7 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]): new_ext = AUDIO_CODEC_TYPES_EXTENSION_MAPPING[self.plugin_options.codec] extracted_audio_file = entry.get_download_file_path().removesuffix(entry.ext) + new_ext - entry.add_kwargs({"ext": new_ext}) + entry.add({v.ext: new_ext}) if not self.is_dry_run: if not os.path.isfile(extracted_audio_file): diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 456994f5..fbd11c09 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -5,12 +5,15 @@ from typing import List from typing import Optional from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import COMMENTS -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS +from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments +from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters from ytdl_sub.utils.file_handler import FileMetadata @@ -19,6 +22,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import ListValidator +v: VariableDefinitions = VARIABLES + SPONSORBLOCK_HIGHLIGHT_CATEGORIES: Set[str] = {"poi_highlight"} SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | { "sponsor", @@ -33,15 +38,11 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | { def _chapters(entry: Entry) -> List[Dict]: - if entry.kwargs_contains("chapters"): - return entry.kwargs("chapters") or [] - return [] + return entry.get(v.chapters, list) def _sponsorblock_chapters(entry: Entry) -> List[Dict]: - if entry.kwargs_contains("sponsorblock_chapters"): - return entry.kwargs("sponsorblock_chapters") or [] - return [] + return entry.get(v.sponsorblock_chapters, list) def _contains_any_chapters(entry: Entry) -> bool: @@ -63,32 +64,29 @@ class ChaptersOptions(OptionsDictValidator): Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" - - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False """ _optional_keys = { @@ -134,23 +132,29 @@ class ChaptersOptions(OptionsDictValidator): @property def embed_chapters(self) -> Optional[bool]: """ - Optional. Embed chapters into the file. Defaults to True. + :expected type: Optional[Boolean] + :description: + Defaults to True. Embed chapters into the file. """ return self._embed_chapters @property def allow_chapters_from_comments(self) -> bool: """ - Optional. If chapters do not exist in the video/description itself, attempt to scrape - comments to find the chapters. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. """ return self._allow_chapters_from_comments @property def remove_chapters_regex(self) -> Optional[List[re.Pattern]]: """ - Optional. List of regex patterns to match chapter titles against and remove them from the - entry. + :expected type: Optional[List[RegexString] + :description: + List of regex patterns to match chapter titles against and remove them from the + entry. """ if self._remove_chapters_regex: return [validator.compiled_regex for validator in self._remove_chapters_regex.list] @@ -159,9 +163,11 @@ class ChaptersOptions(OptionsDictValidator): @property def sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", - "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", - "poi_highlight", or "all" to include all categories. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. """ if self._sponsorblock_categories: category_list = [validator.value for validator in self._sponsorblock_categories.list] @@ -173,9 +179,11 @@ class ChaptersOptions(OptionsDictValidator): @property def remove_sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to remove from the output file. Can only remove - categories that are specified in ``sponsorblock_categories`` or "all", which removes - everything specified in ``sponsorblock_categories``. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. """ if self._remove_sponsorblock_categories: category_list = [ @@ -189,12 +197,21 @@ class ChaptersOptions(OptionsDictValidator): @property def force_key_frames(self) -> bool: """ - Optional. Force keyframes at cuts when removing sections. This is slow due to needing a - re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to - False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. """ return self._force_key_frames + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: + return {PluginOperation.MODIFY_ENTRY: {ytdl_sub_chapters_from_comments.variable_name}} + class ChaptersPlugin(Plugin[ChaptersOptions]): plugin_options_type = ChaptersOptions @@ -300,27 +317,33 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ------- entry """ - chapters = Chapters.from_empty() + has_chapters_from_comments = False # If there are no embedded chapters, and comment chapters are allowed... if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments: + chapters = Chapters.from_empty() + # Try to get chapters from comments - for comment in entry.kwargs_get(COMMENTS, []): + for comment in entry.get(v.comments, list): chapters = Chapters.from_string(comment.get("text", "")) if chapters.contains_any_chapters(): break # If some are actually found, add a special kwarg and embed them if chapters.contains_any_chapters(): - entry.add_kwargs({YTDL_SUB_CUSTOM_CHAPTERS: chapters.to_file_metadata_dict()}) + has_chapters_from_comments = True + entry.add({ytdl_sub_chapters_from_comments: chapters.to_yt_dlp_chapter_metadata()}) if not self.is_dry_run: set_ffmpeg_metadata_chapters( file_path=entry.get_download_file_path(), chapters=chapters, - file_duration_sec=entry.kwargs("duration"), + file_duration_sec=entry.get(v.duration, int), ) + if not has_chapters_from_comments: + entry.add({ytdl_sub_chapters_from_comments: []}) + return entry def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: @@ -334,12 +357,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ------- FileMetadata outlining which chapters/SponsorBlock segments got removed """ - if custom_chapters_metadata := entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS): - title: str = "Chapters from comments" - return FileMetadata.from_dict( - value_dict=custom_chapters_metadata, - title=title, - sort_dict=False, # timestamps + titles are already sorted + if custom_chapters := entry.get(ytdl_sub_chapters_from_comments, list): + return Chapters.from_yt_dlp_chapters(custom_chapters).to_file_metadata( + title="Chapters from comments" ) if self.plugin_options.embed_chapters: @@ -356,9 +376,10 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): if removed_sponsorblock: metadata_dict["Removed SponsorBlock Category Count(s)"] = removed_sponsorblock - # TODO: check if file actually has embedded chapters - return FileMetadata.from_dict( - value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False - ) + # If the entry wasn't split on embedded chapters, report it in the file metadata + if not entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str): + return FileMetadata.from_dict( + value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False + ) return None diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 1d93a483..3948cb1e 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -2,8 +2,8 @@ from typing import List from typing import Optional from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.validators.string_datetime import StringDatetimeValidator @@ -11,16 +11,24 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator class DateRangeOptions(OptionsDictValidator): """ Only download files uploaded within the specified date range. + Dates must adhere to a yt-dlp datetime. From their docs: - Usage: + .. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + + Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. + Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest + granularity possible. + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" """ _optional_keys = {"before", "after"} @@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator): @property def before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos before this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._before @property def after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos after this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._after diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 4886f707..b3ef63e2 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -3,9 +3,8 @@ from typing import Optional import mediafile -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.file_handler import FileHandler @@ -21,19 +20,16 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator): """ Whether to embed thumbnails to the audio/video file or not. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True """ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]): plugin_options_type = EmbedThumbnailOptions - priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT) @property def _embed_thumbnail(self) -> bool: diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index fa6c0c1f..8fccaedb 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -2,12 +2,15 @@ import os from typing import Any from typing import Dict from typing import Optional +from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import EXT +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.ffmpeg import FFMPEG @@ -16,6 +19,9 @@ from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.audo_codec_validator import FileTypeValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +v: VariableDefinitions = VARIABLES class FileConvertWithValidator(StringSelectValidator): @@ -26,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator): """ Converts video files from one extension to another. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" - Supports custom ffmpeg conversions: + Also supports custom ffmpeg conversions: + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text """ _required_keys = {"convert_to"} @@ -83,45 +87,61 @@ class FileConvertOptions(OptionsDictValidator): @property def convert_to(self) -> str: """ - Convert to a desired file type. Supports: + :expected type: String + :description: + Convert to a desired file type. Supports - * Video: avi, flv, mkv, mov, mp4, webm - * Audio: aac, flac, mp3, m4a, opus, vorbis, wav + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav """ return self._convert_to @property def convert_with(self) -> Optional[str]: """ - Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within - yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified - with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - + :expected type: Optional[String] + :description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. """ return self._convert_with @property def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the - form of: + :expected type: Optional[OverridesFormatter] + :description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of - .. code-block:: bash + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext - - The output file will use the extension specified in ``convert_to``. Post-processing args - can still be set with ``convert_with`` set to ``yt-dlp``. + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. """ return self._ffmpeg_post_process_args + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} + class FileConvertPlugin(Plugin[FileConvertOptions]): plugin_options_type = FileConvertOptions - # Perform this after regex - priority: PluginPriority = PluginPriority( - modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 1 - ) + + def __init__( + self, + options: FileConvertOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + # Lookup of entry id to what it was converted from for logging + self._converted_from_lookup: Dict[str, str] = {} def ytdl_options(self) -> Optional[Dict]: """ @@ -193,18 +213,14 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): "file_convert ffmpeg_post_process_args did not produce an output file" ) + FileHandler.delete(input_video_file_path) FileHandler.move(tmp_output_file, converted_video_file_path) FileHandler.delete(tmp_output_file) - FileHandler.delete(input_video_file_path) if original_ext != new_ext: - entry.add_kwargs( - { - "__converted_from": original_ext, - } - ) + self._converted_from_lookup[entry.ytdl_uid()] = original_ext - entry.add_kwargs({EXT: new_ext}) + entry.add({v.ext: new_ext}) return entry @@ -212,7 +228,7 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): """ Add metadata about conversion if it happened """ - if converted_from := entry.kwargs_get("__converted_from"): + if converted_from := self._converted_from_lookup.get(entry.ytdl_uid()): return FileMetadata(f"Converted from {converted_from}") return None diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py new file mode 100644 index 00000000..7f381db1 --- /dev/null +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -0,0 +1,70 @@ +import json +from typing import Dict +from typing import Optional + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +logger = Logger.get("conditional") + + +class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): + """ + Applies a conditional OR on any number of filters comprised of either variables or scripts. + If any filter evaluates to True, the entry will be excluded. + + :Usage: + + .. code-block:: yaml + + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } + """ + + +class FilterExcludePlugin(Plugin[FilterExcludeOptions]): + plugin_options_type = FilterExcludeOptions + + def __init__( + self, + options: FilterExcludeOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + self._evaluated_map: Dict[str, bool] = {} + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + # Already evaluated in modify_entry_metadata, do not recompute + if entry.ytdl_uid() in self._evaluated_map: + return entry + + for formatter in self.plugin_options.list: + out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + if bool(out): + return None + + return entry + + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + try: + output_entry = self.modify_entry(entry=entry) + except StringFormattingException: + # If filtering fails at the metadata stage, try again w/no catch in modify_entry + return entry + + self._evaluated_map[entry.ytdl_uid()] = True + return output_entry diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py new file mode 100644 index 00000000..41a98f78 --- /dev/null +++ b/src/ytdl_sub/plugins/filter_include.py @@ -0,0 +1,78 @@ +import json +from typing import Dict +from typing import Optional + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.logger import Logger +from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + +logger = Logger.get("conditional") + + +class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): + """ + Applies a conditional AND on any number of filters comprised of either variables or scripts. + If all filters evaluate to True, the entry will be included. + + :Usage: + + .. code-block:: yaml + + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + """ + + +class FilterIncludePlugin(Plugin[FilterIncludeOptions]): + plugin_options_type = FilterIncludeOptions + + def __init__( + self, + options: FilterIncludeOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + self._evaluated_map: Dict[str, bool] = {} + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + # Already evaluated in modify_entry_metadata, do not recompute + if entry.ytdl_uid() in self._evaluated_map: + return entry + + for formatter in self.plugin_options.list: + out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + if not bool(out): + return None + + return entry + + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + try: + output_entry = self.modify_entry(entry=entry) + except StringFormattingException: + # If filtering fails at the metadata stage, try again w/no catch in modify_entry + return entry + + self._evaluated_map[entry.ytdl_uid()] = True + return output_entry diff --git a/src/ytdl_sub/plugins/format.py b/src/ytdl_sub/plugins/format.py index 7f1ccc06..6e2d545d 100644 --- a/src/ytdl_sub/plugins/format.py +++ b/src/ytdl_sub/plugins/format.py @@ -1,8 +1,8 @@ from typing import Dict from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.validators.validators import StringValidator @@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator): .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" """ def __init__(self, name, value): diff --git a/src/ytdl_sub/plugins/internal/view.py b/src/ytdl_sub/plugins/internal/view.py index fa8d5471..39685ad3 100644 --- a/src/ytdl_sub/plugins/internal/view.py +++ b/src/ytdl_sub/plugins/internal/view.py @@ -1,10 +1,9 @@ import copy from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -20,7 +19,6 @@ class ViewOptions(OptionsDictValidator): class ViewPlugin(Plugin[ViewOptions]): plugin_options_type = ViewOptions - priority: PluginPriority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT) _MAX_LINE_WIDTH: int = 80 diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index ef4d6d4a..fe10a69b 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -3,8 +3,8 @@ from typing import Any from typing import List from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator @@ -58,32 +58,20 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: class MatchFiltersOptions(OptionsDictValidator): """ - Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. - Uses the same syntax as yt-dlp. + Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. + Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. + For logical AND's between match filters, use the ``&`` operator in a single match filter. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" - - Supports one or multiple filters: - - .. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit<?18" - - "like_count>?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit<?18" - # - "availability=?public" + match_filters: + filters: + - "age_limit<?18 & like_count>?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" """ _optional_keys = {"filters"} diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index e7bb92e4..f6098517 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -6,9 +6,11 @@ from typing import List import mediafile -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -17,6 +19,8 @@ from ytdl_sub.validators.string_formatter_validators import ListFormatterValidat from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator +v: VariableDefinitions = VARIABLES + logger = Logger.get("music_tags") @@ -71,23 +75,22 @@ class MusicTagsOptions(OptionsDictValidator): a full list of tags for various file types in MediaFile's `source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_. - Usage: + :Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" """ _optional_keys = {"tags", "embed_thumbnail"} @@ -132,9 +135,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): """ Tags the entry's audio file using values defined in the metadata options """ - if entry.ext not in AUDIO_CODEC_EXTS: + if (ext := entry.get(v.ext, str)) not in AUDIO_CODEC_EXTS: raise self.plugin_options.validation_exception( - f"music_tags plugin received a video with the extension '{entry.ext}'. Only audio " + f"music_tags plugin received a video with the extension '{ext}'. Only audio " f"files are supported for setting music tags. Ensure you are converting the video " f"to audio using the audio_extract plugin." ) diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index f9b3cc33..728259e1 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -7,8 +7,8 @@ from typing import Dict from typing import List from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata @@ -60,7 +60,9 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def nfo_name(self) -> StringFormatterFileNameValidator: """ - The NFO file name. + :expected type: EntryFormatter + :description: + The NFO file name. """ return self._nfo_name @@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def kodi_safe(self) -> Optional[bool]: """ - Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some - foreign language characters. Setting this to True will replace those characters with '□'. - Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. """ return self._kodi_safe @@ -97,16 +101,18 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC): nfo_tags: Dict[str, List[XmlElement]] = defaultdict(list) for key, string_tags in self.plugin_options.tags.string_tags.items(): - nfo_tags[key].extend( + tags = [ XmlElement( text=self.overrides.apply_formatter(formatter=string_tag, entry=entry), attributes={}, ) for string_tag in string_tags - ) + ] + # Do not add tags with empty text + nfo_tags[key].extend(tag for tag in tags if tag.text) for key, attribute_tags in self.plugin_options.tags.attribute_tags.items(): - nfo_tags[key].extend( + tags = [ XmlElement( text=self.overrides.apply_formatter(formatter=attribute_tag.tag, entry=entry), attributes={ @@ -117,9 +123,12 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC): }, ) for attribute_tag in attribute_tags - ) + ] + # Do not add tags with empty text + nfo_tags[key].extend(tag for tag in tags if tag.text) - return nfo_tags + # Do not add tags with empty lists + return {key: tags for key, tags in nfo_tags.items() if len(tags) > 0} def _create_nfo(self, entry: Entry, save_to_entry: bool = True) -> None: # Write the nfo tags to XML with the nfo_root @@ -128,6 +137,10 @@ class SharedNfoTagsPlugin(Plugin[SharedNfoTagsOptions], ABC): ) nfo_tags = self._get_xml_element_dict(entry=entry) + # If the nfo tags are empty, then stop continuing + if not nfo_tags: + return + if self.plugin_options.kodi_safe: nfo_root = to_max_3_byte_utf8_string(nfo_root) nfo_tags = { @@ -181,22 +194,18 @@ class NfoTagsOptions(SharedNfoTagsOptions): Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False """ _formatter_validator = StringFormatterValidator @@ -206,50 +215,54 @@ class NfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - </episodedetails> + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <episodedetails> + </episodedetails> """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - <title>Awesome Youtube Video - 2022 - 502 - + + + Awesome Youtube Video + 2022 + 502 + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - 2022 - Comedy - Drama + 2022 + Comedy + Drama """ return self._tags diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index de8ae4ca..f754ec44 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -1,6 +1,4 @@ -from typing import Optional - -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.nfo_tags import NfoTagsValidator from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions @@ -39,48 +37,52 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - + + + """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - Sweet youtube TV show - + + + Sweet youtube TV show + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + <title year="2022">Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> """ return self._tags @@ -95,23 +97,16 @@ class OutputDirectoryNfoTagsPlugin(SharedNfoTagsPlugin): enhanced_download_archive: EnhancedDownloadArchive, ): super().__init__(options, overrides, enhanced_download_archive) - self._last_entry: Optional[Entry] = None + self._created_output_nfo = False def post_process_entry(self, entry: Entry) -> None: """ - Tracks the last entry processed - """ - self._last_entry = entry - - def post_process_subscription(self): - """ - Creates an NFO file in the root of the output directory using the last entry + Creates output NFO using the first entry, and only creates it once """ if ( - self.plugin_options.nfo_name is None - or self.plugin_options.nfo_root is None - or self._last_entry is None + not self._created_output_nfo + and self.plugin_options.nfo_name is not None + and self.plugin_options.nfo_root is not None ): - return - - self._create_nfo(entry=self._last_entry, save_to_entry=False) + self._create_nfo(entry=entry, save_to_entry=False) + self._created_output_nfo = True diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index d2228190..47b34262 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -1,17 +1,18 @@ +from collections import defaultdict from typing import Any from typing import Dict from typing import List from typing import Optional +from typing import Set -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS +from ytdl_sub.script.parser import parse +from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import RegexNoMatchException -from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.regex_validator import RegexListValidator from ytdl_sub.validators.source_variable_validator import SourceVariableNameListValidator @@ -19,6 +20,8 @@ from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator +from ytdl_sub.validators.validators import DictValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get(name="regex") @@ -109,11 +112,7 @@ class VariableRegex(StrictDictValidator): return self._capture_group_defaults is not None -class FromSourceVariablesRegex(StrictDictValidator): - - _optional_keys = Entry.source_variables() - _allow_extra_keys = True - +class FromSourceVariablesRegex(DictValidator): def __init__(self, name, value): super().__init__(name, value) self.variable_capture_dict: Dict[str, VariableRegex] = { @@ -123,6 +122,41 @@ class FromSourceVariablesRegex(StrictDictValidator): class RegexOptions(OptionsDictValidator): r""" + .. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -138,51 +172,49 @@ class RegexOptions(OptionsDictValidator): and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" """ _required_keys = {"from"} @@ -204,47 +236,19 @@ class RegexOptions(OptionsDictValidator): key="skip_if_match_fails", validator=BoolValidator, default=True ).value + # Variables added by the regex plugin + self._added_variable_names: Set[str] = set() + @property def skip_if_match_fails(self) -> Optional[bool]: """ - Defaults to True. If True, when any match fails and has no defaults, the entry will be - skipped. If False, ytdl-sub will error and all downloads will not proceed. + :expected type: Optional[Boolean] + :description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. """ return self._skip_if_match_fails - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Ensures each source variable capture group is valid - - Parameters - ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin - """ - for key, regex_options in self.source_variable_capture_dict.items(): - # Ensure each variable getting captured is a source variable - if key not in source_variables and key not in override_variables: - raise self._validation_exception( - f"cannot regex capture '{key}' because it is not a source or override variable" - ) - - # Ensure the capture group names are not existing source/override variables - for capture_group_name in regex_options.capture_group_names: - if capture_group_name in source_variables: - raise self._validation_exception( - f"'{capture_group_name}' cannot be used as a capture group name because it " - f"is a source variable" - ) - if capture_group_name in override_variables: - raise self._validation_exception( - f"'{capture_group_name}' cannot be used as a capture group name because it " - f"is an override variable" - ) - @property def source_variable_capture_dict(self) -> Dict[str, VariableRegex]: """ @@ -254,39 +258,91 @@ class RegexOptions(OptionsDictValidator): """ return self._from.variable_capture_dict - def added_source_variables(self) -> List[str]: + @classmethod + def _can_resolve( + cls, unresolved_variables: Set[str], input_variable_name: str, regex_options: VariableRegex + ) -> bool: + if input_variable_name in unresolved_variables: + return False + for capture_group_default in regex_options.capture_group_defaults or []: + parsed_default = parse(capture_group_default.format_string) + if parsed_default.variables and parsed_default.variables.issubset(unresolved_variables): + return False + return True + + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of new source variables created via regex capture """ - added_source_vars: List[str] = [] - for regex_options in self.source_variable_capture_dict.values(): - added_source_vars.extend(regex_options.capture_group_names) - added_source_vars.extend( - f"{capture_group_name}_sanitized" - for capture_group_name in regex_options.capture_group_names - ) + added_source_vars: Dict[PluginOperation, Set[str]] = { + PluginOperation.MODIFY_ENTRY_METADATA: set(), + PluginOperation.MODIFY_ENTRY: set(), + } + for input_variable_name, regex_options in self.source_variable_capture_dict.items(): + variables_to_add = set(regex_options.capture_group_names) + + if plugin_op != PluginOperation.ANY and input_variable_name not in ( + resolved_variables | unresolved_variables + ): + raise self._validation_exception( + f"cannot regex capture '{input_variable_name}' because it is not a" + " defined variable." + ) + if ( + plugin_op.value >= PluginOperation.MODIFY_ENTRY.value + and input_variable_name in unresolved_variables + ): + raise self._validation_exception( + f"cannot regex capture '{input_variable_name}' because it is not " + f"computed until later in execution." + ) + + if plugin_op == PluginOperation.ANY: + added_source_vars[PluginOperation.MODIFY_ENTRY_METADATA] |= variables_to_add + self._added_variable_names |= variables_to_add + continue + + if not self._can_resolve( + unresolved_variables=unresolved_variables, + input_variable_name=input_variable_name, + regex_options=regex_options, + ): + continue + + for capture_group_name in regex_options.capture_group_names: + if capture_group_name in (resolved_variables - self._added_variable_names): + raise self._validation_exception( + f"cannot use '{capture_group_name}' as a capture group name because it is " + f"an already defined variable." + ) + added_source_vars[plugin_op] |= set(regex_options.capture_group_names) return added_source_vars class RegexPlugin(Plugin[RegexOptions]): plugin_options_type = RegexOptions - priority = PluginPriority( - modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 0, - ) - @classmethod - def _add_processed_regex_variable_name(cls, entry: Entry, source_var: str) -> None: - if not entry.kwargs_contains(YTDL_SUB_REGEX_SOURCE_VARS): - entry.add_kwargs({YTDL_SUB_REGEX_SOURCE_VARS: []}) - - entry.kwargs(YTDL_SUB_REGEX_SOURCE_VARS).append(source_var) - - @classmethod - def _contains_processed_regex_variable(cls, entry: Entry, variable_name: str) -> bool: - return variable_name in entry.kwargs_get(YTDL_SUB_REGEX_SOURCE_VARS, []) + def __init__( + self, + options: RegexOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + # Lookup of entry id to processed regex variables + self._processed_regex_vars: Dict[str, Set[str]] = defaultdict(set) def _try_skip_entry(self, entry: Entry, variable_name: str) -> None: # Skip the entry if toggled @@ -301,34 +357,16 @@ class RegexPlugin(Plugin[RegexOptions]): # Otherwise, error raise RegexNoMatchException(f"Regex failed to match '{variable_name}' from '{entry.title}'") - def _can_process_at_metadata_stage(self, entry: Entry, variable_name: str) -> bool: - # If the variable is an override... - if variable_name in self.overrides.dict: - # Try to see if it can resolve - try: - self.overrides.apply_formatter( - formatter=self.overrides.dict[variable_name], - entry=entry, - ) - # If it can not from missing variables (from post-metadata stage), return False - except StringFormattingVariableNotFoundException: - return False - # If it is a source variable and not present, return false - elif variable_name not in entry.to_dict(): + @classmethod + def _can_process_at_metadata_stage(cls, entry: Entry, variable_name: str) -> bool: + # Try to see if it can resolve + try: + _ = entry.script.get(variable_name) + return True + # If it can not from missing variables (from post-metadata stage), return False + except RuntimeException: return False - return True - - def _get_regex_input_string(self, entry: Entry, variable_name: str) -> str: - # Apply override formatter if it's an override - if variable_name in self.overrides.dict: - return self.overrides.apply_formatter( - formatter=self.overrides.dict[variable_name], - entry=entry, - ) - # Otherwise pluck from the entry's source variable - return entry.to_dict()[variable_name] - def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: """ Parameters @@ -356,7 +394,7 @@ class RegexPlugin(Plugin[RegexOptions]): # Record which regex source variables are processed, to # process as many variables as possible in the metadata stage, then the rest # after the media file has been downloaded. - if self._contains_processed_regex_variable(entry, variable_name): + if variable_name in self._processed_regex_vars[entry.ytdl_uid()]: continue # If it's the metadata stage, and it can't be processed, skip until post-metadata @@ -365,12 +403,8 @@ class RegexPlugin(Plugin[RegexOptions]): ): continue - self._add_processed_regex_variable_name(entry, variable_name) - - regex_input_str = self._get_regex_input_string( - entry=entry, - variable_name=variable_name, - ) + self._processed_regex_vars[entry.ytdl_uid()].add(variable_name) + regex_input_str = str(entry.script.get(variable_name)) if ( regex_options.exclude is not None @@ -388,49 +422,23 @@ class RegexPlugin(Plugin[RegexOptions]): if not regex_options.has_defaults: return self._try_skip_entry(entry=entry, variable_name=variable_name) - # otherwise, use defaults (apply them using the original entry source dict) - source_variables_and_overrides_dict = dict( - entry.to_dict(), **self.overrides.dict_with_format_strings - ) - # add both the default... - entry.add_variables( - variables_to_add={ - regex_options.capture_group_names[i]: default.apply_formatter( - variable_dict=source_variables_and_overrides_dict + entry.add( + { + regex_options.capture_group_names[i]: self.overrides.apply_formatter( + formatter=default, entry=entry ) for i, default in enumerate(regex_options.capture_group_defaults) - }, - ) - # and sanitized default - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - default.apply_formatter( - variable_dict=source_variables_and_overrides_dict - ) - ) - for i, default in enumerate(regex_options.capture_group_defaults) - }, + } ) # There is a capture, add the source variables to the entry as # {source_var}_capture_1, {source_var}_capture_2, ... else: - # Add the value... - entry.add_variables( - variables_to_add={ + entry.add( + { regex_options.capture_group_names[i]: capture for i, capture in enumerate(maybe_capture) - }, - ) - # And the sanitized value - entry.add_variables( - variables_to_add={ - f"{regex_options.capture_group_names[i]}_sanitized": sanitize_filename( - capture - ) - for i, capture in enumerate(maybe_capture) - }, + } ) return entry diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 50717854..b812c0b9 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -1,19 +1,17 @@ -import copy -from pathlib import Path from typing import Any +from typing import Dict from typing import List from typing import Optional +from typing import Set from typing import Tuple -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.config.plugin import SplitPlugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY -from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS -from ytdl_sub.entries.variables.kwargs import UID +from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.chapters import Timestamp from ytdl_sub.utils.exceptions import ValidationException @@ -22,6 +20,8 @@ from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.string_select_validator import StringSelectValidator +v: VariableDefinitions = VARIABLES + def _split_video_ffmpeg_cmd( input_file: str, output_file: str, timestamps: List[Timestamp], idx: int @@ -48,23 +48,22 @@ class WhenNoChaptersValidator(StringSelectValidator): class SplitByChaptersOptions(OptionsDictValidator): """ Splits a file by chapters into multiple files. Each file becomes its own entry with the - new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, - ``chapter_index_padded``, ``chapter_count``. + new variables - If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is - set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" """ _required_keys = {"when_no_chapters"} @@ -81,61 +80,81 @@ class SplitByChaptersOptions(OptionsDictValidator): key="when_no_chapters", validator=WhenNoChaptersValidator ).value - def added_source_variables(self) -> List[str]: - return [ - "chapter_title", - "chapter_title_sanitized", - "chapter_index", - "chapter_index_padded", - "chapter_count", - ] + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: + return { + PluginOperation.MODIFY_ENTRY: { + "chapter_title", + "chapter_title_sanitized", + "chapter_index", + "chapter_index_padded", + "chapter_count", + } + } @property def when_no_chapters(self) -> str: """ - Behavior to perform when no chapters are present. Supports "pass" (continue processing), - "drop" (exclude it from output), and "error" (stop processing for everything). + :expected type: String + :description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. """ return self._when_no_chapters + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + return { + PluginOperation.MODIFY_ENTRY: { + v.uid.variable_name, + ytdl_sub_split_by_chapters_parent_uid.variable_name, + } + } + class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): plugin_options_type = SplitByChaptersOptions + @classmethod + def _non_split_entry(cls, entry: Entry) -> Entry: + entry.add( + { + "chapter_title": f"{{ {v.title.variable_name} }}", + "chapter_index": 1, + "chapter_index_padded": "01", + "chapter_count": 1, + v.uid: entry.uid, + ytdl_sub_split_by_chapters_parent_uid: entry.uid, + } + ) + return entry + def _create_split_entry( - self, source_entry: Entry, title: str, idx: int, chapters: Chapters + self, new_entry: Entry, title: str, idx: int, chapters: Chapters ) -> Tuple[Entry, FileMetadata]: """ Runs ffmpeg to create the split video """ - entry = copy.deepcopy(source_entry) - - entry.add_variables( + new_entry.add( { "chapter_title": title, - "chapter_title_sanitized": sanitize_filename(title), "chapter_index": idx + 1, "chapter_index_padded": f"{(idx + 1):02d}", "chapter_count": len(chapters.timestamps), } ) - # pylint: disable=protected-access - entry.add_kwargs( - { - UID: _split_video_uid(source_uid=entry.uid, idx=idx), - SPLIT_BY_CHAPTERS_PARENT_ENTRY: source_entry._kwargs, - } - ) - - if entry.kwargs_contains(CHAPTERS): - del entry._kwargs[CHAPTERS] - if entry.kwargs_contains(SPONSORBLOCK_CHAPTERS): - del entry._kwargs[SPONSORBLOCK_CHAPTERS] - # pylint: enable=protected-access - timestamp_begin = chapters.timestamps[idx].readable_str - timestamp_end = Timestamp(entry.kwargs("duration")).readable_str + timestamp_end = Timestamp(new_entry.get(v.duration, int)).readable_str if idx + 1 < len(chapters.timestamps): timestamp_end = chapters.timestamps[idx + 1].readable_str @@ -145,7 +164,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): "Warning" ] = "Dry-run assumes embedded chapters with no modifications" - metadata_value_dict["Source Title"] = entry.title + metadata_value_dict["Source Title"] = new_entry.title metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}" metadata = FileMetadata.from_dict( @@ -154,7 +173,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): sort_dict=False, ) - return entry, metadata + return new_entry, metadata def split(self, entry: Entry) -> Optional[List[Tuple[Entry, FileMetadata]]]: """ @@ -166,15 +185,9 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): # If no chapters, do not split anything if not chapters.contains_any_chapters(): if self.plugin_options.when_no_chapters == "pass": - entry.add_variables( - { - "chapter_title": entry.title, - "chapter_index": 1, - "chapter_index_padded": "01", - "chapter_count": 1, - } - ) - return [(entry, FileMetadata())] + # Modify the entry t + return [(self._non_split_entry(entry), FileMetadata())] + if self.plugin_options.when_no_chapters == "drop": return [] @@ -183,18 +196,16 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): ) for idx, title in enumerate(chapters.titles): - new_uid = _split_video_uid(source_uid=entry.uid, idx=idx) + new_entry = Entry.create_split_entry( + entry=entry, new_uid=_split_video_uid(source_uid=entry.uid, idx=idx) + ) if not self.is_dry_run: - # Get the input/output file paths - input_file = entry.get_download_file_path() - output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}") - # Run ffmpeg to create the split the video FFMPEG.run( _split_video_ffmpeg_cmd( - input_file=input_file, - output_file=output_file, + input_file=entry.get_download_file_path(), + output_file=new_entry.get_download_file_path(), timestamps=chapters.timestamps, idx=idx, ) @@ -205,14 +216,13 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): if entry.is_thumbnail_downloaded(): FileHandler.copy( src_file_path=entry.get_download_thumbnail_path(), - dst_file_path=Path(self.working_directory) - / f"{new_uid}.{entry.thumbnail_ext}", + dst_file_path=new_entry.get_download_thumbnail_path(), ) # Format the split video split_videos_and_metadata.append( self._create_split_entry( - source_entry=entry, + new_entry=new_entry, title=title, idx=idx, chapters=chapters, diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 4dff539d..9c53a9bd 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -2,11 +2,15 @@ from pathlib import Path from typing import Dict from typing import List from typing import Optional +from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger @@ -17,6 +21,8 @@ from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import StringListValidator +v: VariableDefinitions = VARIABLES + logger = Logger.get(name="subtitles") @@ -31,18 +37,18 @@ class SubtitleOptions(OptionsDictValidator): ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False """ _optional_keys = { @@ -76,50 +82,65 @@ class SubtitleOptions(OptionsDictValidator): @property def subtitles_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's subtitles if they are present. This can include - directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and - will be placed in the output directory. ``lang`` is dynamic since you can download multiple - subtitles. It will set the respective language to the correct subtitle file. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. """ return self._subtitles_name @property def subtitles_type(self) -> Optional[str]: """ - Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" + :expected type: Optional[String] + :description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". """ return self._subtitles_type @property def embed_subtitles(self) -> Optional[bool]: """ - Optional. Whether to embed the subtitles into the video file. Defaults to False. - NOTE: webm files can only embed "vtt" subtitle types. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. """ return self._embed_subtitles @property def languages(self) -> Optional[List[str]]: """ - Optional. Language code(s) to download for subtitles. Supports a single or list of multiple - language codes. Defaults to "en". + :expected type: Optional[List[String]] + :description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". """ return [lang.value for lang in self._languages] @property def allow_auto_generated_subtitles(self) -> Optional[bool]: """ - Optional. Whether to allow auto generated subtitles. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to allow auto generated subtitles. """ return self._allow_auto_generated_subtitles - def added_source_variables(self) -> List[str]: + def added_variables( + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, + ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of new source variables created by using the subtitles plugin """ - return ["lang", "subtitles_ext"] + return {PluginOperation.MODIFY_ENTRY_METADATA: {"lang", "subtitles_ext"}} class SubtitlesPlugin(Plugin[SubtitleOptions]): @@ -158,18 +179,8 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): return builder.to_dict() - def modify_entry(self, entry: Entry) -> Optional[Entry]: - if not (requested_subtitles := entry.kwargs_get("requested_subtitles", None)): - return entry - - languages = sorted(requested_subtitles.keys()) - entry.add_variables( - variables_to_add={ - "subtitles_ext": self.plugin_options.subtitles_type, - "lang": ",".join(languages), - } - ) - + def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: + entry.add({"subtitles_ext": self.plugin_options.subtitles_type, "lang": ""}) return entry def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: @@ -181,7 +192,7 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): entry: Entry to create subtitles for """ - requested_subtitles = entry.kwargs("requested_subtitles") + requested_subtitles = entry.get(v.requested_subtitles, expected_type=dict) if not requested_subtitles: logger.debug("subtitles not found for %s", entry.title) return None @@ -189,6 +200,10 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): file_metadata: Optional[FileMetadata] = None langs = list(requested_subtitles.keys()) + # HACK to maintain order of languages for fixtures + if len(langs) == len(self.plugin_options.languages): + langs = self.plugin_options.languages + if self.plugin_options.embed_subtitles: file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}") if self.plugin_options.subtitles_name: diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 25b5791b..5295c4f2 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -4,9 +4,9 @@ from typing import List from typing import Optional from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger @@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator): range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. - Usage: + :Usage: .. code-block:: yaml @@ -110,31 +110,39 @@ class ThrottleProtectionOptions(OptionsDictValidator): @property def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each download. Does not include time it takes for - ytdl-sub to perform post-processing. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. """ return self._sleep_per_download_s @property def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each subscription. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each subscription. """ return self._sleep_per_subscription_s @property def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]: """ - Number of downloads to perform per subscription. + :expected type: Optional[Range] + :description: + Number of downloads to perform per subscription. """ return self._max_downloads_per_subscription @property def subscription_download_probability(self) -> Optional[ProbabilityValidator]: """ - Probability to perform any downloads, recomputed for each subscription. This is only - recommended to set if you run ytdl-sub in a cron-job, that way you are statistically - guaranteed over time to eventually download the subscription. + :expected type: Optional[Float] + :description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. """ return self._subscription_download_probability diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index e27af4d8..c3b2e6ec 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -2,8 +2,8 @@ import copy from typing import Any from typing import Dict -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.file_handler import FileMetadata @@ -17,16 +17,14 @@ class VideoTagsOptions(OptionsDictValidator): """ Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" """ _optional_keys = {"tags"} diff --git a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml b/src/ytdl_sub/prebuilt_presets/helpers/common.yaml index 031b2964..9278d012 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/common.yaml @@ -22,12 +22,15 @@ presets: "Only Recent": # Only fetch videos after today minus date_range date_range: - after: "today-{date_range}" + after: "today-{only_recent_date_range}" # Only keep files uploaded after date_range output_options: - keep_files_after: "today-{date_range}" + keep_files_after: "today-{only_recent_date_range}" + keep_max_files: "{only_recent_max_files}" # Set the default date_range to 2 months overrides: - date_range: "2months" \ No newline at end of file + date_range: "2months" # keep for legacy-reasons + only_recent_date_range: "{date_range}" + only_recent_max_files: 0 \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/music/music.yaml b/src/ytdl_sub/prebuilt_presets/music/music.yaml index ee1a290f..7c2f23a3 100644 --- a/src/ytdl_sub/prebuilt_presets/music/music.yaml +++ b/src/ytdl_sub/prebuilt_presets/music/music.yaml @@ -67,12 +67,17 @@ presets: - "_music_base" download: - - "{url}" + - url: "{url}" + include_sibling_metadata: False _albums_from_playlists: preset: - - "Single" + - "_music_base" + + download: + - url: "{url}" + include_sibling_metadata: True overrides: track_album: "{playlist_title}" @@ -107,6 +112,7 @@ presets: # The first URL will be all the artist's tracks. # Treat these as singles - an album with a single track - url: "{url}/tracks" + include_sibling_metadata: False variables: sc_track_album: "{title}" sc_track_number: "1" @@ -117,6 +123,7 @@ presets: # to an album and tracks (in the URL above), it will resolve to this # URL and include the album metadata we set below. - url: "{url}/albums" + include_sibling_metadata: True variables: sc_track_album: "{playlist_title}" sc_track_number: "{playlist_index}" diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml index 53cd842d..bb1c9047 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml @@ -35,7 +35,6 @@ presets: tv_show_name: "{subscription_name}" tv_show_genre: "{subscription_indent_1}" tv_show_content_rating: "{subscription_indent_2}" - season_title: "{season_number_padded}" season_directory_name: "Season {season_number_padded}" episode_title: "{upload_date_standardized} - {title}" episode_plot: "{webpage_url}\n\n{description}" @@ -43,7 +42,7 @@ presets: episode_content_rating: "{tv_show_content_rating}" episode_date_standardized: "{upload_date_standardized}" episode_file_name: "s{season_number_padded}.e{episode_number_padded} - {file_title}" - episode_file_path: "{season_directory_name}/{episode_file_name}" + episode_file_path: "{season_directory_name_sanitized}/{episode_file_name_sanitized}" _episode_video_tags: video_tags: diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml index eb6a1598..308162e2 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml @@ -9,7 +9,7 @@ presets: overrides: tv_show_poster_file_name: "poster.jpg" tv_show_fanart_file_name: "fanart.jpg" - season_poster_file_name: "Season {season_number_padded}/Season{season_number_padded}.jpg" + season_poster_file_name: "{season_directory_name_sanitized}/Season{season_number_padded}.jpg" file_convert: convert_to: "mp4" # webm doesn't play in many plex clients, so use mp4 instead @@ -44,5 +44,6 @@ presets: # addition. Each season sets its own `collection_season_number/_padded` _tv_show_collection: overrides: + collection_season_number_padded: "{ %pad_zero(%int(collection_season_number), 2) }" season_number: "{collection_season_number}" season_number_padded: "{collection_season_number_padded}" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index dc9f7cc9..1b374d33 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -24,7 +24,7 @@ presets: - url: "{collection_season_1_url}" variables: collection_season_number: "1" - collection_season_number_padded: "01" + collection_season_name: "{collection_season_1_name}" playlist_thumbnails: # Use latest_entry first, then see if YT channel artwork exists # ONLY FOR SEASON 1! The channel artwork will be the show's artwork. @@ -47,7 +47,7 @@ presets: - url: "{collection_season_2_url}" variables: collection_season_number: "2" - collection_season_number_padded: "02" + collection_season_name: "{collection_season_2_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -64,7 +64,7 @@ presets: - url: "{collection_season_3_url}" variables: collection_season_number: "3" - collection_season_number_padded: "03" + collection_season_name: "{collection_season_3_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -81,7 +81,7 @@ presets: - url: "{collection_season_4_url}" variables: collection_season_number: "4" - collection_season_number_padded: "04" + collection_season_name: "{collection_season_4_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -98,7 +98,7 @@ presets: - url: "{collection_season_5_url}" variables: collection_season_number: "5" - collection_season_number_padded: "05" + collection_season_name: "{collection_season_5_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -115,7 +115,7 @@ presets: - url: "{collection_season_6_url}" variables: collection_season_number: "6" - collection_season_number_padded: "06" + collection_season_name: "{collection_season_6_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -132,7 +132,7 @@ presets: - url: "{collection_season_7_url}" variables: collection_season_number: "7" - collection_season_number_padded: "07" + collection_season_name: "{collection_season_7_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -149,7 +149,7 @@ presets: - url: "{collection_season_8_url}" variables: collection_season_number: "8" - collection_season_number_padded: "08" + collection_season_name: "{collection_season_8_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -166,7 +166,7 @@ presets: - url: "{collection_season_9_url}" variables: collection_season_number: "9" - collection_season_number_padded: "09" + collection_season_name: "{collection_season_9_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -183,7 +183,7 @@ presets: - url: "{collection_season_10_url}" variables: collection_season_number: "10" - collection_season_number_padded: "10" + collection_season_name: "{collection_season_10_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -200,7 +200,7 @@ presets: - url: "{collection_season_11_url}" variables: collection_season_number: "11" - collection_season_number_padded: "11" + collection_season_name: "{collection_season_11_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -217,7 +217,7 @@ presets: - url: "{collection_season_12_url}" variables: collection_season_number: "12" - collection_season_number_padded: "12" + collection_season_name: "{collection_season_12_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -234,7 +234,7 @@ presets: - url: "{collection_season_13_url}" variables: collection_season_number: "13" - collection_season_number_padded: "13" + collection_season_name: "{collection_season_13_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -251,7 +251,7 @@ presets: - url: "{collection_season_14_url}" variables: collection_season_number: "14" - collection_season_number_padded: "14" + collection_season_name: "{collection_season_14_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -268,7 +268,7 @@ presets: - url: "{collection_season_15_url}" variables: collection_season_number: "15" - collection_season_number_padded: "15" + collection_season_name: "{collection_season_15_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -285,7 +285,7 @@ presets: - url: "{collection_season_16_url}" variables: collection_season_number: "16" - collection_season_number_padded: "16" + collection_season_name: "{collection_season_16_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -302,7 +302,7 @@ presets: - url: "{collection_season_17_url}" variables: collection_season_number: "17" - collection_season_number_padded: "17" + collection_season_name: "{collection_season_17_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -319,7 +319,7 @@ presets: - url: "{collection_season_18_url}" variables: collection_season_number: "18" - collection_season_number_padded: "18" + collection_season_name: "{collection_season_18_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -336,7 +336,7 @@ presets: - url: "{collection_season_19_url}" variables: collection_season_number: "19" - collection_season_number_padded: "19" + collection_season_name: "{collection_season_19_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -353,7 +353,7 @@ presets: - url: "{collection_season_20_url}" variables: collection_season_number: "20" - collection_season_number_padded: "20" + collection_season_name: "{collection_season_20_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -370,7 +370,7 @@ presets: - url: "{collection_season_21_url}" variables: collection_season_number: "21" - collection_season_number_padded: "21" + collection_season_name: "{collection_season_21_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -387,7 +387,7 @@ presets: - url: "{collection_season_22_url}" variables: collection_season_number: "22" - collection_season_number_padded: "22" + collection_season_name: "{collection_season_22_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -404,7 +404,7 @@ presets: - url: "{collection_season_23_url}" variables: collection_season_number: "23" - collection_season_number_padded: "23" + collection_season_name: "{collection_season_23_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -421,7 +421,7 @@ presets: - url: "{collection_season_24_url}" variables: collection_season_number: "24" - collection_season_number_padded: "24" + collection_season_name: "{collection_season_24_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -438,7 +438,7 @@ presets: - url: "{collection_season_25_url}" variables: collection_season_number: "25" - collection_season_number_padded: "25" + collection_season_name: "{collection_season_25_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -455,7 +455,7 @@ presets: - url: "{collection_season_26_url}" variables: collection_season_number: "26" - collection_season_number_padded: "26" + collection_season_name: "{collection_season_26_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -472,7 +472,7 @@ presets: - url: "{collection_season_27_url}" variables: collection_season_number: "27" - collection_season_number_padded: "27" + collection_season_name: "{collection_season_27_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -489,7 +489,7 @@ presets: - url: "{collection_season_28_url}" variables: collection_season_number: "28" - collection_season_number_padded: "28" + collection_season_name: "{collection_season_28_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -506,7 +506,7 @@ presets: - url: "{collection_season_29_url}" variables: collection_season_number: "29" - collection_season_number_padded: "29" + collection_season_name: "{collection_season_29_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -523,7 +523,7 @@ presets: - url: "{collection_season_30_url}" variables: collection_season_number: "30" - collection_season_number_padded: "30" + collection_season_name: "{collection_season_30_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -540,7 +540,7 @@ presets: - url: "{collection_season_31_url}" variables: collection_season_number: "31" - collection_season_number_padded: "31" + collection_season_name: "{collection_season_31_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -557,7 +557,7 @@ presets: - url: "{collection_season_32_url}" variables: collection_season_number: "32" - collection_season_number_padded: "32" + collection_season_name: "{collection_season_32_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -574,7 +574,7 @@ presets: - url: "{collection_season_33_url}" variables: collection_season_number: "33" - collection_season_number_padded: "33" + collection_season_name: "{collection_season_33_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -591,7 +591,7 @@ presets: - url: "{collection_season_34_url}" variables: collection_season_number: "34" - collection_season_number_padded: "34" + collection_season_name: "{collection_season_34_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -608,7 +608,7 @@ presets: - url: "{collection_season_35_url}" variables: collection_season_number: "35" - collection_season_number_padded: "35" + collection_season_name: "{collection_season_35_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -625,7 +625,7 @@ presets: - url: "{collection_season_36_url}" variables: collection_season_number: "36" - collection_season_number_padded: "36" + collection_season_name: "{collection_season_36_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -642,7 +642,7 @@ presets: - url: "{collection_season_37_url}" variables: collection_season_number: "37" - collection_season_number_padded: "37" + collection_season_name: "{collection_season_37_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -659,7 +659,7 @@ presets: - url: "{collection_season_38_url}" variables: collection_season_number: "38" - collection_season_number_padded: "38" + collection_season_name: "{collection_season_38_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -676,7 +676,7 @@ presets: - url: "{collection_season_39_url}" variables: collection_season_number: "39" - collection_season_number_padded: "39" + collection_season_name: "{collection_season_39_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" @@ -693,7 +693,7 @@ presets: - url: "{collection_season_40_url}" variables: collection_season_number: "40" - collection_season_number_padded: "40" + collection_season_name: "{collection_season_40_name}" playlist_thumbnails: - name: "{season_poster_file_name}" uid: "latest_entry" diff --git a/src/ytdl_sub/script/__init__.py b/src/ytdl_sub/script/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/script/functions/__init__.py b/src/ytdl_sub/script/functions/__init__.py new file mode 100644 index 00000000..5b98d24b --- /dev/null +++ b/src/ytdl_sub/script/functions/__init__.py @@ -0,0 +1,82 @@ +from typing import Callable +from typing import Dict + +from ytdl_sub.script.functions.array_functions import ArrayFunctions +from ytdl_sub.script.functions.boolean_functions import BooleanFunctions +from ytdl_sub.script.functions.conditional_functions import ConditionalFunctions +from ytdl_sub.script.functions.date_functions import DateFunctions +from ytdl_sub.script.functions.error_functions import ErrorFunctions +from ytdl_sub.script.functions.json_functions import JsonFunctions +from ytdl_sub.script.functions.map_functions import MapFunctions +from ytdl_sub.script.functions.numeric_functions import NumericFunctions +from ytdl_sub.script.functions.regex_functions import RegexFunctions +from ytdl_sub.script.functions.string_functions import StringFunctions +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExistRuntimeException + + +class Functions( + StringFunctions, + NumericFunctions, + ConditionalFunctions, + ArrayFunctions, + MapFunctions, + BooleanFunctions, + ErrorFunctions, + RegexFunctions, + DateFunctions, + JsonFunctions, +): + _custom_functions: Dict[str, Callable[..., Resolvable]] = {} + + @classmethod + def is_built_in(cls, name: str) -> bool: + """ + Returns + ------- + True if the name exists as a built-in function or custom function. False otherwise. + """ + return hasattr(cls, name) or hasattr(cls, f"{name}_") or name in cls._custom_functions + + @classmethod + def get(cls, name: str) -> Callable[..., Resolvable]: + """ + Returns + ------- + The actual Python callable for the function of the given name. + + Raises + ------ + FunctionDoesNotExistRuntimeException + If the function does not exist. + """ + if hasattr(cls, name): + return getattr(cls, name) + if hasattr(cls, f"{name}_"): + return getattr(cls, f"{name}_") + if name in cls._custom_functions: + return cls._custom_functions[name] + + raise FunctionDoesNotExistRuntimeException(f"The function {name} does not exist") + + @classmethod + def register_function(cls, function: Callable[..., Resolvable]) -> None: + """ + Adds a function to the suite of offered functions. + + Parameters + ---------- + function + A static function whose name will be used as the offered function name. + + Raises + ------ + ValueError + If the name already exists as a function. + """ + if cls.is_built_in(function.__name__): + raise ValueError( + f"Cannot register a function with name {function.__name__} " + f"because it already exists" + ) + cls._custom_functions[function.__name__] = function diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py new file mode 100644 index 00000000..4af6f63c --- /dev/null +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -0,0 +1,218 @@ +import itertools +from typing import List +from typing import Optional + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import LambdaReduce +from ytdl_sub.script.types.resolvable import LambdaTwo +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import ArrayValueDoesNotExist +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException + + +class ArrayFunctions: + @staticmethod + def array(maybe_array: AnyArgument) -> Array: + """ + :description: + Tries to cast an unknown variable type to an Array. + """ + if not isinstance(maybe_array, Array): + raise FunctionRuntimeException( + f"Tried and failed to cast {maybe_array.type_name()} as an Array" + ) + return maybe_array + + @staticmethod + def array_size(array: Array) -> Integer: + """ + :description: + Returns the size of an Array. + """ + return Integer(len(array.value)) + + @staticmethod + def array_extend(*arrays: Array) -> Array: + """ + :description: + Combine multiple Arrays into a single Array. + """ + output: List[Resolvable] = [] + for array in arrays: + output.extend(array.value) + + return Array(output) + + @staticmethod + def array_overlay( + array: Array, overlap: Array, only_missing: Optional[Boolean] = None + ) -> Array: + """ + :description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. + """ + output: List[Resolvable] = [] + output.extend(array.value) + + overlap_only_missing = only_missing and only_missing.value + + for idx, overlap_value in enumerate(overlap.value): + if overlap_only_missing and idx < len(array.value): + continue + + if idx < len(array.value): + output[idx] = overlap_value + else: + output.append(overlap_value) + + return Array(output) + + @staticmethod + def array_at(array: Array, idx: Integer) -> AnyArgument: + """ + :description: + Return the element in the Array at index ``idx``. + """ + return array.value[idx.value] + + @staticmethod + def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: + """ + :description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. + """ + for val in array.value: + if bool(val.value): + return val + + return fallback + + @staticmethod + def array_contains(array: Array, value: AnyArgument) -> Boolean: + """ + :description: + Return True if the value exists in the Array. False otherwise. + """ + return Boolean(value in array.value) + + @staticmethod + def array_index(array: Array, value: AnyArgument) -> Integer: + """ + :description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. + """ + if not ArrayFunctions.array_contains(array=array, value=value): + raise ArrayValueDoesNotExist( + "Tried to get the index of a value in an Array that does not exist" + ) + + if isinstance(value, Resolvable): + return Integer(array.value.index(value)) + + raise UNREACHABLE + + @staticmethod + def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array: + """ + :description: + Returns the slice of the Array. + """ + if end is not None: + return Array(array.value[start.value : end.value]) + return Array(array.value[start.value :]) + + @staticmethod + def array_flatten(array: Array) -> Array: + """ + :description: + Flatten any nested Arrays into a single-dimensional Array. + """ + output: List[Resolvable] = [] + for elem in array.value: + if isinstance(elem, Array): + output.extend(ArrayFunctions.array_flatten(elem).value) + else: + output.append(elem) + + return Array(output) + + @staticmethod + def array_reverse(array: Array) -> Array: + """ + :description: + Reverse an Array. + """ + return Array(list(reversed(array.value))) + + @staticmethod + def array_product(*arrays: Array) -> Array: + """ + :description: + Returns the Cartesian product of elements from different arrays + """ + out: List[Resolvable] = [] + for combo in itertools.product(*[arr.value for arr in arrays]): + out.append(Array(combo)) + + return Array(out) + + # pylint: disable=unused-argument + + @staticmethod + def array_apply(array: Array, lambda_function: Lambda) -> Array: + """ + :description: + Apply a lambda function on every element in the Array. + :usage: + + .. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] + """ + return Array([Array([val]) for val in array.value]) + + @staticmethod + def array_apply_fixed( + array: Array, + fixed_argument: AnyArgument, + lambda2_function: LambdaTwo, + reverse_args: Optional[Boolean] = None, + ) -> Array: + """ + :description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. + """ + if reverse_args and reverse_args.value: + return Array([Array([fixed_argument, val]) for val in array.value]) + + return Array([Array([val, fixed_argument]) for val in array.value]) + + @staticmethod + def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array: + """ + :description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. + """ + return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)]) + + @staticmethod + def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument: + """ + :description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. + """ + return array diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py new file mode 100644 index 00000000..bfb7416f --- /dev/null +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -0,0 +1,109 @@ +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import String + +# pylint: disable=invalid-name + + +class BooleanFunctions: + """ + Comparison functions that output Booleans. + """ + + @staticmethod + def bool(value: AnyArgument) -> Boolean: + """ + :description: + Cast any type to a Boolean. + """ + return Boolean(bool(value.value)) + + @staticmethod + def eq(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + :description: + ``==`` operator. Returns True if left == right. False otherwise. + """ + return Boolean(left.value == right.value) + + @staticmethod + def ne(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + :description: + ``!=`` operator. Returns True if left != right. False otherwise. + """ + return Boolean(left.value != right.value) + + @staticmethod + def lt(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + :description: + ``<`` operator. Returns True if left < right. False otherwise. + """ + return Boolean(left.value < right.value) + + @staticmethod + def lte(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + :description: + ``<=`` operator. Returns True if left <= right. False otherwise. + """ + return Boolean(left.value <= right.value) + + @staticmethod + def gt(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + :description: + ``>`` operator. Returns True if left > right. False otherwise. + """ + return Boolean(left.value > right.value) + + @staticmethod + def gte(left: AnyArgument, right: AnyArgument) -> Boolean: + """ + :description: + ``>=`` operator. Returns True if left >= right. False otherwise. + """ + return Boolean(left.value >= right.value) + + @staticmethod + def and_(*values: AnyArgument) -> Boolean: + """ + :description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. + """ + return Boolean(all(bool(val.value) for val in values)) + + @staticmethod + def or_(*values: AnyArgument) -> Boolean: + """ + :description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. + """ + return Boolean(any(bool(val.value) for val in values)) + + @staticmethod + def xor(*values: AnyArgument) -> Boolean: + """ + :description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + """ + bit_array = [bool(val.value) for val in values] + + return Boolean(sum(bit_array) == 1) + + @staticmethod + def not_(value: Boolean) -> Boolean: + """ + :description: + ``not`` operator. Returns the opposite of value. + """ + return Boolean(not value.value) + + @staticmethod + def is_null(value: AnyArgument) -> Boolean: + """ + :description: + Returns True if a value is null (i.e. an empty string). False otherwise. + """ + return Boolean(isinstance(value, String) and value.value == "") diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py new file mode 100644 index 00000000..e532031b --- /dev/null +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -0,0 +1,33 @@ +from typing import Union + +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import ReturnableArgumentA +from ytdl_sub.script.types.resolvable import ReturnableArgumentB + + +class ConditionalFunctions: + @staticmethod + def if_( + condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB + ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: + """ + :description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. + """ + if condition.value: + return true + return false + + @staticmethod + def if_passthrough( + maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB + ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: + """ + :description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. + """ + if bool(maybe_true_arg.value): + return maybe_true_arg + return else_arg diff --git a/src/ytdl_sub/script/functions/date_functions.py b/src/ytdl_sub/script/functions/date_functions.py new file mode 100644 index 00000000..fea1095c --- /dev/null +++ b/src/ytdl_sub/script/functions/date_functions.py @@ -0,0 +1,14 @@ +from datetime import datetime + +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String + + +class DateFunctions: + @staticmethod + def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String: + """ + :description: + Converts a posix timestamp to a date using strftime formatting. + """ + return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value)) diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py new file mode 100644 index 00000000..3a1be9f5 --- /dev/null +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -0,0 +1,64 @@ +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import ReturnableArgument +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError + + +class ErrorFunctions: + @staticmethod + def throw(error_message: String) -> AnyArgument: + """ + :description: + Explicitly throw an error with the provided error message. + """ + raise UserThrownRuntimeError(error_message) + + @staticmethod + def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument: + """ + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. + """ + if not bool(value.value): + raise UserThrownRuntimeError(assert_message) + return value + + @staticmethod + def assert_then( + value: AnyArgument, ret: ReturnableArgument, assert_message: String + ) -> ReturnableArgument: + """ + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. + """ + if not bool(value.value): + raise UserThrownRuntimeError(assert_message) + return ret + + @staticmethod + def assert_eq( + value: ReturnableArgument, equals: AnyArgument, assert_message: String + ) -> ReturnableArgument: + """ + :description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. + """ + if not value.value == equals.value: + raise UserThrownRuntimeError(assert_message) + return value + + @staticmethod + def assert_ne( + value: ReturnableArgument, equals: AnyArgument, assert_message: String + ) -> ReturnableArgument: + """ + :description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. + """ + if value.value == equals.value: + raise UserThrownRuntimeError(assert_message) + return value diff --git a/src/ytdl_sub/script/functions/json_functions.py b/src/ytdl_sub/script/functions/json_functions.py new file mode 100644 index 00000000..c706a86f --- /dev/null +++ b/src/ytdl_sub/script/functions/json_functions.py @@ -0,0 +1,41 @@ +import json +from typing import Any + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import UNREACHABLE + + +def _from_json(out: Any) -> Resolvable: + # pylint: disable=too-many-return-statements + if out is None: + return String("") + if isinstance(out, int): + return Integer(out) + if isinstance(out, float): + return Float(out) + if isinstance(out, str): + return String(out) + if isinstance(out, bool): + return Boolean(out) + if isinstance(out, list): + return Array(value=[_from_json(arg) for arg in out]) + if isinstance(out, dict): + return Map(value={_from_json(key): _from_json(value) for key, value in out.items()}) + raise UNREACHABLE + + +class JsonFunctions: + @staticmethod + def from_json(argument: String) -> AnyArgument: + """ + :description: + Converts a JSON string into an actual type. + """ + return _from_json(json.loads(argument.value)) diff --git a/src/ytdl_sub/script/functions/map_functions.py b/src/ytdl_sub/script/functions/map_functions.py new file mode 100644 index 00000000..b34fa611 --- /dev/null +++ b/src/ytdl_sub/script/functions/map_functions.py @@ -0,0 +1,104 @@ +from typing import Optional + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Hashable +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import LambdaThree +from ytdl_sub.script.types.resolvable import LambdaTwo +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +class MapFunctions: + @staticmethod + def map(maybe_mapping: AnyArgument) -> Map: + """ + :description: + Tries to cast an unknown variable type to a Map. + """ + if not isinstance(maybe_mapping, Map): + raise FunctionRuntimeException( + f"Tried and failed to cast {maybe_mapping.type_name()} as a Map" + ) + return maybe_mapping + + @staticmethod + def map_size(mapping: Map) -> Integer: + """ + :description: + Returns the size of a Map. + """ + return Integer(len(mapping.value)) + + @staticmethod + def map_contains(mapping: Map, key: AnyArgument) -> Boolean: + """ + :description: + Returns True if the key is in the Map. False otherwise. + """ + if not isinstance(key, Hashable): + raise KeyNotHashableRuntimeException( + f"Tried to use {key.type_name()} as a Map key, but it is not hashable." + ) + + return Boolean(key in mapping.value) + + @staticmethod + def map_get( + mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None + ) -> AnyArgument: + """ + :description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. + """ + if not MapFunctions.map_contains(mapping=mapping, key=key).value: + if default is not None: + return default + + raise KeyDoesNotExistRuntimeException( + f"Tried to call %map_get with key {key.value}, but it does not exist" + ) + return mapping.value[key] + + @staticmethod + def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument: + """ + :description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. + """ + output = MapFunctions.map_get(mapping, key, default) + if isinstance(output, String) and output.value == "": + return default + return output + + # pylint: disable=unused-argument + + @staticmethod + def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array: + """ + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. + """ + return Array([Array([key, value]) for key, value in mapping.value.items()]) + + @staticmethod + def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array: + """ + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. + """ + return Array( + [ + Array([Integer(idx), key_value[0], key_value[1]]) + for idx, key_value in enumerate(mapping.value.items()) + ] + ) diff --git a/src/ytdl_sub/script/functions/numeric_functions.py b/src/ytdl_sub/script/functions/numeric_functions.py new file mode 100644 index 00000000..b25374f9 --- /dev/null +++ b/src/ytdl_sub/script/functions/numeric_functions.py @@ -0,0 +1,98 @@ +import math + +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Numeric + + +def _to_numeric(value: int | float) -> Numeric: + if int(value) == value: + return Integer(value=value) + return Float(value=value) + + +class NumericFunctions: + @staticmethod + def float(value: AnyArgument) -> Float: + """ + :description: + Cast to Float. + """ + return Float(value=float(value.value)) + + @staticmethod + def int(value: AnyArgument) -> Integer: + """ + :description: + Cast to Integer. + """ + return Integer(value=int(value.value)) + + @staticmethod + def add(*values: Numeric) -> Numeric: + """ + :description: + ``+`` operator. Returns the sum of all values. + """ + return _to_numeric(sum(val.value for val in values)) + + @staticmethod + def sub(*values: Numeric) -> Numeric: + """ + :description: + ``-`` operator. Subtracts all values from left to right. + """ + output = values[0].value + for val in values[1:]: + output -= val.value + + return _to_numeric(output) + + @staticmethod + def mul(*values: Numeric) -> Numeric: + """ + :description: + ``*`` operator. Returns the product of all values. + """ + return _to_numeric(math.prod([val.value for val in values])) + + @staticmethod + def pow(base: Numeric, exponent: Numeric) -> Numeric: + """ + :description: + ``**`` operator. Returns the exponential of the base and exponent value. + """ + return _to_numeric(math.pow(base.value, exponent.value)) + + @staticmethod + def div(left: Numeric, right: Numeric) -> Numeric: + """ + :description: + ``/`` operator. Returns ``left / right``. + """ + return _to_numeric(left.value / right.value) + + @staticmethod + def mod(left: Numeric, right: Numeric) -> Numeric: + """ + :description: + ``%`` operator. Returns ``left % right``. + """ + return _to_numeric(value=left.value % right.value) + + @staticmethod + def max(*values: Numeric) -> Numeric: + """ + :description: + Returns max of all values. + """ + return _to_numeric(max(val.value for val in values)) + + @staticmethod + def min(*values: Numeric) -> Numeric: + """ + :description: + Returns min of all values. + """ + return _to_numeric(min(val.value for val in values)) diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py new file mode 100644 index 00000000..f837be32 --- /dev/null +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -0,0 +1,54 @@ +import re +from typing import AnyStr +from typing import Match + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String + + +def _re_output_to_array(re_out: Match[AnyStr] | None) -> Array: + if re_out is None: + return Array([]) + + return Array(list([String(re_out.string)]) + list(String(group) for group in re_out.groups())) + + +class RegexFunctions: + @staticmethod + def regex_match(regex: String, string: String) -> Array: + """ + :description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + """ + return _re_output_to_array(re.match(regex.value, string.value)) + + @staticmethod + def regex_search(regex: String, string: String) -> Array: + """ + :description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + """ + return _re_output_to_array(re.search(regex.value, string.value)) + + @staticmethod + def regex_fullmatch(regex: String, string: String) -> Array: + """ + :description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + """ + return _re_output_to_array(re.fullmatch(regex.value, string.value)) + + @staticmethod + def regex_capture_groups(regex: String) -> Integer: + """ + :description: + Returns number of capture groups in regex + """ + return Integer(re.compile(regex.value).groups) diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py new file mode 100644 index 00000000..1f0280b5 --- /dev/null +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -0,0 +1,113 @@ +from typing import Optional + +from ytdl_sub.script.types.resolvable import AnyArgument +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Numeric +from ytdl_sub.script.types.resolvable import String + + +class StringFunctions: + @staticmethod + def string(value: AnyArgument) -> String: + """ + :description: + Cast to String. + """ + return String(value=str(value.value)) + + @staticmethod + def contains(string: String, contains: String) -> Boolean: + """ + :description: + Returns True if ``contains`` is in ``string``. False otherwise. + """ + return Boolean(contains.value in string.value) + + @staticmethod + def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: + """ + :description: + Returns the slice of the Array. + """ + if end is not None: + return String(string.value[start.value : end.value]) + return String(string.value[start.value :]) + + @staticmethod + def lower(string: String) -> String: + """ + :description: + Lower-case the entire String. + """ + return String(string.value.lower()) + + @staticmethod + def upper(string: String) -> String: + """ + :description: + Upper-case the entire String. + """ + return String(string.value.upper()) + + @staticmethod + def capitalize(string: String) -> String: + """ + :description: + Capitalize the first character in the string. + """ + return String(string.value.capitalize()) + + @staticmethod + def titlecase(string: String) -> String: + """ + :description: + Capitalize each word in the string. + """ + return String(string.value.title()) + + @staticmethod + def replace( + string: String, old: String, new: String, count: Optional[Integer] = None + ) -> String: + """ + :description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. + """ + if count: + return String(string.value.replace(old.value, new.value, count.value)) + + return String(string.value.replace(old.value, new.value)) + + @staticmethod + def concat(*values: String) -> String: + """ + :description: + Concatenate multiple Strings into a single String. + """ + return String("".join(val.value for val in values)) + + @staticmethod + def pad(string: String, length: Integer, char: String) -> String: + """ + :description: + Pads the string to the given length + """ + output = string.value + while len(output) < length.value: + output = f"{char}{output}" + + return String(output) + + @staticmethod + def pad_zero(numeric: Numeric, length: Integer) -> String: + """ + :description: + Pads a numeric with zeros to the given length + """ + return StringFunctions.pad( + string=String(str(numeric.value)), + length=length, + char=String("0"), + ) diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py new file mode 100644 index 00000000..82b66bd9 --- /dev/null +++ b/src/ytdl_sub/script/parser.py @@ -0,0 +1,601 @@ +import json +from enum import Enum +from typing import Dict +from typing import List +from typing import Optional +from typing import Set + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.array import UnresolvedArray +from ytdl_sub.script.types.function import Argument +from ytdl_sub.script.types.function import BuiltInFunction +from ytdl_sub.script.types.function import CustomFunction +from ytdl_sub.script.types.function import Function +from ytdl_sub.script.types.map import UnresolvedMap +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import NonHashable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import FunctionArgument +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exception_formatters import ParserExceptionFormatter +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException +from ytdl_sub.script.utils.exceptions import InvalidVariableName +from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist +from ytdl_sub.script.utils.name_validation import validate_variable_name + +# pylint: disable=invalid-name +# pylint: disable=too-many-branches +# pylint: disable=too-many-return-statements +# pylint: disable=consider-using-ternary + + +class ParsedArgType(Enum): + SCRIPT = "script" + FUNCTION = "function" + ARRAY = "array" + MAP_KEY = "map key" + MAP_VALUE = "map value" + + +BRACKET_NOT_CLOSED = InvalidSyntaxException("Bracket not properly closed") + +NUMERICS_ONLY_ARGS = InvalidSyntaxException( + "Numerics can only be used as arguments to functions, maps, or arrays" +) +NUMERICS_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a numeric") + + +STRINGS_ONLY_ARGS = InvalidSyntaxException( + "Strings can only be used as arguments to functions, maps, or arrays" +) +STRINGS_NOT_CLOSED = InvalidSyntaxException( + "String was not closed properly. " + "Must open and close with the same type of quote (single/double)" +) + +BOOLEAN_ONLY_ARGS = InvalidSyntaxException( + "Booleans can only be used as arguments to functions, maps, or arrays" +) + +CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS = InvalidSyntaxException( + "Custom function arguments can only be used as arguments to functions, maps, or arrays" +) + +FUNCTION_INVALID_CHAR = InvalidSyntaxException("Invalid value when parsing a function") + + +def _UNEXPECTED_CHAR_ARGUMENT(arg_type: ParsedArgType): + return InvalidSyntaxException(f"Unexpected character when parsing {arg_type.value} arguments") + + +def _UNEXPECTED_COMMA_ARGUMENT(arg_type: ParsedArgType): + return InvalidSyntaxException(f"Unexpected comma when parsing {arg_type.value} arguments") + + +MAP_KEY_WITH_NO_VALUE = InvalidSyntaxException("Map has a key with no value") +MAP_KEY_MULTIPLE_VALUES = InvalidSyntaxException( + "Map key has multiple values when there should only be one" +) +MAP_MISSING_KEY = InvalidSyntaxException("Map has a missing key") +MAP_KEY_NOT_HASHABLE = InvalidSyntaxException( + "Map key must be a hashable type (Integer, Float, Boolean, String)" +) + + +def _is_variable_start(char: str) -> bool: + return char.isalpha() and char.islower() + + +def _is_function_name_char(char: str) -> bool: + return (char.isalpha() and char.islower()) or char.isnumeric() or char == "_" + + +def _is_numeric_start(char: str) -> bool: + return char.isnumeric() or char in (".", "-") + + +def _is_string_start_single_char(char: Optional[str]) -> bool: + return char in ["'", '"'] + + +def _is_string_start_multi_char(string: Optional[str]) -> bool: + return string in ["'''", '"""'] + + +def _is_null(string: Optional[str]) -> bool: + return string and string.lower() == "null" + + +def _is_breakable(char: str) -> bool: + return char in ["}", ",", ")", "]", ":"] or char.isspace() + + +def _is_boolean_true(string: Optional[str]) -> bool: + return string and string.lower() == "true" + + +def _is_boolean_false(string: Optional[str]) -> bool: + return string and string.lower() == "false" + + +def _is_custom_function_argument_start(char: str) -> bool: + return char == "$" + + +class _Parser: + def __init__( + self, + text: str, + name: Optional[str], + custom_function_names: Optional[Set[str]], + variable_names: Optional[Set[str]], + ): + self._text = text + self._name = name + self._custom_function_names = custom_function_names + self._variable_names = variable_names + self._pos = 0 + self._error_highlight_pos = 0 + self._ast: List[Argument] = [] + + self._bracket_counter_pos_stack: List[int] = [] + self._bracket_counter = 0 + self._literal_str = "" + + try: + self._syntax_tree = self._parse() + except UserException as exc: + raise ParserExceptionFormatter( + self._text, self._error_highlight_pos, self._pos, exc + ).highlight() from exc + + @property + def ast(self) -> SyntaxTree: + """ + Returns + ------- + Abstract syntax tree of the parsed text + """ + return self._syntax_tree + + def _set_highlight_position(self, pos: Optional[int] = None) -> None: + self._error_highlight_pos = pos if pos is not None else self._pos + + def _read(self, increment_pos: bool = True, length: int = 1) -> Optional[str]: + if isinstance(self._text, int): + pass + if self._pos >= len(self._text): + return None + + try: + ch = self._text[self._pos : (self._pos + length)] + except IndexError: + return None + + if increment_pos: + self._pos += length + return ch + + def _parse_variable(self) -> Variable: + var_name = "" + variable_start_pos = self._pos + while ch := self._read(increment_pos=False): + if ch.isspace() and not var_name: + self._pos += 1 + continue + if _is_breakable(ch): + break + + if not var_name: + variable_start_pos = self._pos + + var_name += ch + self._pos += 1 + + try: + validate_variable_name(var_name) + except InvalidVariableName: + self._set_highlight_position(variable_start_pos) + raise + + if self._variable_names is not None and var_name not in self._variable_names: + self._set_highlight_position(variable_start_pos) + raise VariableDoesNotExist(f"Variable {var_name} does not exist.") + + return Variable(var_name) + + def _parse_custom_function_argument(self) -> FunctionArgument: + """ + Begin parsing function args after the first ``$``, i.e. ``$0`` + """ + var_name = "" + variable_start_pos = self._pos + while ch := self._read(increment_pos=False): + if ch.isspace() and not var_name: + raise InvalidCustomFunctionArgumentName( + "Custom function arguments, denoted by $, cannot have a space proceeding it." + ) + if _is_breakable(ch): + break + + var_name += ch + self._pos += 1 + + if not var_name.isnumeric(): + self._set_highlight_position(variable_start_pos) + raise InvalidCustomFunctionArgumentName( + "Custom function arguments must be numeric and increment starting from zero." + ) + + return FunctionArgument.from_idx(idx=int(var_name), custom_function_name=self._name) + + def _parse_numeric(self) -> Integer | Float: + numeric_string = "" + + if self._read(increment_pos=False) == "-": + numeric_string += "-" + self._pos += 1 + if has_decimal := (self._read(increment_pos=False) == "."): + numeric_string += "." + self._pos += 1 + + while ch := self._read(increment_pos=False): + if ch == "-": + raise NUMERICS_INVALID_CHAR + + if ch == ".": + if has_decimal: + raise NUMERICS_INVALID_CHAR + has_decimal = True + + self._pos += 1 + numeric_string += ch + elif ch.isnumeric(): + self._pos += 1 + numeric_string += ch + elif _is_breakable(ch): + break + else: + self._set_highlight_position() + raise NUMERICS_INVALID_CHAR + + if numeric_string in (".", "-"): + raise NUMERICS_INVALID_CHAR + + try: + numeric_float = float(numeric_string) + except ValueError as exc: + raise UNREACHABLE from exc + + if (numeric_int := int(numeric_float)) == numeric_float: + return Integer(value=numeric_int) + + return Float(value=numeric_float) + + def _parse_string(self, str_open_token: str) -> String: + """ + Begin parsing a string, including the quotation value + """ + self._set_highlight_position() + string_value = "" + + if not _is_string_start_single_char(str_open_token) and not _is_string_start_multi_char( + str_open_token + ): + raise UNREACHABLE + + while ch := self._read(increment_pos=False): + if self._read(increment_pos=False, length=len(str_open_token)) == str_open_token: + self._pos += len(str_open_token) + return String(value=string_value) + + self._pos += 1 + string_value += ch + + raise STRINGS_NOT_CLOSED + + def _parse_function_arg(self, argument_parser: ParsedArgType) -> Argument: + if self._read(increment_pos=False) == "%": + self._pos += 1 + return self._parse_function() + if _is_numeric_start(self._read(increment_pos=False)): + return self._parse_numeric() + if _is_boolean_true(self._read(increment_pos=False, length=4)): + self._pos += 4 + return Boolean(value=True) + if _is_boolean_false(self._read(increment_pos=False, length=5)): + self._pos += 5 + return Boolean(value=False) + if _is_null(self._read(increment_pos=False, length=4)): + self._pos += 4 + return String(value="") + if _is_string_start_multi_char(str_open_token := self._read(increment_pos=False, length=3)): + self._pos += 3 + return self._parse_string(str_open_token) + if _is_string_start_single_char(str_open_token := self._read(increment_pos=False)): + self._pos += 1 + return self._parse_string(str_open_token) + if self._read(increment_pos=False) == "[": + self._pos += 1 + return self._parse_array() + if self._read(increment_pos=False) == "{": + self._pos += 1 + return self._parse_map() + if _is_custom_function_argument_start(self._read(increment_pos=False)): + self._pos += 1 + return self._parse_custom_function_argument() + if _is_variable_start(self._read(increment_pos=False)): + return self._parse_variable() + + self._set_highlight_position() + raise _UNEXPECTED_CHAR_ARGUMENT(arg_type=argument_parser) + + def _parse_args( + self, argument_parser: ParsedArgType, breaking_chars: str = ")" + ) -> List[Argument]: + """ + Begin parsing function args after the first ``(``, i.e. ``function_name(`` + """ + comma_count = 0 + arguments: List[Argument] = [] + while ch := self._read(increment_pos=False): + if ch in breaking_chars: + # i.e. ["arg", ] which is invalid + if arguments and len(arguments) == comma_count: + raise _UNEXPECTED_COMMA_ARGUMENT(argument_parser) + break + + if ch.isspace(): + self._pos += 1 + elif ch == ",": + self._set_highlight_position() + comma_count += 1 + if len(arguments) != comma_count: + raise _UNEXPECTED_COMMA_ARGUMENT(argument_parser) + + self._pos += 1 + else: + arguments.append(self._parse_function_arg(argument_parser=argument_parser)) + + return arguments + + def _parse_function(self) -> Function | Lambda: + """ + Begin parsing a function after reading the first ``%`` + """ + function_name: str = "" + function_args: Optional[List[Argument]] = None + function_start_pos = self._pos + + while ch := self._read(): + if ch == ")": + # Had '(' to indicate there are args + if function_args is not None: + if self._name == function_name: + self._set_highlight_position(function_start_pos) + raise CycleDetected( + f"The custom function %{function_name} cannot call itself." + ) + + if Functions.is_built_in(function_name): + try: + return BuiltInFunction( + name=function_name, args=function_args + ).validate_args() + except IncompatibleFunctionArguments: + self._set_highlight_position(function_start_pos) + raise + + # Is custom function + if ( + self._custom_function_names is not None + and function_name not in self._custom_function_names + ): + self._set_highlight_position(function_start_pos) + raise FunctionDoesNotExist( + f"Function %{function_name} does not exist as a built-in or " + "custom function." + ) + + return CustomFunction( + name=function_name, + args=function_args, + ) + + # Go back one so the parent function can close using the ')' + self._pos -= 1 + return Lambda(value=function_name) + + if _is_function_name_char(ch): + function_name += ch + elif ch == "(": + function_args = self._parse_args(argument_parser=ParsedArgType.FUNCTION) + elif ch.isspace() or ch == ",": + # function with no args, it's a lambda + return Lambda(value=function_name) + else: + break + + self._set_highlight_position(pos=self._pos - 1) + raise FUNCTION_INVALID_CHAR + + def _parse_array(self) -> UnresolvedArray: + """ + Begin parsing an array after reading the first ``[`` + """ + function_args: List[Argument] = [] + + while ch := self._read(increment_pos=False): + if ch == "]": + self._pos += 1 + return UnresolvedArray(value=function_args) + + function_args = self._parse_args( + argument_parser=ParsedArgType.ARRAY, breaking_chars="]" + ) + + raise UNREACHABLE + + def _parse_map(self) -> UnresolvedMap: + """ + Begin parsing a map after reading the first ``{`` + """ + output: Dict[Argument, Argument] = {} + key: Optional[Argument] = None + in_comma = False + + self._set_highlight_position() + while ch := self._read(increment_pos=False): + if ch == "}": + if key is not None: + raise MAP_KEY_WITH_NO_VALUE # key args are parsed immediately + + self._pos += 1 + return UnresolvedMap(value=output) + + if ch == ",": + if in_comma: + raise _UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY) + if key is not None: + raise MAP_KEY_WITH_NO_VALUE # key args are parsed immediately + if not output: + raise _UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY) + in_comma = True + self._pos += 1 + elif key is None: + self._set_highlight_position() + in_comma = False + key_args = self._parse_args( + argument_parser=ParsedArgType.MAP_KEY, breaking_chars=":}" + ) + + if len(key_args) == 0 and self._read(increment_pos=False) == "}": + continue # will return the map next iteration + if len(key_args) == 0: + raise MAP_MISSING_KEY + if len(key_args) > 1: + raise MAP_KEY_MULTIPLE_VALUES + key = key_args[0] + elif key is not None and ch == ":": + self._set_highlight_position() + self._pos += 1 + value_args = self._parse_args( + argument_parser=ParsedArgType.MAP_VALUE, breaking_chars=",}" + ) + if len(value_args) == 0: + raise MAP_KEY_WITH_NO_VALUE + if isinstance(key, NonHashable): + raise MAP_KEY_NOT_HASHABLE + if len(value_args) > 1: + raise MAP_KEY_MULTIPLE_VALUES + + output[key] = value_args[0] + key = None + else: + raise UNREACHABLE + + def _parse_main_loop(self, ch: str) -> bool: + if ch == "}": + if self._bracket_counter == 0: + raise BRACKET_NOT_CLOSED + + del self._bracket_counter_pos_stack[-1] + self._bracket_counter -= 1 + return True + if ch == "{": + self._bracket_counter_pos_stack.append(self._pos - 1) # pos incremented when read + self._bracket_counter += 1 + if self._literal_str: + self._ast.append(String(value=self._literal_str)) + self._literal_str = "" + + # Allow whitespace after bracket opening + while ch1 := self._read(increment_pos=False): + if not ch1.isspace(): + break + self._pos += 1 + + if ch1 is None: + return False # will hit closing bracket error + + if ch1 == "%": + self._pos += 1 + self._ast.append(self._parse_function()) + elif ch1 == "[": + self._pos += 1 + self._ast.append(self._parse_array()) + elif ch1 == "{": + self._pos += 1 + self._ast.append(self._parse_map()) + elif _is_variable_start(ch1): + self._ast.append(self._parse_variable()) + elif _is_numeric_start(ch1): + raise NUMERICS_ONLY_ARGS + elif ( + _is_string_start_single_char(ch1) + or _is_string_start_multi_char(self._read(increment_pos=False, length=3)) + or _is_null(self._read(increment_pos=False, length=4)) + ): + raise STRINGS_ONLY_ARGS + elif _is_boolean_true(self._read(increment_pos=False, length=4)) or _is_boolean_false( + self._read(increment_pos=False, length=5) + ): + raise BOOLEAN_ONLY_ARGS + elif _is_custom_function_argument_start(self._read(increment_pos=False)): + raise CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS + else: + raise _UNEXPECTED_CHAR_ARGUMENT(arg_type=ParsedArgType.SCRIPT) + elif self._bracket_counter == 0: + # Only accumulate literal str if not in brackets + self._literal_str += ch + else: + # Should only be possible to get here if it's a space + assert ch.isspace() + + return True + + def _parse(self) -> SyntaxTree: + + while ch := self._read(): + continue_parse = self._parse_main_loop(ch) + if not continue_parse: + break + + if self._bracket_counter != 0: + self._error_highlight_pos = self._bracket_counter_pos_stack[-1] + raise BRACKET_NOT_CLOSED + + if self._literal_str: + self._ast.append(String(value=self._literal_str)) + + return SyntaxTree(ast=self._ast) + + +def parse( + text: str, + name: Optional[str] = None, + custom_function_names: Optional[Set[str]] = None, + variable_names: Optional[Set[str]] = None, +) -> SyntaxTree: + """ + Entrypoint for parsing ytdl-sub code into a Syntax Tree + """ + return _Parser( + text=json.dumps(text) if not isinstance(text, str) else text, + name=name, + custom_function_names=custom_function_names, + variable_names=variable_names, + ).ast + + +# pylint: enable=invalid-name diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py new file mode 100644 index 00000000..0560984b --- /dev/null +++ b/src/ytdl_sub/script/script.py @@ -0,0 +1,490 @@ +# pylint: disable=missing-raises-doc +from typing import Dict +from typing import List +from typing import Optional +from typing import Set + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.parser import parse +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.script.utils.name_validation import validate_variable_name +from ytdl_sub.script.utils.type_checking import FunctionSpec + + +def _is_function(override_name: str): + return override_name.startswith("%") + + +def _function_name(function_key: str) -> str: + """ + Drop the % in %custom_function + """ + return function_key[1:] + + +def _to_function_definition_name(function_key: str) -> str: + """ + Add % in %custom_function + """ + return f"%{function_key}" + + +class Script: + """ + Takes a dictionary of both + ``{ variable_names: syntax }`` + and + ``{ %custom_function: syntax }`` + """ + + def _ensure_no_cycle( + self, name: str, dep: str, deps: List[str], definitions: Dict[str, SyntaxTree] + ): + if dep not in definitions: + return # does not exist, will throw downstream in parser + + if name in deps + [dep]: + type_name, pre = ( + ("custom functions", "%") if definitions is self._functions else ("variables", "") + ) + cycle_deps = [name] + deps + [dep] + cycle_deps_str = " -> ".join([f"{pre}{name}" for name in cycle_deps]) + + raise CycleDetected(f"Cycle detected within these {type_name}: {cycle_deps_str}") + + def _traverse_variable_dependencies( + self, + variable_name: str, + variable_dependency: SyntaxTree, + deps: List[str], + ) -> None: + for dep in variable_dependency.variables: + self._ensure_no_cycle( + name=variable_name, dep=dep.name, deps=deps, definitions=self._variables + ) + self._traverse_variable_dependencies( + variable_name=variable_name, + variable_dependency=self._variables[dep.name], + deps=deps + [dep.name], + ) + + def _ensure_no_variable_cycles(self, variables: Dict[str, SyntaxTree]): + for variable_name, variable_definition in variables.items(): + self._traverse_variable_dependencies( + variable_name=variable_name, + variable_dependency=variable_definition, + deps=[], + ) + + def _traverse_custom_function_dependencies( + self, + custom_function_name: str, + custom_function_dependency: SyntaxTree, + deps: List[str], + ) -> None: + for dep in custom_function_dependency.custom_functions: + self._ensure_no_cycle( + name=custom_function_name, dep=dep.name, deps=deps, definitions=self._functions + ) + self._traverse_custom_function_dependencies( + custom_function_name=custom_function_name, + custom_function_dependency=self._functions[dep.name], + deps=deps + [dep.name], + ) + + def _ensure_no_custom_function_cycles(self): + for custom_function_name, custom_function in self._functions.items(): + self._traverse_custom_function_dependencies( + custom_function_name=custom_function_name, + custom_function_dependency=custom_function, + deps=[], + ) + + def _ensure_custom_function_arguments_valid(self): + for custom_function_name, custom_function in self._functions.items(): + indices = sorted([arg.index for arg in custom_function.function_arguments]) + if indices != list(range(len(indices))): + if len(indices) == 1: + raise InvalidCustomFunctionArguments( + f"Custom function %{custom_function_name} has invalid function arguments: " + f"The argument must start with $0, not ${indices[0]}." + ) + raise InvalidCustomFunctionArguments( + f"Custom function %{custom_function_name} has invalid function arguments: " + f"{', '.join(sorted(f'${idx}' for idx in indices))} " + f"do not increment from $0 to ${len(indices) - 1}." + ) + + def _ensure_custom_function_usage_num_input_arguments_valid( + self, prefix: str, name: str, definition: SyntaxTree + ): + for nested_custom_function in definition.custom_functions: + if nested_custom_function.num_input_args != ( + expected_num_args := len( + self._functions[nested_custom_function.name].function_arguments + ) + ): + raise InvalidCustomFunctionArguments( + f"{prefix}{name} has invalid usage of the custom " + f"function %{nested_custom_function.name}: Expects {expected_num_args} " + f"argument{'s' if expected_num_args > 1 else ''} but received " + f"{nested_custom_function.num_input_args}" + ) + + def _ensure_lambda_usage_num_input_arguments_valid( + self, prefix: str, name: str, definition: SyntaxTree + ): + for function in definition.built_in_functions: + spec = FunctionSpec.from_callable(Functions.get(function.name)) + if not (lambda_type := spec.is_lambda_like): + return + + lambda_function_names = set( + lamb.value for lamb in SyntaxTree(function.args).lambdas if isinstance(lamb, Lambda) + ) + + # Only case len(lambda_function_names) > 1 is when used in if-statements + for lambda_function_name in lambda_function_names: + if Functions.is_built_in(lambda_function_name): + lambda_spec = FunctionSpec.from_callable(Functions.get(lambda_function_name)) + if not lambda_spec.is_num_args_compatible(lambda_type.num_input_args()): + expected_args_str = str(lambda_spec.num_required_args) + if lambda_spec.num_required_args != len(lambda_spec.args): + expected_args_str = f"{expected_args_str} - {len(lambda_spec.args)}" + + raise IncompatibleFunctionArguments( + f"{prefix}{name} has invalid usage of the " + f"function %{lambda_function_name} as a lambda: " + f"Expects {expected_args_str} " + f"argument{'s' if expected_args_str != '1' else ''} but will " + f"receive {lambda_type.num_input_args()}." + ) + else: # is custom function + if lambda_function_name not in self._functions: + raise UNREACHABLE # Custom function should have been validated + + expected_num_arguments = len( + self._functions[lambda_function_name].function_arguments + ) + if lambda_type.num_input_args() != expected_num_arguments: + raise IncompatibleFunctionArguments( + f"{prefix}{name} has invalid usage of the custom " + f"function %{lambda_function_name} as a lambda: " + f"Expects {expected_num_arguments} " + f"argument{'s' if expected_num_arguments > 1 else ''} but will " + f"receive {lambda_type.num_input_args()}." + ) + + def _validate(self, added_variables: Optional[Set[str]] = None) -> None: + variables = self._variables + if added_variables is not None: + variables = { + name: ast for name, ast in self._variables.items() if name in added_variables + } + + if added_variables is None: + self._ensure_no_custom_function_cycles() + self._ensure_custom_function_arguments_valid() + + self._ensure_no_variable_cycles(variables) + + to_validate = [("Variable ", variables)] + if added_variables is None: + to_validate.append(("Custom function %", self._functions)) + + for prefix, definitions in to_validate: + for name, definition in definitions.items(): + self._ensure_custom_function_usage_num_input_arguments_valid( + prefix=prefix, name=name, definition=definition + ) + self._ensure_lambda_usage_num_input_arguments_valid( + prefix=prefix, name=name, definition=definition + ) + + def __init__(self, script: Dict[str, str]): + function_names: Set[str] = { + _function_name(name) for name in script.keys() if _is_function(name) + } + variable_names: Set[str] = { + validate_variable_name(name) for name in script.keys() if not _is_function(name) + } + + self._functions: Dict[str, SyntaxTree] = { + # custom_function_name must be passed to properly type custom function + # arguments uniquely if they're nested (i.e. $0 to $custom_func___0) + _function_name(function_key): parse( + text=function_value, + name=_function_name(function_key), + custom_function_names=function_names, + variable_names=variable_names, + ) + for function_key, function_value in script.items() + if _is_function(function_key) + } + + self._variables: Dict[str, SyntaxTree] = { + variable_key: parse( + text=variable_value, + name=variable_key, + custom_function_names=function_names, + variable_names=variable_names, + ) + for variable_key, variable_value in script.items() + if not _is_function(variable_key) + } + self._validate() + + def _update_internally(self, resolved_variables: Dict[str, Resolvable]) -> None: + for variable_name, resolved in resolved_variables.items(): + self._variables[variable_name] = SyntaxTree(ast=[resolved]) + + def _resolve( + self, + pre_resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + update: bool = False, + output_filter: Optional[Set[str]] = None, + ) -> ScriptOutput: + """ + Parameters + ---------- + pre_resolved + Optional. Variables that have been resolved elsewhere and could be used in this script + unresolvable + Optional. Variables that cannot be resolved, forcing any variable that depends on it + to not be resolved. + update + Optional. Whether to update the internal representation of variables with their + resolved value (if they get resolved). + + Returns + ------- + Dict of resolved values + + Raises + ------ + ScriptVariableNotResolved + If specifying a filter of variable to resolve, and one of them does not. + """ + resolved: Dict[Variable, Resolvable] = { + Variable(name): value for name, value in (pre_resolved or {}).items() + } + + unresolvable: Set[Variable] = {Variable(name) for name in (unresolvable or {})} + unresolved_filter = set(resolved.keys()).union(unresolvable) + unresolved: Dict[Variable, SyntaxTree] = { + Variable(name): ast + for name, ast in self._variables.items() + if Variable(name) not in unresolved_filter + } + + while unresolved: + unresolved_count: int = len(unresolved) + + for variable in list(unresolved.keys()): + definition = unresolved[variable] + + # If the definition is already a resolvable, mark it as such + if resolvable := definition.maybe_resolvable: + resolved[variable] = resolvable + del unresolved[variable] + + # If the variable's variable dependencies contain an unresolvable variable, + # declare it as unresolvable and continue + elif definition.contains(unresolvable): + unresolvable.add(variable) + del unresolved[variable] + + # Otherwise, if it has dependencies that are all resolved, then + # resolve the definition + elif not definition.is_subset_of(variables=resolved.keys()): + resolved[variable] = unresolved[variable].resolve( + resolved_variables=resolved, + custom_functions=self._functions, + ) + del unresolved[variable] + + if len(unresolved) == unresolved_count: + # Implies a cycle within the variables. Should never reach + # since cycles are detected in __init__ + raise UNREACHABLE + + resolved_variables = { + variable.name: resolvable for variable, resolvable in resolved.items() + } + if update: + self._update_internally(resolved_variables=resolved_variables) + + if output_filter: + for name in output_filter: + if name not in resolved_variables: + raise ScriptVariableNotResolved(f"Specified {name} to resolve, but it did not") + + return ScriptOutput( + { + name: resolvable + for name, resolvable in resolved_variables.items() + if name in output_filter + } + ) + + return ScriptOutput(resolved_variables) + + def resolve( + self, + resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + update: bool = False, + ) -> ScriptOutput: + """ + Resolves the script + + Parameters + ---------- + resolved + Optional. Pre-resolved variables that should be used instead of what is in the script. + unresolvable + Optional. Unresolvable variables that will be ignored in resolution, including all + variables with a dependency to them. + update + Whether to update the script's internal values with the resolved variables instead of + their original definition. This helps avoid re-evaluated the same variables repeatedly. + + Returns + ------- + ScriptOutput + Containing all resolved variables. + """ + return self._resolve( + pre_resolved=resolved, unresolvable=unresolvable, update=update, output_filter=None + ) + + def add(self, variables: Dict[str, str], unresolvable: Optional[Set[str]] = None) -> "Script": + """ + Adds parses and adds new variables to the script. + + Parameters + ---------- + variables + Mapping containing variable name to definition. + unresolvable + Optional. Set of unresolved variables that the new variables may contain, but the + script does not (yet). + + Returns + ------- + Script + self + """ + added_variables_to_validate: Set[str] = set() + for variable_name, variable_definition in variables.items(): + self._variables[variable_name] = parse( + text=variable_definition, + name=variable_name, + custom_function_names=set(self._functions.keys()), + variable_names=set(self._variables.keys()) + .union(variables.keys()) + .union(unresolvable or set()), + ) + + if self._variables[variable_name].maybe_resolvable is None: + added_variables_to_validate.add(variable_name) + + if added_variables_to_validate: + self._validate(added_variables=added_variables_to_validate) + + return self + + def resolve_once( + self, + variable_definitions: Dict[str, str], + resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + ) -> Dict[str, Resolvable]: + """ + Given a new set of variable definitions, resolve them using the Script, but do not + add them to the Script itself. + + Parameters + ---------- + variable_definitions + Variables to resolve, but not store in the Script + resolved + Optional. Pre-resolved variables that should be used instead of what is in the script. + unresolvable + Optional. Unresolvable variables that will be ignored in resolution, including all + variables with a dependency to them. + + Returns + ------- + Dict[str, Resolvable] + Dict containing the variable names to their resolved values. + """ + try: + self.add(variable_definitions) + return self._resolve( + pre_resolved=resolved, + unresolvable=unresolvable, + output_filter=set(list(variable_definitions.keys())), + ).output + finally: + for name in variable_definitions.keys(): + if name in self._variables: + del self._variables[name] + + def get(self, variable_name: str) -> Resolvable: + """ + Parameters + ---------- + variable_name + Name of the resolved variable to get. + + Returns + ------- + Resolvable + The resolved variable of the given name. + + Raises + ------ + RuntimeException + If the variable has not been resolved yet in the Script. + """ + if variable_name not in self._variables: + raise RuntimeException( + f"Tried to get resolved variable {variable_name}, but it does not exist" + ) + + if (resolvable := self._variables[variable_name].maybe_resolvable) is not None: + return resolvable + + raise RuntimeException(f"Tried to get unresolved variable {variable_name}") + + @property + def variable_names(self) -> Set[str]: + """ + Returns + ------- + Set[str] + Names of all the variables within the Script. + """ + return set(list(self._variables.keys())) + + @property + def function_names(self) -> Set[str]: + """ + Returns + ------- + Set[str] + Names of all functions within the Script. + """ + return set(_to_function_definition_name(name) for name in self._functions.keys()) diff --git a/src/ytdl_sub/script/script_output.py b/src/ytdl_sub/script/script_output.py new file mode 100644 index 00000000..d8275e6e --- /dev/null +++ b/src/ytdl_sub/script/script_output.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from typing import Any +from typing import Dict + +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved + + +@dataclass(frozen=True) +class ScriptOutput: + output: Dict[str, Resolvable] + + def as_native(self) -> Dict[str, Any]: + """ + Returns + ------- + The script output as native python types + """ + return {name: out.native for name, out in self.output.items()} + + def get(self, name: str) -> Resolvable: + """ + Returns + ------- + The script output's variable as a resolvable type + + Raises + ------ + ScriptVariableNotResolved + The variable name requested did not resolve + """ + if name not in self.output: + raise ScriptVariableNotResolved( + f"Tried to access resolved variable {name}, but it has not resolved" + ) + return self.output[name] + + def get_native(self, name: str) -> Any: + """ + Returns + ------- + The script output's variable as native python type + """ + return self.get(name).native + + def get_str(self, name: str) -> str: + """ + Returns + ------- + The script output's variable as a string + """ + return str(self.get(name)) diff --git a/src/ytdl_sub/script/types/__init__.py b/src/ytdl_sub/script/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/script/types/array.py b/src/ytdl_sub/script/types/array.py new file mode 100644 index 00000000..3129ffe7 --- /dev/null +++ b/src/ytdl_sub/script/types/array.py @@ -0,0 +1,58 @@ +from abc import ABC +from dataclasses import dataclass +from typing import Any +from typing import Dict +from typing import List +from typing import Type + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import NonHashable +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import ResolvableToJson +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency + + +@dataclass(frozen=True) +class _Array(NonHashable, ABC): + value: List[Resolvable] + + @classmethod + def type_name(cls) -> str: + return "Array" + + +@dataclass(frozen=True) +class UnresolvedArray(_Array, VariableDependency, FutureResolvable): + value: List[Argument] + + @property + def _iterable_arguments(self) -> List[Argument]: + return self.value + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + return Array( + [ + self._resolve_argument_type( + arg=arg, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + for arg in self.value + ] + ) + + def future_resolvable_type(self) -> Type[Resolvable]: + return Array + + +@dataclass(frozen=True) +class Array(_Array, ResolvableToJson): + @property + def native(self) -> Any: + return [val.native for val in self.value] diff --git a/src/ytdl_sub/script/types/function.py b/src/ytdl_sub/script/types/function.py new file mode 100644 index 00000000..7d1cea85 --- /dev/null +++ b/src/ytdl_sub/script/types/function.py @@ -0,0 +1,293 @@ +import copy +import functools +from abc import ABC +from dataclasses import dataclass +from typing import Callable +from typing import Dict +from typing import List +from typing import Type +from typing import Union + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.array import UnresolvedArray +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import FunctionType +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import NamedCustomFunction +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import ReturnableArgument +from ytdl_sub.script.types.resolvable import ReturnableArgumentA +from ytdl_sub.script.types.resolvable import ReturnableArgumentB +from ytdl_sub.script.types.variable import FunctionArgument +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency +from ytdl_sub.script.utils.exception_formatters import FunctionArgumentsExceptionFormatter +from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.script.utils.type_checking import FunctionSpec +from ytdl_sub.script.utils.type_checking import is_union + + +@dataclass(frozen=True) +class Function(FunctionType, VariableDependency, ABC): + @property + def _iterable_arguments(self) -> List[Argument]: + return self.args + + +class CustomFunction(Function, NamedCustomFunction): + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + resolved_args: List[Resolvable] = [ + self._resolve_argument_type( + arg=arg, resolved_variables=resolved_variables, custom_functions=custom_functions + ) + for arg in self.args + ] + + if self.name in custom_functions: + if len(self.args) != len(custom_functions[self.name].function_arguments): + # Should be validated in the Script + raise UNREACHABLE + + resolved_variables_with_args = copy.deepcopy(resolved_variables) + for i, arg in enumerate(resolved_args): + function_arg = FunctionArgument.from_idx(idx=i, custom_function_name=self.name) + + if function_arg in resolved_variables_with_args: + # function args should always be unique since they are only defined once + # in the custom function as %custom_function_name___idx + # and returned as a set from each custom function. + raise UNREACHABLE + + resolved_variables_with_args[function_arg] = arg + + return custom_functions[self.name].resolve( + resolved_variables=resolved_variables_with_args, + custom_functions=custom_functions, + ) + + # Implies the custom function does not exist. This should have + # been checked in the parser with + raise UNREACHABLE + + +class BuiltInFunction(Function, BuiltInFunctionType): + def validate_args(self) -> "BuiltInFunction": + """ + Ensures the args are compatible with the BuiltInFunction. + """ + if not self.function_spec.is_compatible(input_args=self.args): + raise FunctionArgumentsExceptionFormatter( + input_spec=self.function_spec, + function_instance=self, + ).highlight() + + return self + + # pylint: disable=missing-raises-doc + + @property + def callable(self) -> Callable[..., Resolvable]: + """ + Returns + ------- + The actual callable of the BuiltInFunction + """ + try: + return Functions.get(self.name) + except Exception as exc: + # Should be validated in the parser + raise UNREACHABLE from exc + + # pylint: enable=missing-raises-doc + + @functools.cached_property + def function_spec(self) -> FunctionSpec: + """ + Returns + ------- + The FunctionSpec of the BuiltInFunction + """ + return FunctionSpec.from_callable(self.callable) + + @classmethod + def _arg_output_type(cls, arg: Argument) -> Type[Argument]: + if isinstance(arg, BuiltInFunction): + return arg.output_type() + if isinstance(arg, FutureResolvable): + return arg.future_resolvable_type() + return type(arg) + + @classmethod + def _instantiate_lambda(cls, lambda_function_name: str, args: List[Argument]) -> Function: + return ( + BuiltInFunction(name=lambda_function_name, args=args) + if Functions.is_built_in(lambda_function_name) + else CustomFunction(name=lambda_function_name, args=args) + ) + + def _output_type(self, union_args: List[Type[Argument]]) -> Type[Resolvable]: + union_types_list = set() + for union_type in union_args: + possible_output_type = union_type + if union_type in (ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB): + generic_arg_index = self.function_spec.args.index(union_type) + possible_output_type = self._arg_output_type(self.args[generic_arg_index]) + + if is_union(possible_output_type): + union_types_list.update(possible_output_type.__args__) + else: + union_types_list.add(possible_output_type) + + return Union[tuple(union_types_list)] + + def output_type(self) -> Type[Resolvable]: + """ + Returns + ------- + The BuiltInFunction's true output type. + """ + if is_union(self.function_spec.return_type): + return self._output_type(self.function_spec.return_type.__args__) + + return self._output_type([self.function_spec.return_type]) + + def _resolve_lambda_function( + self, + resolved_arguments: List[Resolvable | Lambda], + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + """ + Resolve the lambda function by + 1. Calling the actual built-in function, which actually forms the input args to the + lambda. NOTE: the lambda argument MUST BE the last argument in the input spec + and output a ResolvedArray, where each element is the input args to the lambda. + 2. Preemptively creating the lambda's unresolved output array using output args from (1) + 3. Resolve it like any other syntax + """ + function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)] + if not self.function_spec.is_lambda_function or len(function_input_lambda_args) != 1: + raise UNREACHABLE + + lambda_function_name = function_input_lambda_args[0].value + + try: + lambda_args = self.callable(*resolved_arguments) + except Exception as exc: + raise FunctionRuntimeException( + f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" + ) from exc + + assert isinstance(lambda_args, Array) + + return self._resolve_argument_type( + arg=UnresolvedArray( + [ + self._instantiate_lambda( + lambda_function_name=lambda_function_name, args=lambda_arg.value + ) + for lambda_arg in lambda_args.value + ] + ), + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + def _resolve_lambda_reduce_function( + self, + resolved_arguments: List[Resolvable | Lambda], + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + """ + Resolve the lambda reduce function by + 1. Preemptively create the 'reduce-like' call-stack as unresolvable + 2. Resolve it like any other syntax + """ + function_input_lambda_args = [arg for arg in resolved_arguments if isinstance(arg, Lambda)] + if not self.function_spec.is_lambda_reduce_function or len(function_input_lambda_args) != 1: + raise UNREACHABLE + + lambda_function_name = function_input_lambda_args[0].value + + try: + lambda_array = self.callable(*resolved_arguments) + except Exception as exc: + raise FunctionRuntimeException( + f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" + ) from exc + + assert isinstance(lambda_array, Array) + + if len(lambda_array.value) == 1: + return lambda_array.value[0] + + reduced: Resolvable = self._resolve_argument_type( + arg=self._instantiate_lambda( + lambda_function_name=lambda_function_name, + args=[lambda_array.value[0], lambda_array.value[1]], + ), + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + for idx in range(2, len(lambda_array.value)): + reduced = self._resolve_argument_type( + arg=self._instantiate_lambda( + lambda_function_name=lambda_function_name, + args=[reduced, lambda_array.value[idx]], + ), + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + return reduced + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + # Resolve all non-lambda arguments + resolved_arguments: List[Resolvable | Lambda] = [ + self._resolve_argument_type( + arg=arg, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + for arg in self.args + ] + + # If a lambda is in a function's arg, resolve it differently + if self.function_spec.is_lambda_function: + return self._resolve_lambda_function( + resolved_arguments=resolved_arguments, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + # If a lambda is in a function's arg, resolve it differently + if self.function_spec.is_lambda_reduce_function: + return self._resolve_lambda_reduce_function( + resolved_arguments=resolved_arguments, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + + try: + return self.callable(*resolved_arguments) + except (UserThrownRuntimeError, RuntimeException): + raise + except Exception as exc: + raise FunctionRuntimeException( + f"Runtime error occurred when executing the function %{self.name}: {str(exc)}" + ) from exc diff --git a/src/ytdl_sub/script/types/map.py b/src/ytdl_sub/script/types/map.py new file mode 100644 index 00000000..0fca1cbb --- /dev/null +++ b/src/ytdl_sub/script/types/map.py @@ -0,0 +1,66 @@ +import itertools +from abc import ABC +from dataclasses import dataclass +from typing import Any +from typing import Dict +from typing import List +from typing import Type + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Hashable +from ytdl_sub.script.types.resolvable import NonHashable +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import ResolvableToJson +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +@dataclass(frozen=True) +class _Map(NonHashable, ABC): + value: Dict[Hashable, Resolvable] + + @classmethod + def type_name(cls) -> str: + return "Map" + + +@dataclass(frozen=True) +class UnresolvedMap(_Map, VariableDependency, FutureResolvable): + value: Dict[Argument, Argument] + + @property + def _iterable_arguments(self) -> List[Argument]: + return list(itertools.chain(*self.value.items())) + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, VariableDependency], + ) -> Resolvable: + output: Dict[Hashable, Resolvable] = {} + for key, value in self.value.items(): + resolved_key = self._resolve_argument_type( + arg=key, resolved_variables=resolved_variables, custom_functions=custom_functions + ) + if not isinstance(resolved_key, Hashable): + raise KeyNotHashableRuntimeException( + f"Tried to use {resolved_key.type_name()} as a Map key, but it is not hashable." + ) + + output[resolved_key] = self._resolve_argument_type( + arg=value, resolved_variables=resolved_variables, custom_functions=custom_functions + ) + + return Map(output) + + def future_resolvable_type(self) -> Type[Resolvable]: + return Map + + +@dataclass(frozen=True) +class Map(_Map, ResolvableToJson): + @property + def native(self) -> Any: + return {key.native: value.native for key, value in self.value.items()} diff --git a/src/ytdl_sub/script/types/resolvable.py b/src/ytdl_sub/script/types/resolvable.py new file mode 100644 index 00000000..c042b1f5 --- /dev/null +++ b/src/ytdl_sub/script/types/resolvable.py @@ -0,0 +1,257 @@ +import json +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Any +from typing import Generic +from typing import List +from typing import Type +from typing import TypeVar + +T = TypeVar("T") +NumericT = TypeVar("NumericT", bound=int | float) + + +@dataclass(frozen=True) +class NamedType(ABC): + @classmethod + def type_name(cls) -> str: + """ + Returns + ------- + The type name to present to users. Defaults to the class name. + """ + return cls.__name__ + + +@dataclass(frozen=True) +class Argument(NamedType, ABC): + """ + Any possible argument type that has not been resolved yet + """ + + +@dataclass(frozen=True) +class ValueArgument(Argument, ABC): + """ + Argument that has a value + """ + + value: Any + + +@dataclass(frozen=True) +class NamedArgument(Argument, ABC): + """ + Argument that has an explicit name (i.e. custom function or variable) + """ + + name: str + + +@dataclass(frozen=True) +class ReturnableArgument(ValueArgument, NamedType, ABC): + """ + AnyType to express generics in functions that are part of the return type + """ + + +@dataclass(frozen=True) +class ReturnableArgumentA(ValueArgument, NamedType, ABC): + """ + AnyType to express generics in functions when more than one are present (i.e. `if`) + """ + + +@dataclass(frozen=True) +class ReturnableArgumentB(ValueArgument, NamedType, ABC): + """ + AnyType to express generics in functions when more than one are present (i.e. `if`) + """ + + +@dataclass(frozen=True) +class AnyArgument(ReturnableArgument, ReturnableArgumentA, ReturnableArgumentB, ABC): + """ + Human-readable name for Resolvable + """ + + +@dataclass(frozen=True) +class Resolvable(AnyArgument, ABC): + """ + A type that is resolved into a native Python type (and have no dependencies to other types). + """ + + def __str__(self) -> str: + return str(self.value) + + @property + def native(self) -> Any: + """ + Returns + ------- + The resolvable in its native form + """ + return self.value + + +@dataclass(frozen=True) +class FutureResolvable(AnyArgument, ABC): + """ + Used when parsing, it is an unresolved type that will eventually resolve to a known type + (i.e. Maps, Arrays) + """ + + @abstractmethod + def future_resolvable_type(self) -> Type[Resolvable]: + """ + The resolvable type that this type is known to turn into + """ + + +@dataclass(frozen=True) +class Hashable(Resolvable, ABC): + """ + Resolvable type that can be used as hashes (i.e. in Maps) + """ + + +@dataclass(frozen=True) +class NonHashable(NamedType, ABC): + """ + Type that is known to never be hashable. + """ + + +@dataclass(frozen=True) +class ResolvableToJson(Resolvable, ABC): + """ + Types whose string values should be resolved to JSON (i.e. Maps, Arrays) + """ + + def __str__(self): + return json.dumps(self.native) + + +@dataclass(frozen=True) +class ResolvableT(Hashable, ABC, Generic[T]): + """ + Resolvable types that resolve to the generic T + """ + + value: T + + +@dataclass(frozen=True) +class Numeric(ResolvableT[NumericT], ABC, Generic[NumericT]): + """ + Resolvable numeric types (int/float) + """ + + +@dataclass(frozen=True) +class Integer(Numeric[int], Argument): + """ + Resolved Integer type + """ + + +@dataclass(frozen=True) +class Float(Numeric[float], Argument): + """ + Resolved float type + """ + + +@dataclass(frozen=True) +class Boolean(ResolvableT[bool], Argument): + """ + Resolved bool type + """ + + def __str__(self): + # makes it JSON friendly + return str(self.value).lower() + + +@dataclass(frozen=True) +class String(ResolvableT[str], Argument): + """ + Resolved String type + """ + + +@dataclass(frozen=True) +class NamedCustomFunction(NamedArgument, ABC): + """ + A custom function with a defined name (but unknown args) + """ + + +@dataclass(frozen=True) +class ParsedCustomFunction(NamedCustomFunction): + num_input_args: int + + +@dataclass(frozen=True) +class FunctionType(NamedArgument, ABC): + args: List[Argument] + + +@dataclass(frozen=True) +class BuiltInFunctionType(FunctionType, ABC): + @abstractmethod + def output_type(self) -> Type[Resolvable]: + """ + Returns + ------- + The known Resolvable type that a BuiltInFunction will output + """ + + +@dataclass(frozen=True) +class Lambda(Resolvable): + value: str + + @property + def native(self) -> Any: + return f"%{self.value}" + + @classmethod + def num_input_args(cls) -> int: + """ + Returns + ------- + The number of input args the Lambda function takes + """ + return 1 + + +@dataclass(frozen=True) +class LambdaTwo(Lambda): + """ + Type-hinting for functions that apply lambdas with two inputs per element + """ + + @classmethod + def num_input_args(cls) -> int: + return 2 + + +@dataclass(frozen=True) +class LambdaThree(Lambda): + """ + Type-hinting for functions that apply lambdas with three inputs per element + """ + + @classmethod + def num_input_args(cls) -> int: + return 3 + + +@dataclass(frozen=True) +class LambdaReduce(LambdaTwo): + """ + Type-hinting for functions that apply a reduce-operation using a lambda (two arguments) + """ diff --git a/src/ytdl_sub/script/types/syntax_tree.py b/src/ytdl_sub/script/types/syntax_tree.py new file mode 100644 index 00000000..d058852c --- /dev/null +++ b/src/ytdl_sub/script/types/syntax_tree.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from typing import Dict +from typing import List +from typing import Optional + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.types.variable_dependency import VariableDependency + + +@dataclass(frozen=True) +class SyntaxTree(VariableDependency): + ast: List[Argument] + + @property + def _iterable_arguments(self) -> List[Argument]: + return self.ast + + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, VariableDependency], + ) -> Resolvable: + resolved: List[Resolvable] = [] + for token in self.ast: + resolved.append( + self._resolve_argument_type( + arg=token, + resolved_variables=resolved_variables, + custom_functions=custom_functions, + ) + ) + + # If only one resolvable resides in the AST, return as that + if len(resolved) == 1: + return resolved[0] + + # Otherwise, to concat multiple resolved outputs, we must concat as strings + return String("".join([str(res) for res in resolved])) + + @property + def maybe_resolvable(self) -> Optional[Resolvable]: + """ + Returns + ------- + A resolvable if the AST contains a single type that is resolvable. None otherwise. + """ + if len(self.ast) == 1 and isinstance(self.ast[0], Resolvable): + return self.ast[0] + return None diff --git a/src/ytdl_sub/script/types/variable.py b/src/ytdl_sub/script/types/variable.py new file mode 100644 index 00000000..5471ec30 --- /dev/null +++ b/src/ytdl_sub/script/types/variable.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import Optional + +from ytdl_sub.script.types.resolvable import NamedArgument + + +@dataclass(frozen=True) +class Variable(NamedArgument): + pass + + +@dataclass(frozen=True) +class FunctionArgument(Variable): + """Arguments for custom functions, i.e. $0, $1, etc""" + + index: int + + @classmethod + def from_idx(cls, idx: int, custom_function_name: Optional[str]) -> "FunctionArgument": + """ + Returns + ------- + FunctionArgument whose variable name is the index, and optionally contains the custom + function name its defined in as a prefix. + """ + if custom_function_name: + return FunctionArgument(name=f"${custom_function_name}___{idx}", index=idx) + return FunctionArgument(name=f"${idx}", index=idx) diff --git a/src/ytdl_sub/script/types/variable_dependency.py b/src/ytdl_sub/script/types/variable_dependency.py new file mode 100644 index 00000000..8cd86134 --- /dev/null +++ b/src/ytdl_sub/script/types/variable_dependency.py @@ -0,0 +1,174 @@ +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Dict +from typing import Iterable +from typing import List +from typing import Set +from typing import Type +from typing import TypeVar +from typing import final + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import FunctionType +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import NamedCustomFunction +from ytdl_sub.script.types.resolvable import ParsedCustomFunction +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.variable import FunctionArgument +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import UNREACHABLE + +TypeT = TypeVar("TypeT") + + +@dataclass(frozen=True) +class VariableDependency(ABC): + @property + @abstractmethod + def _iterable_arguments(self) -> List[Argument]: + """ + Returns + ------- + Any arguments in the VariableDependency that may or may not need to be resolved. + """ + + def _recurse_get(self, ttype: Type[TypeT], subclass: bool = False) -> List[TypeT]: + output: List[TypeT] = [] + for arg in self._iterable_arguments: + if subclass and issubclass(type(arg), ttype): + output.append(arg) + elif isinstance(arg, ttype): + output.append(arg) + + if isinstance(arg, VariableDependency): + # pylint: disable=protected-access + output.extend(arg._recurse_get(ttype)) + # pylint: enable=protected-access + + return output + + @final + @property + def variables(self) -> Set[Variable]: + """ + Returns + ------- + All Variables that this depends on. + """ + return set(self._recurse_get(Variable)) + + @final + @property + def built_in_functions(self) -> List[BuiltInFunctionType]: + """ + Returns + ------- + All BuiltInFunctions that this depends on. + """ + return self._recurse_get(BuiltInFunctionType) + + @final + @property + def function_arguments(self) -> Set[FunctionArgument]: + """ + Returns + ------- + All FunctionArguments that this depends on. + """ + return set(self._recurse_get(FunctionArgument)) + + @final + @property + def lambdas(self) -> Set[Lambda]: + """ + Returns + ------- + All Lambdas that this depends on. + """ + return set(self._recurse_get(Lambda, subclass=True)) + + # pylint: disable=missing-raises-doc + + @final + @property + def custom_functions(self) -> Set[ParsedCustomFunction]: + """ + Returns + ------- + All CustomFunctions that this depends on. + """ + output: Set[ParsedCustomFunction] = set() + for arg in self._iterable_arguments: + if isinstance(arg, NamedCustomFunction): + if not isinstance(arg, FunctionType): + # A NamedCustomFunction should also always be a FunctionType + raise UNREACHABLE + + # Custom funcs aren't hashable, so recreate just the base-class portion + output.add(ParsedCustomFunction(name=arg.name, num_input_args=len(arg.args))) + if isinstance(arg, VariableDependency): + output.update(arg.custom_functions) + + return output + + # pylint: enable=missing-raises-doc + + @abstractmethod + def resolve( + self, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + """ + Parameters + ---------- + resolved_variables + Lookup of variables that have been resolved + custom_functions + Lookup of any custom functions that have been parsed + + Returns + ------- + Resolved value + """ + + @classmethod + def _resolve_argument_type( + cls, + arg: Argument, + resolved_variables: Dict[Variable, Resolvable], + custom_functions: Dict[str, "VariableDependency"], + ) -> Resolvable: + if isinstance(arg, Resolvable): + return arg + if isinstance(arg, Variable): + if arg not in resolved_variables: + # All variables should exist and be resolved at this point + raise UNREACHABLE + return resolved_variables[arg] + if isinstance(arg, VariableDependency): + return arg.resolve( + resolved_variables=resolved_variables, custom_functions=custom_functions + ) + + raise UNREACHABLE + + @final + def is_subset_of(self, variables: Iterable[Variable]) -> bool: + """ + Returns + ------- + True if it contains all input variables as a dependency. False otherwise. + """ + return not self.variables.issubset(variables) + + @final + def contains(self, variables: Iterable[Variable]) -> bool: + """ + Returns + ------- + True if it contains any of the input variables. False otherwise. + """ + return len(self.variables.intersection(variables)) > 0 diff --git a/src/ytdl_sub/script/utils/__init__.py b/src/ytdl_sub/script/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/script/utils/exception_formatters.py b/src/ytdl_sub/script/utils/exception_formatters.py new file mode 100644 index 00000000..6e14354e --- /dev/null +++ b/src/ytdl_sub/script/utils/exception_formatters.py @@ -0,0 +1,131 @@ +import sys +from typing import List +from typing import TypeVar + +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.type_checking import FunctionSpec +from ytdl_sub.script.utils.type_checking import is_union + +TUserException = TypeVar("TUserException", bound=UserException) + + +class ParserExceptionFormatter: + def __init__(self, text: str, start: int, end: int, exception: TUserException): + self._text = text + self._start = start + self._end = end + self._exception = exception + + def _exception_text(self, border: int): + """ + Format for single-line exceptions + """ + text_left = max(0, self._start - border) + text_right = min(len(self._text), self._start + border) + relative_start = self._start - text_left + + exception_text: str = "" + if text_left > 3: + exception_text = "… " + relative_start += len(exception_text) + + exception_text += self._text[text_left:text_right] + + if text_right < len(self._text) - 3: + exception_text += " …" + + exception_text += "\n" + exception_text += f"{' ' * relative_start}^" + + return "\n" + exception_text + + @property + def _is_multi_line(self) -> bool: + return "\n" in self._text + + def _exception_text_lines(self, border_lines: int = 0) -> str: + """ + Format for multi-line exceptions + """ + split_text = self._text.split("\n") + + start_line: int = sys.maxsize + end_line: int = -1 + pos: int = 0 + for idx, line in enumerate(split_text): + if self._start <= pos < self._end: + start_line = min(start_line, idx) + end_line = max(end_line, idx + 1) + pos += len(line) + + true_start_line = start_line + start_line = max(0, start_line - border_lines) + end_line = min(len(split_text), end_line + border_lines) + + # Get min leading spaces between all lines to return + min_leading_spaces = sys.maxsize + for line in split_text[start_line:end_line]: + min_leading_spaces = min(min_leading_spaces, len(line) - len(line.lstrip())) + + to_return: List[str] = [] + for idx in range(start_line, end_line): + if idx == true_start_line: + to_return.append(f">>> {split_text[idx][min_leading_spaces:]}") + else: + to_return.append(f" {split_text[idx][min_leading_spaces:]}") + + return "\n" + "\n".join(to_return) + + def highlight(self) -> TUserException: + """ + Returns + ------- + Exception with human-readable error highlighting for invalid syntax + """ + if self._is_multi_line: + invalid_syntax = self._exception_text_lines(border_lines=3) + else: + invalid_syntax = self._exception_text(border=20) + + return self._exception.__class__(f"{invalid_syntax}\n{str(self._exception)}") + + +class FunctionArgumentsExceptionFormatter: + def __init__( + self, + input_spec: FunctionSpec, + function_instance: BuiltInFunctionType, + ): + self._input_spec = input_spec + self._name = function_instance.name + self._input_args = function_instance.args + + def _received_args_str(self) -> str: + received_type_names: List[str] = [] + for arg in self._input_args: + if isinstance(arg, BuiltInFunctionType): + if is_union(arg.output_type()): + readable_type_names = ", ".join( + sorted(type_.type_name() for type_ in arg.output_type().__args__) + ) + received_type_names.append(f"%{arg.name}(...)->Union[{readable_type_names}]") + else: + received_type_names.append(f"%{arg.name}(...)->{arg.output_type().type_name()}") + else: + received_type_names.append(arg.type_name()) + + return f"({', '.join(name for name in received_type_names)})" + + def highlight(self) -> IncompatibleFunctionArguments: + """ + Returns + ------- + Exception with human-readable error highlighting for incompatible function arguments + """ + return IncompatibleFunctionArguments( + f"Incompatible arguments passed to function {self._name}.\n" + f"Expected {self._input_spec.human_readable_input_args()}\n" + f"Received {self._received_args_str()}" + ) diff --git a/src/ytdl_sub/script/utils/exceptions.py b/src/ytdl_sub/script/utils/exceptions.py new file mode 100644 index 00000000..86f18e1b --- /dev/null +++ b/src/ytdl_sub/script/utils/exceptions.py @@ -0,0 +1,96 @@ +from abc import ABC + +from ytdl_sub.utils.exceptions import ValidationException + +################################################################################################### +# USER EXCEPTIONS + + +class UserException(ValidationException, ABC): + """It's the user's fault!""" + + +class InvalidSyntaxException(UserException): + """Syntax is incorrect""" + + +class InvalidVariableName(UserException): + """Variable name is invalid""" + + +class InvalidFunctionName(UserException): + """Custom function name is invalid""" + + +class InvalidCustomFunctionArguments(UserException): + """Custom function arguments are invalid (i.e. they do not increment)""" + + +class InvalidCustomFunctionArgumentName(UserException): + """Custom function argument name (i.e. $0) is invalid""" + + +class IncompatibleFunctionArguments(UserException): + """Function has invalid arguments""" + + +class FunctionDoesNotExist(UserException): + """Tried to use a function that does not exist""" + + +class VariableDoesNotExist(UserException): + """Tried to use a variable that does not exist""" + + +class ScriptBuilderMissingDefinitions(UserException): + """Tried to build an incomplete ScriptBuilder""" + + +class CycleDetected(UserException): + """A cycle exists within a user's script""" + + +class UserThrownRuntimeError(ValidationException): + """An error explicitly thrown by the user via a function""" + + +class _UnreachableSyntaxException(InvalidSyntaxException): + """For use in places where code _should_ never reach, but might from bugs""" + + +UNREACHABLE = _UnreachableSyntaxException( + "If you see this error, you have discovered a bug in the script parser!\n" + "Please upload your config/subscription file(s) to and make a GitHub issue at " + "https://github.com/jmbannon/ytdl-sub/issues" +) + +################################################################################################### +# RUNTIME EXCEPTIONS + + +class RuntimeException(ValueError, ABC): + """Exception thrown at runtime during resolution""" + + +class ScriptVariableNotResolved(RuntimeException): + """Tried to get a variable's resolved value from a script, but has not resolved yet""" + + +class FunctionRuntimeException(RuntimeException): + """Exception thrown when a ytdl-sub function has an error occur at runtime""" + + +class KeyNotHashableRuntimeException(RuntimeException): + """Map tried to use a non-hashable key at runtime""" + + +class ArrayValueDoesNotExist(RuntimeException): + """Tried to get an index of a value in an Array that does not exist""" + + +class FunctionDoesNotExistRuntimeException(RuntimeException): + """Tried to get a function that does not exist""" + + +class KeyDoesNotExistRuntimeException(RuntimeException): + """Tried to access a key on a map that does not exist, with no default""" diff --git a/src/ytdl_sub/script/utils/name_validation.py b/src/ytdl_sub/script/utils/name_validation.py new file mode 100644 index 00000000..de187ec8 --- /dev/null +++ b/src/ytdl_sub/script/utils/name_validation.py @@ -0,0 +1,60 @@ +import re + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.exceptions import InvalidFunctionName +from ytdl_sub.script.utils.exceptions import InvalidVariableName + +_NAME_REGEX_VALIDATOR = re.compile(r"^[a-z][a-z0-9_]*$") + + +def is_valid_name(name: str) -> bool: + """ + Returns + ------- + True if the name adheres to the ``snake_case`` format. False otherwise. + """ + return re.match(_NAME_REGEX_VALIDATOR, name) is not None + + +def validate_variable_name(variable_name: str) -> str: + """ + Raises + ------ + InvalidVariableName + if the variable name is invalid + """ + if not is_valid_name(variable_name): + raise InvalidVariableName( + f"Variable name '{variable_name}' is invalid:" + " Names must be lower_snake_cased and begin with a letter." + ) + + if Functions.is_built_in(variable_name): + raise InvalidVariableName( + f"Variable name '{variable_name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ) + + return variable_name + + +def validate_custom_function_name(custom_function_name: str) -> None: + """ + Raises + ------ + InvalidFunctionName + If the function name is invalid + InvalidVariableName + if the variable name is invalid + """ + if not is_valid_name(custom_function_name): + raise InvalidFunctionName( + f"Custom function name '%{custom_function_name}' is invalid:" + " Names must be %lower_snake_cased and begin with a letter." + ) + + if Functions.is_built_in(custom_function_name): + raise InvalidFunctionName( + f"Custom function name '%{custom_function_name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ) diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py new file mode 100644 index 00000000..2170c3e0 --- /dev/null +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -0,0 +1,282 @@ +# pylint: disable=missing-raises-doc +import inspect +from dataclasses import dataclass +from inspect import FullArgSpec +from typing import Callable +from typing import List +from typing import Optional +from typing import Type +from typing import TypeVar +from typing import Union +from typing import get_origin + +from ytdl_sub.script.types.resolvable import Argument +from ytdl_sub.script.types.resolvable import BuiltInFunctionType +from ytdl_sub.script.types.resolvable import FutureResolvable +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import LambdaReduce +from ytdl_sub.script.types.resolvable import LambdaThree +from ytdl_sub.script.types.resolvable import LambdaTwo +from ytdl_sub.script.types.resolvable import NamedCustomFunction +from ytdl_sub.script.types.resolvable import NamedType +from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import UNREACHABLE + +TLambda = TypeVar("TLambda", bound=Lambda) + + +def is_union(arg_type: Type) -> bool: + """ + Returns + ------- + True if typing is Union. False otherwise. + """ + return get_origin(arg_type) is Union + + +def is_optional(arg_type: Type) -> bool: + """ + Returns + ------- + True if typing is Optional. False otherwise. + """ + return is_union(arg_type) and type(None) in arg_type.__args__ + + +def get_optional_type(optional_type: Type) -> Type[NamedType]: + """ + Returns + ------- + Type within the Optional[Type] + """ + return [arg for arg in optional_type.__args__ if arg != type(None)][0] + + +def _is_type_compatible( + arg_type: Type[NamedType], + expected_arg_type: Type[Resolvable | Optional[Resolvable]], +) -> bool: + """ + Returns + ------- + True if arg is compatible with expected_arg_type. False otherwise. + """ + if is_union(expected_arg_type): + # See if the arg is a valid against the union + valid_type = False + + # if the input arg is a union, do a direct comparison + if is_union(arg_type): + valid_type = arg_type == expected_arg_type + # otherwise, iterate the union to see if it's compatible + else: + for union_type in expected_arg_type.__args__: + if issubclass(arg_type, union_type): + valid_type = True + break + + if not valid_type: + return False + # If the input is a union and the expected type is not, see if + # each possible union input is compatible with the expected type + elif is_union(arg_type): + for union_type in arg_type.__args__: + if not _is_type_compatible(union_type, expected_arg_type): + return False + elif issubclass(arg_type, (NamedCustomFunction, Variable)): + return True # custom-function/variable can be anything, so pass for now + elif issubclass(arg_type, Lambda) and issubclass(expected_arg_type, arg_type): + # lambda, check if expected_arg_type is a subclass + # Do not return on just that to also allow lambdas to be returned as + # ReturnableArguments (i.e in an %if statement) + return True + + elif not issubclass(arg_type, expected_arg_type): + return False + + return True + + +def is_type_compatible( + arg: NamedType, + expected_arg_type: Type[Resolvable | Optional[Resolvable]], +) -> bool: + """ + Returns + ------- + True if arg is compatible with expected_arg_type. False otherwise. + """ + arg_type: Type[NamedType] = arg.__class__ + if isinstance(arg, BuiltInFunctionType): + arg_type = arg.output_type() # built-in function + elif isinstance(arg, FutureResolvable): + arg_type = arg.future_resolvable_type() + + return _is_type_compatible(arg_type, expected_arg_type) + + +@dataclass(frozen=True) +class FunctionSpec: + return_type: Type[Resolvable] + arg_names: List[str] + args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None + varargs: Optional[Type[Resolvable]] = None + + def __post_init__(self): + assert (self.args is None) ^ (self.varargs is None) + + def _is_args_compatible(self, input_args: List[Argument]) -> bool: + assert self.args is not None + + if len(input_args) > len(self.args): + return False + + for idx, arg in enumerate(self.args): + input_arg = input_args[idx] if idx < len(input_args) else None + if not is_type_compatible(arg=input_arg, expected_arg_type=arg): + return False + + return True + + def _is_varargs_compatible(self, input_args: List[Argument]) -> bool: + """ + Returns + ------- + True if the input args are compatible with the spec's varargs. False otherwise. + """ + assert self.varargs is not None + + for input_arg in input_args: + if not is_type_compatible(arg=input_arg, expected_arg_type=self.varargs): + return False + + return True + + def is_compatible(self, input_args: List[Argument]) -> bool: + """ + Returns + ------- + True if input_args is compatible. False otherwise. + """ + if self.args is not None: + return self._is_args_compatible(input_args=input_args) + if self.varargs is not None: + return self._is_varargs_compatible(input_args=input_args) + + raise UNREACHABLE # TODO: functions with no args + + def is_num_args_compatible(self, num_input_args: int) -> bool: + """ + Returns + ------- + True if the number of input args is compatible with the function spec. False otherwise. + """ + if self.args is not None: + return self.num_required_args <= num_input_args <= len(self.args) + return True # varargs can take any number + + @property + def num_required_args(self) -> int: + """ + Returns + ------- + The minimum number of args required to call the function. + """ + if self.args is not None: + return sum(1 for arg in self.args if not is_optional(arg)) + return 0 # varargs can take any number + + @property + def is_lambda_reduce_function(self) -> Optional[Type[LambdaReduce]]: + """ + Returns + ------- + True if the function is a Lambda-reduce function. False otherwise. + """ + return LambdaReduce if LambdaReduce in (self.args or []) else None + + @property + def is_lambda_function(self) -> Optional[Type[Lambda | LambdaTwo | LambdaThree]]: + """ + Returns + ------- + True if the function is a Lambda function (excluding reduce). False otherwise. + """ + if LambdaThree in (self.args or []): + return LambdaThree + if LambdaTwo in (self.args or []): + return LambdaTwo + if Lambda in (self.args or []): + return Lambda + return None + + @property + def is_lambda_like(self) -> Optional[Type[TLambda]]: + """ + Returns + ------- + True if the function is a Lambda type (including reduce). + """ + if l_type := self.is_lambda_reduce_function: + return l_type + if l_type := self.is_lambda_function: + return l_type + return None + + @classmethod + def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: + if is_optional(python_type): + return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" + if is_union(python_type): + args = ", ".join( + sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) + ) + return f"Union[{args}]" + return python_type.type_name() + + def human_readable_input_args(self) -> str: + """ + Returns + ------- + input arg string in human-readable format + """ + if self.args is not None: + args = ", ".join( + f"{name}: {self._to_human_readable_name(type_)}" + for name, type_ in zip(self.arg_names, self.args) + ) + elif self.varargs is not None: + args = f"{self.arg_names[0]}: {self._to_human_readable_name(self.varargs)}, ..." + else: + args = "" + return f"({args})" + + def human_readable_output_type(self) -> str: + """ + Returns + ------- + output type string in human-readable format + """ + return self._to_human_readable_name(self.return_type) + + @classmethod + def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec": + """ + Returns + ------- + FunctionSpec from a built-in function. + """ + arg_spec: FullArgSpec = inspect.getfullargspec(callable_ref) + if arg_spec.varargs: + return FunctionSpec( + return_type=arg_spec.annotations["return"], + arg_names=[arg_spec.varargs], + varargs=arg_spec.annotations[arg_spec.varargs], + ) + + return FunctionSpec( + return_type=arg_spec.annotations["return"], + arg_names=arg_spec.args, + args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args], + ) diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 0c61f7e1..c7872e8a 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -3,10 +3,10 @@ from pathlib import Path from typing import Optional from ytdl_sub.config.config_validator import ConfigOptions +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset import Preset -from ytdl_sub.config.preset import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.utils.file_handler import FileHandlerTransactionLog diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 91989411..50b2f319 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -3,19 +3,16 @@ from pathlib import Path from typing import Any from typing import Dict from typing import List -from typing import Optional from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset from ytdl_sub.subscriptions.subscription_download import SubscriptionDownload from ytdl_sub.subscriptions.subscription_validators import SubscriptionValidator -from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.yaml import load_yaml from ytdl_sub.validators.validators import LiteralDictValidator FILE_PRESET_APPLY_KEY = "__preset__" -FILE_SUBSCRIPTION_VALUE_KEY = "__value__" logger = Logger.get("subscription") @@ -70,36 +67,12 @@ class Subscription(SubscriptionDownload): config=config, ) - @classmethod - def _maybe_get_subscription_value( - cls, config: ConfigFile, subscription_dict: Dict - ) -> Optional[str]: - subscription_value_key: Optional[str] = config.config_options.subscription_value - if FILE_SUBSCRIPTION_VALUE_KEY in subscription_dict: - if not isinstance(subscription_dict[FILE_SUBSCRIPTION_VALUE_KEY], str): - raise ValidationException( - f"Using {FILE_SUBSCRIPTION_VALUE_KEY} in a subscription" - f"must be a string that corresponds to an override variable" - ) - - subscription_value_key = subscription_dict[FILE_SUBSCRIPTION_VALUE_KEY] - - if subscription_value_key is not None: - logger.warning( - "Using %s in a subscription will eventually be deprecated in favor of writing " - "to the override variable `subscription_value`. Please update by Dec 2023.", - FILE_SUBSCRIPTION_VALUE_KEY, - ) - return subscription_value_key - @classmethod def from_file_path( cls, config: ConfigFile, subscription_path: str | Path ) -> List["Subscription"]: """ - Loads subscriptions from a file and applies ``__preset__`` to all of them if present. - If a subscription is in the form of key: value, it will set value to the override - variable defined in ``__value__``. + Loads subscriptions from a file. Parameters ---------- @@ -121,9 +94,6 @@ class Subscription(SubscriptionDownload): subscription_dict = load_yaml(file_path=subscription_path) has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict - file_subscription_value: Optional[str] = cls._maybe_get_subscription_value( - config=config, subscription_dict=subscription_dict - ) # If a file preset is present... if has_file_preset: @@ -138,9 +108,7 @@ class Subscription(SubscriptionDownload): config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict subscriptions_dict: Dict[str, Any] = { - key: obj - for key, obj in subscription_dict.items() - if key not in [FILE_PRESET_APPLY_KEY, FILE_SUBSCRIPTION_VALUE_KEY] + key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY] } subscriptions_dicts = SubscriptionValidator( @@ -149,7 +117,6 @@ class Subscription(SubscriptionDownload): config=config, presets=[], indent_overrides=[], - subscription_value=file_subscription_value, ).subscription_dicts( global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else [] ) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 29bd2165..7bbd09ca 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -7,8 +7,10 @@ from pathlib import Path from typing import List from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions from ytdl_sub.downloaders.source_plugin import SourcePlugin @@ -145,8 +147,18 @@ class SubscriptionDownload(BaseSubscription, ABC): after=self.output_options.keep_files_after, overrides=self.overrides, ) - if date_range_to_keep: - self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep) + + keep_max_files: Optional[int] = None + if self.output_options.keep_max_files: + # validated it can be cast to int within the validator + keep_max_files = int( + self.overrides.apply_formatter(self.output_options.keep_max_files) + ) + + if date_range_to_keep or self.output_options.keep_max_files is not None: + self._enhanced_download_archive.remove_stale_files( + date_range=date_range_to_keep, keep_max_files=keep_max_files + ) self._enhanced_download_archive.save_download_mappings() FileHandler.delete(self._enhanced_download_archive.working_file_path) @@ -196,7 +208,9 @@ class SubscriptionDownload(BaseSubscription, ABC): @classmethod def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]: maybe_entry: Optional[Entry] = entry - for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry_metadata): + for plugin in PluginMapping.order_plugins_by( + plugins, PluginOperation.MODIFY_ENTRY_METADATA + ): if (maybe_entry := plugin.modify_entry_metadata(maybe_entry)) is None: return None @@ -206,7 +220,7 @@ class SubscriptionDownload(BaseSubscription, ABC): self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata ): # Post-process the entry with all plugins - for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.post_process): + for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.POST_PROCESS): optional_plugin_entry_metadata = plugin.post_process_entry(entry) if optional_plugin_entry_metadata: entry_metadata.extend(optional_plugin_entry_metadata) @@ -226,7 +240,7 @@ class SubscriptionDownload(BaseSubscription, ABC): entry_: Optional[Entry] = entry # First, modify the entry with all plugins - for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry): + for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.MODIFY_ENTRY): # Break if it is None, it is indicated to not process any further if (entry_ := plugin.modify_entry(entry_)) is None: break @@ -243,18 +257,10 @@ class SubscriptionDownload(BaseSubscription, ABC): ) -> None: entry_: Optional[Entry] = entry - plugins_pre_split = sorted( - [plugin for plugin in plugins if not plugin.priority.modify_entry_after_split], - key=lambda _plugin: _plugin.priority.modify_entry, - ) - - plugins_post_split = sorted( - [plugin for plugin in plugins if plugin.priority.modify_entry_after_split], - key=lambda _plugin: _plugin.priority.modify_entry, - ) - # First, modify the entry with pre_split plugins - for plugin in plugins_pre_split: + for plugin in PluginMapping.order_plugins_by( + plugins, PluginOperation.MODIFY_ENTRY, before_split=True + ): # Break if it is None, it is indicated to not process any further if (entry_ := plugin.modify_entry(entry_)) is None: break @@ -264,7 +270,9 @@ class SubscriptionDownload(BaseSubscription, ABC): for split_entry, split_entry_metadata in split_plugin.split(entry=entry_): split_entry_: Optional[Entry] = split_entry - for plugin in plugins_post_split: + for plugin in PluginMapping.order_plugins_by( + plugins, PluginOperation.MODIFY_ENTRY, before_split=False + ): # Return if it is None, it is indicated to not process any further. # Break out of the plugin loop if (split_entry_ := plugin.modify_entry(split_entry_)) is None: diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 678a9758..62445552 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -8,12 +8,12 @@ from typing import Optional from typing import final from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE +from ytdl_sub.config.overrides import Overrides from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.validators import DictValidator +from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator @@ -46,19 +46,26 @@ class SubscriptionOutput(Validator, ABC): Subscriptions in the form of ``{ subscription_name: preset_dict }`` """ - @property - def subscription_name(self) -> str: - """ - Returns - ------- - The name of the subscription - """ - return self._leaf_name - -class SubscriptionPresetDictValidator(SubscriptionOutput, DictValidator): - def __init__(self, name, value, presets: List[str], indent_overrides: List[str]): +class NamedSubscriptionValidator(SubscriptionOutput, ABC): + def __init__( + self, name, value, subscription_name: str, presets: List[str], indent_overrides: List[str] + ): super().__init__(name=name, value=value, presets=presets, indent_overrides=indent_overrides) + self.subscription_name = subscription_name + + +class SubscriptionPresetDictValidator(NamedSubscriptionValidator, DictValidator): + def __init__( + self, name, value, subscription_name: str, presets: List[str], indent_overrides: List[str] + ): + super().__init__( + name=name, + value=value, + subscription_name=subscription_name, + presets=presets, + indent_overrides=indent_overrides, + ) _ = self._validate_key_if_present(key="preset", validator=StringListValidator, default=[]) _ = self._validate_key_if_present(key="overrides", validator=Overrides, default={}) @@ -75,21 +82,27 @@ class SubscriptionPresetDictValidator(SubscriptionOutput, DictValidator): output_dict["overrides"] = dict( output_dict.get("overrides", {}), **self._indent_overrides_dict(), - **{SUBSCRIPTION_NAME: self.subscription_name}, ) return {self.subscription_name: output_dict} -class SubscriptionLeafValidator(SubscriptionOutput, ABC): +class SubscriptionLeafValidator(NamedSubscriptionValidator, ABC): def __init__( self, name, value, + subscription_name: str, config: ConfigFile, presets: List[str], indent_overrides: List[str], ): - super().__init__(name=name, value=value, presets=presets, indent_overrides=indent_overrides) + super().__init__( + name=name, + value=value, + subscription_name=subscription_name, + presets=presets, + indent_overrides=indent_overrides, + ) if self.subscription_name in config.presets.keys: raise self._validation_exception( @@ -97,7 +110,7 @@ class SubscriptionLeafValidator(SubscriptionOutput, ABC): f"used as a subscription name" ) - self._overrides_to_add: Dict[str, str] = {SUBSCRIPTION_NAME: self.subscription_name} + self._overrides_to_add: Dict[str, str] = {} @final def subscription_dicts(self, global_presets_to_apply: List[str]) -> Dict[str, Dict]: @@ -117,23 +130,20 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator): self, name, value, + subscription_name: str, config: ConfigFile, presets: List[str], indent_overrides: List[str], - subscription_value: Optional[str], ): super().__init__( name=name, value=value, + subscription_name=subscription_name, config=config, presets=presets, indent_overrides=indent_overrides, ) - - # TODO: Eventually delete in favor of {subscription_value} - if subscription_value: - self._overrides_to_add[subscription_value] = self.value - self._overrides_to_add[SUBSCRIPTION_VALUE] = self.value + self._overrides_to_add[OverrideVariables.subscription_value()] = self.value class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator): @@ -141,6 +151,7 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid self, name, value, + subscription_name: str, config: ConfigFile, presets: List[str], indent_overrides: List[str], @@ -148,6 +159,7 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid super().__init__( name=name, value=value, + subscription_name=subscription_name, config=config, presets=presets, indent_overrides=indent_overrides, @@ -156,7 +168,7 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid for idx, list_value in enumerate(self.list): # Write the first list value into subscription_value as well if idx == 0: - self._overrides_to_add[SUBSCRIPTION_VALUE] = list_value.value + self._overrides_to_add[OverrideVariables.subscription_value()] = list_value.value self._overrides_to_add[ OverrideVariables.subscription_value_i(index=idx) @@ -168,6 +180,7 @@ class SubscriptionWithOverridesValidator(SubscriptionLeafValidator, DictFormatte self, name, value, + subscription_name: str, config: ConfigFile, presets: List[str], indent_overrides: List[str], @@ -175,6 +188,7 @@ class SubscriptionWithOverridesValidator(SubscriptionLeafValidator, DictFormatte super().__init__( name=name, value=value, + subscription_name=subscription_name, config=config, presets=presets, indent_overrides=indent_overrides, @@ -182,15 +196,28 @@ class SubscriptionWithOverridesValidator(SubscriptionLeafValidator, DictFormatte self._overrides_to_add = dict(self.dict_with_format_strings, **self._overrides_to_add) - @property - def subscription_name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - # drop the ~ in "~Subscription Name": - return super().subscription_name[1:] + +class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator): + def __init__( + self, + name, + value, + subscription_name: str, + config: ConfigFile, + presets: List[str], + indent_overrides: List[str], + ): + super().__init__( + name=name, + value=value, + subscription_name=subscription_name, + config=config, + presets=presets, + indent_overrides=indent_overrides, + ) + self._overrides_to_add[OverrideVariables.subscription_map()] = ScriptUtils.to_script( + self.dict + ) class SubscriptionValidator(SubscriptionOutput): @@ -234,7 +261,6 @@ class SubscriptionValidator(SubscriptionOutput): config: ConfigFile, presets: List[str], indent_overrides: List[str], - subscription_value: Optional[str], ): super().__init__(name=name, value=value, presets=presets, indent_overrides=indent_overrides) self._children: List[SubscriptionOutput] = [] @@ -249,10 +275,10 @@ class SubscriptionValidator(SubscriptionOutput): SubscriptionValueValidator( name=obj_name, value=obj, + subscription_name=key, config=config, presets=presets, indent_overrides=indent_overrides, - subscription_value=subscription_value, ) ) # Subscription defined as @@ -264,6 +290,7 @@ class SubscriptionValidator(SubscriptionOutput): SubscriptionListValuesValidator( name=obj_name, value=obj, + subscription_name=key, config=config, presets=presets, indent_overrides=indent_overrides, @@ -279,6 +306,21 @@ class SubscriptionValidator(SubscriptionOutput): SubscriptionWithOverridesValidator( name=obj_name, value=obj, + subscription_name=key[1:].lstrip(), + config=config, + presets=presets, + indent_overrides=indent_overrides, + ) + ) + # Subscription defined as + # "\Sub Name": + # custom_key: "value" + elif key.startswith("+"): + self._children.append( + SubscriptionMapValidator( + name=obj_name, + value=obj, + subscription_name=key[1:].lstrip(), config=config, presets=presets, indent_overrides=indent_overrides, @@ -294,7 +336,6 @@ class SubscriptionValidator(SubscriptionOutput): config=config, presets=presets + preset_indent_key.presets, indent_overrides=indent_overrides + preset_indent_key.indent_overrides, - subscription_value=subscription_value, ) ) else: @@ -302,6 +343,7 @@ class SubscriptionValidator(SubscriptionOutput): SubscriptionPresetDictValidator( name=obj_name, value=obj, + subscription_name=key, presets=presets, indent_overrides=indent_overrides, ) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 6a7a7cea..14916c2e 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -7,7 +7,7 @@ from typing import TypeVar from yt_dlp import match_filter_func -from ytdl_sub.config.plugin import Plugin +from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin diff --git a/src/ytdl_sub/subscriptions/utils.py b/src/ytdl_sub/subscriptions/utils.py deleted file mode 100644 index b82620a0..00000000 --- a/src/ytdl_sub/subscriptions/utils.py +++ /dev/null @@ -1,2 +0,0 @@ -# Key used in configs, should delete at some point -SUBSCRIPTION_VALUE_CONFIG_KEY = "subscription_value" diff --git a/src/ytdl_sub/utils/chapters.py b/src/ytdl_sub/utils/chapters.py index 0875e3bc..9ccfa2c0 100644 --- a/src/ytdl_sub/utils/chapters.py +++ b/src/ytdl_sub/utils/chapters.py @@ -1,14 +1,16 @@ import re from typing import Dict from typing import List -from typing import Optional from typing import Tuple from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS +from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileMetadata +v: VariableDefinitions = VARIABLES + class Timestamp: @@ -157,6 +159,17 @@ class Chapters: """ return self.timestamps[0].timestamp_sec == 0 + def to_yt_dlp_chapter_metadata(self) -> List[Dict[str, str | float]]: + """ + Returns + ------- + Metadata dict + """ + return [ + {"start_time": ts.timestamp_sec, "title": title} + for ts, title in zip(self.timestamps, self.titles) + ] + def to_file_metadata_dict(self) -> Dict: """ Returns @@ -165,7 +178,7 @@ class Chapters: """ return {ts.readable_str: title for ts, title in zip(self.timestamps, self.titles)} - def to_file_metadata(self, title: Optional[str] = None) -> FileMetadata: + def to_file_metadata(self, title: str) -> FileMetadata: """ Parameters ---------- @@ -216,8 +229,23 @@ class Chapters: # If more than 3 timestamps were parsed, return it if len(timestamps) >= 3: return Chapters(timestamps=timestamps, titles=titles) + # Otherwise return empty chapters - return Chapters(timestamps=[], titles=[]) + return cls.from_empty() + + @classmethod + def from_yt_dlp_chapters(cls, chapters: List[Dict[str, str | float]]): + """ + Create a Chapters object from the raw ``chapters`` metadata returned in an info.json + """ + timestamps: List[Timestamp] = [] + titles: List[str] = [] + + for chapter in chapters: + timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"])))) + titles.append(chapter["title"]) + + return Chapters(timestamps=timestamps, titles=titles) @classmethod def from_entry_chapters(cls, entry: Entry) -> "Chapters": @@ -231,19 +259,12 @@ class Chapters: ------- Chapters object """ - timestamps: List[Timestamp] = [] - titles: List[str] = [] + if chapters := ( + entry.try_get(ytdl_sub_chapters_from_comments, list) or entry.get(v.chapters, list) + ): + return cls.from_yt_dlp_chapters(chapters) - if entry.kwargs_contains(CHAPTERS): - for chapter in entry.kwargs_get(CHAPTERS, []): - timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"])))) - titles.append(chapter["title"]) - elif entry.kwargs_contains(YTDL_SUB_CUSTOM_CHAPTERS): - for start_time, title in entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS, {}).items(): - timestamps.append(Timestamp.from_str(start_time)) - titles.append(title) - - return Chapters(timestamps=timestamps, titles=titles) + return cls.from_empty() @classmethod def from_empty(cls) -> "Chapters": diff --git a/src/ytdl_sub/utils/datetime.py b/src/ytdl_sub/utils/datetime.py index 07fcaacf..cce045d7 100644 --- a/src/ytdl_sub/utils/datetime.py +++ b/src/ytdl_sub/utils/datetime.py @@ -3,7 +3,7 @@ from typing import Optional from yt_dlp import DateRange from yt_dlp.utils import datetime_from_str -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.validators.string_datetime import StringDatetimeValidator diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index 7cf5ef2b..646d44e1 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -33,6 +33,13 @@ def get_file_extension(file_name: Path | str) -> str: return file_name.rsplit(".", maxsplit=1)[-1] +def get_md5_hash(contents: str) -> str: + """ + Helper function to compute md5 hash + """ + return hashlib.md5(contents.encode()).hexdigest() + + def get_file_md5_hash(full_file_path: Path | str) -> str: """ Parameters diff --git a/src/ytdl_sub/utils/file_lock.py b/src/ytdl_sub/utils/file_lock.py index e78209dc..a766dd6c 100644 --- a/src/ytdl_sub/utils/file_lock.py +++ b/src/ytdl_sub/utils/file_lock.py @@ -53,7 +53,7 @@ else: "colliding with each other. If you get this error, it typically means it tried to " "create the file in a directory that is not a part of the same filesystem that " "ytdl-sub is running on. See " - "https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl_sub.config.config_validator.ConfigOptions.lock_directory " + "https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html " "on how to change the directory that this lock gets written to." ) from exc # pylint: enable=line-too-long diff --git a/src/ytdl_sub/utils/file_path.py b/src/ytdl_sub/utils/file_path.py new file mode 100644 index 00000000..af8246f1 --- /dev/null +++ b/src/ytdl_sub/utils/file_path.py @@ -0,0 +1,61 @@ +import os +from pathlib import Path +from typing import Tuple + +from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES +from ytdl_sub.utils.file_handler import get_file_extension + + +class FilePathTruncater: + _EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8 + _DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES + + _MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES + + @classmethod + def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None: + """Actually sets the max _base_ file name in bytes (excludes extension)""" + max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES + + # bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES) + max_base_file_name_bytes = max(max_base_file_name_bytes, 16) + max_base_file_name_bytes = min( + max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES + ) + + cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes + + @classmethod + def _is_file_name_too_long(cls, file_name: str) -> bool: + return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES + + @classmethod + def _get_extension_split(cls, file_name: str) -> Tuple[str, str, str]: + if file_name.endswith("-thumb.jpg"): + ext = "-thumb.jpg" + delimiter = "" + else: + ext = get_file_extension(file_name) + delimiter = "." + + return file_name[: -len(ext)], ext, delimiter + + @classmethod + def _truncate_file_name(cls, file_name: str) -> str: + file_sub_name, file_ext, delimiter = cls._get_extension_split(file_name) + + desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 + while len(file_sub_name.encode("utf-8")) > desired_size: + file_sub_name = file_sub_name[:-1] + + return f"{file_sub_name}{delimiter}{file_ext}" + + @classmethod + def maybe_truncate_file_path(cls, file_path: str) -> str: + """Turn into a Path, then a string, to get correct directory separators""" + file_directory, file_name = os.path.split(Path(file_path)) + + if cls._is_file_name_too_long(file_name): + return str(Path(file_directory) / cls._truncate_file_name(file_name)) + + return str(file_path) diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py new file mode 100644 index 00000000..d5d36b87 --- /dev/null +++ b/src/ytdl_sub/utils/script.py @@ -0,0 +1,40 @@ +import json +import re +from typing import Any +from typing import Dict + + +class ScriptUtils: + @classmethod + def add_sanitized_variables(cls, variables: Dict[str, str]) -> Dict[str, str]: + """ + Helper to add sanitized variables to a Script + """ + sanitized_variables = { + f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys() + } + return dict(variables, **sanitized_variables) + + @classmethod + def to_script(cls, value: Any) -> str: + """ + Converts a python value to a script value + """ + if value is None: + out = "" + elif isinstance(value, str): + out = value + elif isinstance(value, int): + out = f"{{%int({value})}}" + elif isinstance(value, float): + out = f"{{%float({value})}}" + elif isinstance(value, bool): + out = f"{{%bool({value})}}" + else: + dumped_json = json.dumps(value, ensure_ascii=False, sort_keys=True) + # Remove triple-single-quotes from JSON to avoid parsing issues + dumped_json = re.sub("'{3,}", "'", dumped_json) + + out = f"{{%from_json('''{dumped_json}''')}}" + + return out diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py new file mode 100644 index 00000000..aba9d1fc --- /dev/null +++ b/src/ytdl_sub/utils/scriptable.py @@ -0,0 +1,68 @@ +import copy +from abc import ABC +from typing import Any +from typing import Dict +from typing import Set + +from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_types import Variable +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.utils.script import ScriptUtils + + +class Scriptable(ABC): + """ + Shared class between Entry and Overrides to manage their underlying Script. + """ + + _BASE_SCRIPT: Script = Script( + ScriptUtils.add_sanitized_variables( + dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) + ) + ) + + def __init__(self): + self.script = copy.deepcopy(Scriptable._BASE_SCRIPT) + self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES) + + def update_script(self) -> None: + """ + Updates any potential variables to a resolvable. This is done + to avoid re-resolving the same variables over-and-over. + """ + self.script.resolve(unresolvable=self.unresolvable, update=True) + + def add(self, values: Dict[str | Variable, Any]) -> None: + """ + Add new values to the script + """ + values_as_str: Dict[str, str] = { + (var.variable_name if isinstance(var, Variable) else var): definition + for var, definition in values.items() + } + + self.unresolvable -= set(list(values_as_str.keys())) + self.script.add( + ScriptUtils.add_sanitized_variables( + { + name: ScriptUtils.to_script(definition) + for name, definition in values_as_str.items() + } + ), + unresolvable=self.unresolvable, + ) + self.update_script() + + for name, definition in values_as_str.items(): + try: + _ = self.script.get(variable_name=name) + except RuntimeException as exc: + raise StringFormattingException( + f"Tried to create the variable with name {name} and definition:\n" + f"{definition}\n" + f"But could not because it has variable dependencies that are not resolved yet" + ) from exc diff --git a/src/ytdl_sub/validators/file_path_validators.py b/src/ytdl_sub/validators/file_path_validators.py index 8d5afe11..c255ed1a 100644 --- a/src/ytdl_sub/validators/file_path_validators.py +++ b/src/ytdl_sub/validators/file_path_validators.py @@ -1,12 +1,8 @@ import os from pathlib import Path from typing import Any -from typing import Dict -from typing import Tuple -from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES -from ytdl_sub.utils.file_handler import get_file_extension -from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS +from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import StringValidator @@ -35,57 +31,8 @@ class FFprobeFileValidator(FFmpegFileValidator): _ffmpeg_dependency = "ffprobe" -class FilePathValidatorMixin: - _EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8 - _DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES - - _MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES - - @classmethod - def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None: - """Actually sets the max _base_ file name in bytes (excludes extension)""" - max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES - - # bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES) - max_base_file_name_bytes = max(max_base_file_name_bytes, 16) - max_base_file_name_bytes = min( - max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES - ) - - cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes - - @classmethod - def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES - - @classmethod - def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: - ext = get_file_extension(file_name) - return file_name[: -len(ext)], ext - - @classmethod - def _truncate_file_name(cls, file_name: str) -> str: - file_sub_name, file_ext = cls._get_extension_split(file_name) - - desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 - while len(file_sub_name.encode("utf-8")) > desired_size: - file_sub_name = file_sub_name[:-1] - - return f"{file_sub_name}.{file_ext}" - - @classmethod - def _maybe_truncate_file_path(cls, file_path: Path) -> str: - """Turn into a Path, then a string, to get correct directory separators""" - file_directory, file_name = os.path.split(Path(file_path)) - - if cls._is_file_name_too_long(file_name): - return str(Path(file_directory) / cls._truncate_file_name(file_name)) - - return str(file_path) - - # pylint: disable=line-too-long -class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidatorMixin): +class StringFormatterFileNameValidator(StringFormatterValidator): """ Same as a :class:`StringFormatterValidator <ytdl_sub.validators.string_formatter_validators.StringFormatterValidator>` @@ -97,51 +44,30 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat _expected_value_type_name = "filepath" - @classmethod - def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES - - @classmethod - def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: - """ - Returns - ------- - file_name, ext (including .) - """ - if file_name.endswith(".info.json"): - ext = ".info.json" - elif file_name.endswith("-thumb.jpg"): - ext = "-thumb.jpg" - elif any(file_name.endswith(f".{subtitle_ext}") for subtitle_ext in SUBTITLE_EXTENSIONS): - file_name_split = file_name.split(".") - ext = file_name_split[-1] - - # Try to capture .lang.ext - if len(file_name_split) > 2 and len(file_name_split[-2]) < 6: - ext = f".{file_name_split[-2]}.{file_name_split[-1]}" - else: - ext = f".{file_name.rsplit('.', maxsplit=1)[-1]}" - - return file_name[: -len(ext)], ext - - @classmethod - def _truncate_file_name(cls, file_name: str) -> str: - file_sub_name, file_ext = cls._get_extension_split(file_name) - - while len(file_sub_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES: - file_sub_name = file_sub_name[:-1] - - return f"{file_sub_name}{file_ext}" - - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - """Turn into a Path, then a string, to get correct directory separators""" - file_path = Path(super().apply_formatter(variable_dict)) - return self._maybe_truncate_file_path(file_path) + def post_process(self, resolved: str) -> str: + return ( + Script( + { + "tmp_var_1": resolved, + "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}", + } + ) + .resolve() + .get_str("tmp_var_2") + ) class OverridesStringFormatterFilePathValidator(OverridesStringFormatterValidator): _expected_value_type_name = "static filepath" - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - """Turn into a Path, then a string, to get correct directory separators""" - return os.path.realpath(super().apply_formatter(variable_dict)) + def post_process(self, resolved: str) -> str: + return ( + Script( + { + "tmp_var_1": resolved, + "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}", + } + ) + .resolve() + .get_str("tmp_var_2") + ) diff --git a/src/ytdl_sub/validators/source_variable_validator.py b/src/ytdl_sub/validators/source_variable_validator.py index 0050779d..ba17c895 100644 --- a/src/ytdl_sub/validators/source_variable_validator.py +++ b/src/ytdl_sub/validators/source_variable_validator.py @@ -1,5 +1,5 @@ +from ytdl_sub.script.utils.name_validation import is_valid_name from ytdl_sub.utils.exceptions import InvalidVariableNameException -from ytdl_sub.validators.string_formatter_validators import is_valid_source_variable_name from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringValidator @@ -9,10 +9,13 @@ class SourceVariableNameValidator(StringValidator): def __init__(self, name, value): super().__init__(name, value) - try: - _ = is_valid_source_variable_name(self.value, raise_exception=True) - except InvalidVariableNameException as exc: - raise self._validation_exception(exc) from exc + + if not is_valid_name(value): + raise self._validation_exception( + f"Variable with name {name} is invalid. Names must be" + " lower_snake_cased and begin with a letter.", + exception_class=InvalidVariableNameException, + ) class SourceVariableNameListValidator(ListValidator[SourceVariableNameValidator]): diff --git a/src/ytdl_sub/validators/string_datetime.py b/src/ytdl_sub/validators/string_datetime.py index 5818f6f4..d50be211 100644 --- a/src/ytdl_sub/validators/string_datetime.py +++ b/src/ytdl_sub/validators/string_datetime.py @@ -1,5 +1,3 @@ -from typing import Dict - from yt_dlp.utils import datetime_from_str from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -21,10 +19,9 @@ class StringDatetimeValidator(OverridesStringFormatterValidator): _expected_value_type_name = "datetime string" - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - output = super().apply_formatter(variable_dict) + def post_process(self, resolved: str) -> str: try: - _ = datetime_from_str(output) + _ = datetime_from_str(resolved) except Exception as exc: raise self._validation_exception(f"Invalid datetime string: {str(exc)}") - return output + return resolved diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 53fbc0dc..7957169d 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -1,53 +1,22 @@ -import re -from collections import OrderedDict -from keyword import iskeyword from typing import Dict -from typing import List +from typing import Set +from typing import Union from typing import final -from yt_dlp.utils import sanitize_filename - -from ytdl_sub.utils.exceptions import InvalidVariableNameException -from ytdl_sub.utils.exceptions import StringFormattingException +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.script.parser import parse +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException +from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import LiteralDictValidator +from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator -_fields_validator = re.compile(r"{([a-z][a-z0-9_]+?)}") -_fields_validator_exception_message: str = ( - "{variable_names} must start with a lowercase letter, should only contain lowercase letters, " - "numbers, underscores, and have a single open and close bracket." -) - - -def is_valid_source_variable_name(input_str: str, raise_exception: bool = False) -> bool: - """ - Parameters - ---------- - input_str - String to see if it can be a source variable - raise_exception - Raise InvalidVariableNameException False. - - Returns - ------- - True if it is. False otherwise. - - Raises - ------ - InvalidVariableNameException - If raise_exception and output is False - """ - # Add brackets around it to pretend its a StringFormatter, see if it captures - is_source_variable_name = len(re.findall(_fields_validator, f"{{{input_str}}}")) > 0 - if not is_source_variable_name and raise_exception: - raise InvalidVariableNameException(_fields_validator_exception_message) - return is_source_variable_name - - -class StringFormatterValidator(Validator): +class StringFormatterValidator(StringValidator): """ String that can use :class:`source variables <ytdl_sub.entries.variables.entry_variables.SourceVariables>` @@ -73,59 +42,17 @@ class StringFormatterValidator(Validator): and would resolve to something like ``sweet_tv_show.s2022.e502.mp4``. """ - _expected_value_type = str _expected_value_type_name = "format string" - _variable_not_found_error_msg_formatter = ( - "Format variable '{variable_name}' does not exist. Available variables: {available_fields}" - ) - - _max_format_recursion = 8 - - def __validate_and_get_format_variables(self) -> List[str]: - """ - Returns - ------- - list[str] - List of format variables in the format string - - Raises - ------ - ValidationException - If the format string contains invalid variable formatting - """ - open_bracket_count = self.format_string.count("{") - close_bracket_count = self.format_string.count("}") - - if open_bracket_count != close_bracket_count: - raise self._validation_exception( - "Brackets are reserved for {variable_names} and should contain " - "a single open and close bracket.", - exception_class=StringFormattingException, - ) - - format_variables: List[str] = list(re.findall(_fields_validator, self.format_string)) - - if len(format_variables) != open_bracket_count: - raise self._validation_exception( - error_message=_fields_validator_exception_message, - exception_class=StringFormattingException, - ) - - for variable in format_variables: - if iskeyword(variable): - raise self._validation_exception( - f"'{variable}' is a Python keyword and cannot be used as a variable.", - exception_class=StringFormattingException, - ) - - return format_variables def __init__(self, name, value: str): super().__init__(name=name, value=value) - self.format_variables = self.__validate_and_get_format_variables() + try: + _ = parse(str(value)) + except UserException as exc: + raise self._validation_exception(exc) from exc - @final @property + @final def format_string(self) -> str: """ Returns @@ -134,67 +61,17 @@ class StringFormatterValidator(Validator): """ return self._value - def _apply_formatter( - self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str] - ) -> "StringFormatterValidator": - # Ensure the variable names exist within the entry and overrides - for variable_name in formatter.format_variables: - # If the variable exists, but is sanitized... - if ( - variable_name.endswith("_sanitized") - and variable_name.removesuffix("_sanitized") in variable_dict - ): - # Resolve just the non-sanitized version, then sanitize it - variable_dict[variable_name] = sanitize_filename( - StringFormatterValidator( - name=self._name, value=f"{{{variable_name.removesuffix('_sanitized')}}}" - ).apply_formatter(variable_dict) - ) - # If the variable doesn't exist, error - elif variable_name not in variable_dict: - available_fields = ", ".join(sorted(variable_dict.keys())) - raise self._validation_exception( - self._variable_not_found_error_msg_formatter.format( - variable_name=variable_name, available_fields=available_fields - ), - exception_class=StringFormattingVariableNotFoundException, - ) + # pylint: disable=no-self-use - return StringFormatterValidator( - name=self._name, - value=formatter.format_string.format(**OrderedDict(variable_dict)), - ) - - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: + def post_process(self, resolved: str) -> str: """ - Calls `format` on the format string using the variable_dict as input kwargs - - Parameters - ---------- - variable_dict - kwargs to pass to the format string - Returns ------- - Format string formatted + Apply any post processing to the resolved value """ - formatter = self - recursion_depth = 0 - max_depth = self._max_format_recursion + return resolved - while formatter.format_variables and recursion_depth < max_depth: - formatter = self._apply_formatter(formatter=formatter, variable_dict=variable_dict) - recursion_depth += 1 - - if formatter.format_variables: - raise self._validation_exception( - f"Attempted to format but failed after reaching max recursion depth of " - f"{max_depth}. Try to keep variables dependent on only one other variable at max. " - f"Unresolved variables: {', '.join(sorted(formatter.format_variables))}", - exception_class=StringFormattingException, - ) - - return formatter.format_string + # pylint: enable=no-self-use # pylint: disable=line-too-long @@ -209,16 +86,24 @@ class OverridesStringFormatterValidator(StringFormatterValidator): :class:`nfo_output_directory <ytdl_sub.plugins.output_directory_nfo_tags.OutputDirectoryNfoTagsOptions>` """ - _variable_not_found_error_msg_formatter = ( - "Override variable '{variable_name}' does not exist. For this field, ensure your override " - "variable does not contain any source variables - it is a requirement that this be a " - "static string. Available override variables: {available_fields}" - ) - # pylint: enable=line-too-long +class OverridesIntegerFormatterValidator(StringFormatterValidator): + _expected_value_type_name = "integer" + + def post_process(self, resolved: str) -> str: + try: + int(resolved) + except Exception as exc: + raise self._validation_exception( + f"Expected an integer, but received '{resolved}'" + ) from exc + + return resolved + + class ListFormatterValidator(ListValidator[StringFormatterValidator]): _inner_list_type = StringFormatterValidator @@ -255,3 +140,63 @@ class OverridesDictFormatterValidator(DictFormatterValidator): """ _key_validator = OverridesStringFormatterValidator + + +def _validate_formatter( + mock_script: Script, + unresolved_variables: Set[str], + formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], +) -> None: + try: + unresolvable = unresolved_variables + if isinstance(formatter_validator, OverridesStringFormatterValidator): + unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + + mock_script.resolve_once( + {"tmp_var": formatter_validator.format_string}, + unresolvable=unresolvable, + ) + except VariableDoesNotExist as exc: + raise StringFormattingVariableNotFoundException(exc) from exc + + +def validate_formatters( + script: Script, + unresolved_variables: Set[str], + validator: Validator, +) -> None: + """ + Ensure all OverridesStringFormatterValidator's only contain variables from the overrides + and resolve. + """ + if isinstance(validator, DictValidator): + # pylint: disable=protected-access + # Usage of protected variables in other validators is fine. The reason to keep + # them protected is for readability when using them in subscriptions. + for validator_value in validator._validator_dict.values(): + validate_formatters( + script=script, + unresolved_variables=unresolved_variables, + validator=validator_value, + ) + # pylint: enable=protected-access + elif isinstance(validator, ListValidator): + for list_value in validator.list: + validate_formatters( + script=script, + unresolved_variables=unresolved_variables, + validator=list_value, + ) + elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)): + _validate_formatter( + mock_script=script, + unresolved_variables=unresolved_variables, + formatter_validator=validator, + ) + elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)): + for validator_value in validator.dict.values(): + _validate_formatter( + mock_script=script, + unresolved_variables=unresolved_variables, + formatter_validator=validator_value, + ) diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index 1220cfca..5bd4dbaa 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -138,6 +138,12 @@ class StringValidator(ValueValidator[str]): _expected_value_type = str _expected_value_type_name = "string" + def __init__(self, name: str, value: Any): + if isinstance(value, (int, float, bool)): + value = str(value) + + super().__init__(name, value) + class FloatValidator(ValueValidator[float]): _expected_value_type = (int, float) diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index ea5777e1..616fe2f9 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -14,7 +14,9 @@ from yt_dlp import DateRange from yt_dlp.utils import make_archive_id from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY +from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata @@ -22,6 +24,8 @@ from ytdl_sub.utils.logger import Logger logger = Logger.get("archive") +v: VariableDefinitions = VARIABLES + @dataclass class DownloadMapping: @@ -71,8 +75,8 @@ class DownloadMapping: DownloadMapping for the entry """ return DownloadMapping( - upload_date=entry.upload_date_standardized, - extractor=entry.extractor, + upload_date=entry.get(v.upload_date_standardized, str), + extractor=entry.download_archive_extractor, file_names=set(), ) @@ -219,10 +223,14 @@ class DownloadMappings: ------- self """ - if entry.uid not in self.entry_ids: - self._entry_mappings[entry.uid] = DownloadMapping.from_entry(entry=entry) + uid = entry.uid + if parent_uid := entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str): + uid = parent_uid - self._entry_mappings[entry.uid].file_names.add(entry_file_path) + if uid not in self.entry_ids: + self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry) + + self._entry_mappings[uid].file_names.add(entry_file_path) return self def remove_entry(self, entry_id: str) -> "DownloadMappings": @@ -537,7 +545,16 @@ class EnhancedDownloadArchive: return self - def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": + def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None: + for file_name in mapping.file_names: + self._file_handler.delete_file_from_output_directory(file_name=file_name) + + self.mapping.remove_entry(entry_id=uid) + self.num_entries_removed += 1 + + def remove_stale_files( + self, date_range: Optional[DateRange], keep_max_files: Optional[int] + ) -> "EnhancedDownloadArchive": """ Checks all entries within the mappings. If any entries' upload dates are not within the provided date range, delete them. @@ -545,22 +562,32 @@ class EnhancedDownloadArchive: Parameters ---------- date_range - Date range the upload date must be in to not get deleted + Optional. Date range the upload date must be in to not get deleted + keep_max_files + Optional. Max number of files to keep Returns ------- self """ - stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( - date_range=date_range - ) + if date_range is not None: + stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( + date_range=date_range + ) - for uid, mapping in stale_mappings.items(): - for file_name in mapping.file_names: - self._file_handler.delete_file_from_output_directory(file_name=file_name) + for uid, mapping in stale_mappings.items(): + self._remove_entry(uid=uid, mapping=mapping) - self.mapping.remove_entry(entry_id=uid) - self.num_entries_removed += 1 + if keep_max_files is not None and keep_max_files > 0: + num_files = 0 + for uid, mapping in sorted( + self.mapping.entry_mappings.items(), + key=lambda kv_: kv_[1].upload_date, + reverse=True, + ): + num_files += 1 + if num_files > keep_max_files: + self._remove_entry(uid=uid, mapping=mapping) return self @@ -627,15 +654,7 @@ class EnhancedDownloadArchive: if output_file_name is None: output_file_name = file_name - # If the entry is created from splitting via chapters, store it to the mapping - # using its parent entry - if entry and entry.kwargs_contains(SPLIT_BY_CHAPTERS_PARENT_ENTRY): - parent_entry = Entry( - entry_dict=entry.kwargs(SPLIT_BY_CHAPTERS_PARENT_ENTRY), - working_directory=entry.working_directory(), - ) - self.mapping.add_entry(parent_entry, entry_file_path=output_file_name) - elif entry: + if entry: self.mapping.add_entry(entry=entry, entry_file_path=output_file_name) is_modified = self._file_handler.move_file_to_output_directory( diff --git a/tests/conftest.py b/tests/conftest.py index f396e28c..b7ec75ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,8 +14,12 @@ from unittest.mock import patch import pytest from expected_download import _get_files_in_directory +from resources import copy_file_fixture +from resources import file_fixture_path +from yt_dlp.utils import sanitize_filename from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.entries.script.custom_functions import CustomFunctions from ytdl_sub.subscriptions.subscription_download import SubscriptionDownload from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger @@ -23,6 +27,14 @@ from ytdl_sub.utils.logger import LoggerLevels from ytdl_sub.utils.yaml import load_yaml +@pytest.fixture(autouse=True) +def register_custom_functions(): + """ + Clean logs after every test + """ + CustomFunctions.register() + + @pytest.fixture(autouse=True) def cleanup_debug_file(): """ @@ -67,13 +79,13 @@ def working_directory() -> str: @pytest.fixture() -def output_directory() -> Path: +def output_directory() -> str: with tempfile.TemporaryDirectory() as temp_dir: yield temp_dir @pytest.fixture() -def reformat_directory() -> Path: +def reformat_directory() -> str: with tempfile.TemporaryDirectory() as temp_dir: yield temp_dir @@ -163,6 +175,38 @@ def preset_dict_to_subscription_yaml_generator() -> Callable: return _preset_dict_to_subscription_yaml_generator +################################################################################################### +# Staging a mock already-existing download + + +@pytest.fixture +def pz_channel_mock_downloaded_with_archive_factory(output_directory: Path) -> Callable: + def _pz_channel_mock_downloaded_with_archive_factory(tv_show_name: str, archive_file_name: str): + subscription_path = Path(output_directory) / sanitize_filename(tv_show_name) + copy_file_fixture( + fixture_name="pz_download_archive.json", + output_file_path=subscription_path / archive_file_name, + ) + + with open( + file_fixture_path("pz_download_archive.json"), "r", encoding="utf-8" + ) as archive_file: + archive_dict = json.load(archive_file) + + assert isinstance(archive_dict, dict) + for uid, metadata in archive_dict.items(): + assert isinstance(metadata, dict) + for filename in metadata["file_names"]: + assert isinstance(filename, str) + output_file_path = subscription_path / filename + if filename.endswith(".mp4"): + copy_file_fixture("sample_vid.mp4", output_file_path) + else: + copy_file_fixture("empty.txt", output_file_path) + + return _pz_channel_mock_downloaded_with_archive_factory + + ################################################################################################### # Example config fixtures diff --git a/tests/e2e/plugins/test_filter.py b/tests/e2e/plugins/test_filter.py new file mode 100644 index 00000000..487c3269 --- /dev/null +++ b/tests/e2e/plugins/test_filter.py @@ -0,0 +1,299 @@ +import re +from typing import Any +from typing import Dict + +import mergedeep +import pytest +from expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def regex_subscription_dict_base(output_directory): + return { + "preset": "Jellyfin Music Videos", + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + "format": "best[height<=480]", # download the worst format so it is fast + "filter_include": ["{%not(%is_null(description_website))}"], + "overrides": { + "in_regex_default": "in regex default", + "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "upload_capture": """{ + %regex_capture_many_with_defaults( + upload_date_standardized, + ["([0-9]+)-([0-9]+)-27"], + ["First", %concat("Second containing ", in_regex_default)] + ) + }""", + "upload_captured_year": "{%array_at(upload_capture, 1)}", + "upload_captured_month": "{%array_at(upload_capture, 2)}", + "description_capture": """{ + %regex_capture_many_with_defaults( + description, + [".*http:\\/\\/(.+).com.*"], + [null] + ) + }""", + "description_website": "{%array_at(description_capture, 1)}", + }, + } + + +@pytest.fixture +def regex_subscription_dict(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "filter_exclude": ["{%is_null(title_type)}"], + "overrides": { + "title_capture_list": """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ], + [ null, null ] + ) + }""", + "title_type": "{%array_at(title_capture_list, 1)}", + "title_date": "{%array_at(title_capture_list, 2)}", + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "filter_exclude": [ + """{ + %regex_search_any( + title, + [ "should not cap", ".*Feb.*" ] + ) + }""" + ] + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude(regex_subscription_dict_base, output_directory): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "title": { + "match": [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]", # should filter out march video + ], + "capture_group_names": ["title_type", "title_date"], + "exclude": [ + "should not cap", + ".*27.*", # should filter out Feb 27th video + ], + }, + }, + }, + "nfo_tags": { + "tags": { + "title_cap_1": "{title_type}", + "title_cap_1_sanitized": "{title_type_sanitized}", + "title_cap_2": "{title_date}", + "desc_cap": "{description_website}", + "upload_date_both_caps": "{upload_captured_year} and {upload_captured_month}", + "override_with_capture_variable": "{contains_regex_default}", + "override_with_capture_variable_sanitized": "{contains_regex_sanitized_default}", + } + }, + "overrides": { + "contains_regex_default": "contains {title_type}", + "contains_regex_sanitized_default": "contains {title_type_sanitized}", + }, + }, + ) + + +@pytest.fixture +def regex_subscription_dict_match_and_exclude_override_variable( + regex_subscription_dict_base, output_directory +): + return mergedeep.merge( + regex_subscription_dict_base, + { + "regex": { + # tests that skip_if_match_fails defaults to True + "from": { + "override_title": { + "exclude": [ + "should not cap", + ".*Feb.*", # should filter out march video + ], + }, + "override_description": { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["override_description_website"], + }, + }, + }, + "overrides": {"override_title": "{title}", "override_description": "{description}"}, + }, + ) + + +@pytest.fixture +def playlist_subscription(default_config, regex_subscription_dict): + return Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_no_match_fails( + default_config: ConfigFile, regex_subscription_dict: Dict[str, Any] +): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_required( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ] + ) + }""" + + return Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + +@pytest.fixture +def playlist_subscription_exclude( + default_config: ConfigFile, regex_subscription_dict_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_exclude_playlist_test", + preset_dict=regex_subscription_dict_exclude, + ) + + +@pytest.fixture +def playlist_subscription_overrides( + default_config: ConfigFile, + regex_subscription_dict_match_and_exclude_override_variable: Dict[str, Any], +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_using_overrides_test", + preset_dict=regex_subscription_dict_match_and_exclude_override_variable, + ) + + +@pytest.fixture +def playlist_subscription_match_and_exclude( + default_config: ConfigFile, regex_subscription_dict_match_and_exclude: Dict[str, Any] +) -> Subscription: + return Subscription.from_dict( + config=default_config, + preset_name="regex_match_and_exclude_playlist_test", + preset_dict=regex_subscription_dict_match_and_exclude, + ) + + +class TestFilter: + def test_regex_success(self, playlist_subscription, output_directory): + # Only dry run is needed to see if capture variables are created + transaction_log = playlist_subscription.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex.txt", + ) + + def test_regex_excludes_success(self, playlist_subscription_exclude, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_exclude.txt", + ) + + def test_regex_match_and_excludes_success( + self, playlist_subscription_match_and_exclude, output_directory + ): + # Should only contain the Feb 1st video + transaction_log = playlist_subscription_match_and_exclude.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_match_and_exclude.txt", + ) + + def test_regex_using_overrides_success(self, playlist_subscription_overrides, output_directory): + # Should only contain the march video + transaction_log = playlist_subscription_overrides.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_regex_overrides.txt", + ) + + def test_regex_fails_no_match(self, playlist_subscription_no_match_fails, output_directory): + with pytest.raises( + UserThrownRuntimeError, + match=re.escape( + "When running %regex_capture_many_required, no regex strings were captured" + ), + ): + _ = playlist_subscription_no_match_fails.download(dry_run=True) + + def test_regex_fails_unequal_defaults(self, regex_subscription_dict, default_config): + regex_subscription_dict["overrides"][ + "title_capture_list" + ] = """{ + %regex_capture_many_with_defaults( + title, + [ "should not cap (.+) - (.+)", ".*\\[(.+) - (Feb.+)]" ], + [ "one default, expects >= 2" ] + ) + }""" + + subscription = Subscription.from_dict( + config=default_config, + preset_name="regex_capture_playlist_test", + preset_dict=regex_subscription_dict, + ) + + with pytest.raises( + UserThrownRuntimeError, + match=re.escape( + "When using %regex_capture_with_defaults, number of regex capture groups must " + "be less than or equal to the number of defaults" + ), + ): + _ = subscription.download(dry_run=True) diff --git a/tests/e2e/plugins/test_nfo_tags.py b/tests/e2e/plugins/test_nfo_tags.py index 7ed7a07e..24e02de2 100644 --- a/tests/e2e/plugins/test_nfo_tags.py +++ b/tests/e2e/plugins/test_nfo_tags.py @@ -19,7 +19,8 @@ def subscription_dict(output_directory): "attributes": {"🎸?": "value\nnewlines 🎸"}, "tag": "the \n tag 🎸🎸", }, - "kodi_safe_multi_title 🎸": ["value 1 🎸", "value 2 🎸"], + # should not show third empty + "kodi_safe_multi_title 🎸": ["value 1 🎸", "value 2 🎸", ""], "kodi_safe_multi_title_with_attrs": [ { "attributes": {"🎸?": "value\nnewlines 🎸"}, @@ -29,7 +30,16 @@ def subscription_dict(output_directory): "attributes": {"🎸?": "value\nnewlines 🎸"}, "tag": "the \n tag 2 🎸🎸", }, + { + "attributes": {"🎸?": "EMPTY TAG SHOULD NOT SHOW"}, + "tag": "", + }, ], + "empty_attribute_tag_SHOULD_NOT_SHOW": { + "attributes": {"🎸?": "value\nnewlines 🎸"}, + "tag": "", + }, + "empty_tag_SHOULD_NOT_SHOW": "", }, }, "output_directory_nfo_tags": { @@ -41,7 +51,8 @@ def subscription_dict(output_directory): "attributes": {"🎸?": "value\nnewlines 🎸"}, "tag": "the \n tag 🎸🎸", }, - "kodi_safe_multi_title 🎸": ["value 1 🎸", "value 2 🎸"], + # should not show third empty + "kodi_safe_multi_title 🎸": ["value 1 🎸", "value 2 🎸", ""], "kodi_safe_multi_title_with_attrs": [ { "attributes": {"🎸?": "value\nnewlines 🎸"}, @@ -51,7 +62,16 @@ def subscription_dict(output_directory): "attributes": {"🎸?": "value\nnewlines 🎸"}, "tag": "the \n tag 2 🎸🎸", }, + { + "attributes": {"🎸?": "EMPTY TAG SHOULD NOT SHOW"}, + "tag": "", + }, ], + "empty_attribute_tag_SHOULD_NOT_SHOW": { + "attributes": {"🎸?": "value\nnewlines 🎸"}, + "tag": "", + }, + "empty_tag_SHOULD_NOT_SHOW": "", }, }, } diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index 504f74bb..63b4be53 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -76,6 +76,21 @@ def regex_subscription_dict(regex_subscription_dict_base, output_directory): } }, "overrides": { + "title_capture_list": f"""{{ + %regex_capture_many_with_defaults( + title, + [ + "should not cap (.+) - (.+)", + ".*\\[(.+) - (Feb.+)]" + ], + [ + "ack", + "ack" + ] + ) + }}""", + "title_capture_list_1": "{%array_at(title_capture_list, 1)}", + "title_capture_list_2": "{%array_at(title_capture_list, 2)}", "contains_regex_default": "contains {title_type}", "contains_regex_sanitized_default": "contains {title_type_sanitized}", }, @@ -302,32 +317,40 @@ class TestRegex: preset_dict=regex_subscription_dict, ) - def test_regex_fails_capture_group_is_source_variable( + def test_regex_fails_capture_group_is_entry_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid" + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["uid"], + } + with pytest.raises( ValidationException, match=re.escape( - "'uid' cannot be used as a capture group name because it is a source variable" + "Cannot use the variable name uid because it exists as a built-in " + "ytdl-sub variable name." ), ): _ = Subscription.from_dict( config=default_config, - preset_name="test_regex_fails_capture_group_is_source_variable", + preset_name="test_regex_fails_capture_group_is_entry_variable", preset_dict=regex_subscription_dict, ) def test_regex_fails_capture_group_is_override_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][ - 0 - ] = "in_regex_default" + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["contains_regex_default"], + } + with pytest.raises( ValidationException, match=re.escape( - "'in_regex_default' cannot be used as a capture group name because it is an override variable" + "Override variable with name contains_regex_default cannot be used since it is " + "added by a plugin." ), ): _ = Subscription.from_dict( @@ -344,9 +367,7 @@ class TestRegex: ) with pytest.raises( ValidationException, - match=re.escape( - "cannot regex capture 'dne' because it is not a source or override variable" - ), + match=re.escape("cannot regex capture 'dne' because it is not a defined variable"), ): _ = Subscription.from_dict( config=default_config, diff --git a/tests/e2e/plugins/test_split_by_chapters.py b/tests/e2e/plugins/test_split_by_chapters.py index 597a1fe4..cb02715e 100644 --- a/tests/e2e/plugins/test_split_by_chapters.py +++ b/tests/e2e/plugins/test_split_by_chapters.py @@ -34,7 +34,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict "from": { # Ensure regex can handle override variables that come from the # post-metadata stage - "override_chapter_title": { + "chapter_title": { "match": r"\d+\. (.+)", "capture_group_names": "captured_track_title", "capture_group_defaults": "{chapter_title}", @@ -56,7 +56,6 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict } }, "overrides": { - "override_chapter_title": "{chapter_title}", "track_title": "{captured_track_title}", "track_album": "{captured_track_album}", "track_artist": "{captured_track_artist}", diff --git a/tests/e2e/soundcloud/test_soundcloud_discography.py b/tests/e2e/soundcloud/test_soundcloud_discography.py index de1f4a4f..8e7f5345 100644 --- a/tests/e2e/soundcloud/test_soundcloud_discography.py +++ b/tests/e2e/soundcloud/test_soundcloud_discography.py @@ -1,7 +1,9 @@ import pytest +from conftest import assert_logs from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches +from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.subscriptions.subscription import Subscription @@ -48,3 +50,19 @@ class TestSoundcloudDiscography: dry_run=dry_run, expected_download_summary_file_name="soundcloud/test_soundcloud_discography.json", ) + + # Ensure another invocation will hit ExistingVideoReached + if not dry_run: + with assert_logs( + logger=YTDLP.logger, + expected_message="ExistingVideoReached, stopping additional downloads", + log_level="debug", + ): + transaction_log = discography_subscription.download() + + assert transaction_log.is_empty + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name="soundcloud/test_soundcloud_discography.json", + ) diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 585abaa4..2b36cf9b 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -1,14 +1,20 @@ +from typing import Callable +from typing import Dict + import pytest from expected_download import assert_expected_downloads from expected_transaction_log import assert_transaction_log_matches +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def channel_preset_dict(output_directory): return { - "preset": "TV Show Full Archive", + "preset": [ + "TV Show Full Archive", + ], "format": "worst[ext=mp4]", # download the worst format so it is fast "ytdl_options": { "max_views": 100000, # do not download the popular PJ concert @@ -65,3 +71,52 @@ class TestChannel: dry_run=dry_run, expected_download_summary_file_name="youtube/test_channel_full.json", ) + + def test_full_channel_existing_archive_downloads_nothing( + self, + pz_channel_mock_downloaded_with_archive_factory: Callable, + tv_show_config: ConfigFile, + channel_preset_dict: Dict, + ): + subscription_name = "pz" + tv_show_name = channel_preset_dict["overrides"]["tv_show_name"] + archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json" + + pz_channel_mock_downloaded_with_archive_factory( + tv_show_name=tv_show_name, archive_file_name=archive_file_name + ) + + full_channel_subscription = Subscription.from_dict( + config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict + ) + transaction_log = full_channel_subscription.download(dry_run=True) + assert transaction_log.is_empty + + def test_full_channel_existing_archive_keep_max_files( + self, + pz_channel_mock_downloaded_with_archive_factory: Callable, + tv_show_config: ConfigFile, + channel_preset_dict: Dict, + output_directory: str, + ): + subscription_name = "pz" + channel_preset_dict["preset"].append("Only Recent") + channel_preset_dict["overrides"]["only_recent_date_range"] = "10years" + channel_preset_dict["overrides"]["only_recent_max_files"] = 1 + + full_channel_subscription = Subscription.from_dict( + config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict + ) + tv_show_name = channel_preset_dict["overrides"]["tv_show_name"] + archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json" + + pz_channel_mock_downloaded_with_archive_factory( + tv_show_name=tv_show_name, archive_file_name=archive_file_name + ) + + transaction_log = full_channel_subscription.download(dry_run=True) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt", + ) diff --git a/tests/expected_transaction_log.py b/tests/expected_transaction_log.py index d6b186a0..3eddf176 100644 --- a/tests/expected_transaction_log.py +++ b/tests/expected_transaction_log.py @@ -11,7 +11,7 @@ _TRANSACTION_LOG_SUMMARY_PATH = RESOURCE_PATH / "transaction_log_summaries" def assert_transaction_log_matches( - output_directory: Path, + output_directory: Path | str, transaction_log: FileHandlerTransactionLog, transaction_log_summary_file_name: str, ): diff --git a/tests/resources.py b/tests/resources.py index 911b330a..8a426c9e 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -1,3 +1,4 @@ +import os import shutil from pathlib import Path @@ -7,5 +8,10 @@ RESOURCE_PATH: Path = Path("tests") / "resources" _FILE_FIXTURE_PATH: Path = RESOURCE_PATH / "file_fixtures" +def file_fixture_path(fixture_name: str) -> Path: + return _FILE_FIXTURE_PATH / fixture_name + + def copy_file_fixture(fixture_name: str, output_file_path: Path) -> None: - shutil.copy(_FILE_FIXTURE_PATH / fixture_name, output_file_path) + os.makedirs(os.path.dirname(output_file_path), exist_ok=True) + shutil.copy(file_fixture_path(fixture_name), output_file_path) diff --git a/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json b/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json index d1dd90b1..9c2ea6ef 100644 --- a/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json +++ b/tests/resources/expected_downloads_summaries/bandcamp/test_artist_url.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-Sithu Aye-download-archive.json": "0c58919b660f699f6aea2ec523c812c5", + ".ytdl-sub-Sithu Aye-download-archive.json": "ca37a404a860a2a6b6b0f38b659c9f17", "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/01 - Double Helix Reimagined.mp3": "56f7ee579031f4795230e68b63b15f6b", "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/02 - Skye Reimagined.mp3": "dfb24e0ef03e203d471bb81f854f5cd3", "Sithu Aye/[2021] 10 Years： Remixes and Reimaginings/03 - Baryofusion.mp3": "95bd9ab2238e5372f445c59daafd0138", diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt index 77140703..4b7fb2b5 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt @@ -1,5 +1,5 @@ { ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475", - "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c", + "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "d53121df33ac8c4a4699ec8919196552", "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json index 890aca91..f077d34c 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json @@ -2,6 +2,6 @@ ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75", "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8", "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b", - "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "b2d6388b4ddf8e3fbca042cb123672c3", + "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975", "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json index 081b7b0f..e0035559 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json @@ -1,5 +1,5 @@ { ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "de33d8f05d743c1f19091ff8b0115d3d", + "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "991b0eb62683c2194c5bdfa9a61ab18e", "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json index 88c93ad0..11201b53 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_best.json @@ -1,5 +1,5 @@ { ".ytdl-sub-single_song_best_test-download-archive.json": "f179ccfe0a9b3a76ae62122a3ccb58fd", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.m4a": "0144f43ed99468d8ef38c6374b8784a0", + "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.m4a": "114b35c29df84146cd98207f807b837b", "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json index f687d984..426abd6c 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single_old_format.json @@ -1,5 +1,5 @@ { ".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4", - "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "7d0473bd154637a151210c5116a63174", + "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/01 - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "11376667a11bb71565b520f8ce5fa303", "YouTube/[2019] YouTube Rewind 2019： For the Record ｜ #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json b/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json index a980f416..3847451d 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json @@ -1,6 +1,6 @@ { ".ytdl-sub-subtitles_embedded_test-download-archive.json": "a74ecea9f7844be23f470bbe702788f3", "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.jpg": "50ee47c80f679029f5d3503bb91b045a", - "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "6366af35ac989142a48af4eb89ed2be6", + "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "d5df2fa121748a54d6954c58e3d5884f", "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "e6ac56ce52c747e2e271f12208f9a538" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json b/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json index e87b531d..56c983a1 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json @@ -3,6 +3,6 @@ "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt": "b343c3bb9257b7ee7ba38f570a115b37", "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt": "fe8c6ee92cae6e059fd80fd61691adbe", "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.jpg": "50ee47c80f679029f5d3503bb91b045a", - "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "6366af35ac989142a48af4eb89ed2be6", + "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "d5df2fa121748a54d6954c58e3d5884f", "JMC/YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "e6ac56ce52c747e2e271f12208f9a538" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 8350d840..74e55e90 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -2,7 +2,7 @@ "Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "6fb0ce965b75035079f82c84ad341e85", + "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "246fa05b6443337785575987904848df", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "INFO_JSON", @@ -10,7 +10,7 @@ "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "43f271ef8d3a19877f0dc9bc5040b42a", + "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "ae1d5e2e3979cea3c96e6a4cfcae8073", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", @@ -34,7 +34,7 @@ "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4": "0101d64db7720efc133c9bf9beb157ab", + "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4": "df6a6901b168cd191c3b6fce2dde8f7a", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo": "e2b078891b27cfee4c791598d046be24", "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg": "e29d49433175de8a761af35c5307791f", "Project ⧸ Zombie/Season 2013/s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index 29dfe4cf..c4fb6cca 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json index 143b2310..c5490aa6 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-JMC-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video.json b/tests/resources/expected_downloads_summaries/youtube/test_video.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json index 04e7c7d9..b425d91d 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json @@ -1,4 +1,4 @@ { - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "718c187e6196c85eea73d16ebd489c91", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d9d2d12feee44ee97729b39ba981c542", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/file_fixtures/empty.txt b/tests/resources/file_fixtures/empty.txt new file mode 100644 index 00000000..7b4d68d7 --- /dev/null +++ b/tests/resources/file_fixtures/empty.txt @@ -0,0 +1 @@ +empty \ No newline at end of file diff --git a/tests/resources/file_fixtures/pz_download_archive.json b/tests/resources/file_fixtures/pz_download_archive.json new file mode 100644 index 00000000..41c4691e --- /dev/null +++ b/tests/resources/file_fixtures/pz_download_archive.json @@ -0,0 +1,123 @@ +{ + "0SVukUyys10": { + "extractor": "youtube", + "file_names": [ + "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg", + "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json", + "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4", + "Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo" + ], + "upload_date": "2011-02-01" + }, + "6sxlggREhMc": { + "extractor": "youtube", + "file_names": [ + "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c-thumb.jpg", + "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c.info.json", + "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c.mp4", + "Season 2011/s2011.e063001 - Project Zombie \uff5cFin\uff5c.nfo" + ], + "upload_date": "2011-06-30" + }, + "DBjFvs6HafU": { + "extractor": "youtube", + "file_names": [ + "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg", + "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json", + "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4", + "Season 2011/s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo" + ], + "upload_date": "2011-03-21" + }, + "HKTNxEqsN3Q": { + "extractor": "youtube", + "file_names": [ + "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1-thumb.jpg", + "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1.info.json", + "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1.mp4", + "Season 2010/s2010.e081301 - Oblivion Mod \uff02Falcor\uff02 p.1.nfo" + ], + "upload_date": "2010-08-13" + }, + "IV9Z4wcA-z8": { + "extractor": "youtube", + "file_names": [ + "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2-thumb.jpg", + "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2.info.json", + "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2.mp4", + "Season 2010/s2010.e120201 - Oblivion Mod \uff02Falcor\uff02 p.2.nfo" + ], + "upload_date": "2010-12-02" + }, + "LN2e6idGluI": { + "extractor": "youtube", + "file_names": [ + "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id-thumb.jpg", + "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.en.srt", + "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.info.json", + "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.mp4", + "Season 2018/s2018.e110201 - Jesse's Minecraft Server \uff5c IP mc.jesse.id.nfo" + ], + "upload_date": "2018-11-02" + }, + "c_PZdc0Zi7M": { + "extractor": "youtube", + "file_names": [ + "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c-thumb.jpg", + "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c.info.json", + "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c.mp4", + "Season 2013/s2013.e071901 - Project Zombie Rewind \uff5cTrailer\uff5c.nfo" + ], + "upload_date": "2013-07-19" + }, + "hP-PL_V2wR8": { + "extractor": "youtube", + "file_names": [ + "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer-thumb.jpg", + "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer.info.json", + "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer.mp4", + "Season 2018/s2018.e102901 - Jesse's Minecraft Server \uff5c Teaser Trailer.nfo" + ], + "upload_date": "2018-10-29" + }, + "hxV9b-1hjfw": { + "extractor": "youtube", + "file_names": [ + "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net)-thumb.jpg", + "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net).info.json", + "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net).mp4", + "Season 2011/s2011.e052901 - Project Zombie \uff5cOfficial Trailer\uff5c (IP\uff1a mc.projectzombie.beastnode.net).nfo" + ], + "upload_date": "2011-05-29" + }, + "qPybBrXspds": { + "extractor": "youtube", + "file_names": [ + "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg", + "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json", + "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4", + "Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo" + ], + "upload_date": "2011-02-27" + }, + "qbMJh2df1M4": { + "extractor": "youtube", + "file_names": [ + "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC]-thumb.jpg", + "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC].info.json", + "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC].mp4", + "Season 2011/s2011.e112101 - Skyrim 'Ultra HD w\u29f8Mods' [PC].nfo" + ], + "upload_date": "2011-11-21" + }, + "y5-3ovwQQ_U": { + "extractor": "youtube", + "file_names": [ + "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c-thumb.jpg", + "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c.info.json", + "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c.mp4", + "Season 2012/s2012.e012301 - Project Zombie \uff5cMap Trailer\uff5c.nfo" + ], + "upload_date": "2012-01-23" + } +} \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt b/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt new file mode 100644 index 00000000..75fd0729 --- /dev/null +++ b/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt @@ -0,0 +1,56 @@ +Files modified: +---------------------------------------- +{output_directory} + .ytdl-sub-pz-download-archive.json + +Files removed: +---------------------------------------- +{output_directory}/Season 2010 + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4 + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4 + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo +{output_directory}/Season 2011 + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4 + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4 + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4 + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4 + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo + s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg + s2011.e063001 - Project Zombie ｜Fin｜.info.json + s2011.e063001 - Project Zombie ｜Fin｜.mp4 + s2011.e063001 - Project Zombie ｜Fin｜.nfo + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4 + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo +{output_directory}/Season 2012 + s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg + s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json + s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4 + s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo +{output_directory}/Season 2013 + s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4 + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo +{output_directory}/Season 2018 + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4 + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo \ No newline at end of file diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 265cf840..74ec03d2 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -5,7 +5,7 @@ from typing import Optional import pytest from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.preset import PRESET_KEYS from ytdl_sub.utils.exceptions import ValidationException @@ -54,6 +54,8 @@ class TestConfigFilePartiallyValidatesPresets: excluded_plugins = [ "embed_thumbnail", # value is bool, not dict "format", # value is string, not dict + "filter_include", # is list + "filter_exclude", # is list ] if plugin not in excluded_plugins: self._partial_validate({plugin: {}}) @@ -71,8 +73,8 @@ class TestConfigFilePartiallyValidatesPresets: preset_dict={"download": {"bad_key": "nope"}}, expected_error_message="Validation error in partial_preset.download.1: " "'partial_preset.download.1' contains the field 'bad_key' which is not allowed. " - "Allowed fields: download_reverse, playlist_thumbnails, source_thumbnails, url, " - "variables", + "Allowed fields: download_reverse, include_sibling_metadata, playlist_thumbnails, " + "source_thumbnails, url, variables", ) @pytest.mark.parametrize( diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index 158b2434..dff51a50 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -129,7 +129,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist.", ): _ = Preset( config=config_file, @@ -145,7 +145,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Override variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -161,7 +161,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -182,7 +182,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -208,22 +208,177 @@ class TestPreset: }, ) - def test_preset_with_multi_url__contains_all_empty_urls_errors( - self, config_file, output_options + @pytest.mark.parametrize( + "entry_variable_name", + [ + "title", + "playlist_uid", + "source_title", + "playlist_max_upload_year", + ], + ) + def test_preset_error_override_variable_collides_with_entry_variable( + self, config_file, output_options, youtube_video, entry_variable_name: str ): with pytest.raises( ValidationException, match=re.escape( - "Validation error in test.download: Must contain at least one " - "url that is non-empty" + f"Override variable with name {entry_variable_name} cannot be used since" + " it is a built-in ytdl-sub entry variable name." ), ): _ = Preset( config=config_file, name="test", value={ - "download": [{"url": "{url}"}, {"url": "{url2}"}], - "output_options": output_options, - "overrides": {"url": "", "url2": ""}, + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, + "overrides": {entry_variable_name: "fail"}, + }, + ) + + def test_preset_error_override_variable_collides_added_variable( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + ValidationException, + match=re.escape( + f"Override variable with name subtitles_ext cannot be used since" + " it is added by a plugin." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "ack"}, + "subtitles": { + "embed_subtitles": True, + }, + "overrides": {"subtitles_ext": "collide"}, + }, + ) + + @pytest.mark.parametrize( + "function_name", + [ + "%extract_field_from_siblings", + "%sanitize", + "%array", + ], + ) + def test_preset_error_override_variable_collides_with_custom_function( + self, config_file, output_options, youtube_video, function_name: str + ): + with pytest.raises( + ValidationException, + match=re.escape( + f"Override function definition with name {function_name} cannot be used since" + " it is a built-in ytdl-sub function name." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, + "overrides": {function_name: "fail"}, + }, + ) + + def test_preset_error_override_added_variable_collides_with_built_in( + self, config_file, output_options + ): + with pytest.raises( + ValidationException, + match=re.escape( + "Cannot use the variable name title because it exists as a " + "built-in ytdl-sub variable name." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"title": "nope"}, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + }, + ) + + def test_preset_error_override_added_variable_collides_with_override( + self, config_file, output_options + ): + with pytest.raises( + ValidationException, + match=re.escape( + "Override variable with name the_bad_one cannot be used since " + "it is added by a plugin." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"the_bad_one": "should error"}, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + "overrides": {"the_bad_one": "ack"}, + }, + ) + + @pytest.mark.parametrize( + "name", ["!ack", "*asfsaf", "1234352", "--234asdf", "___asdf", "1asdfasdfasd"] + ) + @pytest.mark.parametrize("is_function", [True, False]) + def test_preset_error_overrides_invalid_variable_name( + self, config_file, youtube_video, output_options, name: str, is_function: bool + ): + name_type = "function" if is_function else "variable" + name = f"%{name}" if is_function else name + + with pytest.raises( + ValidationException, + match=re.escape( + f"Override {name_type} with name {name} is invalid." + " Names must be lower_snake_cased and begin with a letter." + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": output_options, + "overrides": {name: "ack"}, + }, + ) + + def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options): + with pytest.raises( + ValidationException, + match=re.escape( + "variable the_bad_one cannot use the variables subtitles_ext because it " + "depends on other variables that are computed later in execution" + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"the_bad_one": "{subtitles_ext}"}, + }, + "subtitles": { + "embed_subtitles": True, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, }, ) diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 291079d8..bd7122c5 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -5,23 +5,11 @@ from typing import Dict from unittest.mock import patch import pytest -from mergedeep import mergedeep from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.plugins.nfo_tags import NfoTagsOptions -from ytdl_sub.subscriptions.subscription import FILE_SUBSCRIPTION_VALUE_KEY from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ValidationException -from ytdl_sub.utils.yaml import load_yaml - - -@pytest.fixture -def config_file_with_subscription_value(config_file: ConfigFile): - config_dict = config_file.as_dict() - mergedeep.merge( - config_dict, {"configuration": {"subscription_value": "test_config_subscription_value"}} - ) - return ConfigFile.from_dict(config_dict) @contextmanager @@ -43,8 +31,6 @@ def preset_with_file_preset(youtube_video: Dict, output_options: Dict): "tags": {"key-3": "file_preset"}, }, "overrides": { - "test_file_subscription_value": "original", - "test_config_subscription_value": "original", "subscription_indent_1": "original_1", "subscription_indent_2": "original_2", "current_override": "__preset__", @@ -71,12 +57,11 @@ def preset_with_subscription_value(preset_with_file_preset: Dict): @pytest.fixture -def preset_with_subscription_file_value(preset_with_subscription_value: Dict): +def subscription_with_period_in_name(preset_with_file_preset: Dict): return dict( - preset_with_subscription_value, + preset_with_file_preset, **{ - "__value__": "test_file_subscription_value", - "test_value": "is_overwritten", + "Mr. Beast": "is_overwritten", }, ) @@ -171,7 +156,7 @@ def preset_with_subscription_overrides_tilda( preset_with_subscription_value, **{ "parent_preset_2 | parent_preset_1": { - "~test_2_1": { + "~ test_2_1": { "current_override": "test_2_1", } }, @@ -179,6 +164,27 @@ def preset_with_subscription_overrides_tilda( ) +@pytest.fixture +def preset_with_subscription_overrides_map( + preset_with_subscription_value: Dict, +): + return dict( + preset_with_subscription_value, + **{ + "parent_preset_2 | parent_preset_1": { + "+ test_2_1": { + "custom_key": "custom_value", + "custom_list": [ + "elem1", + "elem2", + "elem3", + ], + } + }, + }, + ) + + @pytest.fixture def preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors( preset_with_subscription_value: Dict, @@ -219,176 +225,114 @@ def test_subscription_file_preset_applies(config_file: ConfigFile, preset_with_f "key-4": "test_preset", } - overrides = preset_sub.overrides.dict_with_format_strings + overrides = preset_sub.overrides.script # preset overrides take precedence over __preset__ - assert overrides.get("current_override") == "test_preset" - - -def test_subscription_file_value_applies( - config_file: ConfigFile, preset_with_subscription_file_value: Dict -): - with mock_load_yaml(preset_dict=preset_with_subscription_file_value): - subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") - assert len(subs) == 2 - - # Test __value__ worked correctly - value_sub = subs[1] - overrides = value_sub.overrides.dict_with_format_strings - assert value_sub.name == "test_value" - - assert overrides.get("test_file_subscription_value") == "is_overwritten" - assert overrides.get("test_file_subscription_value") - assert overrides.get("subscription_value") == "is_overwritten" - assert overrides.get("current_override") == "__preset__" # ensure __preset__ takes precedence - - -def test_subscription_file_value_applies_sub_file_takes_precedence( - config_file_with_subscription_value: ConfigFile, - preset_with_subscription_file_value: Dict, -): - with mock_load_yaml(preset_dict=preset_with_subscription_file_value): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) - assert len(subs) == 2 - - # Test __value__ worked correctly - value_sub = subs[1].overrides.dict_with_format_strings - assert value_sub.get("test_file_subscription_value") == "is_overwritten" - assert value_sub.get("test_config_subscription_value") == "original" - assert value_sub.get("subscription_name") == "test_value" - assert value_sub.get("subscription_name_sanitized") == "test_value" - assert value_sub.get("subscription_value") == "is_overwritten" - assert value_sub.get("current_override") == "__preset__" # ensure __preset__ takes precedence - - -def test_subscription_file_value_applies_from_config( - config_file_with_subscription_value: ConfigFile, preset_with_subscription_value: Dict -): - with mock_load_yaml(preset_dict=preset_with_subscription_value): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) - assert len(subs) == 2 - - # Test __value__ worked correctly from the config - value_sub = subs[1].overrides.dict_with_format_strings - assert value_sub.get("test_file_subscription_value") == "original" - assert value_sub.get("test_config_subscription_value") == "is_overwritten" - assert value_sub.get("subscription_name") == "test_value" - assert value_sub.get("subscription_name_sanitized") == "test_value" - assert value_sub.get("subscription_value") == "is_overwritten" - assert value_sub.get("current_override") == "__preset__" # ensure __preset__ takes precedence - - -def test_subscription_file_value_applies_from_config_and_nested( - config_file_with_subscription_value: ConfigFile, - preset_with_subscription_value_nested_presets: Dict, -): - with mock_load_yaml(preset_dict=preset_with_subscription_value_nested_presets): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) - assert len(subs) == 4 - - # Test __value__ worked correctly from the config - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings - - assert sub_1.get("test_config_subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_name_sanitized") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence - - assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert overrides.get("current_override").native == "test_preset" def test_subscription_list( - config_file_with_subscription_value: ConfigFile, + config_file: ConfigFile, preset_with_subscription_list: Dict, ): with mock_load_yaml(preset_dict=preset_with_subscription_list): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 3 - # Test __value__ worked correctly from the config - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_value_1") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_value_2") == "is_2_1_list_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_value_1").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_value_2").native == "is_2_1_list_2" + assert ( + sub_2_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence def test_subscription_overrides_tilda( - config_file_with_subscription_value: ConfigFile, + config_file: ConfigFile, preset_with_subscription_overrides_tilda: Dict, ): with mock_load_yaml(preset_dict=preset_with_subscription_overrides_tilda): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 3 - # Test __value__ worked correctly from the config - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" - assert sub_2_1.get("current_override") == "test_2_1" # tilda sub takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("current_override").native == "test_2_1" # tilda sub takes precedence + + +def test_subscription_overrides_map( + config_file: ConfigFile, + preset_with_subscription_overrides_map: Dict, +): + with mock_load_yaml(preset_dict=preset_with_subscription_overrides_map): + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") + assert len(subs) == 3 + + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script + + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_map").native == { + "custom_key": "custom_value", + "custom_list": [ + "elem1", + "elem2", + "elem3", + ], + } + + +def test_subscription_with_period_in_name( + config_file: ConfigFile, + subscription_with_period_in_name: Dict, +): + with mock_load_yaml(preset_dict=subscription_with_period_in_name): + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") + assert len(subs) == 2 + + assert subs[1].name == "Mr. Beast" + assert subs[1].overrides.script.get("subscription_name").native == "Mr. Beast" def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables( - config_file_with_subscription_value: ConfigFile, + config_file: ConfigFile, preset_with_subscription_value_nested_presets_and_indent_variables: Dict, ): with mock_load_yaml( preset_dict=preset_with_subscription_value_nested_presets_and_indent_variables ): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 4 - # Test __value__ worked correctly from the config - sub_test_value = [sub for sub in subs if sub.name == "test_value"][ - 0 - ].overrides.dict_with_format_strings - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_test_value = [sub for sub in subs if sub.name == "test_value"][0].overrides.script + sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.script + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_test_value.get("subscription_indent_1") == "original_1" - assert sub_test_value.get("subscription_indent_2") == "original_2" + assert sub_test_value.get("subscription_indent_1").native == "original_1" + assert sub_test_value.get("subscription_indent_2").native == "original_2" - assert sub_1.get("test_config_subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_name_sanitized") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_indent_1") == "INDENT_1" - assert sub_1.get("subscription_indent_2") == "INDENT_2" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_1.get("subscription_name").native == "test_1" + assert sub_1.get("subscription_value").native == "is_1_overwritten" + assert sub_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_1.get("subscription_indent_2").native == "INDENT_2" + assert ( + sub_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence - assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_indent_1") == "INDENT_1" - assert sub_2_1.get("subscription_indent_2") == "original_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_2_1.get("subscription_indent_2").native == "original_2" + assert ( + sub_2_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence @pytest.mark.parametrize("all_same_line", [True, False]) def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables_same_line( - config_file_with_subscription_value: ConfigFile, + config_file: ConfigFile, preset_with_subscription_value_nested_presets_and_indent_variables_same_line: Dict, preset_with_subscription_value_nested_presets_and_indent_variables_all_same_line: Dict, all_same_line: bool, @@ -400,42 +344,35 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia ) with mock_load_yaml(preset_dict=preset_dict): - subs = Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 4 - # Test __value__ worked correctly from the config - sub_test_value = [sub for sub in subs if sub.name == "test_value"][ - 0 - ].overrides.dict_with_format_strings - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_test_value = [sub for sub in subs if sub.name == "test_value"][0].overrides.script + sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.script + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_test_value.get("subscription_indent_1") == "original_1" - assert sub_test_value.get("subscription_indent_2") == "original_2" + assert sub_test_value.get("subscription_indent_1").native == "original_1" + assert sub_test_value.get("subscription_indent_2").native == "original_2" - assert sub_1.get("test_config_subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_name_sanitized") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_indent_1") == "INDENT_1" - assert sub_1.get("subscription_indent_2") == "INDENT_2" - assert sub_1.get("subscription_indent_3") == "INDENT_3" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_1.get("subscription_name").native == "test_1" + assert sub_1.get("subscription_value").native == "is_1_overwritten" + assert sub_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_1.get("subscription_indent_2").native == "INDENT_2" + assert sub_1.get("subscription_indent_3").native == "INDENT_3" + # ensure __preset__ takes precedence + assert sub_1.get("current_override").native == "__preset__" - assert sub_2_1.get("test_config_subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_name_sanitized") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_indent_1") == "INDENT_1" - assert sub_2_1.get("subscription_indent_2") == "original_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence - assert "subscription_indent_3" not in sub_2_1 + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_2_1.get("subscription_indent_2").native == "original_2" + # ensure __preset__ takes precedence + assert sub_2_1.get("current_override").native == "__preset__" + assert "subscription_indent_3" not in sub_2_1.variable_names def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables_same_line_old_format_errors( - config_file_with_subscription_value: ConfigFile, + config_file: ConfigFile, preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors: Dict, ): with mock_load_yaml( @@ -447,28 +384,13 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia "To use as a subscription indent value, define it as '= INDENT_3'" ), ): - Subscription.from_file_path( - config=config_file_with_subscription_value, subscription_path="mocked" - ) - - -def test_subscription_file_bad_value(config_file: ConfigFile): - with mock_load_yaml(preset_dict={"__value__": {"should be": "string"}}), pytest.raises( - ValidationException, - match=re.escape( - f"Using {FILE_SUBSCRIPTION_VALUE_KEY} in a subscription" - f"must be a string that corresponds to an override variable" - ), - ): - _ = Subscription.from_file_path(config=config_file, subscription_path="mocked") + Subscription.from_file_path(config=config_file, subscription_path="mocked") def test_subscription_file_using_conflicting_preset_name(config_file: ConfigFile): with mock_load_yaml( preset_dict={ - "= INDENTS_IN_ERR_MSG ": { - "=ANOTHER": {"jellyfin_tv_show_by_date": "single value, __value__ not defined"} - } + "= INDENTS_IN_ERR_MSG ": {"=ANOTHER": {"jellyfin_tv_show_by_date": "single value"}} } ), pytest.raises( ValidationException, @@ -496,13 +418,15 @@ def test_tv_show_subscriptions(config_file: ConfigFile, tv_show_subscriptions_pa assert len(subs) == 7 assert subs[3].name == "Jake Trains" - jake_train_overrides = subs[3].overrides.dict_with_format_strings + jake_train_overrides = subs[3].overrides.script - assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_name_sanitized"] == "Jake Trains" - assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" - assert jake_train_overrides["subscription_indent_1"] == "Kids" - assert jake_train_overrides["subscription_indent_2"] == "TV-Y" + assert jake_train_overrides.get("subscription_name").native == "Jake Trains" + assert ( + jake_train_overrides.get("subscription_value").native + == "https://www.youtube.com/@JakeTrains" + ) + assert jake_train_overrides.get("subscription_indent_1").native == "Kids" + assert jake_train_overrides.get("subscription_indent_2").native == "TV-Y" def test_advanced_tv_show_subscriptions( @@ -514,22 +438,20 @@ def test_advanced_tv_show_subscriptions( assert len(subs) == 9 assert subs[3].name == "Jake Trains" - jake_train_overrides = subs[3].overrides.dict_with_format_strings + jake_train_overrides = subs[3].overrides.script - assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_name_sanitized"] == "Jake Trains" - assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" - assert jake_train_overrides["subscription_indent_1"] == "Kids" - assert jake_train_overrides["subscription_indent_2"] == "TV-Y" + assert jake_train_overrides.get("subscription_name").native == "Jake Trains" + assert ( + jake_train_overrides.get("subscription_value").native + == "https://www.youtube.com/@JakeTrains" + ) + assert jake_train_overrides.get("subscription_indent_1").native == "Kids" + assert jake_train_overrides.get("subscription_indent_2").native == "TV-Y" assert subs[5].name == "Gardening with Ciscoe" overrides = subs[5].overrides - assert overrides.apply_formatter(overrides.dict["subscription_name"]) == "Gardening with Ciscoe" - assert ( - overrides.apply_formatter(overrides.dict["subscription_name_sanitized"]) - == "Gardening with Ciscoe" - ) + assert overrides.script.get("subscription_name").native == "Gardening with Ciscoe" assert ( overrides.apply_formatter(overrides.dict["url"]) == "https://www.youtube.com/@gardeningwithciscoe4430" @@ -547,12 +469,14 @@ def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_pat assert len(subs) == 14 assert subs[2].name == "Stan Getz" - monk = subs[2].overrides.dict_with_format_strings + monk = subs[2].overrides.script - assert monk["subscription_name"] == "Stan Getz" - assert monk["subscription_name_sanitized"] == "Stan Getz" - assert monk["subscription_value"] == "https://www.youtube.com/@stangetzofficial/releases" - assert monk["subscription_indent_1"] == "Jazz" + assert monk.get("subscription_name").native == "Stan Getz" + assert ( + monk.get("subscription_value").native + == "https://www.youtube.com/@stangetzofficial/releases" + ) + assert monk.get("subscription_indent_1").native == "Jazz" def test_music_video_subscriptions(default_config: ConfigFile, music_video_subscription_path: Path): @@ -562,15 +486,14 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc assert len(subs) == 3 assert subs[1].name == "Michael Jackson" - monk = subs[1].overrides.dict_with_format_strings + monk = subs[1].overrides.script - assert monk["subscription_name"] == "Michael Jackson" - assert monk["subscription_name_sanitized"] == "Michael Jackson" + assert monk.get("subscription_name").native == "Michael Jackson" assert ( - monk["subscription_value"] + monk.get("subscription_value").native == "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" ) - assert monk["subscription_indent_1"] == "Pop" + assert monk.get("subscription_indent_1").native == "Pop" def test_default_docker_config_and_subscriptions(): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index f25845a7..b20b302c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,18 +12,10 @@ from resources import copy_file_fixture from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader from ytdl_sub.downloaders.ytdlp import YTDLP -from ytdl_sub.entries.variables.kwargs import DESCRIPTION -from ytdl_sub.entries.variables.kwargs import EPOCH -from ytdl_sub.entries.variables.kwargs import EXT -from ytdl_sub.entries.variables.kwargs import EXTRACTOR -from ytdl_sub.entries.variables.kwargs import PLAYLIST_COUNT -from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import PLAYLIST_INDEX -from ytdl_sub.entries.variables.kwargs import PLAYLIST_TITLE -from ytdl_sub.entries.variables.kwargs import TITLE -from ytdl_sub.entries.variables.kwargs import UID -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE -from ytdl_sub.entries.variables.kwargs import WEBPAGE_URL +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +v: VariableDefinitions = VARIABLES @pytest.fixture @@ -62,22 +54,23 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable: is_extracted_audio: bool = False, ) -> Dict: entry_dict = { - UID: uid, - EPOCH: 1596878400, - PLAYLIST_TITLE: playlist_title, - PLAYLIST_INDEX: playlist_index, - PLAYLIST_COUNT: playlist_count, - EXTRACTOR: "mock-entry-dict", - TITLE: f"Mock Entry {uid}", - EXT: "mp4", - UPLOAD_DATE: upload_date, - WEBPAGE_URL: f"https://{uid}.com", - PLAYLIST_ENTRY: {"thumbnails": []}, - DESCRIPTION: "The Description", + v.uid.metadata_key: uid, + v.epoch.metadata_key: 1596878400, + v.playlist_title.metadata_key: playlist_title, + v.playlist_index.metadata_key: playlist_index, + v.playlist_count.metadata_key: playlist_count, + v.extractor.metadata_key: "mock-entry-extractor", + v.extractor_key.metadata_key: "mock-entry-dict", + v.title.metadata_key: f"Mock Entry {uid}", + v.ext.metadata_key: "mp4", + v.upload_date.metadata_key: upload_date, + v.webpage_url.metadata_key: f"https://{uid}.com", + v.playlist_metadata.metadata_key: {"thumbnails": []}, + v.description.metadata_key: "The Description", } if is_youtube_channel: - entry_dict[PLAYLIST_ENTRY]["thumbnails"] = [ + entry_dict[v.playlist_metadata.metadata_key]["thumbnails"] = [ { "id": "avatar_uncropped", "url": "https://avatar_uncropped.com", diff --git a/tests/unit/docgen/__init__.py b/tests/unit/docgen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py new file mode 100644 index 00000000..82768347 --- /dev/null +++ b/tests/unit/docgen/test_docgen.py @@ -0,0 +1,30 @@ +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.entry_variables import EntryVariablesDocGen +from tools.docgen.override_variables import OverrideVariablesDocGen +from tools.docgen.plugins import PluginsDocGen +from tools.docgen.scripting_functions import ScriptingFunctionsDocGen +from ytdl_sub.utils.file_handler import get_md5_hash + + +def _test_doc_gen(doc_gen: Type[DocGen]) -> None: + expected_md5_hash = get_md5_hash(doc_gen.generate_and_maybe_write_to_file()) + with open(doc_gen.LOCATION, "r", encoding="utf-8") as file_doc: + md5_hash = get_md5_hash(file_doc.read()) + + assert md5_hash == expected_md5_hash + + +class TestDocGen: + def test_entry_variables_generated(self): + _test_doc_gen(EntryVariablesDocGen) + + def test_override_variables_generated(self): + _test_doc_gen(OverrideVariablesDocGen) + + def test_scripting_functions_generated(self): + _test_doc_gen(ScriptingFunctionsDocGen) + + def test_plugins_generated(self): + _test_doc_gen(PluginsDocGen) diff --git a/tests/unit/docgen/test_docgen_regenerate_disabled.py b/tests/unit/docgen/test_docgen_regenerate_disabled.py new file mode 100644 index 00000000..836b13ea --- /dev/null +++ b/tests/unit/docgen/test_docgen_regenerate_disabled.py @@ -0,0 +1,5 @@ +from tools.docgen.docgen import REGENERATE_DOCS + + +def test_docgen_regenerate_disabled(): + assert REGENERATE_DOCS is False diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 16199c7d..9421d35e 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -163,6 +163,7 @@ def mock_entry_kwargs( "id": uid, "epoch": 1596878400, "extractor": extractor, + "extractor_key": "test_extractor_key", "title": title, "ext": ext, "upload_date": upload_date, @@ -173,20 +174,4 @@ def mock_entry_kwargs( @pytest.fixture def mock_entry(mock_entry_kwargs): - return Entry(entry_dict=mock_entry_kwargs, working_directory=".") - - -@pytest.fixture -def validate_entry_dict_contains_valid_formatters(): - def _validate_entry_dict_contains_valid_formatters(entry: Entry): - for key, value in entry.to_dict().items(): - expected_string = f"test {value} formatting works" - formatter = StringFormatterValidator( - name="test", value=f"test {{{key}}} formatting works" - ) - - assert formatter.apply_formatter(entry.to_dict()) == expected_string - - return True - - return _validate_entry_dict_contains_valid_formatters + return Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script() diff --git a/tests/unit/entries/test_entry.py b/tests/unit/entries/test_entry.py index 4885420d..4797092d 100644 --- a/tests/unit/entries/test_entry.py +++ b/tests/unit/entries/test_entry.py @@ -1,22 +1,19 @@ import pytest +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + +v: VariableDefinitions = VARIABLES + class TestEntry(object): def test_entry_to_dict(self, mock_entry, mock_entry_to_dict): - assert mock_entry.to_dict() == mock_entry_to_dict + out = mock_entry.to_dict() - def test_entry_dict_contains_valid_formatters( - self, mock_entry, validate_entry_dict_contains_valid_formatters - ): - assert validate_entry_dict_contains_valid_formatters(mock_entry) - - def test_entry_missing_kwarg(self, mock_entry): - key = "dne" - expected_error_msg = f"Expected '{key}' in Entry but does not exist." - - assert mock_entry.kwargs_contains(key) is False - with pytest.raises(KeyError, match=expected_error_msg): - mock_entry.kwargs(key) + # HACK: Ensure legacy variables are in new output and equal + for key, expected_value in mock_entry_to_dict.items(): + assert out[key] == expected_value, f"{key} does not equal" @pytest.mark.parametrize( "upload_date, year_rev, month_rev, day_rev, month_rev_pad, day_rev_pad", @@ -26,16 +23,23 @@ class TestEntry(object): ], ) def test_entry_reverse_variables( - self, mock_entry, upload_date, year_rev, month_rev, day_rev, month_rev_pad, day_rev_pad + self, + mock_entry_kwargs, + upload_date, + year_rev, + month_rev, + day_rev, + month_rev_pad, + day_rev_pad, ): - mock_entry._kwargs["upload_date"] = upload_date - assert mock_entry.upload_year_truncated_reversed == year_rev - assert mock_entry.upload_month_reversed == month_rev - assert mock_entry.upload_day_reversed == day_rev - - assert mock_entry.upload_month_reversed_padded == month_rev_pad - assert mock_entry.upload_day_reversed_padded == day_rev_pad + mock_entry_kwargs["upload_date"] = upload_date + entry = Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script() + assert entry.get(v.upload_year_truncated_reversed, int) == year_rev + assert entry.get(v.upload_month_reversed, int) == month_rev + assert entry.get(v.upload_day_reversed, int) == day_rev + assert entry.get(v.upload_month_reversed_padded, str) == month_rev_pad + assert entry.get(v.upload_day_reversed_padded, str) == day_rev_pad @pytest.mark.parametrize( "upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad", @@ -45,11 +49,12 @@ class TestEntry(object): ], ) def test_entry_upload_day_of_year_variables( - self, mock_entry, upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad + self, mock_entry_kwargs, upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad ): - mock_entry._kwargs["upload_date"] = upload_date + mock_entry_kwargs["upload_date"] = upload_date + entry = Entry(entry_dict=mock_entry_kwargs, working_directory=".").initialize_script() - assert mock_entry.upload_day_of_year == day_year - assert mock_entry.upload_day_of_year_reversed == day_year_rev - assert mock_entry.upload_day_of_year_padded == day_year_pad - assert mock_entry.upload_day_of_year_reversed_padded == day_year_rev_pad + assert entry.get(v.upload_day_of_year, int) == day_year + assert entry.get(v.upload_day_of_year_reversed, int) == day_year_rev + assert entry.get(v.upload_day_of_year_padded, str) == day_year_pad + assert entry.get(v.upload_day_of_year_reversed_padded, str) == day_year_rev_pad diff --git a/tests/unit/plugins/test_regex_capture.py b/tests/unit/plugins/test_regex_capture.py deleted file mode 100644 index 8b137891..00000000 --- a/tests/unit/plugins/test_regex_capture.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/unit/prebuilt_presets/test_prebuilt_presets.py b/tests/unit/prebuilt_presets/test_prebuilt_presets.py index e4176333..1a891640 100644 --- a/tests/unit/prebuilt_presets/test_prebuilt_presets.py +++ b/tests/unit/prebuilt_presets/test_prebuilt_presets.py @@ -152,6 +152,7 @@ class TestPrebuiltTVShowPresets: is_youtube_channel: bool, is_many_urls: bool, ): + expected_summary_name = "unit/{}/{}/is_yt_{}{}".format( media_player_preset, tv_show_structure_preset, @@ -204,20 +205,22 @@ class TestPrebuiltTVShowPresets: "_many_urls" if is_many_urls else "", ) - reformatted_subscription = Subscription.from_dict( - config=config, - preset_name=subscription_name, - preset_dict={ - "preset": parent_presets + [reformatted_tv_show_structure_preset], - "output_options": { - "migrated_download_archive_name": ".ytdl-sub-{tv_show_name_sanitized}-download-archive.json" - }, - "overrides": { - "url": "https://your.name.here", - "tv_show_name": "Best Prebuilt TV Show by Date", - "tv_show_directory": output_directory, - }, + reformatted_preset_dict = { + "preset": parent_presets + [reformatted_tv_show_structure_preset], + "output_options": { + "migrated_download_archive_name": ".ytdl-sub-{tv_show_name_sanitized}-download-archive.json" }, + "overrides": { + "url": "https://your.name.here", + "tv_show_name": "Best Prebuilt TV Show by Date", + "tv_show_directory": output_directory, + }, + } + if is_many_urls: + reformatted_preset_dict["overrides"]["url2"] = "https://url.number.2.here" + + reformatted_subscription = Subscription.from_dict( + config=config, preset_name=subscription_name, preset_dict=reformatted_preset_dict ) reformatted_transaction_log = reformatted_subscription.update_with_info_json(dry_run=False) diff --git a/tests/unit/script/__init__.py b/tests/unit/script/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/script/conftest.py b/tests/unit/script/conftest.py new file mode 100644 index 00000000..951f5aff --- /dev/null +++ b/tests/unit/script/conftest.py @@ -0,0 +1,14 @@ +from ytdl_sub.script.script import Script + + +def single_variable_output(script: str): + output = ( + Script( + { + "output": script, + } + ) + .resolve(update=True) + .get_native("output") + ) + return output diff --git a/tests/unit/script/functions/__init__.py b/tests/unit/script/functions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/script/functions/test_array_functions.py b/tests/unit/script/functions/test_array_functions.py new file mode 100644 index 00000000..06a3b150 --- /dev/null +++ b/tests/unit/script/functions/test_array_functions.py @@ -0,0 +1,184 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException + + +class TestArrayFunctions: + def test_array_extend(self): + output = ( + Script( + { + "array1": "{['a']}", + "array2": "{['b']}", + "array3": "{['c']}", + "output": "{%array_extend(array1, array2, array3)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == ["a", "b", "c"] + + def test_array_at(self): + output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}") + assert output == "b" + + def test_array_flatten(self): + output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}") + assert output == ["a", "b", "c"] + + def test_array_contains(self): + output = single_variable_output("{%array_contains(['a', ['b'], [['c']]], [['c']])}") + assert output is True + + def test_array_index(self): + output = single_variable_output("{%array_index(['a', ['b'], [['c']]], [['c']])}") + assert output == 2 + + def test_array_slice(self): + output = single_variable_output("{%array_slice(['a', ['b'], [['c']]], 1, -1)}") + assert output == [["b"]] + + def test_array_reverse(self): + output = single_variable_output("{%array_reverse(['a', 'b', 'c'])}") + assert output == ["c", "b", "a"] + + def test_array_product(self): + output = single_variable_output("{%array_product(['a', 'b', 'c'], ['arg'])}") + assert output == [["a", "arg"], ["b", "arg"], ["c", "arg"]] + + def test_array_apply(self): + output = single_variable_output("{%array_apply(['a', 'b', 'c'], %capitalize)}") + assert output == ["A", "B", "C"] + + def test_array_reduce(self): + output = single_variable_output("{%array_reduce([1, 2, 3, 4], %add)}") + assert output == 10 + + def test_array_reduce_complex(self): + output = ( + Script( + { + "%custom_get": """{ + %if( + %bool(siblings_array), + %array_apply_fixed( + siblings_array, + %string($0), + %map_get + ) + [] + ) + }""", + "siblings_array": """{ + [ + {'upload_date': '20200101'}, + {'upload_date': '19940101'} + ] + }""", + "upload_date": "20230101", + "output": """{ + %array_reduce( + %if_passthrough( + %custom_get('upload_date'), + [ upload_date ] + ), + %max + ) + }""", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == "20200101" + + def test_array_enumerate(self): + output = ( + Script( + { + "%enumerate_output": "{[$0, $1]}", + "array1": "{['a', 'b', 'c']}", + "output": "{%array_enumerate(array1, %enumerate_output)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [[0, "a"], [1, "b"], [2, "c"]] + + def test_cast_array(self): + output = ( + Script( + { + "map_test": "{ {'key': [1, 2, 3]} }", + "output": "{ %array( %map_get(map_test, 'key') )}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [1, 2, 3] + + def test_array_size(self): + output = single_variable_output("{%array_size([1, 2, 3])}") + assert output == 3 + + def test_cast_array_errors_cannot_cast(self): + with pytest.raises( + FunctionRuntimeException, match="Tried and failed to cast Integer as an Array" + ): + single_variable_output("{%array(1)}") + + def test_array_overlay(self): + output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5])}") + assert output == [4, 5, 3] + + output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5, 6, 7, 8])}") + assert output == [4, 5, 6, 7, 8] + + output = single_variable_output("{%array_overlay([1, 2, 3], [4, 5, 6, 7, 8], True)}") + assert output == [1, 2, 3, 7, 8] + + def test_array_first(self): + output = single_variable_output( + "{%array_first(['', false, null, [], {}, 0, 'hi', 'no'], 'fallback')}" + ) + assert output == "hi" + + output = single_variable_output("{%array_first(['', false, null, [], {}, 0], 'fallback')}") + assert output == "fallback" + + def test_array_apply_fixed(self): + output = ( + Script( + { + "map_test": "{ {'key1': 7, 'key2': 8, 'key3': 9} }", + "output": """{ + %array_apply_fixed( ['key1', 'key2', 'key3'], map_test, %map_get, True) + }""", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [7, 8, 9] + + output = ( + Script( + { + "output": "{%array_apply_fixed( ['key1', 'key2', 'key3'], '3', %contains)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [False, False, True] diff --git a/tests/unit/script/functions/test_boolean_functions.py b/tests/unit/script/functions/test_boolean_functions.py new file mode 100644 index 00000000..dfdd79c6 --- /dev/null +++ b/tests/unit/script/functions/test_boolean_functions.py @@ -0,0 +1,142 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script + + +class TestBooleanFunctions: + @pytest.mark.parametrize( + "lhs, rhs, expected_output", + [ + ("'abc'", "'abc'", True), + ("123", "123", True), + ("3.14", "3.14", True), + ("True", "True", True), + ("False", "False", True), + ("[1, 2, 3]", "[1, 2, 3]", True), + ("{'key': 'value'}", "{'key': 'value'}", True), + ("{'key': 'value'}", 5, False), + ("{'key': 'value'}", "[1, 2, 3]", False), + ], + ) + @pytest.mark.parametrize("is_ne", [True, False]) + def test_eq_ne(self, lhs: str, rhs: str, expected_output: bool, is_ne: bool): + op = "ne" if is_ne else "eq" + output = single_variable_output(f"{{%{op}({lhs}, {rhs})}}") + + if is_ne: + assert output != expected_output + else: + assert output == expected_output + + @pytest.mark.parametrize( + "lhs, rhs, expected_output", + [ + ("'abc'", "'abc'", True), + ("123", "123", True), + ("3.14", "3.14", True), + ("3.14", "4.0", True), + ], + ) + @pytest.mark.parametrize("is_gt", [True, False]) + def test_lte_gt(self, lhs: str, rhs: str, expected_output: bool, is_gt: bool): + op = "gt" if is_gt else "lte" + output = single_variable_output(f"{{%{op}({lhs}, {rhs})}}") + + if is_gt: + assert output != expected_output + else: + assert output == expected_output + + @pytest.mark.parametrize( + "lhs, rhs, expected_output", + [ + ("'abc'", "'abc'", True), + ("123", "123", True), + ("3.14", "3.14", True), + ("5.32", "4", True), + ("3.14", "4.0", False), + ], + ) + @pytest.mark.parametrize("is_lt", [True, False]) + def test_gte_lt(self, lhs: str, rhs: str, expected_output: bool, is_lt: bool): + op = "lt" if is_lt else "gte" + output = single_variable_output(f"{{%{op}({lhs}, {rhs})}}") + + if is_lt: + assert output != expected_output + else: + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("True", True), + ("True, True", True), + ("True, True, True", True), + ("False", False), + ("False, True", False), + ("True, False, True", False), + ], + ) + def test_and(self, values: str, expected_output: bool): + output = single_variable_output(f"{{%and({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("True", True), + ("True, True", True), + ("True, True, True", True), + ("False", False), + ("False, True", True), + ("True, False, True", True), + ("False, False, False", False), + ], + ) + def test_or(self, values: str, expected_output: bool): + output = single_variable_output(f"{{%or({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("True", True), + ("True, True", False), + ("True, True, True", False), + ("False", False), + ("False, True", True), + ("True, False, True", False), + ("False, False, False", False), + ("False, True, False", True), + ], + ) + def test_xor(self, values: str, expected_output: bool): + output = single_variable_output(f"{{%xor({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "value, expected_output", + [ + ("True", False), + ("False", True), + ], + ) + def test_not(self, value: str, expected_output: bool): + output = single_variable_output(f"{{%not({value})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "value, expected_output", + [ + ("null", True), + ("''", True), + ("0", False), + ("{}", False), + ("'h'", False), + ], + ) + def test_is_null(self, value: str, expected_output: bool): + output = single_variable_output(f"{{%is_null({value})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_conditional_functions.py b/tests/unit/script/functions/test_conditional_functions.py new file mode 100644 index 00000000..f7217915 --- /dev/null +++ b/tests/unit/script/functions/test_conditional_functions.py @@ -0,0 +1,35 @@ +import pytest +from unit.script.conftest import single_variable_output + + +class TestConditionalFunction: + @pytest.mark.parametrize( + "function_str, expected_output", + [ + ("{%if(True, True, False)}", True), + ("{%if(False, True, False)}", False), + ], + ) + def test_if_function(self, function_str: str, expected_output: bool): + output = single_variable_output(function_str) + assert output == expected_output + + def test_nested_if_function(self): + output = single_variable_output( + """{ + %if( + True, + %if( + True, + %if( + True, + "winner", + True + ), + True + ), + True + ) + }""" + ) + assert output == "winner" diff --git a/tests/unit/script/functions/test_date_functions.py b/tests/unit/script/functions/test_date_functions.py new file mode 100644 index 00000000..1c826ea4 --- /dev/null +++ b/tests/unit/script/functions/test_date_functions.py @@ -0,0 +1,17 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "timestamp, date_format, expected_output", + [ + (1596877200, "'%Y%m%d'", "20200808"), + (1596877200, "'%m'", "08"), + ], + ) + def test_datetime_strftime(self, timestamp: int, date_format: str, expected_output: str): + output = single_variable_output(f"{{%datetime_strftime({timestamp}, {date_format})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_error_functions.py b/tests/unit/script/functions/test_error_functions.py new file mode 100644 index 00000000..be2d90c2 --- /dev/null +++ b/tests/unit/script/functions/test_error_functions.py @@ -0,0 +1,105 @@ +import re + +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError + + +class TestErrorFunctions: + def test_user_throw(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + Script({"throw_error": "{%throw('test this error message')}"}).resolve() + + def test_user_assert_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + Script({"throw_error": "{%assert(False, 'test this error message')}"}).resolve() + + def test_user_assert_passthrough(self): + output = single_variable_output("{%assert(['a'], 'test this error message')}") + assert output == ["a"] + + def test_user_assert_passthrough_as_arg(self): + output = single_variable_output("{%int(%assert('123', 'test this error message'))}") + assert output == 123 + + def test_user_assert_eq(self): + output = single_variable_output( + """{ + %int( + %assert_eq( + '123', + %array_at(['123'], 0), + 'test this error message' + ) + ) + }""" + ) + assert output == 123 + + def test_user_assert_eq_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + single_variable_output( + """{ + %int( + %assert_eq( + '123', + %array_at(['no'], 0), + 'test this error message' + ) + ) + }""" + ) + + def test_user_assert_ne(self): + output = single_variable_output( + """{ + %int( + %assert_ne( + '123', + %array_at(['nope'], 0), + 'test this error message' + ) + ) + }""" + ) + assert output == 123 + + def test_user_assert_ne_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + single_variable_output( + """{ + %int( + %assert_ne( + '123', + %array_at(['123'], 0), + 'test this error message' + ) + ) + }""" + ) + + def test_user_assert_then(self): + output = single_variable_output( + """{ + %assert_then( + '123', + %array_at(['nope'], 0), + 'test this error message' + ) + }""" + ) + assert output == "nope" + + def test_user_assert_then_raises(self): + with pytest.raises(UserThrownRuntimeError, match=re.escape("test this error message")): + single_variable_output( + """{ + %assert_then( + {}, + %array_at(['nope'], 0), + 'test this error message' + ) + }""" + ) diff --git a/tests/unit/script/functions/test_json_functions.py b/tests/unit/script/functions/test_json_functions.py new file mode 100644 index 00000000..b2477ac5 --- /dev/null +++ b/tests/unit/script/functions/test_json_functions.py @@ -0,0 +1,31 @@ +import json + +import pytest +from unit.script.conftest import single_variable_output + + +class TestJsonFunctions: + @pytest.mark.parametrize("str_token", ["'''", '"""']) + def test_from_json(self, str_token: str): + json_dict = { + "string": "value", + "quotes": "has '' and \"\"", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + "nested_dict": { + "string": "value", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + }, + } + + output = single_variable_output( + f"{{ %from_json({str_token}{json.dumps(json_dict)}{str_token}) }}" + ) + assert output == json_dict diff --git a/tests/unit/script/functions/test_map_functions.py b/tests/unit/script/functions/test_map_functions.py new file mode 100644 index 00000000..709445e6 --- /dev/null +++ b/tests/unit/script/functions/test_map_functions.py @@ -0,0 +1,140 @@ +import re + +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import KeyDoesNotExistRuntimeException +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +class TestMapFunctions: + def test_map_get(self): + output = ( + Script( + { + "input_map": "{{'key': 'value'}}", + "output": "{%map_get(input_map, 'key')}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == "value" + + def test_map_get_optional(self): + output = ( + Script( + { + "input_map": "{{'key': 'value'}}", + "output": "{%map_get(input_map, 'dne', 'optional_value')}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == "optional_value" + + def test_map_get_errors_missing_key(self): + with pytest.raises( + KeyDoesNotExistRuntimeException, + match=re.escape("Tried to call %map_get with key dne, but it does not exist"), + ): + Script( + { + "input_map": "{{'key': 'value'}}", + "output": "{%map_get(input_map, 'dne')}", + } + ).resolve() + + def test_map_get_errors_key_not_hashable(self): + with pytest.raises( + KeyNotHashableRuntimeException, + match=re.escape("Tried to use Array as a Map key, but it is not hashable."), + ): + Script( + { + "non_hashable_key": "{%array([1, 2, ['nest']])}", + "input_map": "{{'key': 'value'}}", + "output": "{ %map_get( input_map, %array_at(non_hashable_key, 2)) }", + } + ).resolve() + + @pytest.mark.parametrize( + "contains_value, expected_value", + [ + ("'key'", True), + ("'dne'", False), + ("%string(%array_at(['dne', 'key'], 1))", True), + ], + ) + def test_map_contains(self, contains_value: str, expected_value: bool): + output = ( + Script( + { + "input_map": "{{'key': 'value'}}", + "output": f"{{%map_contains(input_map, {contains_value})}}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == expected_value + + def test_map_apply(self): + output = ( + Script( + { + "%custom_func": "{[%upper($0), %lower($1)]}", + "map1": "{{'Key1': 'Value1', 'Key2': 'Value2'}}", + "output": "{%map_apply(map1, %custom_func)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [["KEY1", "value1"], ["KEY2", "value2"]] + + def test_map_enumerate(self): + output = ( + Script( + { + "%custom_func": "{[$0, %upper($1), %lower($2)]}", + "map1": "{{'Key1': 'Value1', 'Key2': 'Value2'}}", + "output": "{%map_enumerate(map1, %custom_func)}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == [[0, "KEY1", "value1"], [1, "KEY2", "value2"]] + + def test_map_size(self): + output = single_variable_output("{%map_size({'key': 'value', 1: 3})}") + assert output == 2 + + def test_cast_map(self): + output = ( + Script( + { + "array_test": "{ [1, 2, {'key': 'value'}] }", + "output": "{ %map( %array_at(array_test, 2) )}", + } + ) + .resolve(update=True) + .get("output") + .native + ) + assert output == {"key": "value"} + + def test_cast_map_errors_cannot_cast(self): + with pytest.raises( + FunctionRuntimeException, match="Tried and failed to cast Integer as a Map" + ): + single_variable_output("{%map(1)}") diff --git a/tests/unit/script/functions/test_numeric_functions.py b/tests/unit/script/functions/test_numeric_functions.py new file mode 100644 index 00000000..c41f4c59 --- /dev/null +++ b/tests/unit/script/functions/test_numeric_functions.py @@ -0,0 +1,78 @@ +import pytest +from unit.script.conftest import single_variable_output + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "values, expected_output", [("1, 2, 3", 6), ("1", 1), ("-1, -2, -3", -6), ("1.1, 1.2", 2.3)] + ) + def test_add(self, values: str, expected_output: int): + output = single_variable_output(f"{{%add({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", [("1, 2, 3", -4), ("1", 1), ("-1, -2, -3", 4), ("1.5, 2.5", -1)] + ) + def test_sub(self, values: str, expected_output: int): + output = single_variable_output(f"{{%sub({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [("1, 2, 3", 6), ("1", 1), ("-1, -2, -3", -6), ("1.5, 2.5", 3.75)], + ) + def test_mul(self, values: str, expected_output: int): + output = single_variable_output(f"{{%mul({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", [("2, 2", 1), ("10, 5", 2), ("4.5, 0.5", 9), ("-3.5, -2", 1.75)] + ) + def test_div(self, values: str, expected_output: int): + output = single_variable_output(f"{{%div({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("8, 3", 2), + ("1, 1", 0), + ], + ) + def test_mod(self, values: str, expected_output: int): + output = single_variable_output(f"{{%mod({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("8, 3, 0.3, 0.2, 1.4, 99.9", 99.9), + ("1, 1, 0, 1", 1), + ], + ) + def test_max(self, values: str, expected_output: int): + output = single_variable_output(f"{{%max({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("8, 3, 0.3, 0.2, 1.4, 99.9", 0.2), + ("1, 1, 0, 1", 0), + ], + ) + def test_min(self, values: str, expected_output: int): + output = single_variable_output(f"{{%min({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("2, 2", 2**2), + ("9, 0.5", 9**0.5), + ("4.4, 2.2", 4.4**2.2), + ], + ) + def test_pow(self, values: str, expected_output: float): + output = single_variable_output(f"{{ %pow({values}) }}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_regex_functions.py b/tests/unit/script/functions/test_regex_functions.py new file mode 100644 index 00000000..b0ca2e16 --- /dev/null +++ b/tests/unit/script/functions/test_regex_functions.py @@ -0,0 +1,45 @@ +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.script import Script + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'ow', 'lower'", []), + ("'.*ow.*', 'lower'", ["lower"]), + ("'.*(ow).*', 'lower'", ["lower", "ow"]), + ("'(.*)(ow)(.*)', 'lower'", ["lower", "l", "ow", "er"]), + ], + ) + def test_regex_match(self, values: str, expected_output: str): + output = single_variable_output(f"{{%regex_match({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'ow', 'lower'", ["lower"]), + ("'.*ow.*', 'lower'", ["lower"]), + ("'.*(ow).*', 'lower'", ["lower", "ow"]), + ("'(.*)(ow)(.*)', 'lower'", ["lower", "l", "ow", "er"]), + ], + ) + def test_regex_search(self, values: str, expected_output: str): + output = single_variable_output(f"{{%regex_search({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'ow', 'lower'", []), + ("'.*ow.*', 'lower'", ["lower"]), + ("'.*(ow).*', 'lower'", ["lower", "ow"]), + ("'(.*)(ow)(.*)', 'lower'", ["lower", "l", "ow", "er"]), + ], + ) + def test_regex_fullmatch(self, values: str, expected_output: str): + output = single_variable_output(f"{{%regex_fullmatch({values})}}") + assert output == expected_output diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py new file mode 100644 index 00000000..9a48079a --- /dev/null +++ b/tests/unit/script/functions/test_string_functions.py @@ -0,0 +1,116 @@ +import pytest +from unit.script.conftest import single_variable_output + + +class TestNumericFunctions: + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower'", "LOWER"), + ("'UPPER'", "UPPER"), + ], + ) + def test_upper(self, values: str, expected_output: str): + output = single_variable_output(f"{{%upper({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower'", "lower"), + ("'UPPER'", "upper"), + ], + ) + def test_lower(self, values: str, expected_output: str): + output = single_variable_output(f"{{%lower({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower First word'", "Lower first word"), + ("'UPPER First word'", "Upper first word"), + ], + ) + def test_capitalize(self, values: str, expected_output: str): + output = single_variable_output(f"{{%capitalize({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower First word'", "Lower First Word"), + ("'UPPER First word'", "Upper First Word"), + ], + ) + def test_titlecase(self, values: str, expected_output: str): + output = single_variable_output(f"{{%titlecase({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ( + "'lower First word second word', 'word', 'string'", + "lower First string second string", + ), + ( + "'lower First word second word', 'word', 'string', 1", + "lower First string second word", + ), + ], + ) + def test_replace(self, values: str, expected_output: str): + output = single_variable_output(f"{{%replace({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'lower First word'", "lower First word"), + ("'lower First word', ' second', ' string'", "lower First word second string"), + ], + ) + def test_concat(self, values: str, expected_output: str): + output = single_variable_output(f"{{%concat({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'HI', 6, '.'", "....HI"), + ("'HI', 2, '.'", "HI"), + ], + ) + def test_pad(self, values: str, expected_output: str): + output = single_variable_output(f"{{%pad({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("2012, 6", "002012"), + ("2012, 2", "2012"), + ], + ) + def test_pad_zero(self, values: str, expected_output: str): + output = single_variable_output(f"{{%pad_zero({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "values, expected_output", + [ + ("'2012', 2", "12"), + ("'2012', 1, 3", "01"), + ], + ) + def test_slice(self, values, expected_output): + output = single_variable_output(f"{{%slice({values})}}") + assert output == expected_output + + @pytest.mark.parametrize( + "value, expected_output", [("a", True), ("nope", False), ("dog", True)] + ) + def test_contains(self, value, expected_output): + output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}") + assert output == expected_output diff --git a/tests/unit/script/test_parser.py b/tests/unit/script/test_parser.py new file mode 100644 index 00000000..196ae8f2 --- /dev/null +++ b/tests/unit/script/test_parser.py @@ -0,0 +1,210 @@ +import re +from typing import Optional +from typing import Union + +import pytest + +from ytdl_sub.script.parser import _UNEXPECTED_CHAR_ARGUMENT +from ytdl_sub.script.parser import BRACKET_NOT_CLOSED +from ytdl_sub.script.parser import ParsedArgType +from ytdl_sub.script.parser import parse +from ytdl_sub.script.types.array import UnresolvedArray +from ytdl_sub.script.types.function import BuiltInFunction +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import Lambda +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.types.variable import Variable +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestParser: + def test_simple(self): + parsed = parse("hello world") + assert parsed == SyntaxTree([String(value="hello world")]) + assert parsed.variables == set() + + def test_single_function_one_arg(self): + parsed = parse("hello {%capitalize('hi mom')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction(name="capitalize", args=[String(value="hi mom")]), + ] + ) + + def test_conditional(self): + parsed = parse("hello {%if(True, 'hi', 3.4)}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="if", args=[Boolean(value=True), String(value="hi"), Float(value=3.4)] + ), + ] + ) + assert parsed.ast[1].output_type() == Union[String, Float] + + def test_conditional_as_input_same_outputs(self): + parsed = parse("hello {%concat(%if(True, 'hi', 'mom'), 'and dad')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="concat", + args=[ + BuiltInFunction( + name="if", args=[Boolean(value=True), String("hi"), String("mom")] + ), + String(value="and dad"), + ], + ), + ] + ) + + def test_conditional_as_input_different_outputs(self): + parsed = parse("hello {%string(%if(True, 'hi', 4))}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="string", + args=[ + BuiltInFunction(name="if", args=[Boolean(True), String("hi"), Integer(4)]), + ], + ), + ] + ) + + def test_nested_if_output_type(self): + parsed = parse( + """{ + %if( + True, + %if( + True, + %if( + True, + "winner", + True + ), + True + ), + True + ) + }""" + ) + assert len(parsed.ast) == 1 + token = parsed.ast[0] + assert isinstance(token, BuiltInFunction) + assert token.output_type() == Union[String, Boolean] + + def test_single_function_one_vararg(self): + parsed = parse("hello {%concat('hi mom')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction(name="concat", args=[String(value="hi mom")]), + ] + ) + + def test_single_function_many_vararg(self): + parsed = parse("hello {%concat('hi', 'mom')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction(name="concat", args=[String(value="hi"), String(value="mom")]), + ] + ) + + def test_single_function_many_args_with_optional_none(self): + parsed = parse("hello {%replace('hi mom', 'hi', '')}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="replace", + args=[String(value="hi mom"), String(value="hi"), String(value="")], + ), + ] + ) + + def test_single_function_many_args_with_optional_provided(self): + parsed = parse("hello {%replace('hi mom', 'hi', '', 1)}") + assert parsed == SyntaxTree( + [ + String("hello "), + BuiltInFunction( + name="replace", + args=[ + String(value="hi mom"), + String(value="hi"), + String(value=""), + Integer(value=1), + ], + ), + ] + ) + + @pytest.mark.parametrize("whitespace", [None, " ", " ", "\n", " \n "]) + def test_single_function_multiple_args(self, whitespace: Optional[str]): + s = whitespace + if s is None: + s = "" + + input_str = ( + f"hello{s}{{{s}%concat({s}'string'{s},{s}%string(1){s},{s}%string(2.4){s}," + f"{s}%string(True){s},{s}%string(variable_name){s},{s}%capitalize({s}'hi'{s}){s})}}" + f"{s}" + ) + parsed = parse(input_str) + assert parsed == SyntaxTree( + [ + String(value=f"hello{s}"), + BuiltInFunction( + name="concat", + args=[ + String(value="string"), + BuiltInFunction(name="string", args=[Integer(value=1)]), + BuiltInFunction(name="string", args=[Float(value=2.4)]), + BuiltInFunction(name="string", args=[Boolean(value=True)]), + BuiltInFunction(name="string", args=[Variable(name="variable_name")]), + BuiltInFunction(name="capitalize", args=[String(value="hi")]), + ], + ), + ] + + ([String(value=s)] if s else []) + ) + assert parsed.variables == {Variable(name="variable_name")} + + def test_lambda_function(self): + assert parse("{%array_apply([1], %times_two)}") == SyntaxTree( + [ + BuiltInFunction( + name="array_apply", + args=[ + UnresolvedArray(value=[Integer(1)]), + Lambda(value="times_two"), + ], + ) + ] + ) + + +class TestParserBracketFailures: + def test_bracket_open(self): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BRACKET_NOT_CLOSED))): + parse("{") + + def test_bracket_close(self): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BRACKET_NOT_CLOSED))): + parse("}") + + def test_bracket_in_function(self): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_CHAR_ARGUMENT(ParsedArgType.MAP_KEY))), + ): + parse("hello {%capitalize({as_arg)}") diff --git a/tests/unit/script/test_script.py b/tests/unit/script/test_script.py new file mode 100644 index 00000000..0b86e2be --- /dev/null +++ b/tests/unit/script/test_script.py @@ -0,0 +1,63 @@ +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import String + + +class TestScript: + def test_pre_resolved(self): + assert Script( + { + "%custom_func": "return {[$0, $1]}", + "aa": "a", + "bb": "b", + "cc": "{%custom_func(aa, bb)}", + } + ).resolve(resolved={"bb": String("bb_override")}) == ScriptOutput( + { + "aa": String("a"), + "bb": String("bb_override"), + "cc": String('return ["a", "bb_override"]'), + } + ) + + def test_partial_resolve(self): + assert Script( + { + "%custom_func": "return {[$0, $1]}", + "aa": "a", + "bb": "b", + "cc": "{%custom_func(aa, bb)}", + } + ).resolve(unresolvable={"bb"}) == ScriptOutput({"aa": String("a")}) + + def test_partial_update_script(self): + # to be resolved later + entry_map = Map({String("title"): String("the title")}) + + script = Script( + { + "entry": "{%throw('entry has not been populated yet')}", + "title": "{%map_get(entry, 'title')}", + "override": "hi", + "resolved_override": "{override} mom", + } + ) + + script.resolve(unresolvable={"entry"}, update=True) + assert script.get("override") == String("hi") + assert script.get("resolved_override") == String("hi mom") + + script.add( + { + "new_variable_titlecase": "{%titlecase(new_variable_upper)}", + "new_variable": "{resolved_override} {title}", + "new_variable_upper": "{%upper(new_variable)}", + } + ).resolve(resolved={"entry": entry_map}, update=True) + + assert script.get("title") == String("the title") + assert script.get("new_variable") == String("hi mom the title") + assert script.get("new_variable_upper") == String("HI MOM THE TITLE") + assert script.get("new_variable_titlecase") == String("Hi Mom The Title") + assert script.get("entry") == entry_map diff --git a/tests/unit/script/types/__init__.py b/tests/unit/script/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/script/types/test_array.py b/tests/unit/script/types/test_array.py new file mode 100644 index 00000000..f816b1bd --- /dev/null +++ b/tests/unit/script/types/test_array.py @@ -0,0 +1,119 @@ +import re + +import pytest + +from ytdl_sub.script.parser import _UNEXPECTED_CHAR_ARGUMENT +from ytdl_sub.script.parser import _UNEXPECTED_COMMA_ARGUMENT +from ytdl_sub.script.parser import ParsedArgType +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestArray: + def test_return(self): + assert Script({"arr": "{['a', 3.14]}"}).resolve() == ScriptOutput( + {"arr": Array([String("a"), Float(3.14)])} + ) + + def test_return_as_str(self): + assert Script({"arr": "str: {['a', 3.14]}"}).resolve() == ScriptOutput( + {"arr": String('str: ["a", 3.14]')} + ) + + def test_nested_array(self): + assert Script( + {"arr": "{['level1', ['level2', ['level3', 'level3'], 'level2'], 'level1']}"} + ).resolve() == ScriptOutput( + { + "arr": Array( + [ + String("level1"), + Array( + [ + String("level2"), + Array([String("level3"), String("level3")]), + String("level2"), + ], + ), + String("level1"), + ] + ) + } + ) + + @pytest.mark.parametrize( + "array", + [ + "{[]}", + "{ [] }", + "{ [ ] }", + "{[\n]}", + ], + ) + def test_empty(self, array: str): + assert Script({"arr": array}).resolve() == ScriptOutput({"arr": Array([])}) + + @pytest.mark.parametrize( + "array", + [ + "{[,]}", + "{[ ,]}", + "{ [ , ]}", + "{ ['test',] }", + "{ [ 'test', ] }", + "{[\n,\n]}", + ], + ) + def test_unexpected_comma(self, array: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.ARRAY))), + ): + Script({"arr": array}).resolve() + + @pytest.mark.parametrize( + "array", + [ + "{[}", + "{[ }", + "{[\n}", + "{['key'}", + "{[ 'key' }", + ], + ) + def test_array_not_closed(self, array: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_CHAR_ARGUMENT(ParsedArgType.ARRAY))), + ): + assert Script({"arr": array}).resolve() + + @pytest.mark.parametrize( + "array", + [ + "{]}" "{ ]}", + "{\n]}", + ], + ) + def test_array_not_opened(self, array: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_CHAR_ARGUMENT(ParsedArgType.SCRIPT))), + ): + assert Script({"arr": array}).resolve() + + def test_custom_function(self): + assert Script( + { + "%custom_func": "return {[$0, $1]}", + "aa": "a", + "bb": "b", + "cc": "{%custom_func(aa, bb)}", + } + ).resolve() == ScriptOutput( + {"aa": String("a"), "bb": String("b"), "cc": String('return ["a", "b"]')} + ) diff --git a/tests/unit/script/types/test_bool.py b/tests/unit/script/types/test_bool.py new file mode 100644 index 00000000..5de7ddb1 --- /dev/null +++ b/tests/unit/script/types/test_bool.py @@ -0,0 +1,66 @@ +import re + +import pytest + +from ytdl_sub.script.parser import BOOLEAN_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestBool: + @pytest.mark.parametrize( + "boolean", + [ + "{True}", + "{ True }", + "{False}", + "{ False }", + ], + ) + def test_boolean_not_arg(self, boolean: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BOOLEAN_ONLY_ARGS))): + Script({"boolean": boolean}).resolve() + + @pytest.mark.parametrize( + "boolean, expected_boolean", + [ + ("{%bool(True)}", True), + ("{%bool(False)}", False), + ("{%bool( True )}", True), + ("{%bool( False )}", False), + ], + ) + def test_boolean(self, boolean: bool, expected_boolean: bool): + assert Script( + {"boolean": boolean, "as_string": "{%string(boolean)}"} + ).resolve() == ScriptOutput( + { + "boolean": Boolean(expected_boolean), + "as_string": String(str(expected_boolean)), + } + ) + + @pytest.mark.parametrize( + "to_cast, expected_bool", + [ + ("{%bool(False)}", False), + ("{%bool(True)}", True), + ("{%bool(0)}", False), + ("{%bool(1)}", True), + ("{%bool(0.0)}", False), + ("{%bool(0.1)}", True), + ("{%bool('')}", False), + ("{%bool('false')}", True), + ("{%bool([])}", False), + ("{%bool([False])}", True), + ("{%bool({})}", False), + ("{%bool({'key': 'value'})}", True), + ], + ) + def test_cast_as_bool(self, to_cast: str, expected_bool: bool): + assert Script({"as_bool": to_cast}).resolve() == ScriptOutput( + {"as_bool": Boolean(expected_bool)} + ) diff --git a/tests/unit/script/types/test_custom_function.py b/tests/unit/script/types/test_custom_function.py new file mode 100644 index 00000000..e7b76e80 --- /dev/null +++ b/tests/unit/script/types/test_custom_function.py @@ -0,0 +1,235 @@ +import re + +import pytest + +from ytdl_sub.script.parser import CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArgumentName +from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestCustomFunction: + def test_custom_function_use_input_param_multiple_times(self): + assert Script( + { + "%custom_square": "{%mul($0, $0)}", + "output": "{%custom_square(3)}", + } + ).resolve() == ScriptOutput({"output": Integer(9)}) + + def test_custom_function_cycle(self): + with pytest.raises( + CycleDetected, match=re.escape("The custom function %cycle_func cannot call itself.") + ): + Script( + {"%cycle_func": "{%mul(%cycle_func(1), $0)}", "output": "{%cycle_func(1)}"} + ).resolve() + + def test_custom_function_chained_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape( + "Cycle detected within these custom functions: " + "%cycle_func1 -> %cycle_func0 -> %cycle_func1" + ), + ): + Script( + { + "%cycle_func1": "{%mul(%cycle_func0(1), $0)}", + "%cycle_func0": "{%mul(%cycle_func1(1), $0)}", + "output": "{%cycle_func0(1)}", + } + ).resolve() + + def test_custom_function_deep_chained_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape( + "Cycle detected within these custom functions: " + "%cycle_func4 -> " + "%cycle_func0 -> " + "%cycle_func1 -> " + "%cycle_func2 -> " + "%cycle_func3 -> " + "%cycle_func4" + ), + ): + Script( + { + "%nested_safe_func": "{%mul($0, 1)}", + "%safe_func": "{%nested_safe_func($0, 1)}", + "%cycle_func4": "{%mul(%cycle_func0(1), %safe_func($0))}", + "%cycle_func3": "{%mul(%cycle_func4(1), %safe_func($0))}", + "%cycle_func2": "{%mul(%cycle_func3(1), %safe_func($0))}", + "%cycle_func1": "{%mul(%cycle_func2(1), %safe_func($0))}", + "%cycle_func0": "{%mul(%cycle_func1(1), %safe_func($0))}", + "output": "{%cycle_func0(1)}", + } + ).resolve() + + def test_custom_function_uses_non_existent_function(self): + with pytest.raises( + FunctionDoesNotExist, + match=re.escape("Function %lolnope does not exist as a built-in or custom function."), + ): + Script( + { + "%func1": "{%mul(%lolnope(1), $0)}", + "%func0": "{%mul(%func1(1), $0)}", + "output": "{%func(1)}", + } + ).resolve() + + @pytest.mark.parametrize( + "name", + [ + "$00invalid", + "$abc", + "$3.14", + ], + ) + def test_custom_function_invalid_function_argument_names(self, name: str): + with pytest.raises( + InvalidCustomFunctionArgumentName, + match=re.escape( + "Custom function arguments must be numeric and increment starting from zero." + ), + ): + Script( + { + "%func1": f"{{%mul(1, {name})}}", + "%func0": "{%mul(%func1(1), $0)}", + "output": "{%func0(1)}", + } + ).resolve() + + def test_custom_function_function_argument_usage_in_brackets(self): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(CUSTOM_FUNCTION_ARGUMENTS_ONLY_ARGS)), + ): + Script({"%func1": "{$0}"}).resolve() + + @pytest.mark.parametrize( + "argument", + [ + "$1", + "$2", + "$3", + ], + ) + def test_custom_function_invalid_function_argument_single(self, argument: str): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + f"Custom function %func1 has invalid function arguments: " + f"The argument must start with $0, not {argument}." + ), + ): + Script( + { + "%func1": f"{{[{argument}]}}", + } + ).resolve() + + @pytest.mark.parametrize( + "arguments", + [ + "$0, $2", + "$1, $2, $3", + ], + ) + def test_custom_function_invalid_function_argument_out_of_order(self, arguments: str): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + f"Custom function %func1 has invalid function arguments: " + f"{arguments} do not increment from $0 to ${len(arguments.split(',')) - 1}." + ), + ): + Script( + { + "%func1": f"{{[{arguments}]}}", + } + ).resolve() + + def test_custom_function_uses_custom_function_wrong_number_of_arguments(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Custom function %func0 has invalid usage of the custom function %func1: " + "Expects 1 argument but received 2" + ), + ): + Script( + { + "%func1": "{%mul(1, $0)}", + "%func0": "{%mul(%func1(1, 2), $0)}", + "output": "{%func0(1)}", + } + ).resolve() + + def test_custom_function_uses_custom_function_wrong_number_of_arguments_plural(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Custom function %func0 has invalid usage of the custom function %func1: " + "Expects 2 arguments but received 1" + ), + ): + Script( + { + "%func1": "{%mul($1, $0)}", + "%func0": "{%mul(%func1(1), $0)}", + "output": "{%func0(1)}", + } + ).resolve() + + def test_variable_uses_custom_function_wrong_number_of_arguments(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the custom function %func1: " + "Expects 1 argument but received 2" + ), + ): + Script( + { + "%func1": "{%mul(1, $0)}", + "output": "{%mul(%func1(1, 2), 1)}", + } + ).resolve() + + def test_variable_uses_custom_function_wrong_number_of_arguments_plural(self): + with pytest.raises( + InvalidCustomFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the custom function %func1: " + "Expects 2 arguments but received 1" + ), + ): + Script( + { + "%func1": "{%mul($1, $0)}", + "output": "{%mul(%func1(1), 1)}", + } + ).resolve() + + def test_function_argument_errors_has_spaces(self): + with pytest.raises( + InvalidCustomFunctionArgumentName, + match=re.escape( + "Custom function arguments, denoted by $, cannot have a space proceeding it." + ), + ): + Script( + { + "%func1": "{%mul($ 1, $0)}", + "output": "{%mul(%func1(1), 1)}", + } + ) diff --git a/tests/unit/script/types/test_float.py b/tests/unit/script/types/test_float.py new file mode 100644 index 00000000..e285ce17 --- /dev/null +++ b/tests/unit/script/types/test_float.py @@ -0,0 +1,81 @@ +import re + +import pytest + +from ytdl_sub.script.parser import NUMERICS_INVALID_CHAR +from ytdl_sub.script.parser import NUMERICS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestFloat: + @pytest.mark.parametrize( + "integer", + [ + "{1.0}", + "{ 1.2 }", + "{-1.4}", + "{ -1.5 }", + "{0001.2}", + "{ 0001.5 }", + ], + ) + def test_float_not_arg(self, integer: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_ONLY_ARGS))): + Script({"out": integer}).resolve() + + @pytest.mark.parametrize( + "float_, expected_float", + [ + ("{%float(1.1)}", 1.1), + ("{%float( 1.2345 )}", 1.2345), + ("{%float(-1.34)}", -1.34), + ("{%float( -1.535 )}", -1.535), + ("{%float(0001.)}", 1.0), + ("{%float( 0001. )}", 1.0), + ("{%float(.2)}", 0.2), + ("{%float(-.1)}", -0.1), + ], + ) + def test_float(self, float_: str, expected_float: int): + assert Script({"out": float_, "as_string": "{%string(out)}"}).resolve() == ScriptOutput( + { + "out": Float(expected_float), + "as_string": String(str(expected_float)), + } + ) + + @pytest.mark.parametrize( + "float_", + [ + "{%add(0, --1.0)}", + "{%add(0, 1-.0 )}", + "{%add(0,-1.0.)}", + "{%add(0, -1.0. )}", + "{%add(0,0001.a)}", + "{%add(0, 0001.- )}", + "{%add(0, ..3)}", + ], + ) + def test_invalid_float(self, float_: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_INVALID_CHAR))): + Script({"out": float_}).resolve() + + @pytest.mark.parametrize( + "to_cast, expected_float", + [ + ("{%float(5)}", 5.0), + ("{%float(0.9)}", 0.9), + ("{%float(-3.00)}", -3.0), + ("{%float(True)}", 1.0), + ("{%float(False)}", 0.0), + ("{%float('142.43')}", 142.43), + ], + ) + def test_cast_as_float(self, to_cast: str, expected_float: float): + assert Script({"as_float": to_cast}).resolve() == ScriptOutput( + {"as_float": Float(expected_float)} + ) diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py new file mode 100644 index 00000000..014e4a85 --- /dev/null +++ b/tests/unit/script/types/test_function.py @@ -0,0 +1,97 @@ +import re + +import pytest +from unit.script.conftest import single_variable_output + +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.parser import FUNCTION_INVALID_CHAR +from ytdl_sub.script.script import Script +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import FunctionDoesNotExist +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +def _incompatible_arguments_match(expected: str, recieved: str) -> str: + return re.escape(f"Expected ({expected})\nReceived ({recieved})") + + +def mock_register_function(integer: Integer) -> Integer: + return Integer(integer.value + 100) + + +class TestFunction: + def test_nested_if_function_incompatible(self): + function_str = """{ + %map_get( + %if( + True, + %if( + True, + {}, + [] + ), + {} + ), + "key" + ) + }""" + with pytest.raises( + IncompatibleFunctionArguments, + match=_incompatible_arguments_match( + expected="mapping: Map, key: AnyArgument, default: Optional[AnyArgument]", + recieved="%if(...)->Union[Array, Map], String", + ), + ): + Script({"func": function_str}).resolve() + + @pytest.mark.parametrize( + "function_str, expected_types, received_types", + [ + ("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), + ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), + ( + "{%replace('hi mom', 'mom', 'dad', 1, 0)}", + "string: String, old: String, new: String, count: Optional[Integer]", + "String, String, String, Integer, Integer", + ), + ], + ) + def test_incompatible_types(self, function_str: str, expected_types: str, received_types: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=_incompatible_arguments_match(expected=expected_types, recieved=received_types), + ): + Script({"func": function_str}).resolve() + + def test_runtime_error(self): + with pytest.raises( + FunctionRuntimeException, + match=re.escape( + "Runtime error occurred when executing the function %div: division by zero" + ), + ): + Script({"divide_by_zero": "{%div(8820, 0)}"}).resolve() + + def test_function_does_not_exist(self): + with pytest.raises( + FunctionDoesNotExist, + match=re.escape("Function %lolnope does not exist as a built-in or custom function."), + ): + Script({"dne": "{%lolnope(False, 'test this error message')}"}).resolve() + + def test_function_does_not_close(self): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(FUNCTION_INVALID_CHAR)), + ): + Script({"dne": "{%throw}"}).resolve() + + def test_register_function(self): + try: + Functions.register_function(function=mock_register_function) + output = single_variable_output(f"{{%mock_register_function(10)}}") + assert output == 110 + finally: + del Functions._custom_functions[mock_register_function.__name__] diff --git a/tests/unit/script/types/test_integer.py b/tests/unit/script/types/test_integer.py new file mode 100644 index 00000000..18747e11 --- /dev/null +++ b/tests/unit/script/types/test_integer.py @@ -0,0 +1,81 @@ +import re + +import pytest + +from ytdl_sub.script.parser import NUMERICS_INVALID_CHAR +from ytdl_sub.script.parser import NUMERICS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import Boolean +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestInteger: + @pytest.mark.parametrize( + "integer", + [ + "{1}", + "{ 1 }", + "{-1}", + "{ -1 }", + "{0001}", + "{ 0001 }", + ], + ) + def test_integer_not_arg(self, integer: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_ONLY_ARGS))): + Script({"integer": integer}).resolve() + + @pytest.mark.parametrize( + "integer, expected_integer", + [ + ("{%int(1)}", 1), + ("{%int( 1 )}", 1), + ("{%int(-1)}", -1), + ("{%int( -1 )}", -1), + ("{%int(0001)}", 1), + ("{%int( 0001 )}", 1), + ], + ) + def test_integer(self, integer: str, expected_integer: int): + assert Script( + {"integer": integer, "as_string": "{%string(integer)}"} + ).resolve() == ScriptOutput( + { + "integer": Integer(expected_integer), + "as_string": String(str(expected_integer)), + } + ) + + @pytest.mark.parametrize( + "integer", + [ + "{%add(0, --1)}", + "{%add(0, 1- )}", + "{%add(0,-1-)}", + "{%add(0, -1 - )}", + "{%add(0,0001a)}", + "{%add(0, 0001b )}", + ], + ) + def test_invalid_integer(self, integer: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(NUMERICS_INVALID_CHAR))): + Script({"integer": integer}).resolve() + + @pytest.mark.parametrize( + "to_cast, expected_int", + [ + ("{%int(5)}", 5), + ("{%int(0.9)}", 0), + ("{%int(-3.00)}", -3), + ("{%int(True)}", 1), + ("{%int(False)}", 0), + ("{%int('142')}", 142), + ], + ) + def test_cast_as_integer(self, to_cast: str, expected_int: int): + assert Script({"as_int": to_cast}).resolve() == ScriptOutput( + {"as_int": Integer(expected_int)} + ) diff --git a/tests/unit/script/types/test_lambda_function.py b/tests/unit/script/types/test_lambda_function.py new file mode 100644 index 00000000..379c2207 --- /dev/null +++ b/tests/unit/script/types/test_lambda_function.py @@ -0,0 +1,127 @@ +import re + +import pytest + +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments + + +class TestLambdaFunction: + def test_lambda_with_custom_function(self): + assert Script( + {"%times_two": "{%mul($0, 2)}", "wip": "{%array_apply([1, 2, 3], %times_two)}"} + ).resolve() == ScriptOutput({"wip": Array([Integer(2), Integer(4), Integer(6)])}) + + def test_conditional_lambda_with_custom_functions(self): + assert Script( + { + "%times_three": "{%mul($0, 3)}", + "%times_two": "{%mul($0, 2)}", + "wip": "{%array_apply([1, 2, 3], %if(False, %times_two, %times_three))}", + } + ).resolve() == ScriptOutput({"wip": Array([Integer(3), Integer(6), Integer(9)])}) + + def test_nested_custom_functions(self): + assert Script( + { + "%times_three": "{%mul($0, 3)}", + "%times_two": "{%mul($0, 2)}", + "identity": "{%times_three(%times_two(1))}", + } + ).resolve() == ScriptOutput({"identity": Integer(6)}) + + def test_nested_custom_functions_within_custom_functions(self): + assert Script( + { + "%power_2": "{%mul($0, 2)}", + "%power_3": "{%mul(%power_2($0), 2)}", + "%power_4": "{%mul(%power_3($0), 2)}", + "power_of_4": "{%power_4(2)}", + } + ).resolve() == ScriptOutput({"power_of_4": Integer(16)}) + + def test_nested_lambda_custom_functions_within_custom_functions(self): + assert Script( + { + "%nest4": "{%mul($0, 2)}", + "%nest3": "{%array_at(%array_apply([$0], %nest4), 0)}", + "%nest2": "{%array_at(%array_apply([$0], %nest3), 0)}", + "%nest1": "{%array_at(%array_apply([$0], %nest2), 0)}", + "output": "{%array_at(%array_apply([2], %nest1), 0)}", + } + ).resolve() == ScriptOutput({"output": Integer(4)}) + + +class TestLambdaFunctionIncompatibleNumArguments: + @pytest.mark.parametrize( + "lambda_value", ["%enumerate_output", "%if(False, %capitalize, %enumerate_output)"] + ) + def test_custom_function_lambda_in_variable(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the custom function " + "%enumerate_output as a lambda: Expects 2 arguments but will receive 1." + ), + ): + Script( + { + "%enumerate_output": "{[$0, $1]}", + "array1": "{['a', 'b', 'c']}", + "output": f"{{%array_apply(array1, {lambda_value})}}", + } + ) + + @pytest.mark.parametrize("lambda_value", ["%replace", "%if(False, %capitalize, %replace)"]) + def test_function_lambda_in_variable(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Variable output has invalid usage of the function %replace as a lambda: " + "Expects 3 - 4 arguments but will receive 1." + ), + ): + Script( + { + "array1": "{['a', 'b', 'c']}", + "output": f"{{%array_apply(array1, {lambda_value})}}", + } + ) + + @pytest.mark.parametrize( + "lambda_value", ["%enumerate_output", "%if(False, %concat, %enumerate_output)"] + ) + def test_custom_function_lambda_in_custom_function(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Custom function %output has invalid usage of the custom function " + "%enumerate_output as a lambda: Expects 3 arguments but will receive 2." + ), + ): + Script( + { + "%enumerate_output": "{[$0, $1, $2]}", + "array1": "{['a', 'b', 'c']}", + "%output": f"{{%array_enumerate(array1, {lambda_value})}}", + } + ) + + @pytest.mark.parametrize("lambda_value", ["%replace", "%if(False, %concat, %replace)"]) + def test_function_lambda_in_custom_function(self, lambda_value: str): + with pytest.raises( + IncompatibleFunctionArguments, + match=re.escape( + "Custom function %output has invalid usage of the function " + "%replace as a lambda: Expects 3 - 4 arguments but will receive 2." + ), + ): + Script( + { + "array1": "{['a', 'b', 'c']}", + "%output": f"{{%array_enumerate(array1, {lambda_value})}}", + } + ) diff --git a/tests/unit/script/types/test_map.py b/tests/unit/script/types/test_map.py new file mode 100644 index 00000000..3a6608cf --- /dev/null +++ b/tests/unit/script/types/test_map.py @@ -0,0 +1,200 @@ +import re + +import pytest + +from ytdl_sub.script.parser import _UNEXPECTED_COMMA_ARGUMENT +from ytdl_sub.script.parser import BRACKET_NOT_CLOSED +from ytdl_sub.script.parser import MAP_KEY_MULTIPLE_VALUES +from ytdl_sub.script.parser import MAP_KEY_NOT_HASHABLE +from ytdl_sub.script.parser import MAP_KEY_WITH_NO_VALUE +from ytdl_sub.script.parser import MAP_MISSING_KEY +from ytdl_sub.script.parser import ParsedArgType +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import Float +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException +from ytdl_sub.script.utils.exceptions import KeyNotHashableRuntimeException + + +class TestMap: + def test_return(self): + assert Script({"dict": "{{'a': 3.14}}"}).resolve() == ScriptOutput( + {"dict": Map({String("a"): Float(3.14)})} + ) + + def test_return_as_str(self): + assert Script({"dict": "json: {{'a': 3.14}}"}).resolve() == ScriptOutput( + {"dict": String('json: {"a": 3.14}')} + ) + + def test_nested_map(self): + map_str = """{ + { + 'level1': { + 'level2': { + 'level3': { + 'level4_key': 'level4_value' + }, + 'level3_key': 'level3_value' + }, + 'level2_key': 'level2_value' + }, + 'level1_key': 'level1_value' + } + }""" + + assert Script({"dict": map_str}).resolve() == ScriptOutput( + { + "dict": Map( + { + String("level1"): Map( + { + String("level2"): Map( + { + String("level3"): Map( + {String("level4_key"): String("level4_value")} + ), + String("level3_key"): String("level3_value"), + } + ), + String("level2_key"): String("level2_value"), + } + ), + String("level1_key"): String("level1_value"), + } + ) + } + ) + + @pytest.mark.parametrize( + "empty_map", + [ + "{{}}", + "{{ }}", + "{{ }}", + "{{\n}}", + ], + ) + def test_empty_map(self, empty_map: str): + assert Script({"dict": empty_map}).resolve() == ScriptOutput({"dict": Map({})}) + + @pytest.mark.parametrize( + "map", + [ + "{{}", + "{{ }", + "{{\n}", + "{{'key': 'value'}", + "{{ 'key' : 'value' }", + "{{ }", + ], + ) + def test_map_not_closed(self, map: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(BRACKET_NOT_CLOSED))): + Script({"dict": map}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{'key': }}", + "{{'key':}}", + "{{'key': 'value', 'key2':}}", + "{{'key1': 'value1','value2'}}", + "{{'key': 'value', 'key2': }}", + "{{ 'key': 'value', 'key2':\n}}", + "{{ 'key': ,\n}}", + ], + ) + def test_key_has_no_value(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_KEY_WITH_NO_VALUE))): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{,}}", + "{{ , }}", + "{{'key':'value',,}}", + "{{'key': 'value', ,}}", + ], + ) + def test_map_unexpected_comma(self, value: str): + with pytest.raises( + InvalidSyntaxException, + match=re.escape(str(_UNEXPECTED_COMMA_ARGUMENT(ParsedArgType.MAP_KEY))), + ): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{'key1','key2'}}", + "{{'key1' , 'key2'}}", + "{{ 'key1', 'key2': 'value' }}", + ], + ) + def test_map_multiple_keys(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_KEY_MULTIPLE_VALUES))): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{'key1': 'value1',}}", + "{{'key1': 'value1' , }}", + ], + ) + def test_map_trailing_comma_okay(self, value: str): + assert Script({"dict": value}).resolve().get_native("dict") == {"key1": "value1"} + + @pytest.mark.parametrize( + "value", + [ + "{{:}}", + "{{ : }}", + "{{ : 'value' }}", + ], + ) + def test_map_missing_key(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_MISSING_KEY))): + Script({"dict": value}).resolve() + + @pytest.mark.parametrize( + "value", + [ + "{{{}:'value'}}", + "{{ {} : 'value' }}", + "{{[]:'value'}}", + "{{ [] : 'value' }}", + ], + ) + def test_map_key_not_hashable(self, value: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(MAP_KEY_NOT_HASHABLE))): + Script({"dict": value}).resolve() + + def test_map_key_is_hashable_variable(self): + assert Script( + { + "dict": "{{key_variable : 'value' }}", + "key_variable": "hashable", + } + ).resolve() == ScriptOutput( + { + "key_variable": String("hashable"), + "dict": Map({String("hashable"): String("value")}), + } + ) + + def test_map_key_is_non_hashable_variable(self): + with pytest.raises( + KeyNotHashableRuntimeException, + match=re.escape("Tried to use Array as a Map key, but it is not hashable."), + ): + Script( + { + "dict": "{{key_variable : 'value' }}", + "key_variable": "{['non-hashable']}", + } + ).resolve() diff --git a/tests/unit/script/types/test_string.py b/tests/unit/script/types/test_string.py new file mode 100644 index 00000000..c3749845 --- /dev/null +++ b/tests/unit/script/types/test_string.py @@ -0,0 +1,67 @@ +import re + +import pytest + +from ytdl_sub.script.parser import STRINGS_NOT_CLOSED +from ytdl_sub.script.parser import STRINGS_ONLY_ARGS +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import InvalidSyntaxException + + +class TestString: + @pytest.mark.parametrize( + "string", + [ + "{'323'}", + '{ "4253" }', + '{"hi"}', + '{ "asfsd" }', + '{"sdfasf"}', + "{ '3fsdf' }", + ], + ) + def test_string_not_arg(self, string: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(STRINGS_ONLY_ARGS))): + Script({"out": string}).resolve() + + @pytest.mark.parametrize( + "string, expected_string", + [ + ("{%string('')}", ""), + ('{%string("")}', ""), + ('{%string("\\")}', "\\"), + ("{%string('323')}", "323"), + ('{%string( "4253" )}', "4253"), + ('{%string("hi")}', "hi"), + ('{%string( "asfsd" )}', "asfsd"), + ('{%string("sdfasf")}', "sdfasf"), + ("{%string( '3fsdf' )}", "3fsdf"), + ("{%string('newlines \n newlines')}", "newlines \n newlines"), + ("{%string('in function')} out of function", "in function out of function"), + ("{%string('supports \" in string')}", 'supports " in string'), + ('{%string("supports \' in string")}', "supports ' in string"), + ("{%string('backslash \\\\')}", "backslash \\\\"), + ("{%string('''triple quote with \" ' \\''')}", "triple quote with \" ' \\"), + ('{%string("""triple quote with " \' \\""")}', "triple quote with \" ' \\"), + ], + ) + def test_string(self, string: str, expected_string: str): + assert Script({"out": string}).resolve() == ScriptOutput({"out": String(expected_string)}) + + def test_null_is_empty_string(self): + assert Script({"out": "{%string(null)}"}).resolve() == ScriptOutput({"out": String("")}) + + @pytest.mark.parametrize( + "string", + [ + "{%string('open only single)}", + '{%string( "open only double )}', + "{%string(\"open double close single ')}", + "{%string( 'open single close double\" )}", + ], + ) + def test_string_not_closed_properly(self, string: str): + with pytest.raises(InvalidSyntaxException, match=re.escape(str(STRINGS_NOT_CLOSED))): + Script({"out": string}).resolve() diff --git a/tests/unit/script/types/test_variable.py b/tests/unit/script/types/test_variable.py new file mode 100644 index 00000000..526e1476 --- /dev/null +++ b/tests/unit/script/types/test_variable.py @@ -0,0 +1,130 @@ +import re + +import pytest + +from ytdl_sub.script.script import Script +from ytdl_sub.script.script_output import ScriptOutput +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.utils.exceptions import CycleDetected +from ytdl_sub.script.utils.exceptions import InvalidVariableName +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist + + +class TestVariable: + def test_simple(self): + assert Script({"a": "a", "b": "{b_}", "b_": "b"}).resolve() == ScriptOutput( + { + "a": String("a"), + "b": String("b"), + "b_": String("b"), + } + ) + + def test_multiple_variables(self): + assert Script({"a": "a", "b": "b", "b_": " {a} {b} "}).resolve() == ScriptOutput( + { + "a": String("a"), + "b": String("b"), + "b_": String(" a b "), + } + ) + + def test_simple_with_function(self): + assert Script({"a": "a", "b": "{%capitalize(b_)}", "b_": "b"}).resolve() == ScriptOutput( + { + "a": String("a"), + "b": String("B"), + "b_": String("b"), + } + ) + + def test_simple_cycle(self): + with pytest.raises( + CycleDetected, + match=re.escape("Cycle detected within these variables: a -> b -> a"), + ): + Script({"a": "{b}", "b": "{a}"}).resolve() + + def test_simple_cycle_with_function(self): + with pytest.raises( + CycleDetected, + match=re.escape("Cycle detected within these variables: b -> b_ -> b"), + ): + Script({"b": "{%capitalize(b_)}", "b_": "{b}"}).resolve() + + def test_undefined_variable(self): + with pytest.raises( + VariableDoesNotExist, + match=re.escape("Variable c does not exist."), + ): + Script({"a": "a", "b": "{c}"}).resolve() + + @pytest.mark.parametrize( + "name", + [ + "vali_LOL_INVALID", + "name!!", + "na(", + "na[", + ], + ) + def test_invalid_variable_name_inline(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " Names must be lower_snake_cased and begin with a letter." + ), + ): + Script({"a": f"{{{name}}}"}).resolve() + + @pytest.mark.parametrize( + "name", + ["float", "bool", "mul"], + ) + def test_invalid_variable_name_inline_is_built_in(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ), + ): + Script({"a": f"{{{name}}}"}).resolve() + + @pytest.mark.parametrize( + "name", + [ + "vali_LOL_INVALID", + "name!!", + "na(", + "na[", + "$2232", + "1245", + "CAN_CATCH_MORE", + "{brackets_in_definition}", + ], + ) + def test_invalid_variable_name_definition(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " Names must be lower_snake_cased and begin with a letter." + ), + ): + Script({f"{name}": "value"}).resolve() + + @pytest.mark.parametrize( + "name", + ["float", "bool", "mul"], + ) + def test_invalid_variable_name_definition_is_built_in(self, name: str): + with pytest.raises( + InvalidVariableName, + match=re.escape( + f"Variable name '{name}' is invalid:" + " The name is used by a built-in function and cannot be overwritten." + ), + ): + Script({f"{name}": "value"}).resolve() diff --git a/tests/unit/utils/test_script_utils.py b/tests/unit/utils/test_script_utils.py new file mode 100644 index 00000000..8b2b0e90 --- /dev/null +++ b/tests/unit/utils/test_script_utils.py @@ -0,0 +1,33 @@ +import copy + +from unit.script.conftest import single_variable_output + +from ytdl_sub.utils.script import ScriptUtils + + +class TestScriptUtils: + def test_dict_to_script(self): + json_dict = { + "string": "value", + "quotes": "has '' and \"\"", + "triple-single-quote": "right here! '''''''''''''''''''''''''''''' ack '''''''", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + "nested_dict": { + "string": "value", + "int": 1, + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "float": 3.14, + }, + } + + expected_output = copy.deepcopy(json_dict) + expected_output["triple-single-quote"] = "right here! ' ack '" + + output = single_variable_output(ScriptUtils.to_script(json_dict)) + assert output == expected_output diff --git a/tests/unit/validators/test_dict_validator.py b/tests/unit/validators/test_dict_validator.py index 544c67eb..e1eb2aeb 100644 --- a/tests/unit/validators/test_dict_validator.py +++ b/tests/unit/validators/test_dict_validator.py @@ -42,7 +42,7 @@ class TestDictValidator: ): _ = dict_validator._validate_key(key="key_name", validator=StringValidator) - @pytest.mark.parametrize("bad_value", [True, None, {}]) + @pytest.mark.parametrize("bad_value", [None, {}, []]) def test_dict_validator_validate_key_errors_bad_validation(self, bad_value): dict_validator = DictValidator(name="parent", value={"child": bad_value}) with pytest.raises( @@ -87,7 +87,7 @@ class TestDictValidator: assert out is None - @pytest.mark.parametrize("bad_value", [True, None, {}]) + @pytest.mark.parametrize("bad_value", [None, {}, []]) def test_dict_validator_validate_key_errors_none_bad_validation(self, bad_value): dict_validator = DictValidator(name="parent", value={"child": bad_value}) with pytest.raises( diff --git a/tests/unit/validators/test_file_path_validators.py b/tests/unit/validators/test_file_path_validators.py index 7cf7b9f2..7ff62143 100644 --- a/tests/unit/validators/test_file_path_validators.py +++ b/tests/unit/validators/test_file_path_validators.py @@ -5,11 +5,13 @@ import pytest from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES +from ytdl_sub.script.script import Script +from ytdl_sub.utils.file_path import FilePathTruncater from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS -from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator +@pytest.mark.usefixtures("register_custom_functions") class TestStringFormatterFilePathValidator: @pytest.mark.parametrize( "ext", @@ -28,7 +30,9 @@ class TestStringFormatterFilePathValidator: file_path = str(Path(temp_dir) / file_name) formatter = StringFormatterFileNameValidator(name="test", value=str(file_path)) - truncated_file_path = formatter.apply_formatter({}) + truncated_file_path = formatter.post_process( + Script({"file_name": formatter.format_string}).resolve().get_str("file_name") + ) assert truncated_file_path.count(".") == ext.count(".") assert str(Path(temp_dir)) in truncated_file_path @@ -56,7 +60,9 @@ class TestStringFormatterFilePathValidator: file_path = str(Path(temp_dir) / f"{base_file_name}{ext}") formatter = StringFormatterFileNameValidator(name="test", value=str(file_path)) - truncated_file_path = formatter.apply_formatter({}) + truncated_file_path = formatter.post_process( + Script({"file_name": formatter.format_string}).resolve().get_str("file_name") + ) assert truncated_file_path == str( Path(temp_dir) @@ -74,16 +80,16 @@ class TestStringFormatterFilePathValidator: @pytest.mark.parametrize( "file_name_max_bytes, expected_max", [ - (50, 50 - FilePathValidatorMixin._EXTENSION_BYTES), + (50, 50 - FilePathTruncater._EXTENSION_BYTES), (0, 16), - (10000, MAX_FILE_NAME_BYTES - FilePathValidatorMixin._EXTENSION_BYTES), + (10000, MAX_FILE_NAME_BYTES - FilePathTruncater._EXTENSION_BYTES), ], ) def test_config_changes_max_file_name_bytes(self, file_name_max_bytes: int, expected_max: int): # Ensure the default is set assert ( - FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES - == FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES + FilePathTruncater._MAX_BASE_FILE_NAME_BYTES + == FilePathTruncater._DEFAULT_MAX_BASE_FILE_NAME_BYTES ) try: @@ -98,8 +104,8 @@ class TestStringFormatterFilePathValidator: } ) - assert FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES == expected_max + assert FilePathTruncater._MAX_BASE_FILE_NAME_BYTES == expected_max finally: - FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES = ( - FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES + FilePathTruncater._MAX_BASE_FILE_NAME_BYTES = ( + FilePathTruncater._DEFAULT_MAX_BASE_FILE_NAME_BYTES ) diff --git a/tests/unit/validators/test_string_formatter_validator.py b/tests/unit/validators/test_string_formatter_validator.py index fbafda4d..0928fc24 100644 --- a/tests/unit/validators/test_string_formatter_validator.py +++ b/tests/unit/validators/test_string_formatter_validator.py @@ -9,22 +9,6 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -@pytest.fixture -def error_message_unequal_brackets_str(): - return ( - "Brackets are reserved for {variable_names} and " - "should contain a single open and close bracket." - ) - - -@pytest.fixture -def error_message_unequal_regex_matches_str(): - return ( - "{variable_names} must start with a lowercase letter, should only contain lowercase " - "letters, numbers, underscores, and have a single open and close bracket." - ) - - @pytest.mark.parametrize( "string_formatter_class", [StringFormatterValidator, OverridesStringFormatterValidator] ) @@ -34,16 +18,6 @@ class TestStringFormatterValidator(object): validator = string_formatter_class(name="test_format_variables", value=format_string) assert validator.format_string == format_string - assert validator.format_variables == ["var_one1", "var_two"] - - def test_format_variables(self, string_formatter_class): - format_string = "No vars 💩" - assert ( - string_formatter_class( - name="test_format_variables_empty", value=format_string - ).format_variables - == [] - ) @pytest.mark.parametrize( "format_string", @@ -54,12 +28,8 @@ class TestStringFormatterValidator(object): "Try }var_one} and {var_one}", ], ) - def test_validate_fail_uneven_brackets( - self, string_formatter_class, format_string, error_message_unequal_brackets_str - ): - expected_error_msg = f"Validation error in fail: {error_message_unequal_brackets_str}" - - with pytest.raises(ValidationException, match=expected_error_msg): + def test_validate_fail_uneven_brackets(self, string_formatter_class, format_string): + with pytest.raises(ValidationException, match="Validation error in fail:"): _ = string_formatter_class(name="fail", value=format_string) @pytest.mark.parametrize( @@ -74,124 +44,10 @@ class TestStringFormatterValidator(object): "Try {} empty", ], ) - def test_validate_fail_bad_variable( - self, string_formatter_class, format_string, error_message_unequal_regex_matches_str - ): - expected_error_msg = f"Validation error in fail: {error_message_unequal_regex_matches_str}" - - with pytest.raises(ValidationException, match=expected_error_msg): + def test_validate_fail_bad_variable(self, string_formatter_class, format_string): + with pytest.raises(ValidationException, match="Validation error in fail:"): _ = string_formatter_class(name="fail", value=format_string) - @pytest.mark.parametrize( - "format_string, bad_variable", - [ - ("keyword {while}", "while"), - ("{try} {valid_var}", "try"), - ], - ) - def test_validate_fail_variable_keyword_or_not_identifier( - self, string_formatter_class, format_string, bad_variable - ): - expected_error_msg = ( - f"Validation error in fail: " - f"'{bad_variable}' is a Python keyword and cannot be used as a variable." - ) - - with pytest.raises(ValidationException, match=expected_error_msg): - _ = string_formatter_class(name="fail", value=format_string) - - def test_entry_formatter_fails_missing_field(self, string_formatter_class): - format_string = string_formatter_class(name="test", value=f"prefix {{bah_humbug}} suffix") - variable_dict = {"varb": "a", "vara": "b"} - expected_error_msg = ( - f"Validation error in test: Format variable 'bah_humbug' does not exist. " - f"Available variables: {', '.join(sorted(variable_dict.keys()))}" - ) - if string_formatter_class == OverridesStringFormatterValidator: - expected_error_msg = ( - f"Validation error in test: Override variable 'bah_humbug' does not exist. " - f"For this field, ensure your override variable does not contain any source " - f"variables - it is a requirement that this be a static string. " - f"Available override variables: {', '.join(sorted(variable_dict.keys()))}" - ) - - with pytest.raises(StringFormattingException, match=expected_error_msg): - assert format_string.apply_formatter(variable_dict=variable_dict) - - def test_string_formatter_single_field(self, string_formatter_class): - uid = "this uid" - format_string = string_formatter_class(name="test", value=f"prefix {{uid}} suffix") - expected_string = f"prefix {uid} suffix" - - assert format_string.apply_formatter(variable_dict={"uid": uid}) == expected_string - - def test_entry_formatter_duplicate_fields(self, string_formatter_class): - upload_year = "2022" - format_string = string_formatter_class( - name="test", value=f"prefix {{upload_year}} {{upload_year}} suffix" - ) - expected_string = f"prefix {upload_year} {upload_year} suffix" - - assert ( - format_string.apply_formatter(variable_dict={"upload_year": upload_year}) - == expected_string - ) - - def test_entry_formatter_override_recursive(self, string_formatter_class): - variable_dict = { - "level_a": "level a", - "level_b": "level b and {level_a}", - "level_c": "level c and {level_b}", - } - - format_string = string_formatter_class(name="test", value="level d and {level_c}") - expected_string = "level d and level c and level b and level a" - - assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string - - def test_entry_formatter_override_sanitized_recursive(self, string_formatter_class): - variable_dict = { - "level_a": "level a", - "level_b": "level b ? {level_a}", - "level_c": "level c and {level_b}", - } - - format_string = string_formatter_class(name="test", value="level d and {level_c_sanitized}") - expected_string = "level d and " + sanitize_filename("level c and level b ? level a") - - assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string - - def test_entry_formatter_override_sanitized_recursive_inner(self, string_formatter_class): - variable_dict = { - "level_a": "level a ?", - "level_b": "level b ? {level_a_sanitized}", - "level_c": "level c and {level_b_sanitized}", - } - - format_string = string_formatter_class(name="test", value="level d and {level_c}") - expected_string = "level d and level c and " + sanitize_filename("level b ? level a ?") - - assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string - - def test_entry_formatter_override_recursive_fail_cycle(self, string_formatter_class): - variable_dict = { - "level_a": "{level_b}", - "level_b": "{level_a}", - } - - # Max depth is 3 so should go level_a -(0)-> level_b -(1)-> level_a -(2)-> level_b - expected_error_msg = ( - "Validation error in test: Attempted to format but failed after reaching max recursion " - "depth of 3. Try to keep variables dependent on only one other variable at max. " - "Unresolved variables: level_b" - ) - - format_string = string_formatter_class(name="test", value="{level_a}") - format_string._max_format_recursion = 3 - - with pytest.raises(StringFormattingException, match=expected_error_msg): - _ = format_string.apply_formatter(variable_dict=variable_dict) - class TestDictFormatterValidator(object): @pytest.mark.parametrize( @@ -215,9 +71,6 @@ class TestDictFormatterValidator(object): assert validator.dict["key1"].format_string == key1_format_string assert validator.dict["key2"].format_string == key2_format_string - assert validator.dict["key1"].format_variables == ["variable"] - assert validator.dict["key2"].format_variables == [] - assert validator.dict_with_format_strings == { "key1": key1_format_string, "key2": key2_format_string, diff --git a/tests/unit/validators/test_string_validator.py b/tests/unit/validators/test_string_validator.py index b495bdef..ea336b4a 100644 --- a/tests/unit/validators/test_string_validator.py +++ b/tests/unit/validators/test_string_validator.py @@ -4,16 +4,16 @@ from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.validators import StringValidator -@pytest.mark.parametrize("value", ["a", "unicode 💩", ""]) +@pytest.mark.parametrize("value", ["a", "unicode 💩", "", 1, 3.14, True, False]) def test_string_validator(value): string_validator = StringValidator(name="good_str_validator", value=value) assert string_validator._name == "good_str_validator" - assert string_validator._value == value - assert string_validator.value == value + assert string_validator._value == str(value) + assert string_validator.value == str(value) -@pytest.mark.parametrize("value", [None, {}, True, 0]) -def test_bool_validator_fails_bad_value(value): +@pytest.mark.parametrize("value", [None, dict(), [], set()]) +def test_string_validator_fails_bad_value(value): with pytest.raises( ValidationException, match="Validation error in fail: should be of type string" ): diff --git a/tools/docgen/docgen.py b/tools/docgen/docgen.py new file mode 100644 index 00000000..f1307b29 --- /dev/null +++ b/tools/docgen/docgen.py @@ -0,0 +1,32 @@ +from abc import abstractmethod +from pathlib import Path + +REGENERATE_DOCS: bool = False + + +class DocGen: + """ + Home-made auto doc generation + """ + + LOCATION: Path + + @classmethod + @abstractmethod + def generate(cls) -> str: + """ + Generate the docs as a single string + """ + + @classmethod + def generate_and_maybe_write_to_file(cls) -> str: + """ + Maybe writes the docs to their file if the global is set to True, and returns + the generated docs + """ + contents = cls.generate() + if REGENERATE_DOCS: + with open(cls.LOCATION, "w", encoding="utf-8") as out: + out.write(contents) + + return contents diff --git a/tools/docgen/entry_variables.py b/tools/docgen/entry_variables.py new file mode 100644 index 00000000..df53b13a --- /dev/null +++ b/tools/docgen/entry_variables.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import cached_properties +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import line_section +from tools.docgen.utils import section +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + + +def _variable_class_to_name(obj: Type[Any]) -> str: + assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions" + return ( + camel_case_to_human(obj.__name__) + .replace("Variable Definitions", "Variables") + .replace("Ytdl Sub", "Ytdl-Sub") + ) + + +class EntryVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/entry_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Entry Variables", level=0) + + parent_objs: Dict[str, Type[Any]] = { + _variable_class_to_name(obj): obj for obj in VariableDefinitions.__bases__ + } + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for variable_function_name in cached_properties(parent_objs[name]): + docs += get_function_docs( + function_name=variable_function_name, + obj=parent_objs[name], + pre_docstring=f":type: ``{getattr(VARIABLES, variable_function_name).human_readable_type()}``\n", + level=2, + ) + + return docs diff --git a/tools/docgen/override_variables.py b/tools/docgen/override_variables.py new file mode 100644 index 00000000..e919877c --- /dev/null +++ b/tools/docgen/override_variables.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.variables.override_variables import OverrideVariables + + +class OverrideVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Override Variables", level=0) + + for name in static_methods(OverrideVariables): + docs += get_function_docs( + function_name=name, + obj=OverrideVariables, + level=1, + ) + + return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py new file mode 100644 index 00000000..10323066 --- /dev/null +++ b/tools/docgen/plugins.py @@ -0,0 +1,91 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import line_section +from tools.docgen.utils import properties +from tools.docgen.utils import section +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator + + +def should_filter_all_properties(plugin_name: str) -> bool: + return plugin_name in ( + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "video_tags", + "download", + ) + + +def should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + ) + + +def get_function_docs(function_name: str, obj: Any, level: int) -> str: + docs = f"\n``{function_name}``\n\n" + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n\n" + return docs + + +def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: + docs = "" + docs += section(name, level=offset + 0) + + docs += inspect.cleandoc(options.__doc__) + docs += "\n" + + if should_filter_all_properties(name): + return docs + + property_names = [prop for prop in properties(options) if not should_filter_property(prop)] + for property_name in sorted(property_names): + docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) + + return docs + + +class PluginsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/plugins.rst") + + @classmethod + def generate(cls): + options_dict: Dict[str, Type[OptionsValidator]] = { + "output_options": OutputOptions, + "ytdl_options": YTDLOptions, + "overrides": Overrides, + "download": MultiUrlValidator, + } + for plugin_name, plugin_type in PluginMapping._MAPPING.items(): + if plugin_name.startswith("_"): + continue + options_dict[plugin_name] = plugin_type.plugin_options_type + + docs = section("Plugins", level=0) + for idx, name in enumerate(sorted(options_dict.keys())): + docs += line_section(section_idx=idx) + docs += generate_plugin_docs(name, options_dict[name], offset=1) + + return docs diff --git a/tools/docgen/scripting_functions.py b/tools/docgen/scripting_functions.py new file mode 100644 index 00000000..c4663477 --- /dev/null +++ b/tools/docgen/scripting_functions.py @@ -0,0 +1,82 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import line_section +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.type_checking import FunctionSpec + + +def maybe_get_function_name(function_name: str) -> Optional[str]: + if function_name in ["register"]: + return None + + if function_name.endswith("_"): + return function_name[:-1] + return function_name + + +def function_class_to_name(obj: Type[Any]) -> str: + assert "Functions" in obj.__name__ + return camel_case_to_human(obj.__name__) + + +def function_type_hinting(display_function_name: str, function: Any) -> str: + spec = FunctionSpec.from_callable(function) + out = ":spec: ``" + out += display_function_name + out += spec.human_readable_input_args() + out += " -> " + out += spec.human_readable_output_type() + out += "``\n\n" + return out + + +def get_function_docstring( + function_name: str, function: Any, level: int, display_function_name: Optional[str] = None +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + + docs += function_type_hinting(display_function_name=display_function_name, function=function) + docs += inspect.cleandoc(function.__doc__) + docs += "\n" + return docs + + +class ScriptingFunctionsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/scripting_functions.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Scripting Functions", level=0) + + parent_objs: Dict[str, Type[Any]] = { + function_class_to_name(obj): obj for obj in Functions.__bases__ + } + parent_objs["Ytdl-Sub Functions"] = CustomFunctions + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for function_name in static_methods(parent_objs[name]): + if display_function_name := maybe_get_function_name(function_name): + docs += get_function_docstring( + function_name=function_name, + display_function_name=display_function_name, + function=getattr(parent_objs[name], function_name), + level=2, + ) + + return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py new file mode 100644 index 00000000..81ddc3fa --- /dev/null +++ b/tools/docgen/utils.py @@ -0,0 +1,66 @@ +import inspect +from functools import cached_property +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Type + +LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} + + +def section(name: str, level: int, as_code: bool = False) -> str: + if as_code: + name = f"``{name}``" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" + + +def properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property)) + + +def cached_properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), cached_property)) + + +def static_methods(obj: Type[Any]) -> List[str]: + return sorted( + name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod) + ) + + +def camel_case_to_human(string: str) -> str: + output_str = string[0] + for char in string[1:]: + if char.islower(): + output_str += char + else: + output_str += f" {char}" + + return output_str + + +def get_function_docs( + function_name: str, + obj: Any, + level: int, + display_function_name: Optional[str] = None, + pre_docstring: Optional[str] = None, +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + docs += pre_docstring or "" + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n" + return docs + + +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return "" diff --git a/tools/linter b/tools/linter index 28394065..b3a4fb7f 100755 --- a/tools/linter +++ b/tools/linter @@ -3,11 +3,9 @@ if [[ $1 = "check" ]]; then isort . --check-only --diff \ && black . --check \ - && pylint src/ \ - && pydocstyle src/* + && pylint src/ else isort . black . pylint src/ - pydocstyle src/* fi \ No newline at end of file diff --git a/ytdl-sub.spec b/ytdl-sub.spec index efecd408..a035093c 100644 --- a/ytdl-sub.spec +++ b/ytdl-sub.spec @@ -9,10 +9,7 @@ a = Analysis( pathex=[], binaries=[], datas=[ - ('src/ytdl_sub/prebuilt_presets/helpers/*.yaml', 'ytdl_sub/prebuilt_presets/helpers'), - ('src/ytdl_sub/prebuilt_presets/internal/*.yaml', 'ytdl_sub/prebuilt_presets/internal'), - ('src/ytdl_sub/prebuilt_presets/music_videos/*.yaml', 'ytdl_sub/prebuilt_presets/music_videos'), - ('src/ytdl_sub/prebuilt_presets/tv_show/*.yaml', 'ytdl_sub/prebuilt_presets/tv_show'), + ('src/ytdl_sub/prebuilt_presets', 'ytdl_sub/prebuilt_presets'), ], hiddenimports=[], hookspath=[],