From 731d4e444c53f4aa868d2aa90d20083445356cd4 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 28 Dec 2023 22:39:59 -0800 Subject: [PATCH 1/5] [DOCS] Tooling to auto-generate custom sphinx docs (#849) Tooling + updating docs --- .gitignore | 2 + docs/source/_static/custom.css | 0 docs/source/conf.py | 1 + docs/source/config_reference/config_yaml.rst | 46 +- docs/source/config_reference/plugins.rst | 1094 +++++++------ .../scripting/config_types.rst | 17 - .../scripting/entry_variables.rst | 15 +- .../config_reference/scripting/index.rst | 9 +- .../scripting/override_variables.rst | 2 +- .../scripting/scripting_functions.rst | 443 ++++-- docs/source/prebuilt_presets/tv_shows.rst | 2 +- src/ytdl_sub/config/overrides.py | 14 +- src/ytdl_sub/config/preset_options.py | 94 +- src/ytdl_sub/downloaders/url/multi_url.py | 37 - src/ytdl_sub/downloaders/url/url.py | 25 - src/ytdl_sub/downloaders/url/validators.py | 49 +- .../entries/script/variable_definitions.py | 1348 +++++++---------- .../entries/variables/override_variables.py | 18 +- src/ytdl_sub/plugins/audio_extract.py | 22 +- src/ytdl_sub/plugins/chapters.py | 80 +- src/ytdl_sub/plugins/date_range.py | 28 +- src/ytdl_sub/plugins/embed_thumbnail.py | 6 +- src/ytdl_sub/plugins/file_convert.py | 61 +- src/ytdl_sub/plugins/filter_exclude.py | 12 +- src/ytdl_sub/plugins/filter_include.py | 29 +- src/ytdl_sub/plugins/format.py | 4 +- src/ytdl_sub/plugins/match_filters.py | 32 +- src/ytdl_sub/plugins/music_tags.py | 21 +- src/ytdl_sub/plugins/nfo_tags.py | 94 +- .../plugins/output_directory_nfo_tags.py | 56 +- src/ytdl_sub/plugins/regex.py | 115 +- src/ytdl_sub/plugins/split_by_chapters.py | 29 +- src/ytdl_sub/plugins/subtitles.py | 48 +- src/ytdl_sub/plugins/throttle_protection.py | 24 +- src/ytdl_sub/plugins/video_tags.py | 12 +- .../script/functions/array_functions.py | 72 +- .../script/functions/boolean_functions.py | 36 +- .../script/functions/conditional_functions.py | 10 +- .../script/functions/date_functions.py | 3 +- .../script/functions/error_functions.py | 23 +- .../script/functions/json_functions.py | 3 +- .../script/functions/map_functions.py | 34 +- .../script/functions/numeric_functions.py | 30 +- .../script/functions/regex_functions.py | 24 +- .../script/functions/string_functions.py | 35 +- .../script/utils/exception_formatters.py | 28 +- src/ytdl_sub/script/utils/type_checking.py | 39 + src/ytdl_sub/utils/file_handler.py | 7 + ...ters_with_regex_no_chapters_video_pass.txt | 2 +- .../plugins/test_audio_extract_playlist.json | 2 +- .../youtube/test_channel_full.json | 4 +- .../youtube/test_playlist.json | 2 +- .../test_playlist_archive_migrated.json | 2 +- .../youtube/test_video.json | 2 +- .../youtube/test_video_cli.json | 2 +- .../youtube/test_video_missing_thumb.json | 2 +- tests/unit/docgen/__init__.py | 0 tests/unit/docgen/test_docgen.py | 30 + .../docgen/test_docgen_regenerate_disabled.py | 5 + tests/unit/script/types/test_function.py | 8 +- tools/docgen/docgen.py | 32 + tools/docgen/entry_variables.py | 47 + tools/docgen/override_variables.py | 25 + tools/docgen/plugins.py | 91 ++ tools/docgen/scripting_functions.py | 82 + tools/docgen/utils.py | 56 + 66 files changed, 2660 insertions(+), 1967 deletions(-) create mode 100644 docs/source/_static/custom.css delete mode 100644 docs/source/config_reference/scripting/config_types.rst delete mode 100644 src/ytdl_sub/downloaders/url/multi_url.py delete mode 100644 src/ytdl_sub/downloaders/url/url.py create mode 100644 tests/unit/docgen/__init__.py create mode 100644 tests/unit/docgen/test_docgen.py create mode 100644 tests/unit/docgen/test_docgen_regenerate_disabled.py create mode 100644 tools/docgen/docgen.py create mode 100644 tools/docgen/entry_variables.py create mode 100644 tools/docgen/override_variables.py create mode 100644 tools/docgen/plugins.py create mode 100644 tools/docgen/scripting_functions.py create mode 100644 tools/docgen/utils.py diff --git a/.gitignore b/.gitignore index 20d1b188..e18f90d3 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,5 @@ docker/testing/volumes ffmpeg.exe ffprobe.exe + +tools/docgen/out \ No newline at end of file diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/conf.py b/docs/source/conf.py index 6a48e5f0..7e551b67 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,6 +53,7 @@ html_theme_options = { } html_static_path = ["_static"] +html_css_files = ["custom.css"] # Make sure the all autosectionlabel targets are unique diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst index 521e1f0a..8e4097c3 100644 --- a/docs/source/config_reference/config_yaml.rst +++ b/docs/source/config_reference/config_yaml.rst @@ -53,51 +53,7 @@ presets ~~~~~~~ ``presets`` define a `formula` for how to format downloaded media and metadata. -download_strategy -""""""""""""""""" -Download strategies dictate what is getting downloaded from a source. Each -download strategy has its own set of parameters. - -.. _url: - -url -''' -.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions() - :members: url, playlist_thumbnails, source_thumbnails, download_reverse - :member-order: bysource - -multi_url -''''''''' -.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions() - :members: urls, variables - -------------------------------------------------------------------------------- - -output_options -"""""""""""""" - -.. autoclass:: ytdl_sub.config.preset_options.OutputOptions() - :members: - :member-order: bysource - :exclude-members: get_upload_date_range_to_keep, partial_validate - -------------------------------------------------------------------------------- - -.. _ytdl_options: - -ytdl_options -"""""""""""" -.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions() - -------------------------------------------------------------------------------- - -.. _overrides: - -overrides -""""""""" -.. autoclass:: ytdl_sub.config.overrides.Overrides() - -.. _parent preset: +This section is work-in-progress! preset """""" diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 451a0ed8..69512e8f 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -1,3 +1,4 @@ + Plugins ======= @@ -5,175 +6,315 @@ audio_extract ------------- Extracts audio from a video file. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 -codec -~~~~~ -The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, -opus, vorbis, wav, and best to grab the best possible format at runtime. +``codec`` -quality -~~~~~~~ -Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` -(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. +:expected type: String +:description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. + + +``quality`` + +:expected type: Float +:description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + + +---------------------------------------------------------------------------------------------------- chapters -------- Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False -allow_chapters_from_comments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. If chapters do not exist in the video/description itself, attempt to scrape -comments to find the chapters. Defaults to False. +``allow_chapters_from_comments`` -embed_chapters -~~~~~~~~~~~~~~ -Optional. Embed chapters into the file. Defaults to True. +:expected type: Optional[Boolean] +:description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. -force_key_frames -~~~~~~~~~~~~~~~~ -Optional. Force keyframes at cuts when removing sections. This is slow due to needing a -re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to -False. -remove_chapters_regex -~~~~~~~~~~~~~~~~~~~~~ -Optional. List of regex patterns to match chapter titles against and remove them from the -entry. +``embed_chapters`` -remove_sponsorblock_categories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. List of SponsorBlock categories to remove from the output file. Can only remove -categories that are specified in ``sponsorblock_categories`` or "all", which removes -everything specified in ``sponsorblock_categories``. +:expected type: Optional[Boolean] +:description: + Defaults to True. Embed chapters into the file. -sponsorblock_categories -~~~~~~~~~~~~~~~~~~~~~~~ -Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", -"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", -"poi_highlight", or "all" to include all categories. + +``force_key_frames`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. + + +``remove_chapters_regex`` + +:expected type: Optional[List[RegexString] +:description: + List of regex patterns to match chapter titles against and remove them from the + entry. + + +``remove_sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. + + +``sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. + + +---------------------------------------------------------------------------------------------------- date_range ---------- Only download files uploaded within the specified date range. +Dates must adhere to a yt-dlp datetime. From their docs: -Usage: +.. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + +Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. +Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest +granularity possible. + +:Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" -after -~~~~~ -Optional. Only download videos after this datetime. +``after`` -before -~~~~~~ -Optional. Only download videos before this datetime. +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +``before`` + +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +---------------------------------------------------------------------------------------------------- + +download +-------- +Sets the URL(s) to download from. Can be used in many forms, including + +:Single URL: + +.. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + +:Multi URL: + +.. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + +:Thumbnails + Variables: + +All variables must be defined for the top-most url. All subsequent URL variables can be either +overwritten or default to the top-most value. + +If an entry is returned from more than one URL, it will use the variables in the bottom-most +URL. + +.. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + +---------------------------------------------------------------------------------------------------- embed_thumbnail --------------- Whether to embed thumbnails to the audio/video file or not. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True + +---------------------------------------------------------------------------------------------------- file_convert ------------ Converts video files from one extension to another. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" -Supports custom ffmpeg conversions: +Also supports custom ffmpeg conversions: + +:Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text -convert_to -~~~~~~~~~~ -Convert to a desired file type. Supports: +``convert_to`` -* Video: avi, flv, mkv, mov, mp4, webm -* Audio: aac, flac, mp3, m4a, opus, vorbis, wav +:expected type: String +:description: + Convert to a desired file type. Supports -convert_with -~~~~~~~~~~~~ -Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within -yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified -with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav -ffmpeg_post_process_args -~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the -form of: -.. code-block:: bash +``convert_with`` - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext +:expected type: Optional[String] +:description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. -The output file will use the extension specified in ``convert_to``. Post-processing args -can still be set with ``convert_with`` set to ``yt-dlp``. + +``ffmpeg_post_process_args`` + +:expected type: Optional[OverridesFormatter] +:description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of + + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. + + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. + + +---------------------------------------------------------------------------------------------------- + +filter_exclude +-------------- +Applies a conditional OR on any number of filters comprised of either variables or scripts. +If any filter evaluates to True, the entry will be excluded. + +:Usage: + +.. code-block:: yaml + + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } + +---------------------------------------------------------------------------------------------------- + +filter_include +-------------- +Applies a conditional AND on any number of filters comprised of either variables or scripts. +If all filters evaluate to True, the entry will be included. + +:Usage: + +.. code-block:: yaml + + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + +---------------------------------------------------------------------------------------------------- format ------ @@ -184,48 +325,28 @@ Usage: .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" -format -~~~~~~ -yt-dlp format, uses same syntax as yt-dlp. +---------------------------------------------------------------------------------------------------- match_filters ------------- -Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. -Uses the same syntax as yt-dlp. +Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. +Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. +For logical AND's between match filters, use the ``&`` operator in a single match filter. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" + match_filters: + filters: + - "age_limit?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" -Supports one or multiple filters: - -.. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit`_. -Usage: +:Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" -embed_thumbnail -~~~~~~~~~~~~~~~ -Optional. Whether to embed the thumbnail into the audio file. - -tags -~~~~ -Key, values of tag names, tag values. Supports source and override variables. -Supports lists which will get written to MP3s as id3v2.4 multi-tags. +---------------------------------------------------------------------------------------------------- nfo_tags -------- Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False -kodi_safe -~~~~~~~~~ -Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some -foreign language characters. Setting this to True will replace those characters with '□'. -Defaults to False. +``kodi_safe`` -nfo_name -~~~~~~~~ -The NFO file name. +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. -nfo_root -~~~~~~~~ -The root tag of the NFO's XML. In the usage above, it would look like -.. code-block:: xml +``nfo_name`` - - - +:expected type: EntryFormatter +:description: + The NFO file name. -tags -~~~~ -Tags within the nfo_root tag. In the usage above, it would look like -.. code-block:: xml +``nfo_root`` - - - Awesome Youtube Video - 2022 - 502 - +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like -Also supports xml attributes and duplicate keys: + .. code-block:: xml -.. code-block:: yaml + + + - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" -Which translates to +``tags`` -.. code-block:: xml +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like - 2022 - Comedy - Drama + .. code-block:: xml + + + + Awesome Youtube Video + 2022 + 502 + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + 2022 + Comedy + Drama + + +---------------------------------------------------------------------------------------------------- output_directory_nfo_tags ------------------------- @@ -361,63 +484,77 @@ Usage: # optional kodi_safe: False -kodi_safe -~~~~~~~~~ -Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some -foreign language characters. Setting this to True will replace those characters with '□'. -Defaults to False. +``kodi_safe`` -nfo_name -~~~~~~~~ -The NFO file name. +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. -nfo_root -~~~~~~~~ -The root tag of the NFO's XML. In the usage above, it would look like -.. code-block:: xml +``nfo_name`` - - - +:expected type: EntryFormatter +:description: + The NFO file name. -tags -~~~~ -Tags within the nfo_root tag. In the usage above, it would look like -.. code-block:: xml +``nfo_root`` - - - Sweet youtube TV show - +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like -Also supports xml attributes and duplicate keys: + .. code-block:: xml -.. code-block:: yaml + + + - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" -Which translates to +``tags`` -.. code-block:: xml +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + .. code-block:: xml + + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <tvshow> + <title>Sweet youtube TV show + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> + + +---------------------------------------------------------------------------------------------------- output_options -------------- Defines where to output files and thumbnails after all post-processing has completed. -Usage: +:Usage: .. code-block:: yaml @@ -436,79 +573,113 @@ Usage: keep_files_before: now keep_files_after: 19000101 -download_archive_name -~~~~~~~~~~~~~~~~~~~~~ -Optional. The file name to store a subscriptions download archive placed relative to -the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` +``download_archive_name`` -file_name -~~~~~~~~~ -Required. The file name for the media file. This can include directories such as -``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. +:expected type: Optional[OverridesFormatter] +:description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` -info_json_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's info json file. This can include directories such -as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output -directory. Can be set to empty string or `null` to disable info json writes. -keep_files_after -~~~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. +``file_name`` -Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep -files after ``19000101``, which implies all files. Can be used in conjunction with -``keep_max_files``. +:expected type: EntryFormatter +:description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. -keep_files_before -~~~~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. -Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep -files before ``now``, which implies all files. Can be used in conjunction with -``keep_max_files``. +``info_json_name`` -keep_max_files -~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. -Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be -applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. -maintain_download_archive -~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Maintains a download archive file in the output directory for a subscription. -It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the -output directory. +``keep_files_after`` -The download archive contains a mapping of ytdl IDs to downloaded files. This is used to -create a ytdl download-archive file when invoking a download on a subscription. This will -prevent ytdl from redownloading media already downloaded. +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. -Defaults to False. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. -migrated_download_archive_name -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Intended to be used if you are migrating a subscription with either a new -subscription name or output directory. It will try to load the archive file using this name -first, and fallback to ``download_archive_name``. It will always save to this file -and remove the original ``download_archive_name``. -output_directory -~~~~~~~~~~~~~~~~ -Required. The output directory to store all media files downloaded. +``keep_files_before`` -thumbnail_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's thumbnail image. This can include directories such -as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output -directory. Can be set to empty string or `null` to disable thumbnail writes. +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. + + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. + + +``keep_max_files`` + +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + + +``maintain_download_archive`` + +:expected type: Optional[Boolean] +:description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. + + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. + + Defaults to False. + + +``migrated_download_archive_name`` + +:expected type: Optional[OverridesFormatter] +:description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. + + +``output_directory`` + +:expected type: OverridesFormatter +:description: + The output directory to store all media files downloaded. + + +``thumbnail_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. + + +---------------------------------------------------------------------------------------------------- overrides --------- -Optional. This section allows you to define variables that can be used in any string formatter. -For example, if you want your file and thumbnail files to match without copy-pasting a large -format string, you can define something like: +Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + +:Usage: .. code-block:: yaml @@ -531,8 +702,45 @@ In addition, any override variable defined will automatically create a ``sanitiz for use. In the example above, ``output_directory_sanitized`` will exist and perform sanitization on the value when used. +---------------------------------------------------------------------------------------------------- + regex ----- +.. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -548,82 +756,95 @@ For example, creating the override variable ``"title_and_description": "{title} and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" -skip_if_match_fails -~~~~~~~~~~~~~~~~~~~ -Defaults to True. If True, when any match fails and has no defaults, the entry will be -skipped. If False, ytdl-sub will error and all downloads will not proceed. +``skip_if_match_fails`` + +:expected type: Optional[Boolean] +:description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. + + +---------------------------------------------------------------------------------------------------- split_by_chapters ----------------- Splits a file by chapters into multiple files. Each file becomes its own entry with the -new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, -``chapter_index_padded``, ``chapter_count``. +new variables -If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is -set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" -when_no_chapters -~~~~~~~~~~~~~~~~ -Behavior to perform when no chapters are present. Supports "pass" (continue processing), -"drop" (exclude it from output), and "error" (stop processing for everything). +``when_no_chapters`` + +:expected type: String +:description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + + +---------------------------------------------------------------------------------------------------- subtitles --------- @@ -631,43 +852,60 @@ Defines how to download and store subtitles. Using this plugin creates two new v ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False -allow_auto_generated_subtitles -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Whether to allow auto generated subtitles. Defaults to False. +``allow_auto_generated_subtitles`` -embed_subtitles -~~~~~~~~~~~~~~~ -Optional. Whether to embed the subtitles into the video file. Defaults to False. -NOTE: webm files can only embed "vtt" subtitle types. +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to allow auto generated subtitles. -languages -~~~~~~~~~ -Optional. Language code(s) to download for subtitles. Supports a single or list of multiple -language codes. Defaults to "en". -subtitles_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's subtitles if they are present. This can include -directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and -will be placed in the output directory. ``lang`` is dynamic since you can download multiple -subtitles. It will set the respective language to the correct subtitle file. +``embed_subtitles`` -subtitles_type -~~~~~~~~~~~~~~ -Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. + + +``languages`` + +:expected type: Optional[List[String]] +:description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". + + +``subtitles_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. + + +``subtitles_type`` + +:expected type: Optional[String] +:description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". + + +---------------------------------------------------------------------------------------------------- throttle_protection ------------------- @@ -675,7 +913,7 @@ Provides options to make ytdl-sub look more 'human-like' to protect from throttl range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. -Usage: +:Usage: .. code-block:: yaml @@ -693,52 +931,62 @@ Usage: max: 36 subscription_download_probability: 1.0 -max_downloads_per_subscription -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Number of downloads to perform per subscription. +``max_downloads_per_subscription`` -sleep_per_download_s -~~~~~~~~~~~~~~~~~~~~ -Number in seconds to sleep between each download. Does not include time it takes for -ytdl-sub to perform post-processing. +:expected type: Optional[Range] +:description: + Number of downloads to perform per subscription. -sleep_per_subscription_s -~~~~~~~~~~~~~~~~~~~~~~~~ -Number in seconds to sleep between each subscription. -subscription_download_probability -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Probability to perform any downloads, recomputed for each subscription. This is only -recommended to set if you run ytdl-sub in a cron-job, that way you are statistically -guaranteed over time to eventually download the subscription. +``sleep_per_download_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. + + +``sleep_per_subscription_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each subscription. + + +``subscription_download_probability`` + +:expected type: Optional[Float] +:description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. + + +---------------------------------------------------------------------------------------------------- video_tags ---------- Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" -tags -~~~~ -Key/values of tag names/values. Supports source and override variables. +---------------------------------------------------------------------------------------------------- ytdl_options ------------ -Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. +Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. -ytdl_options should be formatted like: +:Usage: .. code-block:: yaml diff --git a/docs/source/config_reference/scripting/config_types.rst b/docs/source/config_reference/scripting/config_types.rst deleted file mode 100644 index 31273cde..00000000 --- a/docs/source/config_reference/scripting/config_types.rst +++ /dev/null @@ -1,17 +0,0 @@ -================== -Config Field Types -================== - -The ``config.yaml`` uses various types for its configurable fields. Below is a definition for each type. - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator() - -.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator() diff --git a/docs/source/config_reference/scripting/entry_variables.rst b/docs/source/config_reference/scripting/entry_variables.rst index cbd3c0cf..c4467330 100644 --- a/docs/source/config_reference/scripting/entry_variables.rst +++ b/docs/source/config_reference/scripting/entry_variables.rst @@ -1,3 +1,4 @@ + Entry Variables =============== @@ -111,6 +112,8 @@ webpage_url ~~~~~~~~~~~ The url to the webpage. +---------------------------------------------------------------------------------------------------- + Metadata Variables ------------------ @@ -130,6 +133,8 @@ source_metadata ~~~~~~~~~~~~~~~ Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) +---------------------------------------------------------------------------------------------------- + Playlist Variables ------------------ @@ -210,6 +215,8 @@ playlist_webpage_url ~~~~~~~~~~~~~~~~~~~~ The playlist webpage url if it exists. Otherwise, returns the entry webpage url. +---------------------------------------------------------------------------------------------------- + Release Date Variables ---------------------- @@ -285,6 +292,8 @@ release_year_truncated_reversed The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. 2022 returns ``100 - 22`` = ``78`` +---------------------------------------------------------------------------------------------------- + Source Variables ---------------- @@ -332,6 +341,8 @@ source_webpage_url ~~~~~~~~~~~~~~~~~~ The source webpage url if it exists, otherwise returns the playlist webpage url. +---------------------------------------------------------------------------------------------------- + Upload Date Variables --------------------- @@ -406,6 +417,8 @@ upload_year_truncated_reversed The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. 2022 returns ``100 - 22`` = ``78`` +---------------------------------------------------------------------------------------------------- + Ytdl-Sub Variables ------------------ @@ -436,4 +449,4 @@ The upload_date_index padded two digits ytdl_sub_input_url ~~~~~~~~~~~~~~~~~~ -The input URL used in ytdl-sub to create this entry. \ No newline at end of file +The input URL used in ytdl-sub to create this entry. diff --git a/docs/source/config_reference/scripting/index.rst b/docs/source/config_reference/scripting/index.rst index c6ae4532..c99105f8 100644 --- a/docs/source/config_reference/scripting/index.rst +++ b/docs/source/config_reference/scripting/index.rst @@ -2,8 +2,11 @@ Scripting ========= -.. toctree:: +Work in progress! Explanation of how to define scripts/variables will be added here. + +.. toctree:: + :maxdepth: 1 + entry_variables override_variables - scripting_functions - config_types \ No newline at end of file + scripting_functions \ No newline at end of file diff --git a/docs/source/config_reference/scripting/override_variables.rst b/docs/source/config_reference/scripting/override_variables.rst index 7cb23e4a..3b02c4c7 100644 --- a/docs/source/config_reference/scripting/override_variables.rst +++ b/docs/source/config_reference/scripting/override_variables.rst @@ -1,3 +1,4 @@ + Override Variables ================== @@ -41,4 +42,3 @@ For subscriptions in the form of ``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...`` and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to ``subscription_value``. - diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index b462acf0..d757f07c 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -7,414 +7,571 @@ Array Functions array ~~~~~ -``array(maybe_array: AnyArgument) -> Array`` +:spec: ``array(maybe_array: AnyArgument) -> Array`` -Tries to cast an unknown variable type to an Array. +:description: + Tries to cast an unknown variable type to an Array. array_apply ~~~~~~~~~~~ -``array_apply(array: Array, lambda_function: Lambda) -> Array`` +:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array`` -Apply a lambda function on every element in the Array. +:description: + Apply a lambda function on every element in the Array. +:usage: + +.. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] + +array_apply_fixed +~~~~~~~~~~~~~~~~~ +:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array`` + +:description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. array_at ~~~~~~~~ -``array_at(array: Array, idx: Integer) -> AnyArgument`` +:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` -Return the element in the Array at index ``idx``. +:description: + Return the element in the Array at index ``idx``. array_contains ~~~~~~~~~~~~~~ -``array_contains(array: Array, value: AnyArgument) -> Boolean`` +:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean`` -Return True if the value exists in the Array. False otherwise. +:description: + Return True if the value exists in the Array. False otherwise. array_enumerate ~~~~~~~~~~~~~~~ -``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` +:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` -Apply a lambda function on every element in the Array, where each arg -passed to the lambda function is ``idx, element`` as two separate args. +:description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. array_extend ~~~~~~~~~~~~ -``array_extend(arrays: Array, ...) -> Array`` +:spec: ``array_extend(arrays: Array, ...) -> Array`` -Combine multiple Arrays into a single Array. +:description: + Combine multiple Arrays into a single Array. + +array_first +~~~~~~~~~~~ +:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument`` + +:description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. array_flatten ~~~~~~~~~~~~~ -``array_flatten(array: Array) -> Array`` +:spec: ``array_flatten(array: Array) -> Array`` -Flatten any nested Arrays into a single-dimensional Array. +:description: + Flatten any nested Arrays into a single-dimensional Array. array_index ~~~~~~~~~~~ -``array_index(array: Array, value: AnyArgument) -> Integer`` +:spec: ``array_index(array: Array, value: AnyArgument) -> Integer`` -Return the index of the value within the Array if it exists. If it does not, it will -throw an error. +:description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. + +array_overlay +~~~~~~~~~~~~~ +:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array`` + +:description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. array_product ~~~~~~~~~~~~~ -``array_product(arrays: Array, ...) -> Array`` +:spec: ``array_product(arrays: Array, ...) -> Array`` -Returns the Cartesian product of elements from different arrays +:description: + Returns the Cartesian product of elements from different arrays array_reduce ~~~~~~~~~~~~ -``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` +:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` -Apply a reduce function on pairs of elements in the Array, until one element remains. -Executes using the left-most and reduces in the right direction. +:description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. array_reverse ~~~~~~~~~~~~~ -``array_reverse(array: Array) -> Array`` +:spec: ``array_reverse(array: Array) -> Array`` -Reverse an Array. +:description: + Reverse an Array. array_size ~~~~~~~~~~ -``array_size(array: Array) -> Integer`` +:spec: ``array_size(array: Array) -> Integer`` -Returns the size of an Array. +:description: + Returns the size of an Array. array_slice ~~~~~~~~~~~ -``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` +:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` -Returns the slice of the Array. +:description: + Returns the slice of the Array. + +---------------------------------------------------------------------------------------------------- Boolean Functions ----------------- and ~~~ -``and(values: AnyArgument, ...) -> Boolean`` +:spec: ``and(values: AnyArgument, ...) -> Boolean`` -``and`` operator. Returns True if all values evaluate to True. False otherwise. +:description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. bool ~~~~ -``bool(value: AnyArgument) -> Boolean`` +:spec: ``bool(value: AnyArgument) -> Boolean`` -Cast any type to a Boolean. +:description: + Cast any type to a Boolean. eq ~~ -``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` -``==`` operator. Returns True if left == right. False otherwise. +:description: + ``==`` operator. Returns True if left == right. False otherwise. gt ~~ -``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` -``>`` operator. Returns True if left > right. False otherwise. +:description: + ``>`` operator. Returns True if left > right. False otherwise. gte ~~~ -``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` -``>=`` operator. Returns True if left >= right. False otherwise. +:description: + ``>=`` operator. Returns True if left >= right. False otherwise. + +is_null +~~~~~~~ +:spec: ``is_null(value: AnyArgument) -> Boolean`` + +:description: + Returns True if a value is null (i.e. an empty string). False otherwise. lt ~~ -``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` -``<`` operator. Returns True if left < right. False otherwise. +:description: + ``<`` operator. Returns True if left < right. False otherwise. lte ~~~ -``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` -``<=`` operator. Returns True if left <= right. False otherwise. +:description: + ``<=`` operator. Returns True if left <= right. False otherwise. ne ~~ -``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` -``!=`` operator. Returns True if left != right. False otherwise. +:description: + ``!=`` operator. Returns True if left != right. False otherwise. not ~~~ -``not(value: Boolean) -> Boolean`` +:spec: ``not(value: Boolean) -> Boolean`` -``not`` operator. Returns the opposite of value. +:description: + ``not`` operator. Returns the opposite of value. or ~~ -``or(values: AnyArgument, ...) -> Boolean`` +:spec: ``or(values: AnyArgument, ...) -> Boolean`` -``or`` operator. Returns True if any value evaluates to True. False otherwise. +:description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. xor ~~~ -``xor(values: AnyArgument, ...) -> Boolean`` +:spec: ``xor(values: AnyArgument, ...) -> Boolean`` -``^`` operator. Returns True if exactly one value is set to True. False otherwise. +:description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + +---------------------------------------------------------------------------------------------------- Conditional Functions --------------------- if ~~ -``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` +:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` -Conditional ``if`` statement that returns the ``true`` or ``false`` parameter -depending on the ``condition`` value. +:description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. if_passthrough ~~~~~~~~~~~~~~ -``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` +:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` -Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, -otherwise returns ``else_arg``. +:description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. + +---------------------------------------------------------------------------------------------------- Date Functions -------------- datetime_strftime ~~~~~~~~~~~~~~~~~ -``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` +:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` -Converts a posix timestamp to a date using strftime formatting. +:description: + Converts a posix timestamp to a date using strftime formatting. + +---------------------------------------------------------------------------------------------------- Error Functions --------------- assert ~~~~~~ -``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` +:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` -Explicitly throw an error with the provided assert message if ``value`` evaluates to False. -If it evaluates to True, it will return ``value``. +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. + +assert_eq +~~~~~~~~~ +:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. + +assert_ne +~~~~~~~~~ +:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. + +assert_then +~~~~~~~~~~~ +:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. throw ~~~~~ -``throw(error_message: String) -> AnyArgument`` +:spec: ``throw(error_message: String) -> AnyArgument`` -Explicitly throw an error with the provided error message. +:description: + Explicitly throw an error with the provided error message. + +---------------------------------------------------------------------------------------------------- Json Functions -------------- from_json ~~~~~~~~~ -``from_json(argument: String) -> AnyArgument`` +:spec: ``from_json(argument: String) -> AnyArgument`` -Converts a JSON string into an actual type. +:description: + Converts a JSON string into an actual type. + +---------------------------------------------------------------------------------------------------- Map Functions ------------- map ~~~ -``map(maybe_mapping: AnyArgument) -> Map`` +:spec: ``map(maybe_mapping: AnyArgument) -> Map`` -Tries to cast an unknown variable type to a Map. +:description: + Tries to cast an unknown variable type to a Map. map_apply ~~~~~~~~~ -``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` +:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` -Apply a lambda function on the Map, where each arg -passed to the lambda function is ``key, value`` as two separate args. +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. map_contains ~~~~~~~~~~~~ -``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` +:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` -Returns True if the key is in the Map. False otherwise. +:description: + Returns True if the key is in the Map. False otherwise. map_enumerate ~~~~~~~~~~~~~ -``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` +:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` -Apply a lambda function on the Map, where each arg -passed to the lambda function is ``idx, key, value`` as three separate args. +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. map_get ~~~~~~~ -``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` +:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` -Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is -provided, it will return ``default``. Otherwise, will error. +:description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. map_get_non_empty ~~~~~~~~~~~~~~~~~ -``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` +:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` -Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, -return ``default``. Otherwise, will error. +:description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. map_size ~~~~~~~~ -``map_size(mapping: Map) -> Integer`` +:spec: ``map_size(mapping: Map) -> Integer`` -Returns the size of a Map. +:description: + Returns the size of a Map. + +---------------------------------------------------------------------------------------------------- Numeric Functions ----------------- add ~~~ -``add(values: Numeric, ...) -> Numeric`` +:spec: ``add(values: Numeric, ...) -> Numeric`` -``+`` operator. Returns the sum of all values. +:description: + ``+`` operator. Returns the sum of all values. div ~~~ -``div(left: Numeric, right: Numeric) -> Numeric`` +:spec: ``div(left: Numeric, right: Numeric) -> Numeric`` -``/`` operator. Returns ``left / right``. +:description: + ``/`` operator. Returns ``left / right``. float ~~~~~ -``float(value: AnyArgument) -> Float`` +:spec: ``float(value: AnyArgument) -> Float`` -Cast to Float. +:description: + Cast to Float. int ~~~ -``int(value: AnyArgument) -> Integer`` +:spec: ``int(value: AnyArgument) -> Integer`` -Cast to Integer. +:description: + Cast to Integer. max ~~~ -``max(values: Numeric, ...) -> Numeric`` +:spec: ``max(values: Numeric, ...) -> Numeric`` -Returns max of all values. +:description: + Returns max of all values. min ~~~ -``min(values: Numeric, ...) -> Numeric`` +:spec: ``min(values: Numeric, ...) -> Numeric`` -Returns min of all values. +:description: + Returns min of all values. mod ~~~ -``mod(left: Numeric, right: Numeric) -> Numeric`` +:spec: ``mod(left: Numeric, right: Numeric) -> Numeric`` -``%`` operator. Returns ``left % right``. +:description: + ``%`` operator. Returns ``left % right``. mul ~~~ -``mul(values: Numeric, ...) -> Numeric`` +:spec: ``mul(values: Numeric, ...) -> Numeric`` -``*`` operator. Returns the product of all values. +:description: + ``*`` operator. Returns the product of all values. pow ~~~ -``pow(base: Numeric, exponent: Numeric) -> Numeric`` +:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric`` -``**`` operator. Returns the exponential of the base and exponent value. +:description: + ``**`` operator. Returns the exponential of the base and exponent value. sub ~~~ -``sub(values: Numeric, ...) -> Numeric`` +:spec: ``sub(values: Numeric, ...) -> Numeric`` -``-`` operator. Subtracts all values from left to right. +:description: + ``-`` operator. Subtracts all values from left to right. + +---------------------------------------------------------------------------------------------------- Regex Functions --------------- +regex_capture_groups +~~~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_groups(regex: String) -> Integer`` + +:description: + Returns number of capture groups in regex + regex_fullmatch ~~~~~~~~~~~~~~~ -``regex_fullmatch(regex: String, string: String) -> Array`` +:spec: ``regex_fullmatch(regex: String, string: String) -> Array`` -Checks for entire string to be a match. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. regex_match ~~~~~~~~~~~ -``regex_match(regex: String, string: String) -> Array`` +:spec: ``regex_match(regex: String, string: String) -> Array`` -Checks for a match only at the beginning of the string. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. regex_search ~~~~~~~~~~~~ -``regex_search(regex: String, string: String) -> Array`` +:spec: ``regex_search(regex: String, string: String) -> Array`` -Checks for a match anywhere in the string. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + +---------------------------------------------------------------------------------------------------- String Functions ---------------- capitalize ~~~~~~~~~~ -``capitalize(string: String) -> String`` +:spec: ``capitalize(string: String) -> String`` -Capitalize the first character in the string. +:description: + Capitalize the first character in the string. concat ~~~~~~ -``concat(values: String, ...) -> String`` +:spec: ``concat(values: String, ...) -> String`` -Concatenate multiple Strings into a single String. +:description: + Concatenate multiple Strings into a single String. + +contains +~~~~~~~~ +:spec: ``contains(string: String, contains: String) -> Boolean`` + +:description: + Returns True if ``contains`` is in ``string``. False otherwise. lower ~~~~~ -``lower(string: String) -> String`` +:spec: ``lower(string: String) -> String`` -Lower-case the entire String. +:description: + Lower-case the entire String. pad ~~~ -``pad(string: String, length: Integer, char: String) -> String`` +:spec: ``pad(string: String, length: Integer, char: String) -> String`` -Pads the string to the given length +:description: + Pads the string to the given length pad_zero ~~~~~~~~ -``pad_zero(numeric: Numeric, length: Integer) -> String`` +:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String`` -Pads a numeric with zeros to the given length +:description: + Pads a numeric with zeros to the given length replace ~~~~~~~ -``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` +:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` -Replace the ``old`` part of the String with the ``new``. Optionally only replace it -``count`` number of times. +:description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. slice ~~~~~ -``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` +:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` -Returns the slice of the Array. +:description: + Returns the slice of the Array. string ~~~~~~ -``string(value: AnyArgument) -> String`` +:spec: ``string(value: AnyArgument) -> String`` -Cast to String. +:description: + Cast to String. titlecase ~~~~~~~~~ -``titlecase(string: String) -> String`` +:spec: ``titlecase(string: String) -> String`` -Capitalize each word in the string. +:description: + Capitalize each word in the string. upper ~~~~~ -``upper(string: String) -> String`` +:spec: ``upper(string: String) -> String`` -Upper-case the entire String. +:description: + Upper-case the entire String. + +---------------------------------------------------------------------------------------------------- Ytdl-Sub Functions ------------------ legacy_bracket_safety ~~~~~~~~~~~~~~~~~~~~~ -``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` +:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not interfere with its legacy variable scripting system. This function replicates that @@ -422,14 +579,14 @@ behavior. sanitize ~~~~~~~~ -``sanitize(value: AnyArgument) -> String`` +:spec: ``sanitize(value: AnyArgument) -> String`` Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use for file/directory names on any OS. sanitize_plex_episode ~~~~~~~~~~~~~~~~~~~~~ -``sanitize_plex_episode(string: String) -> String`` +:spec: ``sanitize_plex_episode(string: String) -> String`` Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the @@ -437,7 +594,7 @@ season and/or episode. to_date_metadata ~~~~~~~~~~~~~~~~ -``to_date_metadata(yyyymmdd: String) -> Map`` +:spec: ``to_date_metadata(yyyymmdd: String) -> Map`` Takes a date in the form of YYYYMMDD and returns a Map containing: @@ -461,14 +618,14 @@ Takes a date in the form of YYYYMMDD and returns a Map containing: to_native_filepath ~~~~~~~~~~~~~~~~~~ -``to_native_filepath(filepath: String) -> String`` +:spec: ``to_native_filepath(filepath: String) -> String`` Convert any unix-based path separators ('/') with the OS's native separator. truncate_filepath_if_too_long ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``truncate_filepath_if_too_long(filepath: String) -> String`` +:spec: ``truncate_filepath_if_too_long(filepath: String) -> String`` If a file-path is too long for the OS, this function will truncate it while preserving the extension. diff --git a/docs/source/prebuilt_presets/tv_shows.rst b/docs/source/prebuilt_presets/tv_shows.rst index cded4940..f2b1da1d 100644 --- a/docs/source/prebuilt_presets/tv_shows.rst +++ b/docs/source/prebuilt_presets/tv_shows.rst @@ -18,7 +18,7 @@ Jellyfin Kodi ~~~~ * Everything that the Jellyfin version does -* Turns on :ref:`config_reference/plugins:kodi_safe`, replacing characters that would break kodi with safer characters +* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□`` Plex ~~~~ diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 8f6903d2..b576e1ad 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -8,7 +8,7 @@ import mergedeep from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.entries.variables.override_variables import OverrideHelpers from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved @@ -23,9 +23,9 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid class Overrides(DictFormatterValidator, Scriptable): """ - Optional. This section allows you to define variables that can be used in any string formatter. - For example, if you want your file and thumbnail files to match without copy-pasting a large - format string, you can define something like: + Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + + :Usage: .. code-block:: yaml @@ -89,7 +89,7 @@ class Overrides(DictFormatterValidator, Scriptable): """ Ensures the variable name does not collide with any entry variables or built-in functions. """ - if not OverrideVariables.is_valid_name(name): + if not OverrideHelpers.is_valid_name(name): override_type = "function" if name.startswith("%") else "variable" raise self._validation_exception( f"Override {override_type} with name {name} is invalid. Names must be" @@ -97,14 +97,14 @@ class Overrides(DictFormatterValidator, Scriptable): exception_class=InvalidVariableNameException, ) - if OverrideVariables.is_entry_variable_name(name): + if OverrideHelpers.is_entry_variable_name(name): raise self._validation_exception( f"Override variable with name {name} cannot be used since it is a" " built-in ytdl-sub entry variable name.", exception_class=InvalidVariableNameException, ) - if OverrideVariables.is_function_name(name): + if OverrideHelpers.is_function_name(name): raise self._validation_exception( f"Override function definition with name {name} cannot be used since it is" " a built-in ytdl-sub function name.", diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index d6b2bc2e..8cc4697c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -15,12 +15,12 @@ from ytdl_sub.validators.validators import LiteralDictValidator class YTDLOptions(LiteralDictValidator): """ - Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. + Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. - ytdl_options should be formatted like: + :Usage: .. code-block:: yaml @@ -58,7 +58,7 @@ class OutputOptions(StrictDictValidator): """ Defines where to output files and thumbnails after all post-processing has completed. - Usage: + :Usage: .. code-block:: yaml @@ -156,97 +156,119 @@ class OutputOptions(StrictDictValidator): @property def output_directory(self) -> OverridesStringFormatterValidator: """ - Required. The output directory to store all media files downloaded. + :expected type: OverridesFormatter + :description: + The output directory to store all media files downloaded. """ return self._output_directory @property def file_name(self) -> StringFormatterValidator: """ - Required. The file name for the media file. This can include directories such as - ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. + :expected type: EntryFormatter + :description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. """ return self._file_name @property def thumbnail_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's thumbnail image. This can include directories such - as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable thumbnail writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. """ return self._thumbnail_name @property def info_json_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's info json file. This can include directories such - as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable info json writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. """ return self._info_json_name @property def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. The file name to store a subscriptions download archive placed relative to - the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` + :expected type: Optional[OverridesFormatter] + :description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` """ return self._download_archive_name @property def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. Intended to be used if you are migrating a subscription with either a new - subscription name or output directory. It will try to load the archive file using this name - first, and fallback to ``download_archive_name``. It will always save to this file - and remove the original ``download_archive_name``. + :expected type: Optional[OverridesFormatter] + :description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. """ return self._migrated_download_archive_name @property def maintain_download_archive(self) -> bool: """ - Optional. Maintains a download archive file in the output directory for a subscription. - It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the - output directory. + :expected type: Optional[Boolean] + :description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. - The download archive contains a mapping of ytdl IDs to downloaded files. This is used to - create a ytdl download-archive file when invoking a download on a subscription. This will - prevent ytdl from redownloading media already downloaded. + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. - Defaults to False. + Defaults to False. """ return self._maintain_download_archive.value @property def keep_files_before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep - files before ``now``, which implies all files. Can be used in conjunction with - ``keep_max_files``. + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_before @property def keep_files_after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep - files after ``19000101``, which implies all files. Can be used in conjunction with - ``keep_max_files``. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_after @property def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. - Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be - applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. """ return self._keep_max_files diff --git a/src/ytdl_sub/downloaders/url/multi_url.py b/src/ytdl_sub/downloaders/url/multi_url.py deleted file mode 100644 index b55b8eed..00000000 --- a/src/ytdl_sub/downloaders/url/multi_url.py +++ /dev/null @@ -1,37 +0,0 @@ -from ytdl_sub.downloaders.url.validators import MultiUrlValidator - - -# TODO: Remove later - keep for docstring -class MultiUrlDownloadOptions(MultiUrlValidator): - """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - urls: - - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "1" - season_name: "Uploads" - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "2" - season_name: "Playlist as Season" - playlist_thumbnails: - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - """ diff --git a/src/ytdl_sub/downloaders/url/url.py b/src/ytdl_sub/downloaders/url/url.py deleted file mode 100644 index e5589645..00000000 --- a/src/ytdl_sub/downloaders/url/url.py +++ /dev/null @@ -1,25 +0,0 @@ -from ytdl_sub.downloaders.url.validators import UrlValidator - - -# TODO: Remove later - keep for docstring -class UrlDownloadOptions(UrlValidator): - """ - Downloads from a single URL supported by yt-dlp. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - # optional - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - download_reverse: True - """ diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index eb999b15..631ee979 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -215,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]): class MultiUrlValidator(OptionsValidator): """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. + Sets the URL(s) to download from. Can be used in many forms, including + + :Single URL: + + .. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + + :Multi URL: + + .. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + + :Thumbnails + Variables: + + All variables must be defined for the top-most url. All subsequent URL variables can be either + overwritten or default to the top-most value. + + If an entry is returned from more than one URL, it will use the variables in the bottom-most + URL. + + .. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" """ @classmethod diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 6c576db4..e711a214 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1,3 +1,4 @@ +from abc import ABC from dataclasses import dataclass # This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion @@ -35,51 +36,567 @@ class SiblingMetadata(MetadataVariable): pass -class VariableDefinitions: +class MetadataVariableDefinitions(ABC): @property - def entry_metadata(self) -> Metadata: + def entry_metadata(self: "VariableDefinitions") -> Metadata: """ The entry's info.json """ return Metadata("entry_metadata") @property - def playlist_metadata(self) -> RelativeMetadata: + def playlist_metadata(self: "VariableDefinitions") -> RelativeMetadata: """ Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) """ return RelativeMetadata("playlist_metadata", metadata_key="playlist_metadata") @property - def source_metadata(self) -> RelativeMetadata: + def source_metadata(self: "VariableDefinitions") -> RelativeMetadata: """ Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) """ return RelativeMetadata("source_metadata", metadata_key="source_metadata") @property - def sibling_metadata(self) -> SiblingMetadata: + def sibling_metadata(self: "VariableDefinitions") -> SiblingMetadata: """ Metadata from any sibling entries that reside in the same playlist as this entry. """ return SiblingMetadata("sibling_metadata", metadata_key="sibling_metadata") + +class PlaylistVariableDefinitions(ABC): @property - def uid(self) -> MetadataVariable: + def playlist_uid(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist unique ID if it exists, otherwise return the entry unique ID. + """ + return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") + + @property + def playlist_title(self: "VariableDefinitions") -> MetadataVariable: + """ + Name of its parent playlist/channel if it exists, otherwise returns its title. + """ + return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") + + @property + def playlist_index(self: "VariableDefinitions") -> MetadataVariable: + """ + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") + + @property + def playlist_index_reversed(self: "VariableDefinitions") -> Variable: + """ + Playlist index reversed via ``playlist_count - playlist_index + 1`` + """ + return Variable("playlist_index_reversed") + + @property + def playlist_index_padded(self: "VariableDefinitions") -> Variable: + """ + playlist_index padded two digits + """ + return Variable("playlist_index_padded") + + @property + def playlist_index_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + playlist_index_reversed padded two digits + """ + return Variable("playlist_index_reversed_padded") + + @property + def playlist_index_padded6(self: "VariableDefinitions") -> Variable: + """ + playlist_index padded six digits. + """ + return Variable("playlist_index_padded6") + + @property + def playlist_index_reversed_padded6(self: "VariableDefinitions") -> Variable: + """ + playlist_index_reversed padded six digits. + """ + return Variable("playlist_index_reversed_padded6") + + @property + def playlist_count(self: "VariableDefinitions") -> MetadataVariable: + """ + Playlist count if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") + + @property + def playlist_description(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist description if it exists, otherwise returns the entry's description. + """ + return MetadataVariable( + variable_name="playlist_description", metadata_key=self.description.metadata_key + ) + + @property + def playlist_webpage_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + """ + return MetadataVariable( + variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key + ) + + @property + def playlist_max_upload_date(self: "VariableDefinitions") -> Variable: + """ + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` + """ + return Variable("playlist_max_upload_date") + + @property + def playlist_max_upload_year(self: "VariableDefinitions") -> Variable: + """ + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + """ + # override in EntryParent + return Variable("playlist_max_upload_year") + + @property + def playlist_max_upload_year_truncated(self: "VariableDefinitions") -> Variable: + """ + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + """ + return Variable("playlist_max_upload_year_truncated") + + @property + def playlist_uploader_id(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist uploader id if it exists, otherwise returns the entry uploader ID. + """ + return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") + + @property + def playlist_uploader(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist uploader if it exists, otherwise return the entry uploader. + """ + return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) + + @property + def playlist_uploader_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + """ + return MetadataVariable( + "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key + ) + + +class SourceVariableDefinitions(ABC): + @property + def source_title(self: "VariableDefinitions") -> MetadataVariable: + """ + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + """ + return MetadataVariable("source_title", metadata_key=self.title.metadata_key) + + @property + def source_uid(self: "VariableDefinitions") -> MetadataVariable: + """ + The source unique id if it exists, otherwise returns the playlist unique ID. + """ + return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) + + @property + def source_index(self: "VariableDefinitions") -> MetadataVariable: + """ + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + """ + return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) + + @property + def source_index_padded(self: "VariableDefinitions") -> Variable: + """ + The source index, padded. + """ + return Variable("source_index_padded") + + @property + def source_count(self: "VariableDefinitions") -> MetadataVariable: + """ + The source count if it exists, otherwise returns the playlist count. + """ + return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) + + @property + def source_webpage_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The source webpage url if it exists, otherwise returns the playlist webpage url. + """ + return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) + + @property + def source_description(self: "VariableDefinitions") -> MetadataVariable: + """ + The source description if it exists, otherwise returns the playlist description. + """ + return MetadataVariable("source_description", metadata_key=self.description.metadata_key) + + @property + def source_uploader_id(self: "VariableDefinitions") -> MetadataVariable: + """ + The source uploader id if it exists, otherwise returns the playlist_uploader_id + """ + return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) + + @property + def source_uploader(self: "VariableDefinitions") -> MetadataVariable: + """ + The source uploader if it exists, otherwise return the playlist_uploader + """ + return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) + + @property + def source_uploader_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The source uploader url if it exists, otherwise returns the source webpage_url. + """ + return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) + + +class UploadDateVariableDefinitions(ABC): + @property + def upload_date(self: "VariableDefinitions") -> MetadataVariable: + """ + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + """ + return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") + + @property + def upload_year(self: "VariableDefinitions") -> Variable: + """ + The entry's upload year + """ + return Variable("upload_year") + + @property + def upload_year_truncated(self: "VariableDefinitions") -> Variable: + """ + The last two digits of the upload year, i.e. 22 in 2022 + """ + return Variable("upload_year_truncated") + + @property + def upload_year_truncated_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return Variable("upload_year_truncated_reversed") + + @property + def upload_month_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + """ + return Variable("upload_month_reversed") + + @property + def upload_month_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed upload month, but padded. i.e. November returns "02" + """ + return Variable("upload_month_reversed_padded") + + @property + def upload_month_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return Variable("upload_month_padded") + + @property + def upload_day_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return Variable("upload_day_padded") + + @property + def upload_month(self: "VariableDefinitions") -> Variable: + """ + The upload month as an integer (no padding). + """ + return Variable("upload_month") + + @property + def upload_day(self: "VariableDefinitions") -> Variable: + """ + The upload day as an integer (no padding). + """ + return Variable("upload_day") + + @property + def upload_day_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return Variable("upload_day_reversed") + + @property + def upload_day_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return Variable("upload_day_reversed_padded") + + @property + def upload_day_of_year(self: "VariableDefinitions") -> Variable: + """ + The day of the year, i.e. February 1st returns ``32`` + """ + return Variable("upload_day_of_year") + + @property + def upload_day_of_year_padded(self: "VariableDefinitions") -> Variable: + """ + The upload day of year, but padded i.e. February 1st returns "032" + """ + return Variable("upload_day_of_year_padded") + + @property + def upload_day_of_year_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return Variable("upload_day_of_year_reversed") + + @property + def upload_day_of_year_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return Variable("upload_day_of_year_reversed_padded") + + @property + def upload_date_standardized(self: "VariableDefinitions") -> Variable: + """ + The uploaded date formatted as YYYY-MM-DD + """ + return Variable("upload_date_standardized") + + +class ReleaseDateVariableDefinitions(ABC): + @property + def release_date(self: "VariableDefinitions") -> MetadataVariable: + """ + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + """ + return MetadataVariable(variable_name="release_date", metadata_key="release_date") + + @property + def release_year(self: "VariableDefinitions") -> Variable: + """ + The entry's release year + """ + return Variable("release_year") + + @property + def release_year_truncated(self: "VariableDefinitions") -> Variable: + """ + The last two digits of the release year, i.e. 22 in 2022 + """ + return Variable("release_year_truncated") + + @property + def release_year_truncated_reversed(self: "VariableDefinitions") -> Variable: + """ + The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return Variable("release_year_truncated_reversed") + + @property + def release_month_reversed(self: "VariableDefinitions") -> Variable: + """ + The release month, but reversed + using ``13 - {release_month}``, i.e. March returns ``10`` + """ + return Variable("release_month_reversed") + + @property + def release_month_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed release month, but padded. i.e. November returns "02" + """ + return Variable("release_month_reversed_padded") + + @property + def release_month_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's release month padded to two digits, i.e. March returns "03" + """ + return Variable("release_month_padded") + + @property + def release_day_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's release day padded to two digits, i.e. the fifth returns "05" + """ + return Variable("release_day_padded") + + @property + def release_month(self: "VariableDefinitions") -> Variable: + """ + The release month as an integer (no padding). + """ + return Variable("release_month") + + @property + def release_day(self: "VariableDefinitions") -> Variable: + """ + The release day as an integer (no padding). + """ + return Variable("release_day") + + @property + def release_day_reversed(self: "VariableDefinitions") -> Variable: + """ + The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return Variable("release_day_reversed") + + @property + def release_day_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed release day, but padded. i.e. August 30th returns "02". + """ + return Variable("release_day_reversed_padded") + + @property + def release_day_of_year(self: "VariableDefinitions") -> Variable: + """ + The day of the year, i.e. February 1st returns ``32`` + """ + return Variable("release_day_of_year") + + @property + def release_day_of_year_padded(self: "VariableDefinitions") -> Variable: + """ + The release day of year, but padded i.e. February 1st returns "032" + """ + return Variable("release_day_of_year_padded") + + @property + def release_day_of_year_reversed(self: "VariableDefinitions") -> Variable: + """ + The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return Variable("release_day_of_year_reversed") + + @property + def release_day_of_year_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed release day of year, but padded i.e. December 31st returns "001" + """ + return Variable("release_day_of_year_reversed_padded") + + @property + def release_date_standardized(self: "VariableDefinitions") -> Variable: + """ + The release date formatted as YYYY-MM-DD + """ + return Variable("release_date_standardized") + + +class YtdlSubVariableDefinitions(ABC): + @property + def ytdl_sub_input_url(self: "VariableDefinitions") -> Variable: + """ + The input URL used in ytdl-sub to create this entry. + """ + return Variable("ytdl_sub_input_url") + + @property + def download_index(self: "VariableDefinitions") -> Variable: + """ + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. + """ + return Variable(variable_name="download_index") + + @property + def download_index_padded6(self: "VariableDefinitions") -> Variable: + """ + The download_index padded six digits + """ + return Variable("download_index_padded6") + + @property + def upload_date_index(self: "VariableDefinitions") -> Variable: + """ + The i'th entry downloaded with this upload date. + """ + return Variable(variable_name="upload_date_index") + + @property + def upload_date_index_padded(self: "VariableDefinitions") -> Variable: + """ + The upload_date_index padded two digits + """ + return Variable("upload_date_index_padded") + + @property + def upload_date_index_reversed(self: "VariableDefinitions") -> Variable: + """ + 100 - upload_date_index + """ + return Variable("upload_date_index_reversed") + + @property + def upload_date_index_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The upload_date_index padded two digits + """ + return Variable("upload_date_index_reversed_padded") + + +class EntryVariableDefinitions(ABC): + @property + def uid(self: "VariableDefinitions") -> MetadataVariable: """ The entry's unique ID """ return MetadataVariable(metadata_key="id", variable_name="uid") @property - def duration(self) -> MetadataVariable: + def duration(self: "VariableDefinitions") -> MetadataVariable: """ The duration of the entry in seconds """ return MetadataVariable("duration", metadata_key="duration") @property - def uid_sanitized_plex(self) -> Variable: + def uid_sanitized_plex(self: "VariableDefinitions") -> Variable: """ The sanitized uid with additional sanitizing for Plex. Replaces numbers with fixed-width numbers so Plex does not recognize them as season or episode numbers. @@ -87,56 +604,56 @@ class VariableDefinitions: return Variable("uid_sanitized_plex") @property - def ie_key(self) -> MetadataVariable: + def ie_key(self: "VariableDefinitions") -> MetadataVariable: """ The ie_key, used in legacy yt-dlp things as the 'info-extractor key' """ return MetadataVariable(metadata_key="ie_key", variable_name="ie_key") @property - def extractor_key(self) -> MetadataVariable: + def extractor_key(self: "VariableDefinitions") -> MetadataVariable: """ The yt-dlp extractor key """ return MetadataVariable(metadata_key="extractor_key", variable_name="extractor_key") @property - def extractor(self) -> MetadataVariable: + def extractor(self: "VariableDefinitions") -> MetadataVariable: """ The yt-dlp extractor name """ return MetadataVariable(variable_name="extractor", metadata_key="extractor") @property - def epoch(self) -> MetadataVariable: + def epoch(self: "VariableDefinitions") -> MetadataVariable: """ The unix epoch of when the metadata was scraped by yt-dlp. """ return MetadataVariable(metadata_key="epoch", variable_name="epoch") @property - def epoch_date(self) -> Variable: + def epoch_date(self: "VariableDefinitions") -> Variable: """ The epoch's date, in YYYYMMDD format. """ return Variable("epoch_date") @property - def epoch_hour(self) -> Variable: + def epoch_hour(self: "VariableDefinitions") -> Variable: """ The epoch's hour """ return Variable("epoch_hour") @property - def title(self) -> MetadataVariable: + def title(self: "VariableDefinitions") -> MetadataVariable: """ The title of the entry. If a title does not exist, returns its unique ID. """ return MetadataVariable(variable_name="title", metadata_key="title") @property - def title_sanitized_plex(self) -> Variable: + def title_sanitized_plex(self: "VariableDefinitions") -> Variable: """ The sanitized title with additional sanitizing for Plex. It replaces numbers with fixed-width numbers so Plex does not recognize them as season or episode numbers. @@ -144,847 +661,122 @@ class VariableDefinitions: return Variable("title_sanitized_plex") @property - def webpage_url(self) -> MetadataVariable: + def webpage_url(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The url to the webpage. + The url to the webpage. """ return MetadataVariable(metadata_key="webpage_url", variable_name="webpage_url") @property - def info_json_ext(self) -> Variable: + def info_json_ext(self: "VariableDefinitions") -> Variable: """ - Returns - ------- - str - The "info.json" extension + The "info.json" extension """ return Variable("info_json_ext") @property - def description(self) -> MetadataVariable: + def description(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The description if it exists. Otherwise, returns an emtpy string. + The description if it exists. Otherwise, returns an emtpy string. """ return MetadataVariable(variable_name="description", metadata_key="description") @property - def uploader_id(self) -> MetadataVariable: + def uploader_id(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The uploader id if it exists, otherwise return the unique ID. + The uploader id if it exists, otherwise return the unique ID. """ return MetadataVariable(variable_name="uploader_id", metadata_key="uploader_id") @property - def uploader(self) -> MetadataVariable: + def uploader(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The uploader if it exists, otherwise return the uploader ID. + The uploader if it exists, otherwise return the uploader ID. """ return MetadataVariable(variable_name="uploader", metadata_key="uploader") @property - def uploader_url(self) -> MetadataVariable: + def uploader_url(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The uploader url if it exists, otherwise returns the webpage_url. + The uploader url if it exists, otherwise returns the webpage_url. """ return MetadataVariable("uploader_url", metadata_key="uploader_url") @property - def source_title(self) -> MetadataVariable: + def creator(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - Name of the source (i.e. channel with multiple playlists) if it exists, otherwise - returns its playlist_title. - """ - return MetadataVariable("source_title", metadata_key=self.title.metadata_key) - - @property - def source_uid(self) -> MetadataVariable: - """ - Returns - ------- - str - The source unique id if it exists, otherwise returns the playlist unique ID. - """ - return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) - - @property - def source_index(self) -> MetadataVariable: - """ - Returns - ------- - int - Source index if it exists, otherwise returns ``1``. - - It is recommended to not use this unless you know the source will never add new content - (it is easy for this value to change). - """ - return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) - - @property - def source_index_padded(self) -> Variable: - """ - Returns - ------- - int - The source index, padded. - """ - return Variable("source_index_padded") - - @property - def source_count(self) -> MetadataVariable: - """ - Returns - ------- - int - The source count if it exists, otherwise returns the playlist count. - """ - return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) - - @property - def source_webpage_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The source webpage url if it exists, otherwise returns the playlist webpage url. - """ - return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) - - @property - def source_description(self) -> MetadataVariable: - """ - Returns - ------- - str - The source description if it exists, otherwise returns the playlist description. - """ - return MetadataVariable("source_description", metadata_key=self.description.metadata_key) - - @property - def playlist_uid(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist unique ID if it exists, otherwise return the entry unique ID. - """ - return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") - - @property - def playlist_title(self) -> MetadataVariable: - """ - Returns - ------- - str - Name of its parent playlist/channel if it exists, otherwise returns its title. - """ - return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") - - @property - def playlist_index(self) -> MetadataVariable: - """ - Returns - ------- - int - Playlist index if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") - - @property - def playlist_index_reversed(self) -> Variable: - """ - Returns - ------- - int - Playlist index reversed via ``playlist_count - playlist_index + 1`` - """ - return Variable("playlist_index_reversed") - - @property - def playlist_index_padded(self) -> Variable: - """ - Returns - ------- - str - playlist_index padded two digits - """ - return Variable("playlist_index_padded") - - @property - def playlist_index_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - playlist_index_reversed padded two digits - """ - return Variable("playlist_index_reversed_padded") - - @property - def playlist_index_padded6(self) -> Variable: - """ - Returns - ------- - str - playlist_index padded six digits. - """ - return Variable("playlist_index_padded6") - - @property - def playlist_index_reversed_padded6(self) -> Variable: - """ - Returns - ------- - str - playlist_index_reversed padded six digits. - """ - return Variable("playlist_index_reversed_padded6") - - @property - def playlist_count(self) -> MetadataVariable: - """ - Returns - ------- - int - Playlist count if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") - - @property - def playlist_description(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist description if it exists, otherwise returns the entry's description. - """ - return MetadataVariable( - variable_name="playlist_description", metadata_key=self.description.metadata_key - ) - - @property - def playlist_webpage_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist webpage url if it exists. Otherwise, returns the entry webpage url. - """ - return MetadataVariable( - variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key - ) - - @property - def playlist_max_upload_date(self) -> Variable: - """ - Returns - ------- - Max upload_date for all entries in this entry's playlist if it exists, otherwise returns - ``upload_date`` - """ - return Variable("playlist_max_upload_date") - - @property - def playlist_max_upload_year(self) -> Variable: - """ - Returns - ------- - int - Max upload_year for all entries in this entry's playlist if it exists, otherwise returns - ``upload_year`` - """ - # override in EntryParent - return Variable("playlist_max_upload_year") - - @property - def playlist_max_upload_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The max playlist truncated upload year for all entries in this entry's playlist if it - exists, otherwise returns ``upload_year_truncated``. - """ - return Variable("playlist_max_upload_year_truncated") - - @property - def playlist_uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader id if it exists, otherwise returns the entry uploader ID. - """ - return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") - - @property - def playlist_uploader(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader if it exists, otherwise return the entry uploader. - """ - return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) - - @property - def playlist_uploader_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader url if it exists, otherwise returns the playlist webpage_url. - """ - return MetadataVariable( - "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key - ) - - @property - def source_uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader id if it exists, otherwise returns the playlist_uploader_id - """ - return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) - - @property - def source_uploader(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader if it exists, otherwise return the playlist_uploader - """ - return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) - - @property - def source_uploader_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader url if it exists, otherwise returns the source webpage_url. - """ - return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) - - @property - def creator(self) -> MetadataVariable: - """ - Returns - ------- - str - The creator name if it exists, otherwise returns the channel. + The creator name if it exists, otherwise returns the channel. """ return MetadataVariable(variable_name="creator", metadata_key="creator") @property - def channel(self) -> MetadataVariable: + def channel(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The channel name if it exists, otherwise returns the uploader. + The channel name if it exists, otherwise returns the uploader. """ return MetadataVariable(variable_name="channel", metadata_key="channel") @property - def channel_id(self) -> MetadataVariable: + def channel_id(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The channel id if it exists, otherwise returns the entry uploader ID. + The channel id if it exists, otherwise returns the entry uploader ID. """ return MetadataVariable(variable_name="channel_id", metadata_key="channel_id") @property - def ext(self) -> MetadataVariable: + def ext(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The downloaded entry's file extension + The downloaded entry's file extension """ return MetadataVariable(variable_name="ext", metadata_key="ext") @property - def thumbnail_ext(self) -> Variable: + def thumbnail_ext(self: "VariableDefinitions") -> Variable: """ - Returns - ------- - str - The download entry's thumbnail extension. Will always return 'jpg'. Until there is a - need to support other image types, we always convert to jpg. + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. """ return Variable("thumbnail_ext") @property - def comments(self) -> MetadataVariable: + def comments(self: "VariableDefinitions") -> MetadataVariable: """ Comments if they are requested """ return MetadataVariable("comments", "comments") @property - def chapters(self) -> MetadataVariable: + def chapters(self: "VariableDefinitions") -> MetadataVariable: """ Chapters if they exist """ return MetadataVariable("chapters", "chapters") @property - def sponsorblock_chapters(self) -> MetadataVariable: + def sponsorblock_chapters(self: "VariableDefinitions") -> MetadataVariable: """ Sponsorblock Chapters if they are requested and exist """ return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") @property - def requested_subtitles(self) -> MetadataVariable: + def requested_subtitles(self: "VariableDefinitions") -> MetadataVariable: """ Subtitles if they are requested and exist """ return MetadataVariable("requested_subtitles", "requested_subtitles") - @property - def ytdl_sub_input_url(self) -> Variable: - """ - The input URL used in ytdl-sub to create this entry. - """ - return Variable("ytdl_sub_input_url") - @property - def download_index(self) -> Variable: - """ - Returns - ------- - int - The i'th entry downloaded. NOTE that this is fetched dynamically from the download - archive. - """ - return Variable(variable_name="download_index") - - @property - def download_index_padded6(self) -> Variable: - """ - Returns - ------- - str - The download_index padded six digits - """ - return Variable("download_index_padded6") - - @property - def upload_date_index(self) -> Variable: - """ - Returns - ------- - int - The i'th entry downloaded with this upload date. - """ - return Variable(variable_name="upload_date_index") - - @property - def upload_date_index_padded(self) -> Variable: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return Variable("upload_date_index_padded") - - @property - def upload_date_index_reversed(self) -> Variable: - """ - Returns - ------- - int - 100 - upload_date_index - """ - return Variable("upload_date_index_reversed") - - @property - def upload_date_index_reversed_padded(self) -> Variable: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return Variable("upload_date_index_reversed_padded") - - @property - def upload_date(self) -> MetadataVariable: - """ - Returns - ------- - str - The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. - """ - return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") - - @property - def upload_year(self) -> Variable: - """ - Returns - ------- - int - The entry's upload year - """ - return Variable("upload_year") - - @property - def upload_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The last two digits of the upload year, i.e. 22 in 2022 - """ - return Variable("upload_year_truncated") - - @property - def upload_year_truncated_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return Variable("upload_year_truncated_reversed") - - @property - def upload_month_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` - """ - return Variable("upload_month_reversed") - - @property - def upload_month_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload month, but padded. i.e. November returns "02" - """ - return Variable("upload_month_reversed_padded") - - @property - def upload_month_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's upload month padded to two digits, i.e. March returns "03" - """ - return Variable("upload_month_padded") - - @property - def upload_day_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's upload day padded to two digits, i.e. the fifth returns "05" - """ - return Variable("upload_day_padded") - - @property - def upload_month(self) -> Variable: - """ - Returns - ------- - int - The upload month as an integer (no padding). - """ - return Variable("upload_month") - - @property - def upload_day(self) -> Variable: - """ - Returns - ------- - int - The upload day as an integer (no padding). - """ - return Variable("upload_day") - - @property - def upload_day_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, - i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - return Variable("upload_day_reversed") - - @property - def upload_day_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload day, but padded. i.e. August 30th returns "02". - """ - return Variable("upload_day_reversed_padded") - - @property - def upload_day_of_year(self) -> Variable: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - return Variable("upload_day_of_year") - - @property - def upload_day_of_year_padded(self) -> Variable: - """ - Returns - ------- - str - The upload day of year, but padded i.e. February 1st returns "032" - """ - return Variable("upload_day_of_year_padded") - - @property - def upload_day_of_year_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, - i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - return Variable("upload_day_of_year_reversed") - - @property - def upload_day_of_year_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload day of year, but padded i.e. December 31st returns "001" - """ - return Variable("upload_day_of_year_reversed_padded") - - @property - def upload_date_standardized(self) -> Variable: - """ - Returns - ------- - str - The uploaded date formatted as YYYY-MM-DD - """ - return Variable("upload_date_standardized") - - @property - def release_date(self) -> MetadataVariable: - """ - Returns - ------- - str - The entry’s release date, in YYYYMMDD format. If not present, return the upload date. - """ - return MetadataVariable(variable_name="release_date", metadata_key="release_date") - - @property - def release_year(self) -> Variable: - """ - Returns - ------- - int - The entry's release year - """ - return Variable("release_year") - - @property - def release_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The last two digits of the release year, i.e. 22 in 2022 - """ - return Variable("release_year_truncated") - - @property - def release_year_truncated_reversed(self) -> Variable: - """ - Returns - ------- - int - The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return Variable("release_year_truncated_reversed") - - @property - def release_month_reversed(self) -> Variable: - """ - Returns - ------- - int - The release month, but reversed - using ``13 - {release_month}``, i.e. March returns ``10`` - """ - return Variable("release_month_reversed") - - @property - def release_month_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release month, but padded. i.e. November returns "02" - """ - return Variable("release_month_reversed_padded") - - @property - def release_month_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's release month padded to two digits, i.e. March returns "03" - """ - return Variable("release_month_padded") - - @property - def release_day_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's release day padded to two digits, i.e. the fifth returns "05" - """ - return Variable("release_day_padded") - - @property - def release_month(self) -> Variable: - """ - Returns - ------- - int - The release month as an integer (no padding). - """ - return Variable("release_month") - - @property - def release_day(self) -> Variable: - """ - Returns - ------- - int - The release day as an integer (no padding). - """ - return Variable("release_day") - - @property - def release_day_reversed(self) -> Variable: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, - i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - return Variable("release_day_reversed") - - @property - def release_day_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release day, but padded. i.e. August 30th returns "02". - """ - return Variable("release_day_reversed_padded") - - @property - def release_day_of_year(self) -> Variable: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - return Variable("release_day_of_year") - - @property - def release_day_of_year_padded(self) -> Variable: - """ - Returns - ------- - str - The release day of year, but padded i.e. February 1st returns "032" - """ - return Variable("release_day_of_year_padded") - - @property - def release_day_of_year_reversed(self) -> Variable: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, - i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - return Variable("release_day_of_year_reversed") - - @property - def release_day_of_year_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release day of year, but padded i.e. December 31st returns "001" - """ - return Variable("release_day_of_year_reversed_padded") - - @property - def release_date_standardized(self) -> Variable: - """ - Returns - ------- - str - The release date formatted as YYYY-MM-DD - """ - return Variable("release_date_standardized") +class VariableDefinitions( + EntryVariableDefinitions, + MetadataVariableDefinitions, + PlaylistVariableDefinitions, + SourceVariableDefinitions, + UploadDateVariableDefinitions, + ReleaseDateVariableDefinitions, + YtdlSubVariableDefinitions, +): + pass # Singleton to use externally diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index a798dd18..04af57b0 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -10,15 +10,15 @@ SUBSCRIPTION_ARRAY = "subscription_array" class OverrideVariables: - @classmethod - def subscription_name(cls) -> str: + @staticmethod + def subscription_name() -> str: """ Name of the subscription """ return SUBSCRIPTION_NAME - @classmethod - def subscription_value(cls) -> str: + @staticmethod + def subscription_value() -> str: """ For subscriptions in the form of @@ -30,8 +30,8 @@ class OverrideVariables: """ return SUBSCRIPTION_VALUE - @classmethod - def subscription_indent_i(cls, index: int) -> str: + @staticmethod + def subscription_indent_i(index: int) -> str: """ For subscriptions in the form of @@ -46,8 +46,8 @@ class OverrideVariables: """ return f"subscription_indent_{index + 1}" - @classmethod - def subscription_value_i(cls, index: int) -> str: + @staticmethod + def subscription_value_i(index: int) -> str: """ For subscriptions in the form of @@ -63,6 +63,8 @@ class OverrideVariables: """ return f"subscription_value_{index + 1}" + +class OverrideHelpers: @classmethod def is_entry_variable_name(cls, name: str) -> bool: """ diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index e728f392..644fa1c1 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -25,15 +25,13 @@ class AudioExtractOptions(OptionsDictValidator): """ Extracts audio from a video file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 """ _required_keys = {"codec"} @@ -56,16 +54,20 @@ class AudioExtractOptions(OptionsDictValidator): @property def codec(self) -> str: """ - The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, - opus, vorbis, wav, and best to grab the best possible format at runtime. + :expected type: String + :description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. """ return self._codec @property def quality(self) -> Optional[float]: """ - Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` - (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + :expected type: Float + :description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. """ if self._quality is not None: return self._quality.value diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index bbb10ddd..fbd11c09 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -64,32 +64,29 @@ class ChaptersOptions(OptionsDictValidator): Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" - - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False """ _optional_keys = { @@ -135,23 +132,29 @@ class ChaptersOptions(OptionsDictValidator): @property def embed_chapters(self) -> Optional[bool]: """ - Optional. Embed chapters into the file. Defaults to True. + :expected type: Optional[Boolean] + :description: + Defaults to True. Embed chapters into the file. """ return self._embed_chapters @property def allow_chapters_from_comments(self) -> bool: """ - Optional. If chapters do not exist in the video/description itself, attempt to scrape - comments to find the chapters. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. """ return self._allow_chapters_from_comments @property def remove_chapters_regex(self) -> Optional[List[re.Pattern]]: """ - Optional. List of regex patterns to match chapter titles against and remove them from the - entry. + :expected type: Optional[List[RegexString] + :description: + List of regex patterns to match chapter titles against and remove them from the + entry. """ if self._remove_chapters_regex: return [validator.compiled_regex for validator in self._remove_chapters_regex.list] @@ -160,9 +163,11 @@ class ChaptersOptions(OptionsDictValidator): @property def sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", - "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", - "poi_highlight", or "all" to include all categories. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. """ if self._sponsorblock_categories: category_list = [validator.value for validator in self._sponsorblock_categories.list] @@ -174,9 +179,11 @@ class ChaptersOptions(OptionsDictValidator): @property def remove_sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to remove from the output file. Can only remove - categories that are specified in ``sponsorblock_categories`` or "all", which removes - everything specified in ``sponsorblock_categories``. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. """ if self._remove_sponsorblock_categories: category_list = [ @@ -190,9 +197,10 @@ class ChaptersOptions(OptionsDictValidator): @property def force_key_frames(self) -> bool: """ - Optional. Force keyframes at cuts when removing sections. This is slow due to needing a - re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to - False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. """ return self._force_key_frames diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 092d1054..3948cb1e 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -11,16 +11,24 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator class DateRangeOptions(OptionsDictValidator): """ Only download files uploaded within the specified date range. + Dates must adhere to a yt-dlp datetime. From their docs: - Usage: + .. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + + Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. + Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest + granularity possible. + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" """ _optional_keys = {"before", "after"} @@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator): @property def before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos before this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._before @property def after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos after this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._after diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 64703870..b3ef63e2 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -20,13 +20,11 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator): """ Whether to embed thumbnails to the audio/video file or not. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True """ diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 3b56e1c1..8fccaedb 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -32,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator): """ Converts video files from one extension to another. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" - Supports custom ffmpeg conversions: + Also supports custom ffmpeg conversions: + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text """ _required_keys = {"convert_to"} @@ -89,35 +87,38 @@ class FileConvertOptions(OptionsDictValidator): @property def convert_to(self) -> str: """ - Convert to a desired file type. Supports: + :expected type: String + :description: + Convert to a desired file type. Supports - * Video: avi, flv, mkv, mov, mp4, webm - * Audio: aac, flac, mp3, m4a, opus, vorbis, wav + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav """ return self._convert_to @property def convert_with(self) -> Optional[str]: """ - Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within - yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified - with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - + :expected type: Optional[String] + :description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. """ return self._convert_with @property def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the - form of: + :expected type: Optional[OverridesFormatter] + :description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of - .. code-block:: bash + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext - - The output file will use the extension specified in ``convert_to``. Post-processing args - can still be set with ``convert_with`` set to ``yt-dlp``. + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. """ return self._ffmpeg_post_process_args diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index cceb4700..7f381db1 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -19,15 +19,15 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): Applies a conditional OR on any number of filters comprised of either variables or scripts. If any filter evaluates to True, the entry will be excluded. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - filter_exclude: - - { %contains( %lower(title), '#short' ) } - - { %contains( %lower(description), '#short' ) } + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } """ diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 988426fe..41a98f78 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -19,24 +19,23 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): Applies a conditional AND on any number of filters comprised of either variables or scripts. If all filters evaluate to True, the entry will be included. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - filter_include: - - {description} - - >- - { - %regex_search_any( - title, - [ - "Full Episode", - "FULL", - ] - ) - } + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } """ diff --git a/src/ytdl_sub/plugins/format.py b/src/ytdl_sub/plugins/format.py index 3b5c334f..6e2d545d 100644 --- a/src/ytdl_sub/plugins/format.py +++ b/src/ytdl_sub/plugins/format.py @@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator): .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" """ def __init__(self, name, value): diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 4e1ad498..fe10a69b 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -58,32 +58,20 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: class MatchFiltersOptions(OptionsDictValidator): """ - Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. - Uses the same syntax as yt-dlp. + Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. + Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. + For logical AND's between match filters, use the ``&`` operator in a single match filter. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" - - Supports one or multiple filters: - - .. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit<?18" - - "like_count>?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit<?18" - # - "availability=?public" + match_filters: + filters: + - "age_limit<?18 & like_count>?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" """ _optional_keys = {"filters"} diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index ec99d50f..f6098517 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -75,23 +75,22 @@ class MusicTagsOptions(OptionsDictValidator): a full list of tags for various file types in MediaFile's `source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_. - Usage: + :Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" """ _optional_keys = {"tags", "embed_thumbnail"} diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 92e2b968..728259e1 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -60,7 +60,9 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def nfo_name(self) -> StringFormatterFileNameValidator: """ - The NFO file name. + :expected type: EntryFormatter + :description: + The NFO file name. """ return self._nfo_name @@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def kodi_safe(self) -> Optional[bool]: """ - Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some - foreign language characters. Setting this to True will replace those characters with '□'. - Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. """ return self._kodi_safe @@ -190,22 +194,18 @@ class NfoTagsOptions(SharedNfoTagsOptions): Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False """ _formatter_validator = StringFormatterValidator @@ -215,50 +215,54 @@ class NfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - </episodedetails> + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <episodedetails> + </episodedetails> """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - <title>Awesome Youtube Video - 2022 - 502 - + + + Awesome Youtube Video + 2022 + 502 + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - 2022 - Comedy - Drama + 2022 + Comedy + Drama """ return self._tags diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 72f1828d..f754ec44 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -37,48 +37,52 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - + + + """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - Sweet youtube TV show - + + + Sweet youtube TV show + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + <title year="2022">Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> """ return self._tags diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 325649ea..47b34262 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -122,6 +122,41 @@ class FromSourceVariablesRegex(DictValidator): class RegexOptions(OptionsDictValidator): r""" + .. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -137,51 +172,49 @@ class RegexOptions(OptionsDictValidator): and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" """ _required_keys = {"from"} @@ -209,8 +242,10 @@ class RegexOptions(OptionsDictValidator): @property def skip_if_match_fails(self) -> Optional[bool]: """ - Defaults to True. If True, when any match fails and has no defaults, the entry will be - skipped. If False, ytdl-sub will error and all downloads will not proceed. + :expected type: Optional[Boolean] + :description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. """ return self._skip_if_match_fails diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index fc83a07c..b812c0b9 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -48,23 +48,22 @@ class WhenNoChaptersValidator(StringSelectValidator): class SplitByChaptersOptions(OptionsDictValidator): """ Splits a file by chapters into multiple files. Each file becomes its own entry with the - new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, - ``chapter_index_padded``, ``chapter_count``. + new variables - If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is - set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" """ _required_keys = {"when_no_chapters"} @@ -100,8 +99,16 @@ class SplitByChaptersOptions(OptionsDictValidator): @property def when_no_chapters(self) -> str: """ - Behavior to perform when no chapters are present. Supports "pass" (continue processing), - "drop" (exclude it from output), and "error" (stop processing for everything). + :expected type: String + :description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. """ return self._when_no_chapters diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 445a00a9..9c53a9bd 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -37,18 +37,18 @@ class SubtitleOptions(OptionsDictValidator): ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False """ _optional_keys = { @@ -82,40 +82,50 @@ class SubtitleOptions(OptionsDictValidator): @property def subtitles_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's subtitles if they are present. This can include - directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and - will be placed in the output directory. ``lang`` is dynamic since you can download multiple - subtitles. It will set the respective language to the correct subtitle file. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. """ return self._subtitles_name @property def subtitles_type(self) -> Optional[str]: """ - Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" + :expected type: Optional[String] + :description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". """ return self._subtitles_type @property def embed_subtitles(self) -> Optional[bool]: """ - Optional. Whether to embed the subtitles into the video file. Defaults to False. - NOTE: webm files can only embed "vtt" subtitle types. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. """ return self._embed_subtitles @property def languages(self) -> Optional[List[str]]: """ - Optional. Language code(s) to download for subtitles. Supports a single or list of multiple - language codes. Defaults to "en". + :expected type: Optional[List[String]] + :description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". """ return [lang.value for lang in self._languages] @property def allow_auto_generated_subtitles(self) -> Optional[bool]: """ - Optional. Whether to allow auto generated subtitles. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to allow auto generated subtitles. """ return self._allow_auto_generated_subtitles diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 5bab172c..5295c4f2 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator): range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. - Usage: + :Usage: .. code-block:: yaml @@ -110,31 +110,39 @@ class ThrottleProtectionOptions(OptionsDictValidator): @property def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each download. Does not include time it takes for - ytdl-sub to perform post-processing. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. """ return self._sleep_per_download_s @property def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each subscription. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each subscription. """ return self._sleep_per_subscription_s @property def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]: """ - Number of downloads to perform per subscription. + :expected type: Optional[Range] + :description: + Number of downloads to perform per subscription. """ return self._max_downloads_per_subscription @property def subscription_download_probability(self) -> Optional[ProbabilityValidator]: """ - Probability to perform any downloads, recomputed for each subscription. This is only - recommended to set if you run ytdl-sub in a cron-job, that way you are statistically - guaranteed over time to eventually download the subscription. + :expected type: Optional[Float] + :description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. """ return self._subscription_download_probability diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 9ce3d8fd..c3b2e6ec 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -17,16 +17,14 @@ class VideoTagsOptions(OptionsDictValidator): """ Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" """ _optional_keys = {"tags"} diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py index 5dc0e653..4af6f63c 100644 --- a/src/ytdl_sub/script/functions/array_functions.py +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -19,12 +19,8 @@ class ArrayFunctions: @staticmethod def array(maybe_array: AnyArgument) -> Array: """ - Tries to cast an unknown variable type to an Array. - - Raises - ------ - FunctionRuntimeException - If the input type is not actually an Array. + :description: + Tries to cast an unknown variable type to an Array. """ if not isinstance(maybe_array, Array): raise FunctionRuntimeException( @@ -35,14 +31,16 @@ class ArrayFunctions: @staticmethod def array_size(array: Array) -> Integer: """ - Returns the size of an Array. + :description: + Returns the size of an Array. """ return Integer(len(array.value)) @staticmethod def array_extend(*arrays: Array) -> Array: """ - Combine multiple Arrays into a single Array. + :description: + Combine multiple Arrays into a single Array. """ output: List[Resolvable] = [] for array in arrays: @@ -55,7 +53,8 @@ class ArrayFunctions: array: Array, overlap: Array, only_missing: Optional[Boolean] = None ) -> Array: """ - Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. + :description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. """ output: List[Resolvable] = [] output.extend(array.value) @@ -76,15 +75,17 @@ class ArrayFunctions: @staticmethod def array_at(array: Array, idx: Integer) -> AnyArgument: """ - Return the element in the Array at index ``idx``. + :description: + Return the element in the Array at index ``idx``. """ return array.value[idx.value] @staticmethod def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: """ - Returns the first element whose boolean conversion is True. Returns fallback - if all elements evaluate to False. + :description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. """ for val in array.value: if bool(val.value): @@ -95,15 +96,17 @@ class ArrayFunctions: @staticmethod def array_contains(array: Array, value: AnyArgument) -> Boolean: """ - Return True if the value exists in the Array. False otherwise. + :description: + Return True if the value exists in the Array. False otherwise. """ return Boolean(value in array.value) @staticmethod def array_index(array: Array, value: AnyArgument) -> Integer: """ - Return the index of the value within the Array if it exists. If it does not, it will - throw an error. + :description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. """ if not ArrayFunctions.array_contains(array=array, value=value): raise ArrayValueDoesNotExist( @@ -118,7 +121,8 @@ class ArrayFunctions: @staticmethod def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array: """ - Returns the slice of the Array. + :description: + Returns the slice of the Array. """ if end is not None: return Array(array.value[start.value : end.value]) @@ -127,7 +131,8 @@ class ArrayFunctions: @staticmethod def array_flatten(array: Array) -> Array: """ - Flatten any nested Arrays into a single-dimensional Array. + :description: + Flatten any nested Arrays into a single-dimensional Array. """ output: List[Resolvable] = [] for elem in array.value: @@ -141,14 +146,16 @@ class ArrayFunctions: @staticmethod def array_reverse(array: Array) -> Array: """ - Reverse an Array. + :description: + Reverse an Array. """ return Array(list(reversed(array.value))) @staticmethod def array_product(*arrays: Array) -> Array: """ - Returns the Cartesian product of elements from different arrays + :description: + Returns the Cartesian product of elements from different arrays """ out: List[Resolvable] = [] for combo in itertools.product(*[arr.value for arr in arrays]): @@ -161,7 +168,17 @@ class ArrayFunctions: @staticmethod def array_apply(array: Array, lambda_function: Lambda) -> Array: """ - Apply a lambda function on every element in the Array. + :description: + Apply a lambda function on every element in the Array. + :usage: + + .. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] """ return Array([Array([val]) for val in array.value]) @@ -173,8 +190,9 @@ class ArrayFunctions: reverse_args: Optional[Boolean] = None, ) -> Array: """ - Apply a lambda function on every element in the Array, with ``fixed_argument`` - passed as a second argument to every invocation. + :description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. """ if reverse_args and reverse_args.value: return Array([Array([fixed_argument, val]) for val in array.value]) @@ -184,15 +202,17 @@ class ArrayFunctions: @staticmethod def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array: """ - Apply a lambda function on every element in the Array, where each arg - passed to the lambda function is ``idx, element`` as two separate args. + :description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. """ return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)]) @staticmethod def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument: """ - Apply a reduce function on pairs of elements in the Array, until one element remains. - Executes using the left-most and reduces in the right direction. + :description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. """ return array diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py index f1bb9a35..bfb7416f 100644 --- a/src/ytdl_sub/script/functions/boolean_functions.py +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -13,70 +13,80 @@ class BooleanFunctions: @staticmethod def bool(value: AnyArgument) -> Boolean: """ - Cast any type to a Boolean. + :description: + Cast any type to a Boolean. """ return Boolean(bool(value.value)) @staticmethod def eq(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``==`` operator. Returns True if left == right. False otherwise. + :description: + ``==`` operator. Returns True if left == right. False otherwise. """ return Boolean(left.value == right.value) @staticmethod def ne(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``!=`` operator. Returns True if left != right. False otherwise. + :description: + ``!=`` operator. Returns True if left != right. False otherwise. """ return Boolean(left.value != right.value) @staticmethod def lt(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``<`` operator. Returns True if left < right. False otherwise. + :description: + ``<`` operator. Returns True if left < right. False otherwise. """ return Boolean(left.value < right.value) @staticmethod def lte(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``<=`` operator. Returns True if left <= right. False otherwise. + :description: + ``<=`` operator. Returns True if left <= right. False otherwise. """ return Boolean(left.value <= right.value) @staticmethod def gt(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``>`` operator. Returns True if left > right. False otherwise. + :description: + ``>`` operator. Returns True if left > right. False otherwise. """ return Boolean(left.value > right.value) @staticmethod def gte(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``>=`` operator. Returns True if left >= right. False otherwise. + :description: + ``>=`` operator. Returns True if left >= right. False otherwise. """ return Boolean(left.value >= right.value) @staticmethod def and_(*values: AnyArgument) -> Boolean: """ - ``and`` operator. Returns True if all values evaluate to True. False otherwise. + :description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. """ return Boolean(all(bool(val.value) for val in values)) @staticmethod def or_(*values: AnyArgument) -> Boolean: """ - ``or`` operator. Returns True if any value evaluates to True. False otherwise. + :description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. """ return Boolean(any(bool(val.value) for val in values)) @staticmethod def xor(*values: AnyArgument) -> Boolean: """ - ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + :description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. """ bit_array = [bool(val.value) for val in values] @@ -85,13 +95,15 @@ class BooleanFunctions: @staticmethod def not_(value: Boolean) -> Boolean: """ - ``not`` operator. Returns the opposite of value. + :description: + ``not`` operator. Returns the opposite of value. """ return Boolean(not value.value) @staticmethod def is_null(value: AnyArgument) -> Boolean: """ - Returns True if a value is null (i.e. an empty string). False otherwise. + :description: + Returns True if a value is null (i.e. an empty string). False otherwise. """ return Boolean(isinstance(value, String) and value.value == "") diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py index 8caaae02..e532031b 100644 --- a/src/ytdl_sub/script/functions/conditional_functions.py +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -11,8 +11,9 @@ class ConditionalFunctions: condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: """ - Conditional ``if`` statement that returns the ``true`` or ``false`` parameter - depending on the ``condition`` value. + :description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. """ if condition.value: return true @@ -23,8 +24,9 @@ class ConditionalFunctions: maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: """ - Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, - otherwise returns ``else_arg``. + :description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. """ if bool(maybe_true_arg.value): return maybe_true_arg diff --git a/src/ytdl_sub/script/functions/date_functions.py b/src/ytdl_sub/script/functions/date_functions.py index 8d68f052..fea1095c 100644 --- a/src/ytdl_sub/script/functions/date_functions.py +++ b/src/ytdl_sub/script/functions/date_functions.py @@ -8,6 +8,7 @@ class DateFunctions: @staticmethod def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String: """ - Converts a posix timestamp to a date using strftime formatting. + :description: + Converts a posix timestamp to a date using strftime formatting. """ return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value)) diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py index 517c6141..3a1be9f5 100644 --- a/src/ytdl_sub/script/functions/error_functions.py +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -8,15 +8,17 @@ class ErrorFunctions: @staticmethod def throw(error_message: String) -> AnyArgument: """ - Explicitly throw an error with the provided error message. + :description: + Explicitly throw an error with the provided error message. """ raise UserThrownRuntimeError(error_message) @staticmethod def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` evaluates to False. - If it evaluates to True, it will return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) @@ -27,8 +29,9 @@ class ErrorFunctions: value: AnyArgument, ret: ReturnableArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` evaluates to False. - If it evaluates to True, it will return ``ret``. + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) @@ -39,8 +42,9 @@ class ErrorFunctions: value: ReturnableArgument, equals: AnyArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` does not equal - ``equals``. If they do equal, then return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. """ if not value.value == equals.value: raise UserThrownRuntimeError(assert_message) @@ -51,8 +55,9 @@ class ErrorFunctions: value: ReturnableArgument, equals: AnyArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` equals - ``equals``. If they do equal, then return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. """ if value.value == equals.value: raise UserThrownRuntimeError(assert_message) diff --git a/src/ytdl_sub/script/functions/json_functions.py b/src/ytdl_sub/script/functions/json_functions.py index 915fedf1..c706a86f 100644 --- a/src/ytdl_sub/script/functions/json_functions.py +++ b/src/ytdl_sub/script/functions/json_functions.py @@ -35,6 +35,7 @@ class JsonFunctions: @staticmethod def from_json(argument: String) -> AnyArgument: """ - Converts a JSON string into an actual type. + :description: + Converts a JSON string into an actual type. """ return _from_json(json.loads(argument.value)) diff --git a/src/ytdl_sub/script/functions/map_functions.py b/src/ytdl_sub/script/functions/map_functions.py index 45ca13f1..b34fa611 100644 --- a/src/ytdl_sub/script/functions/map_functions.py +++ b/src/ytdl_sub/script/functions/map_functions.py @@ -18,12 +18,8 @@ class MapFunctions: @staticmethod def map(maybe_mapping: AnyArgument) -> Map: """ - Tries to cast an unknown variable type to a Map. - - Raises - ------ - FunctionRuntimeException - If the input type is not actually a Map. + :description: + Tries to cast an unknown variable type to a Map. """ if not isinstance(maybe_mapping, Map): raise FunctionRuntimeException( @@ -34,14 +30,16 @@ class MapFunctions: @staticmethod def map_size(mapping: Map) -> Integer: """ - Returns the size of a Map. + :description: + Returns the size of a Map. """ return Integer(len(mapping.value)) @staticmethod def map_contains(mapping: Map, key: AnyArgument) -> Boolean: """ - Returns True if the key is in the Map. False otherwise. + :description: + Returns True if the key is in the Map. False otherwise. """ if not isinstance(key, Hashable): raise KeyNotHashableRuntimeException( @@ -55,8 +53,9 @@ class MapFunctions: mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None ) -> AnyArgument: """ - Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is - provided, it will return ``default``. Otherwise, will error. + :description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. """ if not MapFunctions.map_contains(mapping=mapping, key=key).value: if default is not None: @@ -70,8 +69,9 @@ class MapFunctions: @staticmethod def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument: """ - Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, - return ``default``. Otherwise, will error. + :description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. """ output = MapFunctions.map_get(mapping, key, default) if isinstance(output, String) and output.value == "": @@ -83,16 +83,18 @@ class MapFunctions: @staticmethod def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array: """ - Apply a lambda function on the Map, where each arg - passed to the lambda function is ``key, value`` as two separate args. + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. """ return Array([Array([key, value]) for key, value in mapping.value.items()]) @staticmethod def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array: """ - Apply a lambda function on the Map, where each arg - passed to the lambda function is ``idx, key, value`` as three separate args. + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. """ return Array( [ diff --git a/src/ytdl_sub/script/functions/numeric_functions.py b/src/ytdl_sub/script/functions/numeric_functions.py index 5211c690..b25374f9 100644 --- a/src/ytdl_sub/script/functions/numeric_functions.py +++ b/src/ytdl_sub/script/functions/numeric_functions.py @@ -16,28 +16,32 @@ class NumericFunctions: @staticmethod def float(value: AnyArgument) -> Float: """ - Cast to Float. + :description: + Cast to Float. """ return Float(value=float(value.value)) @staticmethod def int(value: AnyArgument) -> Integer: """ - Cast to Integer. + :description: + Cast to Integer. """ return Integer(value=int(value.value)) @staticmethod def add(*values: Numeric) -> Numeric: """ - ``+`` operator. Returns the sum of all values. + :description: + ``+`` operator. Returns the sum of all values. """ return _to_numeric(sum(val.value for val in values)) @staticmethod def sub(*values: Numeric) -> Numeric: """ - ``-`` operator. Subtracts all values from left to right. + :description: + ``-`` operator. Subtracts all values from left to right. """ output = values[0].value for val in values[1:]: @@ -48,41 +52,47 @@ class NumericFunctions: @staticmethod def mul(*values: Numeric) -> Numeric: """ - ``*`` operator. Returns the product of all values. + :description: + ``*`` operator. Returns the product of all values. """ return _to_numeric(math.prod([val.value for val in values])) @staticmethod def pow(base: Numeric, exponent: Numeric) -> Numeric: """ - ``**`` operator. Returns the exponential of the base and exponent value. + :description: + ``**`` operator. Returns the exponential of the base and exponent value. """ return _to_numeric(math.pow(base.value, exponent.value)) @staticmethod def div(left: Numeric, right: Numeric) -> Numeric: """ - ``/`` operator. Returns ``left / right``. + :description: + ``/`` operator. Returns ``left / right``. """ return _to_numeric(left.value / right.value) @staticmethod def mod(left: Numeric, right: Numeric) -> Numeric: """ - ``%`` operator. Returns ``left % right``. + :description: + ``%`` operator. Returns ``left % right``. """ return _to_numeric(value=left.value % right.value) @staticmethod def max(*values: Numeric) -> Numeric: """ - Returns max of all values. + :description: + Returns max of all values. """ return _to_numeric(max(val.value for val in values)) @staticmethod def min(*values: Numeric) -> Numeric: """ - Returns min of all values. + :description: + Returns min of all values. """ return _to_numeric(min(val.value for val in values)) diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py index e38b3139..f837be32 100644 --- a/src/ytdl_sub/script/functions/regex_functions.py +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -18,33 +18,37 @@ class RegexFunctions: @staticmethod def regex_match(regex: String, string: String) -> Array: """ - Checks for a match only at the beginning of the string. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.match(regex.value, string.value)) @staticmethod def regex_search(regex: String, string: String) -> Array: """ - Checks for a match anywhere in the string. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.search(regex.value, string.value)) @staticmethod def regex_fullmatch(regex: String, string: String) -> Array: """ - Checks for entire string to be a match. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.fullmatch(regex.value, string.value)) @staticmethod def regex_capture_groups(regex: String) -> Integer: """ - Returns number of capture groups in regex + :description: + Returns number of capture groups in regex """ return Integer(re.compile(regex.value).groups) diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index 807a5dd4..1f0280b5 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -11,21 +11,24 @@ class StringFunctions: @staticmethod def string(value: AnyArgument) -> String: """ - Cast to String. + :description: + Cast to String. """ return String(value=str(value.value)) @staticmethod def contains(string: String, contains: String) -> Boolean: """ - Returns True if ``contains`` is in ``string``. False otherwise. + :description: + Returns True if ``contains`` is in ``string``. False otherwise. """ return Boolean(contains.value in string.value) @staticmethod def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: """ - Returns the slice of the Array. + :description: + Returns the slice of the Array. """ if end is not None: return String(string.value[start.value : end.value]) @@ -34,28 +37,32 @@ class StringFunctions: @staticmethod def lower(string: String) -> String: """ - Lower-case the entire String. + :description: + Lower-case the entire String. """ return String(string.value.lower()) @staticmethod def upper(string: String) -> String: """ - Upper-case the entire String. + :description: + Upper-case the entire String. """ return String(string.value.upper()) @staticmethod def capitalize(string: String) -> String: """ - Capitalize the first character in the string. + :description: + Capitalize the first character in the string. """ return String(string.value.capitalize()) @staticmethod def titlecase(string: String) -> String: """ - Capitalize each word in the string. + :description: + Capitalize each word in the string. """ return String(string.value.title()) @@ -64,8 +71,9 @@ class StringFunctions: string: String, old: String, new: String, count: Optional[Integer] = None ) -> String: """ - Replace the ``old`` part of the String with the ``new``. Optionally only replace it - ``count`` number of times. + :description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. """ if count: return String(string.value.replace(old.value, new.value, count.value)) @@ -75,14 +83,16 @@ class StringFunctions: @staticmethod def concat(*values: String) -> String: """ - Concatenate multiple Strings into a single String. + :description: + Concatenate multiple Strings into a single String. """ return String("".join(val.value for val in values)) @staticmethod def pad(string: String, length: Integer, char: String) -> String: """ - Pads the string to the given length + :description: + Pads the string to the given length """ output = string.value while len(output) < length.value: @@ -93,7 +103,8 @@ class StringFunctions: @staticmethod def pad_zero(numeric: Numeric, length: Integer) -> String: """ - Pads a numeric with zeros to the given length + :description: + Pads a numeric with zeros to the given length """ return StringFunctions.pad( string=String(str(numeric.value)), diff --git a/src/ytdl_sub/script/utils/exception_formatters.py b/src/ytdl_sub/script/utils/exception_formatters.py index fb3b5e6e..6e14354e 100644 --- a/src/ytdl_sub/script/utils/exception_formatters.py +++ b/src/ytdl_sub/script/utils/exception_formatters.py @@ -1,16 +1,11 @@ import sys from typing import List -from typing import Type from typing import TypeVar -from typing import Union from ytdl_sub.script.types.resolvable import BuiltInFunctionType -from ytdl_sub.script.types.resolvable import NamedType from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.type_checking import FunctionSpec -from ytdl_sub.script.utils.type_checking import get_optional_type -from ytdl_sub.script.utils.type_checking import is_optional from ytdl_sub.script.utils.type_checking import is_union TUserException = TypeVar("TUserException", bound=UserException) @@ -103,28 +98,10 @@ class FunctionArgumentsExceptionFormatter: input_spec: FunctionSpec, function_instance: BuiltInFunctionType, ): - self._args = input_spec.args - self._varargs = input_spec.varargs + self._input_spec = input_spec self._name = function_instance.name self._input_args = function_instance.args - @classmethod - def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: - if is_optional(python_type): - return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" - if is_union(python_type): - return ", ".join( - sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) - ) - return python_type.type_name() - - def _expected_args_str(self) -> str: - if self._args is not None: - return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})" - if self._varargs is not None: - return f"({self._to_human_readable_name(self._varargs)}, ...)" - return "()" - def _received_args_str(self) -> str: received_type_names: List[str] = [] for arg in self._input_args: @@ -149,5 +126,6 @@ class FunctionArgumentsExceptionFormatter: """ return IncompatibleFunctionArguments( f"Incompatible arguments passed to function {self._name}.\n" - f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}" + f"Expected {self._input_spec.human_readable_input_args()}\n" + f"Received {self._received_args_str()}" ) diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py index cc7d87ff..2170c3e0 100644 --- a/src/ytdl_sub/script/utils/type_checking.py +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -119,6 +119,7 @@ def is_type_compatible( @dataclass(frozen=True) class FunctionSpec: return_type: Type[Resolvable] + arg_names: List[str] args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None varargs: Optional[Type[Resolvable]] = None @@ -223,6 +224,42 @@ class FunctionSpec: return l_type return None + @classmethod + def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: + if is_optional(python_type): + return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" + if is_union(python_type): + args = ", ".join( + sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) + ) + return f"Union[{args}]" + return python_type.type_name() + + def human_readable_input_args(self) -> str: + """ + Returns + ------- + input arg string in human-readable format + """ + if self.args is not None: + args = ", ".join( + f"{name}: {self._to_human_readable_name(type_)}" + for name, type_ in zip(self.arg_names, self.args) + ) + elif self.varargs is not None: + args = f"{self.arg_names[0]}: {self._to_human_readable_name(self.varargs)}, ..." + else: + args = "" + return f"({args})" + + def human_readable_output_type(self) -> str: + """ + Returns + ------- + output type string in human-readable format + """ + return self._to_human_readable_name(self.return_type) + @classmethod def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec": """ @@ -234,10 +271,12 @@ class FunctionSpec: if arg_spec.varargs: return FunctionSpec( return_type=arg_spec.annotations["return"], + arg_names=[arg_spec.varargs], varargs=arg_spec.annotations[arg_spec.varargs], ) return FunctionSpec( return_type=arg_spec.annotations["return"], + arg_names=arg_spec.args, args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args], ) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index 7cf5ef2b..646d44e1 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -33,6 +33,13 @@ def get_file_extension(file_name: Path | str) -> str: return file_name.rsplit(".", maxsplit=1)[-1] +def get_md5_hash(contents: str) -> str: + """ + Helper function to compute md5 hash + """ + return hashlib.md5(contents.encode()).hexdigest() + + def get_file_md5_hash(full_file_path: Path | str) -> str: """ Parameters diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt index 77140703..4b7fb2b5 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt @@ -1,5 +1,5 @@ { ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475", - "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c", + "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "d53121df33ac8c4a4699ec8919196552", "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json index 890aca91..f077d34c 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json @@ -2,6 +2,6 @@ ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75", "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8", "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b", - "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "b2d6388b4ddf8e3fbca042cb123672c3", + "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975", "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 30f23eb5..74e55e90 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -2,7 +2,7 @@ "Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "6fb0ce965b75035079f82c84ad341e85", + "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "246fa05b6443337785575987904848df", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "INFO_JSON", @@ -10,7 +10,7 @@ "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "43f271ef8d3a19877f0dc9bc5040b42a", + "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "ae1d5e2e3979cea3c96e6a4cfcae8073", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index 29dfe4cf..c4fb6cca 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json index 143b2310..c5490aa6 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-JMC-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video.json b/tests/resources/expected_downloads_summaries/youtube/test_video.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json index 04e7c7d9..b425d91d 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json @@ -1,4 +1,4 @@ { - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "718c187e6196c85eea73d16ebd489c91", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d9d2d12feee44ee97729b39ba981c542", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/unit/docgen/__init__.py b/tests/unit/docgen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py new file mode 100644 index 00000000..82768347 --- /dev/null +++ b/tests/unit/docgen/test_docgen.py @@ -0,0 +1,30 @@ +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.entry_variables import EntryVariablesDocGen +from tools.docgen.override_variables import OverrideVariablesDocGen +from tools.docgen.plugins import PluginsDocGen +from tools.docgen.scripting_functions import ScriptingFunctionsDocGen +from ytdl_sub.utils.file_handler import get_md5_hash + + +def _test_doc_gen(doc_gen: Type[DocGen]) -> None: + expected_md5_hash = get_md5_hash(doc_gen.generate_and_maybe_write_to_file()) + with open(doc_gen.LOCATION, "r", encoding="utf-8") as file_doc: + md5_hash = get_md5_hash(file_doc.read()) + + assert md5_hash == expected_md5_hash + + +class TestDocGen: + def test_entry_variables_generated(self): + _test_doc_gen(EntryVariablesDocGen) + + def test_override_variables_generated(self): + _test_doc_gen(OverrideVariablesDocGen) + + def test_scripting_functions_generated(self): + _test_doc_gen(ScriptingFunctionsDocGen) + + def test_plugins_generated(self): + _test_doc_gen(PluginsDocGen) diff --git a/tests/unit/docgen/test_docgen_regenerate_disabled.py b/tests/unit/docgen/test_docgen_regenerate_disabled.py new file mode 100644 index 00000000..836b13ea --- /dev/null +++ b/tests/unit/docgen/test_docgen_regenerate_disabled.py @@ -0,0 +1,5 @@ +from tools.docgen.docgen import REGENERATE_DOCS + + +def test_docgen_regenerate_disabled(): + assert REGENERATE_DOCS is False diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py index 1331d485..014e4a85 100644 --- a/tests/unit/script/types/test_function.py +++ b/tests/unit/script/types/test_function.py @@ -40,7 +40,7 @@ class TestFunction: with pytest.raises( IncompatibleFunctionArguments, match=_incompatible_arguments_match( - expected="Map, AnyArgument, Optional[AnyArgument]", + expected="mapping: Map, key: AnyArgument, default: Optional[AnyArgument]", recieved="%if(...)->Union[Array, Map], String", ), ): @@ -49,11 +49,11 @@ class TestFunction: @pytest.mark.parametrize( "function_str, expected_types, received_types", [ - ("{%array_at({'a': 'dict?'}, 1)}", "Array, Integer", "Map, Integer"), - ("{%array_extend('not', 'array')}", "Array, ...", "String, String"), + ("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), + ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), ( "{%replace('hi mom', 'mom', 'dad', 1, 0)}", - "String, String, String, Optional[Integer]", + "string: String, old: String, new: String, count: Optional[Integer]", "String, String, String, Integer, Integer", ), ], diff --git a/tools/docgen/docgen.py b/tools/docgen/docgen.py new file mode 100644 index 00000000..f1307b29 --- /dev/null +++ b/tools/docgen/docgen.py @@ -0,0 +1,32 @@ +from abc import abstractmethod +from pathlib import Path + +REGENERATE_DOCS: bool = False + + +class DocGen: + """ + Home-made auto doc generation + """ + + LOCATION: Path + + @classmethod + @abstractmethod + def generate(cls) -> str: + """ + Generate the docs as a single string + """ + + @classmethod + def generate_and_maybe_write_to_file(cls) -> str: + """ + Maybe writes the docs to their file if the global is set to True, and returns + the generated docs + """ + contents = cls.generate() + if REGENERATE_DOCS: + with open(cls.LOCATION, "w", encoding="utf-8") as out: + out.write(contents) + + return contents diff --git a/tools/docgen/entry_variables.py b/tools/docgen/entry_variables.py new file mode 100644 index 00000000..02cca731 --- /dev/null +++ b/tools/docgen/entry_variables.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import line_section +from tools.docgen.utils import properties +from tools.docgen.utils import section +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + + +def _variable_class_to_name(obj: Type[Any]) -> str: + assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions" + return ( + camel_case_to_human(obj.__name__) + .replace("Variable Definitions", "Variables") + .replace("Ytdl Sub", "Ytdl-Sub") + ) + + +class EntryVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/entry_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Entry Variables", level=0) + + parent_objs: Dict[str, Type[Any]] = { + _variable_class_to_name(obj): obj for obj in VariableDefinitions.__bases__ + } + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for variable_function_name in properties(parent_objs[name]): + docs += get_function_docs( + function_name=variable_function_name, + obj=parent_objs[name], + level=2, + ) + + return docs diff --git a/tools/docgen/override_variables.py b/tools/docgen/override_variables.py new file mode 100644 index 00000000..e919877c --- /dev/null +++ b/tools/docgen/override_variables.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.variables.override_variables import OverrideVariables + + +class OverrideVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Override Variables", level=0) + + for name in static_methods(OverrideVariables): + docs += get_function_docs( + function_name=name, + obj=OverrideVariables, + level=1, + ) + + return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py new file mode 100644 index 00000000..10323066 --- /dev/null +++ b/tools/docgen/plugins.py @@ -0,0 +1,91 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import line_section +from tools.docgen.utils import properties +from tools.docgen.utils import section +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator + + +def should_filter_all_properties(plugin_name: str) -> bool: + return plugin_name in ( + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "video_tags", + "download", + ) + + +def should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + ) + + +def get_function_docs(function_name: str, obj: Any, level: int) -> str: + docs = f"\n``{function_name}``\n\n" + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n\n" + return docs + + +def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: + docs = "" + docs += section(name, level=offset + 0) + + docs += inspect.cleandoc(options.__doc__) + docs += "\n" + + if should_filter_all_properties(name): + return docs + + property_names = [prop for prop in properties(options) if not should_filter_property(prop)] + for property_name in sorted(property_names): + docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) + + return docs + + +class PluginsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/plugins.rst") + + @classmethod + def generate(cls): + options_dict: Dict[str, Type[OptionsValidator]] = { + "output_options": OutputOptions, + "ytdl_options": YTDLOptions, + "overrides": Overrides, + "download": MultiUrlValidator, + } + for plugin_name, plugin_type in PluginMapping._MAPPING.items(): + if plugin_name.startswith("_"): + continue + options_dict[plugin_name] = plugin_type.plugin_options_type + + docs = section("Plugins", level=0) + for idx, name in enumerate(sorted(options_dict.keys())): + docs += line_section(section_idx=idx) + docs += generate_plugin_docs(name, options_dict[name], offset=1) + + return docs diff --git a/tools/docgen/scripting_functions.py b/tools/docgen/scripting_functions.py new file mode 100644 index 00000000..c4663477 --- /dev/null +++ b/tools/docgen/scripting_functions.py @@ -0,0 +1,82 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import line_section +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.type_checking import FunctionSpec + + +def maybe_get_function_name(function_name: str) -> Optional[str]: + if function_name in ["register"]: + return None + + if function_name.endswith("_"): + return function_name[:-1] + return function_name + + +def function_class_to_name(obj: Type[Any]) -> str: + assert "Functions" in obj.__name__ + return camel_case_to_human(obj.__name__) + + +def function_type_hinting(display_function_name: str, function: Any) -> str: + spec = FunctionSpec.from_callable(function) + out = ":spec: ``" + out += display_function_name + out += spec.human_readable_input_args() + out += " -> " + out += spec.human_readable_output_type() + out += "``\n\n" + return out + + +def get_function_docstring( + function_name: str, function: Any, level: int, display_function_name: Optional[str] = None +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + + docs += function_type_hinting(display_function_name=display_function_name, function=function) + docs += inspect.cleandoc(function.__doc__) + docs += "\n" + return docs + + +class ScriptingFunctionsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/scripting_functions.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Scripting Functions", level=0) + + parent_objs: Dict[str, Type[Any]] = { + function_class_to_name(obj): obj for obj in Functions.__bases__ + } + parent_objs["Ytdl-Sub Functions"] = CustomFunctions + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for function_name in static_methods(parent_objs[name]): + if display_function_name := maybe_get_function_name(function_name): + docs += get_function_docstring( + function_name=function_name, + display_function_name=display_function_name, + function=getattr(parent_objs[name], function_name), + level=2, + ) + + return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py new file mode 100644 index 00000000..94cab101 --- /dev/null +++ b/tools/docgen/utils.py @@ -0,0 +1,56 @@ +import inspect +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Type + +LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} + + +def section(name: str, level: int, as_code: bool = False) -> str: + if as_code: + name = f"``{name}``" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" + + +def properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property)) + + +def static_methods(obj: Type[Any]) -> List[str]: + return sorted( + name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod) + ) + + +def camel_case_to_human(string: str) -> str: + output_str = string[0] + for char in string[1:]: + if char.islower(): + output_str += char + else: + output_str += f" {char}" + + return output_str + + +def get_function_docs( + function_name: str, obj: Any, level: int, display_function_name: Optional[str] = None +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n" + return docs + + +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return "" From 189e8978882a81d4b4efe72f04a7b95313127360 Mon Sep 17 00:00:00 2001 From: Jesse Bannon <jbann1994@gmail.com> Date: Fri, 29 Dec 2023 08:55:12 -0800 Subject: [PATCH 2/5] [FEATURE] `subscription_map` value (#859) To be used later for advanced presets. For subscriptions in the form of (+ as prefix) ``` + Subscription Name: Music Videos: - "https://url1.com/..." Concerts: - "https://url2.com/..." ``` Stores all the contents under the subscription name into the override variable ``subscription_map`` as a Map value. The above example is stored as: ``` { "Music Videos": [ "https://url1.com/..." ], "Concerts: [ "https://url2.com/..." ] } ``` Advanced scripting is needed to dissect this Map variable into a usable preset --- .../scripting/override_variables.rst | 26 +++++++++++ .../entries/variables/override_variables.py | 29 ++++++++++++ .../subscriptions/subscription_validators.py | 40 ++++++++++++++++- tests/unit/config/test_subscription.py | 44 ++++++++++++++++++- 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/docs/source/config_reference/scripting/override_variables.rst b/docs/source/config_reference/scripting/override_variables.rst index 3b02c4c7..c7a9991b 100644 --- a/docs/source/config_reference/scripting/override_variables.rst +++ b/docs/source/config_reference/scripting/override_variables.rst @@ -15,6 +15,32 @@ For subscriptions in the form of ``subscription_indent_1`` and ``subscription_indent_2`` get set to ``Indent Value 1`` and ``Indent Value 2``. +subscription_map +---------------- +For subscriptions in the form of + +.. code-block:: yaml + + + Subscription Name: + Music Videos: + - "https://url1.com/..." + Concerts: + - "https://url2.com/..." + +Stores all the contents under the subscription name into the override variable +``subscription_map`` as a Map value. The above example is stored as: + +.. code-block:: python + + { + "Music Videos": [ + "https://url1.com/..." + ], + "Concerts: [ + "https://url2.com/..." + ] + } + subscription_name ----------------- Name of the subscription diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 04af57b0..4e06d441 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -63,6 +63,35 @@ class OverrideVariables: """ return f"subscription_value_{index + 1}" + @staticmethod + def subscription_map() -> str: + """ + For subscriptions in the form of + + .. code-block:: yaml + + + Subscription Name: + Music Videos: + - "https://url1.com/..." + Concerts: + - "https://url2.com/..." + + Stores all the contents under the subscription name into the override variable + ``subscription_map`` as a Map value. The above example is stored as: + + .. code-block:: python + + { + "Music Videos": [ + "https://url1.com/..." + ], + "Concerts: [ + "https://url2.com/..." + ] + } + """ + return SUBSCRIPTION_MAP + class OverrideHelpers: @classmethod diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 55e1ea25..5dbe4d2b 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -9,11 +9,14 @@ from typing import final from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.overrides import Overrides +from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_MAP from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.validators import DictValidator +from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator @@ -198,6 +201,27 @@ class SubscriptionWithOverridesValidator(SubscriptionLeafValidator, DictFormatte self._overrides_to_add = dict(self.dict_with_format_strings, **self._overrides_to_add) +class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator): + def __init__( + self, + name, + value, + subscription_name: str, + config: ConfigFile, + presets: List[str], + indent_overrides: List[str], + ): + super().__init__( + name=name, + value=value, + subscription_name=subscription_name, + config=config, + presets=presets, + indent_overrides=indent_overrides, + ) + self._overrides_to_add = {SUBSCRIPTION_MAP: ScriptUtils.to_script(self.dict)} + + class SubscriptionValidator(SubscriptionOutput): """ Top-level subscription validator @@ -284,7 +308,21 @@ class SubscriptionValidator(SubscriptionOutput): SubscriptionWithOverridesValidator( name=obj_name, value=obj, - subscription_name=key[1:], + subscription_name=key[1:].lstrip(), + config=config, + presets=presets, + indent_overrides=indent_overrides, + ) + ) + # Subscription defined as + # "\Sub Name": + # custom_key: "value" + elif key.startswith("+"): + self._children.append( + SubscriptionMapValidator( + name=obj_name, + value=obj, + subscription_name=key[1:].lstrip(), config=config, presets=presets, indent_overrides=indent_overrides, diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 730ad6eb..2a3da732 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -156,7 +156,7 @@ def preset_with_subscription_overrides_tilda( preset_with_subscription_value, **{ "parent_preset_2 | parent_preset_1": { - "~test_2_1": { + "~ test_2_1": { "current_override": "test_2_1", } }, @@ -164,6 +164,27 @@ def preset_with_subscription_overrides_tilda( ) +@pytest.fixture +def preset_with_subscription_overrides_map( + preset_with_subscription_value: Dict, +): + return dict( + preset_with_subscription_value, + **{ + "parent_preset_2 | parent_preset_1": { + "+ test_2_1": { + "custom_key": "custom_value", + "custom_list": [ + "elem1", + "elem2", + "elem3", + ], + } + }, + }, + ) + + @pytest.fixture def preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors( preset_with_subscription_value: Dict, @@ -240,6 +261,27 @@ def test_subscription_overrides_tilda( assert sub_2_1.get("current_override") == "test_2_1" # tilda sub takes precedence +def test_subscription_overrides_map( + config_file: ConfigFile, + preset_with_subscription_overrides_map: Dict, +): + with mock_load_yaml(preset_dict=preset_with_subscription_overrides_map): + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") + assert len(subs) == 3 + + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script + + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_map").native == { + "custom_key": "custom_value", + "custom_list": [ + "elem1", + "elem2", + "elem3", + ], + } + + def test_subscription_with_period_in_name( config_file: ConfigFile, subscription_with_period_in_name: Dict, From f2d55fdb23425b39600811731892049fdc0c703f Mon Sep 17 00:00:00 2001 From: Jesse Bannon <jbann1994@gmail.com> Date: Fri, 29 Dec 2023 14:39:19 -0800 Subject: [PATCH 3/5] [REFACTOR] Organize variable definitions (#860) --- .../scripting/entry_variables.rst | 435 ++++-- .../scripting/scripting_functions.rst | 2 +- .../config/validators/variable_validation.py | 2 +- .../info_json/info_json_downloader.py | 5 +- src/ytdl_sub/entries/entry.py | 12 +- src/ytdl_sub/entries/entry_parent.py | 9 +- .../entries/script/custom_functions.py | 4 +- .../entries/script/variable_definitions.py | 1337 +++++++++++------ .../entries/script/variable_scripts.py | 296 ---- src/ytdl_sub/entries/script/variable_types.py | 321 ++++ .../entries/variables/override_variables.py | 2 +- src/ytdl_sub/utils/scriptable.py | 6 +- tools/docgen/entry_variables.py | 6 +- tools/docgen/utils.py | 12 +- 14 files changed, 1532 insertions(+), 917 deletions(-) delete mode 100644 src/ytdl_sub/entries/script/variable_scripts.py create mode 100644 src/ytdl_sub/entries/script/variable_types.py diff --git a/docs/source/config_reference/scripting/entry_variables.rst b/docs/source/config_reference/scripting/entry_variables.rst index c4467330..8aa85ca6 100644 --- a/docs/source/config_reference/scripting/entry_variables.rst +++ b/docs/source/config_reference/scripting/entry_variables.rst @@ -7,110 +7,163 @@ Entry Variables channel ~~~~~~~ -The channel name if it exists, otherwise returns the uploader. +:type: ``String`` +:description: + The channel name if it exists, otherwise returns the uploader. channel_id ~~~~~~~~~~ -The channel id if it exists, otherwise returns the entry uploader ID. +:type: ``String`` +:description: + The channel id if it exists, otherwise returns the entry uploader ID. chapters ~~~~~~~~ -Chapters if they exist +:type: ``Array`` +:description: + Chapters if they exist comments ~~~~~~~~ -Comments if they are requested +:type: ``Array`` +:description: + Comments if they are requested creator ~~~~~~~ -The creator name if it exists, otherwise returns the channel. +:type: ``String`` +:description: + The creator name if it exists, otherwise returns the channel. description ~~~~~~~~~~~ -The description if it exists. Otherwise, returns an emtpy string. +:type: ``String`` +:description: + The description if it exists. Otherwise, returns an emtpy string. duration ~~~~~~~~ -The duration of the entry in seconds +:type: ``Integer`` +:description: + The duration of the entry in seconds if it exists. Defaults to zero otherwise. epoch ~~~~~ -The unix epoch of when the metadata was scraped by yt-dlp. +:type: ``Integer`` +:description: + The unix epoch of when the metadata was scraped by yt-dlp. epoch_date ~~~~~~~~~~ -The epoch's date, in YYYYMMDD format. +:type: ``String`` +:description: + The epoch's date, in YYYYMMDD format. epoch_hour ~~~~~~~~~~ -The epoch's hour +:type: ``String`` +:description: + The epoch's hour ext ~~~ -The downloaded entry's file extension +:type: ``String`` +:description: + The downloaded entry's file extension extractor ~~~~~~~~~ -The yt-dlp extractor name +:type: ``String`` +:description: + The yt-dlp extractor name extractor_key ~~~~~~~~~~~~~ -The yt-dlp extractor key +:type: ``String`` +:description: + The yt-dlp extractor key ie_key ~~~~~~ -The ie_key, used in legacy yt-dlp things as the 'info-extractor key' +:type: ``String`` +:description: + The ie_key, used in legacy yt-dlp things as the 'info-extractor key'. + If it does not exist, return ``extractor_key`` info_json_ext ~~~~~~~~~~~~~ -The "info.json" extension +:type: ``String`` +:description: + The "info.json" extension requested_subtitles ~~~~~~~~~~~~~~~~~~~ -Subtitles if they are requested and exist +:type: ``Map`` +:description: + Subtitles if they are requested and exist sponsorblock_chapters ~~~~~~~~~~~~~~~~~~~~~ -Sponsorblock Chapters if they are requested and exist +:type: ``Array`` +:description: + Sponsorblock Chapters if they are requested and exist thumbnail_ext ~~~~~~~~~~~~~ -The download entry's thumbnail extension. Will always return 'jpg'. Until there is a -need to support other image types, we always convert to jpg. +:type: ``String`` +:description: + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. title ~~~~~ -The title of the entry. If a title does not exist, returns its unique ID. +:type: ``String`` +:description: + The title of the entry. If a title does not exist, returns its unique ID. title_sanitized_plex ~~~~~~~~~~~~~~~~~~~~ -The sanitized title with additional sanitizing for Plex. It replaces numbers with -fixed-width numbers so Plex does not recognize them as season or episode numbers. +:type: ``String`` +:description: + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. uid ~~~ -The entry's unique ID +:type: ``String`` +:description: + The entry's unique ID uid_sanitized_plex ~~~~~~~~~~~~~~~~~~ -The sanitized uid with additional sanitizing for Plex. Replaces numbers with -fixed-width numbers so Plex does not recognize them as season or episode numbers. +:type: ``String`` +:description: + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. uploader ~~~~~~~~ -The uploader if it exists, otherwise return the uploader ID. +:type: ``String`` +:description: + The uploader if it exists, otherwise return the uploader ID. uploader_id ~~~~~~~~~~~ -The uploader id if it exists, otherwise return the unique ID. +:type: ``String`` +:description: + The uploader id if it exists, otherwise return the unique ID. uploader_url ~~~~~~~~~~~~ -The uploader url if it exists, otherwise returns the webpage_url. +:type: ``String`` +:description: + The uploader url if it exists, otherwise returns the webpage_url. webpage_url ~~~~~~~~~~~ -The url to the webpage. +:type: ``String`` +:description: + The url to the webpage. ---------------------------------------------------------------------------------------------------- @@ -119,19 +172,28 @@ Metadata Variables entry_metadata ~~~~~~~~~~~~~~ -The entry's info.json +:type: ``Map`` +:description: + The entry's info.json playlist_metadata ~~~~~~~~~~~~~~~~~ -Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) +:type: ``Map`` +:description: + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) sibling_metadata ~~~~~~~~~~~~~~~~ -Metadata from any sibling entries that reside in the same playlist as this entry. +:type: ``Array`` +:description: + Metadata from any sibling entries that reside in the same playlist as this entry. source_metadata ~~~~~~~~~~~~~~~ -Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) +:type: ``Map`` +:description: + Metadata from the source + (i.e. the grandparent metadata, like channel -> playlist -> entry) ---------------------------------------------------------------------------------------------------- @@ -140,80 +202,114 @@ Playlist Variables playlist_count ~~~~~~~~~~~~~~ -Playlist count if it exists, otherwise returns ``1``. +:type: ``Integer`` +:description: + Playlist count if it exists, otherwise returns ``1``. -Note that for channels/playlists, any change (i.e. adding or removing a video) will make -this value change. Use with caution. + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. playlist_description ~~~~~~~~~~~~~~~~~~~~ -The playlist description if it exists, otherwise returns the entry's description. +:type: ``String`` +:description: + The playlist description if it exists, otherwise returns the entry's description. playlist_index ~~~~~~~~~~~~~~ -Playlist index if it exists, otherwise returns ``1``. +:type: ``Integer`` +:description: + Playlist index if it exists, otherwise returns ``1``. -Note that for channels/playlists, any change (i.e. adding or removing a video) will make -this value change. Use with caution. + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. playlist_index_padded ~~~~~~~~~~~~~~~~~~~~~ -playlist_index padded two digits +:type: ``String`` +:description: + playlist_index padded two digits playlist_index_padded6 ~~~~~~~~~~~~~~~~~~~~~~ -playlist_index padded six digits. +:type: ``String`` +:description: + playlist_index padded six digits. playlist_index_reversed ~~~~~~~~~~~~~~~~~~~~~~~ -Playlist index reversed via ``playlist_count - playlist_index + 1`` +:type: ``Integer`` +:description: + Playlist index reversed via ``playlist_count - playlist_index + 1`` playlist_index_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -playlist_index_reversed padded two digits +:type: ``String`` +:description: + playlist_index_reversed padded two digits playlist_index_reversed_padded6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -playlist_index_reversed padded six digits. +:type: ``String`` +:description: + playlist_index_reversed padded six digits. playlist_max_upload_date ~~~~~~~~~~~~~~~~~~~~~~~~ -Max upload_date for all entries in this entry's playlist if it exists, otherwise returns -``upload_date`` +:type: ``String`` +:description: + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` playlist_max_upload_year ~~~~~~~~~~~~~~~~~~~~~~~~ -Max upload_year for all entries in this entry's playlist if it exists, otherwise returns -``upload_year`` +:type: ``Integer`` +:description: + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` playlist_max_upload_year_truncated ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The max playlist truncated upload year for all entries in this entry's playlist if it -exists, otherwise returns ``upload_year_truncated``. +:type: ``Integer`` +:description: + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. playlist_title ~~~~~~~~~~~~~~ -Name of its parent playlist/channel if it exists, otherwise returns its title. +:type: ``String`` +:description: + Name of its parent playlist/channel if it exists, otherwise returns its title. playlist_uid ~~~~~~~~~~~~ -The playlist unique ID if it exists, otherwise return the entry unique ID. +:type: ``String`` +:description: + The playlist unique ID if it exists, otherwise return the entry unique ID. playlist_uploader ~~~~~~~~~~~~~~~~~ -The playlist uploader if it exists, otherwise return the entry uploader. +:type: ``String`` +:description: + The playlist uploader if it exists, otherwise return the entry uploader. playlist_uploader_id ~~~~~~~~~~~~~~~~~~~~ -The playlist uploader id if it exists, otherwise returns the entry uploader ID. +:type: ``String`` +:description: + The playlist uploader id if it exists, otherwise returns the entry uploader ID. playlist_uploader_url ~~~~~~~~~~~~~~~~~~~~~ -The playlist uploader url if it exists, otherwise returns the playlist webpage_url. +:type: ``String`` +:description: + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. playlist_webpage_url ~~~~~~~~~~~~~~~~~~~~ -The playlist webpage url if it exists. Otherwise, returns the entry webpage url. +:type: ``String`` +:description: + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. ---------------------------------------------------------------------------------------------------- @@ -222,75 +318,108 @@ Release Date Variables release_date ~~~~~~~~~~~~ -The entry’s release date, in YYYYMMDD format. If not present, return the upload date. +:type: ``String`` +:description: + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. release_date_standardized ~~~~~~~~~~~~~~~~~~~~~~~~~ -The release date formatted as YYYY-MM-DD +:type: ``String`` +:description: + The uploaded date formatted as YYYY-MM-DD release_day ~~~~~~~~~~~ -The release day as an integer (no padding). +:type: ``Integer`` +:description: + The upload day as an integer (no padding). release_day_of_year ~~~~~~~~~~~~~~~~~~~ -The day of the year, i.e. February 1st returns ``32`` +:type: ``Integer`` +:description: + The day of the year, i.e. February 1st returns ``32`` release_day_of_year_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The release day of year, but padded i.e. February 1st returns "032" +:type: ``String`` +:description: + The upload day of year, but padded i.e. February 1st returns "032" release_day_of_year_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, -i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` release_day_of_year_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed release day of year, but padded i.e. December 31st returns "001" +:type: ``String`` +:description: + The reversed upload day of year, but padded i.e. December 31st returns "001" release_day_padded ~~~~~~~~~~~~~~~~~~ -The entry's release day padded to two digits, i.e. the fifth returns "05" +:type: ``String`` +:description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" release_day_reversed ~~~~~~~~~~~~~~~~~~~~ -The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, -i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` release_day_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed release day, but padded. i.e. August 30th returns "02". +:type: ``String`` +:description: + The reversed upload day, but padded. i.e. August 30th returns "02". release_month ~~~~~~~~~~~~~ -The release month as an integer (no padding). +:type: ``Integer`` +:description: + The upload month as an integer (no padding). release_month_padded ~~~~~~~~~~~~~~~~~~~~ -The entry's release month padded to two digits, i.e. March returns "03" +:type: ``String`` +:description: + The entry's upload month padded to two digits, i.e. March returns "03" release_month_reversed ~~~~~~~~~~~~~~~~~~~~~~ -The release month, but reversed -using ``13 - {release_month}``, i.e. March returns ``10`` +:type: ``Integer`` +:description: + The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10`` release_month_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed release month, but padded. i.e. November returns "02" +:type: ``String`` +:description: + The reversed upload month, but padded. i.e. November returns "02" release_year ~~~~~~~~~~~~ -The entry's release year +:type: ``Integer`` +:description: + The entry's upload year release_year_truncated ~~~~~~~~~~~~~~~~~~~~~~ -The last two digits of the release year, i.e. 22 in 2022 +:type: ``Integer`` +:description: + The last two digits of the upload year, i.e. 22 in 2022 release_year_truncated_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. -2022 returns ``100 - 22`` = ``78`` +:type: ``Integer`` +:description: + The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` ---------------------------------------------------------------------------------------------------- @@ -299,47 +428,67 @@ Source Variables source_count ~~~~~~~~~~~~ -The source count if it exists, otherwise returns the playlist count. +:type: ``Integer`` +:description: + The source count if it exists, otherwise returns ``1``. source_description ~~~~~~~~~~~~~~~~~~ -The source description if it exists, otherwise returns the playlist description. +:type: ``String`` +:description: + The source description if it exists, otherwise returns the playlist description. source_index ~~~~~~~~~~~~ -Source index if it exists, otherwise returns ``1``. +:type: ``Integer`` +:description: + Source index if it exists, otherwise returns ``1``. -It is recommended to not use this unless you know the source will never add new content -(it is easy for this value to change). + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). source_index_padded ~~~~~~~~~~~~~~~~~~~ -The source index, padded. +:type: ``String`` +:description: + The source index, padded two digits. source_title ~~~~~~~~~~~~ -Name of the source (i.e. channel with multiple playlists) if it exists, otherwise -returns its playlist_title. +:type: ``String`` +:description: + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. source_uid ~~~~~~~~~~ -The source unique id if it exists, otherwise returns the playlist unique ID. +:type: ``String`` +:description: + The source unique id if it exists, otherwise returns the playlist unique ID. source_uploader ~~~~~~~~~~~~~~~ -The source uploader if it exists, otherwise return the playlist_uploader +:type: ``String`` +:description: + The source uploader if it exists, otherwise return the playlist_uploader source_uploader_id ~~~~~~~~~~~~~~~~~~ -The source uploader id if it exists, otherwise returns the playlist_uploader_id +:type: ``String`` +:description: + The source uploader id if it exists, otherwise returns the playlist_uploader_id source_uploader_url ~~~~~~~~~~~~~~~~~~~ -The source uploader url if it exists, otherwise returns the source webpage_url. +:type: ``String`` +:description: + The source uploader url if it exists, otherwise returns the source webpage_url. source_webpage_url ~~~~~~~~~~~~~~~~~~ -The source webpage url if it exists, otherwise returns the playlist webpage url. +:type: ``String`` +:description: + The source webpage url if it exists, otherwise returns the playlist webpage url. ---------------------------------------------------------------------------------------------------- @@ -348,74 +497,108 @@ Upload Date Variables upload_date ~~~~~~~~~~~ -The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. +:type: ``String`` +:description: + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. upload_date_standardized ~~~~~~~~~~~~~~~~~~~~~~~~ -The uploaded date formatted as YYYY-MM-DD +:type: ``String`` +:description: + The uploaded date formatted as YYYY-MM-DD upload_day ~~~~~~~~~~ -The upload day as an integer (no padding). +:type: ``Integer`` +:description: + The upload day as an integer (no padding). upload_day_of_year ~~~~~~~~~~~~~~~~~~ -The day of the year, i.e. February 1st returns ``32`` +:type: ``Integer`` +:description: + The day of the year, i.e. February 1st returns ``32`` upload_day_of_year_padded ~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload day of year, but padded i.e. February 1st returns "032" +:type: ``String`` +:description: + The upload day of year, but padded i.e. February 1st returns "032" upload_day_of_year_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, -i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` upload_day_of_year_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed upload day of year, but padded i.e. December 31st returns "001" +:type: ``String`` +:description: + The reversed upload day of year, but padded i.e. December 31st returns "001" upload_day_padded ~~~~~~~~~~~~~~~~~ -The entry's upload day padded to two digits, i.e. the fifth returns "05" +:type: ``String`` +:description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" upload_day_reversed ~~~~~~~~~~~~~~~~~~~ -The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, -i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` upload_day_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed upload day, but padded. i.e. August 30th returns "02". +:type: ``String`` +:description: + The reversed upload day, but padded. i.e. August 30th returns "02". upload_month ~~~~~~~~~~~~ -The upload month as an integer (no padding). +:type: ``Integer`` +:description: + The upload month as an integer (no padding). upload_month_padded ~~~~~~~~~~~~~~~~~~~ -The entry's upload month padded to two digits, i.e. March returns "03" +:type: ``String`` +:description: + The entry's upload month padded to two digits, i.e. March returns "03" upload_month_reversed ~~~~~~~~~~~~~~~~~~~~~ -The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` +:type: ``Integer`` +:description: + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` upload_month_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed upload month, but padded. i.e. November returns "02" +:type: ``String`` +:description: + The reversed upload month, but padded. i.e. November returns "02" upload_year ~~~~~~~~~~~ -The entry's upload year +:type: ``Integer`` +:description: + The entry's upload year upload_year_truncated ~~~~~~~~~~~~~~~~~~~~~ -The last two digits of the upload year, i.e. 22 in 2022 +:type: ``Integer`` +:description: + The last two digits of the upload year, i.e. 22 in 2022 upload_year_truncated_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. -2022 returns ``100 - 22`` = ``78`` +:type: ``Integer`` +:description: + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` ---------------------------------------------------------------------------------------------------- @@ -424,29 +607,43 @@ Ytdl-Sub Variables download_index ~~~~~~~~~~~~~~ -The i'th entry downloaded. NOTE that this is fetched dynamically from the download -archive. +:type: ``Integer`` +:description: + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. download_index_padded6 ~~~~~~~~~~~~~~~~~~~~~~ -The download_index padded six digits +:type: ``String`` +:description: + The download_index padded six digits upload_date_index ~~~~~~~~~~~~~~~~~ -The i'th entry downloaded with this upload date. +:type: ``Integer`` +:description: + The i'th entry downloaded with this upload date. upload_date_index_padded ~~~~~~~~~~~~~~~~~~~~~~~~ -The upload_date_index padded two digits +:type: ``String`` +:description: + The upload_date_index padded two digits upload_date_index_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~ -100 - upload_date_index +:type: ``Integer`` +:description: + 100 - upload_date_index upload_date_index_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload_date_index padded two digits +:type: ``String`` +:description: + The upload_date_index padded two digits ytdl_sub_input_url ~~~~~~~~~~~~~~~~~~ -The input URL used in ytdl-sub to create this entry. +:type: ``String`` +:description: + The input URL used in ytdl-sub to create this entry. diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index d757f07c..c271669e 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -603,7 +603,7 @@ Takes a date in the form of YYYYMMDD and returns a Map containing: - year (Integer) - month (Integer) - day (Integer) -- year_truncated (String, YY from YY[YY]) +- year_truncated (Integer, YY from YY[YY]) - month_padded (String) - day_padded (String) - year_truncated_reversed (Integer, 100 - year_truncated) diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index d52489a7..af8a777f 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -13,7 +13,7 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import validate_formatters diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index f1b29aa4..509a4f8d 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -11,10 +11,9 @@ from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions -from ytdl_sub.entries.script.variable_scripts import DOWNLOADER_INJECTED_VARIABLES -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import get_file_extension @@ -114,7 +113,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): inj.variable_name, VARIABLE_SCRIPTS[inj.variable_name], ) - for inj in DOWNLOADER_INJECTED_VARIABLES + for inj in v.injected_variables() } ) entries.append(entry) diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index b996c20b..c7442992 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -12,8 +12,10 @@ from typing import final from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_definitions import Variable from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.entries.script.variable_types import ArrayVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.scriptable import Scriptable @@ -23,8 +25,12 @@ from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS v: VariableDefinitions = VARIABLES _YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables" -ytdl_sub_chapters_from_comments = Variable("ytdl_sub_chapters_from_comments") -ytdl_sub_split_by_chapters_parent_uid = Variable("ytdl_sub_split_by_chapters_parent_uid") +ytdl_sub_chapters_from_comments = ArrayVariable( + "ytdl_sub_chapters_from_comments", definition="{ [] }" +) +ytdl_sub_split_by_chapters_parent_uid = StringVariable( + "ytdl_sub_split_by_chapters_parent_uid", definition="{ %string('') }" +) TypeT = TypeVar("TypeT") diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 21ff7d18..6cf7dcb1 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -3,15 +3,14 @@ from typing import Any from typing import Dict from typing import List from typing import Optional +from typing import Set from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import TBaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_definitions import MetadataVariable from ytdl_sub.entries.script.variable_definitions import VariableDefinitions -from ytdl_sub.entries.script.variable_scripts import ENTRY_DEFAULT_VARIABLES -from ytdl_sub.entries.script.variable_scripts import ENTRY_REQUIRED_VARIABLES +from ytdl_sub.entries.script.variable_types import MetadataVariable v: VariableDefinitions = VARIABLES @@ -53,8 +52,8 @@ class EntryParent(BaseEntry): def _sibling_entry_metadata(self) -> List[Dict[str, Any]]: sibling_entry_metadata: List[Dict[str, Any]] = [] - variable_filter: List[MetadataVariable] = list(ENTRY_REQUIRED_VARIABLES.keys()) + list( - ENTRY_DEFAULT_VARIABLES.keys() + variable_filter: Set[MetadataVariable] = ( + v.required_entry_variables() | v.default_entry_variables() ) for entry in self.entry_children(): sibling_entry_metadata.append( diff --git a/src/ytdl_sub/entries/script/custom_functions.py b/src/ytdl_sub/entries/script/custom_functions.py index a7440b9d..690095de 100644 --- a/src/ytdl_sub/entries/script/custom_functions.py +++ b/src/ytdl_sub/entries/script/custom_functions.py @@ -101,7 +101,7 @@ class CustomFunctions: - year (Integer) - month (Integer) - day (Integer) - - year_truncated (String, YY from YY[YY]) + - year_truncated (Integer, YY from YY[YY]) - month_padded (String) - day_padded (String) - year_truncated_reversed (Integer, 100 - year_truncated) @@ -149,7 +149,7 @@ class CustomFunctions: String("year"): Integer(year), String("month"): Integer(month), String("day"): Integer(day), - String("year_truncated"): String(year_truncated), + String("year_truncated"): Integer(year_truncated), String("month_padded"): String(month_padded), String("day_padded"): String(day_padded), String("year_truncated_reversed"): Integer(100 - year_truncated), diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index e711a214..d06b015f 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1,770 +1,1065 @@ from abc import ABC -from dataclasses import dataclass +from functools import cache +from functools import cached_property +from typing import Dict +from typing import Set + +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.entries.script.variable_types import ArrayMetadataVariable +from ytdl_sub.entries.script.variable_types import IntegerMetadataVariable +from ytdl_sub.entries.script.variable_types import IntegerVariable +from ytdl_sub.entries.script.variable_types import MapMetadataVariable +from ytdl_sub.entries.script.variable_types import MapVariable +from ytdl_sub.entries.script.variable_types import MetadataVariable +from ytdl_sub.entries.script.variable_types import StringDateMetadataVariable +from ytdl_sub.entries.script.variable_types import StringDateVariable +from ytdl_sub.entries.script.variable_types import StringMetadataVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable # This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion # pylint: disable=no-member # pylint: disable=too-many-public-methods - - -@dataclass(frozen=True) -class Variable: - variable_name: str - - -@dataclass(frozen=True) -class InternalVariable(Variable): - pass - - -@dataclass(frozen=True) -class Metadata(Variable): - pass - - -@dataclass(frozen=True) -class MetadataVariable(Variable): - metadata_key: str - - -@dataclass(frozen=True) -class RelativeMetadata(MetadataVariable, Metadata): - pass - - -@dataclass(frozen=True) -class SiblingMetadata(MetadataVariable): - pass +# pylint: disable=too-many-lines class MetadataVariableDefinitions(ABC): - @property - def entry_metadata(self: "VariableDefinitions") -> Metadata: + @cached_property + def entry_metadata(self: "VariableDefinitions") -> MapVariable: """ - The entry's info.json + :description: + The entry's info.json """ - return Metadata("entry_metadata") + return MapVariable(variable_name="entry_metadata", definition="{ {} }") - @property - def playlist_metadata(self: "VariableDefinitions") -> RelativeMetadata: + @cached_property + def playlist_metadata(self: "VariableDefinitions") -> MapMetadataVariable: """ - Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) + :description: + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) """ - return RelativeMetadata("playlist_metadata", metadata_key="playlist_metadata") + return MapMetadataVariable.from_entry( + metadata_key="playlist_metadata", + default={}, + ) - @property - def source_metadata(self: "VariableDefinitions") -> RelativeMetadata: + @cached_property + def source_metadata(self: "VariableDefinitions") -> MapMetadataVariable: """ - Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) + :description: + Metadata from the source + (i.e. the grandparent metadata, like channel -> playlist -> entry) """ - return RelativeMetadata("source_metadata", metadata_key="source_metadata") + return MapMetadataVariable.from_entry( + metadata_key="source_metadata", + default={}, + ) - @property - def sibling_metadata(self: "VariableDefinitions") -> SiblingMetadata: + @cached_property + def sibling_metadata(self: "VariableDefinitions") -> ArrayMetadataVariable: """ - Metadata from any sibling entries that reside in the same playlist as this entry. + :description: + Metadata from any sibling entries that reside in the same playlist as this entry. """ - return SiblingMetadata("sibling_metadata", metadata_key="sibling_metadata") + return ArrayMetadataVariable.from_entry( + metadata_key="sibling_metadata", + default=[], + ) class PlaylistVariableDefinitions(ABC): - @property - def playlist_uid(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def playlist_uid(self: "VariableDefinitions") -> StringMetadataVariable: """ - The playlist unique ID if it exists, otherwise return the entry unique ID. + :description: + The playlist unique ID if it exists, otherwise return the entry unique ID. """ - return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") - - @property - def playlist_title(self: "VariableDefinitions") -> MetadataVariable: - """ - Name of its parent playlist/channel if it exists, otherwise returns its title. - """ - return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") - - @property - def playlist_index(self: "VariableDefinitions") -> MetadataVariable: - """ - Playlist index if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") - - @property - def playlist_index_reversed(self: "VariableDefinitions") -> Variable: - """ - Playlist index reversed via ``playlist_count - playlist_index + 1`` - """ - return Variable("playlist_index_reversed") - - @property - def playlist_index_padded(self: "VariableDefinitions") -> Variable: - """ - playlist_index padded two digits - """ - return Variable("playlist_index_padded") - - @property - def playlist_index_reversed_padded(self: "VariableDefinitions") -> Variable: - """ - playlist_index_reversed padded two digits - """ - return Variable("playlist_index_reversed_padded") - - @property - def playlist_index_padded6(self: "VariableDefinitions") -> Variable: - """ - playlist_index padded six digits. - """ - return Variable("playlist_index_padded6") - - @property - def playlist_index_reversed_padded6(self: "VariableDefinitions") -> Variable: - """ - playlist_index_reversed padded six digits. - """ - return Variable("playlist_index_reversed_padded6") - - @property - def playlist_count(self: "VariableDefinitions") -> MetadataVariable: - """ - Playlist count if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") - - @property - def playlist_description(self: "VariableDefinitions") -> MetadataVariable: - """ - The playlist description if it exists, otherwise returns the entry's description. - """ - return MetadataVariable( - variable_name="playlist_description", metadata_key=self.description.metadata_key + return StringMetadataVariable.from_playlist( + metadata_key="playlist_id", variable_name="playlist_uid", default=self.uid ) - @property - def playlist_webpage_url(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def playlist_title(self: "VariableDefinitions") -> StringMetadataVariable: """ - The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + :description: + Name of its parent playlist/channel if it exists, otherwise returns its title. """ - return MetadataVariable( - variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key + return StringMetadataVariable.from_entry(metadata_key="playlist_title", default=self.title) + + @cached_property + def playlist_index(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return IntegerMetadataVariable.from_entry(metadata_key="playlist_index", default=1) + + @cached_property + def playlist_index_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + Playlist index reversed via ``playlist_count - playlist_index + 1`` + """ + return IntegerVariable( + variable_name="playlist_index_reversed", + definition=f"""{{ + %sub( + {self.playlist_count.variable_name}, + {self.playlist_index.variable_name}, + -1 + ) + }}""", ) - @property - def playlist_max_upload_date(self: "VariableDefinitions") -> Variable: + @cached_property + def playlist_index_padded(self: "VariableDefinitions") -> StringVariable: """ - Max upload_date for all entries in this entry's playlist if it exists, otherwise returns - ``upload_date`` + :description: + playlist_index padded two digits """ - return Variable("playlist_max_upload_date") + return self.playlist_index.to_padded_int(variable_name="playlist_index_padded", pad=2) - @property - def playlist_max_upload_year(self: "VariableDefinitions") -> Variable: + @cached_property + def playlist_index_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - Max upload_year for all entries in this entry's playlist if it exists, otherwise returns - ``upload_year`` + :description: + playlist_index_reversed padded two digits """ - # override in EntryParent - return Variable("playlist_max_upload_year") + return self.playlist_index_reversed.to_padded_int( + variable_name="playlist_index_reversed_padded", + pad=2, + ) - @property - def playlist_max_upload_year_truncated(self: "VariableDefinitions") -> Variable: + @cached_property + def playlist_index_padded6(self: "VariableDefinitions") -> StringVariable: """ - The max playlist truncated upload year for all entries in this entry's playlist if it - exists, otherwise returns ``upload_year_truncated``. + :description: + playlist_index padded six digits. """ - return Variable("playlist_max_upload_year_truncated") + return self.playlist_index.to_padded_int(variable_name="playlist_index_padded6", pad=6) - @property - def playlist_uploader_id(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def playlist_index_reversed_padded6(self: "VariableDefinitions") -> StringVariable: """ - The playlist uploader id if it exists, otherwise returns the entry uploader ID. + :description: + playlist_index_reversed padded six digits. """ - return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") + return self.playlist_index_reversed.to_padded_int( + variable_name="playlist_index_reversed_padded6", pad=6 + ) - @property - def playlist_uploader(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def playlist_count(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - The playlist uploader if it exists, otherwise return the entry uploader. - """ - return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) + :description: + Playlist count if it exists, otherwise returns ``1``. - @property - def playlist_uploader_url(self: "VariableDefinitions") -> MetadataVariable: + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. """ - The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + return IntegerMetadataVariable.from_entry(metadata_key="playlist_count", default=1) + + @cached_property + def playlist_description(self: "VariableDefinitions") -> StringMetadataVariable: """ - return MetadataVariable( - "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key + :description: + The playlist description if it exists, otherwise returns the entry's description. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.description.metadata_key, + variable_name="playlist_description", + default=self.description, + ) + + @cached_property + def playlist_webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.webpage_url.metadata_key, + variable_name="playlist_webpage_url", + default=self.webpage_url, + ) + + @cached_property + def playlist_max_upload_date(self: "VariableDefinitions") -> StringDateVariable: + """ + :description: + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` + """ + return StringVariable( + variable_name="playlist_max_upload_date", + definition=f"""{{ + %array_reduce( + %if_passthrough( + %extract_field_from_siblings('{self.upload_date.variable_name}'), + [{self.upload_date.variable_name}] + ), + %max + ) + }}""", + ).as_date_variable() + + @cached_property + def playlist_max_upload_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + """ + return self.playlist_max_upload_date.get_integer_date_metadata( + date_metadata_key="year", + variable_name="playlist_max_upload_year", + ) + + @cached_property + def playlist_max_upload_year_truncated(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + """ + return self.playlist_max_upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="playlist_max_upload_year_truncated" + ) + + @cached_property + def playlist_uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist uploader id if it exists, otherwise returns the entry uploader ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="playlist_uploader_id", + default=self.uploader_id, + ) + + @cached_property + def playlist_uploader(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist uploader if it exists, otherwise return the entry uploader. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.uploader.metadata_key, + variable_name="playlist_uploader", + default=self.uploader, + ) + + @cached_property + def playlist_uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + """ + return StringMetadataVariable.from_playlist( + metadata_key=self.uploader_url.metadata_key, + variable_name="playlist_uploader_url", + default=self.webpage_url, ) class SourceVariableDefinitions(ABC): - @property - def source_title(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_title(self: "VariableDefinitions") -> StringMetadataVariable: """ - Name of the source (i.e. channel with multiple playlists) if it exists, otherwise - returns its playlist_title. + :description: + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. """ - return MetadataVariable("source_title", metadata_key=self.title.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.title.metadata_key, + variable_name="source_title", + default=self.playlist_title, + ) - @property - def source_uid(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_uid(self: "VariableDefinitions") -> StringMetadataVariable: """ - The source unique id if it exists, otherwise returns the playlist unique ID. + :description: + The source unique id if it exists, otherwise returns the playlist unique ID. """ - return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.uid.metadata_key, + variable_name="source_uid", + default=self.playlist_uid, + ) - @property - def source_index(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_index(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - Source index if it exists, otherwise returns ``1``. + :description: + Source index if it exists, otherwise returns ``1``. - It is recommended to not use this unless you know the source will never add new content - (it is easy for this value to change). + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). """ - return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) + return IntegerMetadataVariable.from_playlist( + metadata_key=self.playlist_index.metadata_key, + variable_name="source_index", + default=1, + ) - @property - def source_index_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def source_index_padded(self: "VariableDefinitions") -> StringVariable: """ - The source index, padded. + :description: + The source index, padded two digits. """ - return Variable("source_index_padded") + return self.source_index.to_padded_int(variable_name="source_index_padded", pad=2) - @property - def source_count(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_count(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - The source count if it exists, otherwise returns the playlist count. + :description: + The source count if it exists, otherwise returns ``1``. """ - return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) + return IntegerMetadataVariable.from_playlist( + metadata_key=self.playlist_count.metadata_key, + variable_name="source_count", + default=1, + ) - @property - def source_webpage_url(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: """ - The source webpage url if it exists, otherwise returns the playlist webpage url. + :description: + The source webpage url if it exists, otherwise returns the playlist webpage url. """ - return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.webpage_url.metadata_key, + variable_name="source_webpage_url", + default=self.playlist_webpage_url, + ) - @property - def source_description(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_description(self: "VariableDefinitions") -> StringMetadataVariable: """ - The source description if it exists, otherwise returns the playlist description. + :description: + The source description if it exists, otherwise returns the playlist description. """ - return MetadataVariable("source_description", metadata_key=self.description.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.description.metadata_key, + variable_name="source_description", + default=self.playlist_description, + ) - @property - def source_uploader_id(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: """ - The source uploader id if it exists, otherwise returns the playlist_uploader_id + :description: + The source uploader id if it exists, otherwise returns the playlist_uploader_id """ - return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.uploader_id.metadata_key, + variable_name="source_uploader_id", + default=self.playlist_uploader_id, + ) - @property - def source_uploader(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_uploader(self: "VariableDefinitions") -> StringMetadataVariable: """ - The source uploader if it exists, otherwise return the playlist_uploader + :description: + The source uploader if it exists, otherwise return the playlist_uploader """ - return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.uploader.metadata_key, + variable_name="source_uploader", + default=self.playlist_uploader, + ) - @property - def source_uploader_url(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def source_uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: """ - The source uploader url if it exists, otherwise returns the source webpage_url. + :description: + The source uploader url if it exists, otherwise returns the source webpage_url. """ - return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) + return StringMetadataVariable.from_source( + metadata_key=self.uploader_url.metadata_key, + variable_name="source_uploader_url", + default=self.source_webpage_url, + ) class UploadDateVariableDefinitions(ABC): - @property - def upload_date(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def upload_date(self: "VariableDefinitions") -> StringDateMetadataVariable: """ - The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + :description: + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. """ - return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") + return StringDateMetadataVariable.from_entry(metadata_key="upload_date").as_date_variable() - @property - def upload_year(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_year(self: "VariableDefinitions") -> IntegerVariable: """ - The entry's upload year + :description: + The entry's upload year """ - return Variable("upload_year") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year", variable_name="upload_year" + ) - @property - def upload_year_truncated(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_year_truncated(self: "VariableDefinitions") -> IntegerVariable: """ - The last two digits of the upload year, i.e. 22 in 2022 + :description: + The last two digits of the upload year, i.e. 22 in 2022 """ - return Variable("upload_year_truncated") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="upload_year_truncated" + ) - @property - def upload_year_truncated_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_year_truncated_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` + :description: + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` """ - return Variable("upload_year_truncated_reversed") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated_reversed", + variable_name="upload_year_truncated_reversed", + ) - @property - def upload_month_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_month_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + :description: + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` """ - return Variable("upload_month_reversed") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="month_reversed", variable_name="upload_month_reversed" + ) - @property - def upload_month_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_month_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The reversed upload month, but padded. i.e. November returns "02" + :description: + The reversed upload month, but padded. i.e. November returns "02" """ - return Variable("upload_month_reversed_padded") + return self.upload_date.get_string_date_metadata( + date_metadata_key="month_reversed_padded", variable_name="upload_month_reversed_padded" + ) - @property - def upload_month_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_month_padded(self: "VariableDefinitions") -> StringVariable: """ - The entry's upload month padded to two digits, i.e. March returns "03" + :description: + The entry's upload month padded to two digits, i.e. March returns "03" """ - return Variable("upload_month_padded") + return self.upload_date.get_string_date_metadata( + date_metadata_key="month_padded", variable_name="upload_month_padded" + ) - @property - def upload_day_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_padded(self: "VariableDefinitions") -> StringVariable: """ - The entry's upload day padded to two digits, i.e. the fifth returns "05" + :description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" """ - return Variable("upload_day_padded") + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_padded", variable_name="upload_day_padded" + ) - @property - def upload_month(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_month(self: "VariableDefinitions") -> IntegerVariable: """ - The upload month as an integer (no padding). + :description: + The upload month as an integer (no padding). """ - return Variable("upload_month") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="month", variable_name="upload_month" + ) - @property - def upload_day(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day(self: "VariableDefinitions") -> IntegerVariable: """ - The upload day as an integer (no padding). + :description: + The upload day as an integer (no padding). """ - return Variable("upload_day") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day", variable_name="upload_day" + ) - @property - def upload_day_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, - i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + :description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` """ - return Variable("upload_day_reversed") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_reversed", variable_name="upload_day_reversed" + ) - @property - def upload_day_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The reversed upload day, but padded. i.e. August 30th returns "02". + :description: + The reversed upload day, but padded. i.e. August 30th returns "02". """ - return Variable("upload_day_reversed_padded") + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_reversed_padded", variable_name="upload_day_reversed_padded" + ) - @property - def upload_day_of_year(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_of_year(self: "VariableDefinitions") -> IntegerVariable: """ - The day of the year, i.e. February 1st returns ``32`` + :description: + The day of the year, i.e. February 1st returns ``32`` """ - return Variable("upload_day_of_year") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_of_year", variable_name="upload_day_of_year" + ) - @property - def upload_day_of_year_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_of_year_padded(self: "VariableDefinitions") -> StringVariable: """ - The upload day of year, but padded i.e. February 1st returns "032" + :description: + The upload day of year, but padded i.e. February 1st returns "032" """ - return Variable("upload_day_of_year_padded") + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_of_year_padded", variable_name="upload_day_of_year_padded" + ) - @property - def upload_day_of_year_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_of_year_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, - i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + :description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` """ - return Variable("upload_day_of_year_reversed") + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_of_year_reversed", variable_name="upload_day_of_year_reversed" + ) - @property - def upload_day_of_year_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_day_of_year_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The reversed upload day of year, but padded i.e. December 31st returns "001" + :description: + The reversed upload day of year, but padded i.e. December 31st returns "001" """ - return Variable("upload_day_of_year_reversed_padded") + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_of_year_reversed_padded", + variable_name="upload_day_of_year_reversed_padded", + ) - @property - def upload_date_standardized(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_date_standardized(self: "VariableDefinitions") -> StringVariable: """ - The uploaded date formatted as YYYY-MM-DD + :description: + The uploaded date formatted as YYYY-MM-DD """ - return Variable("upload_date_standardized") + return self.upload_date.get_string_date_metadata( + date_metadata_key="date_standardized", variable_name="upload_date_standardized" + ) class ReleaseDateVariableDefinitions(ABC): - @property - def release_date(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def release_date(self: "VariableDefinitions") -> StringDateMetadataVariable: """ - The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + :description: + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. """ - return MetadataVariable(variable_name="release_date", metadata_key="release_date") + return StringMetadataVariable.from_entry( + metadata_key="release_date", default=self.upload_date + ).as_date_variable() - @property - def release_year(self: "VariableDefinitions") -> Variable: + @cached_property + def release_year(self: "VariableDefinitions") -> IntegerVariable: """ - The entry's release year + :description: + The entry's upload year """ - return Variable("release_year") + return self.release_date.get_integer_date_metadata( + date_metadata_key="year", variable_name="release_year" + ) - @property - def release_year_truncated(self: "VariableDefinitions") -> Variable: + @cached_property + def release_year_truncated(self: "VariableDefinitions") -> IntegerVariable: """ - The last two digits of the release year, i.e. 22 in 2022 + :description: + The last two digits of the upload year, i.e. 22 in 2022 """ - return Variable("release_year_truncated") + return self.release_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="release_year_truncated" + ) - @property - def release_year_truncated_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def release_year_truncated_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` + :description: + The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` """ - return Variable("release_year_truncated_reversed") + return self.release_date.get_integer_date_metadata( + date_metadata_key="year_truncated_reversed", + variable_name="release_year_truncated_reversed", + ) - @property - def release_month_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def release_month_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The release month, but reversed - using ``13 - {release_month}``, i.e. March returns ``10`` + :description: + The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10`` """ - return Variable("release_month_reversed") + return self.release_date.get_integer_date_metadata( + date_metadata_key="month_reversed", variable_name="release_month_reversed" + ) - @property - def release_month_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def release_month_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The reversed release month, but padded. i.e. November returns "02" + :description: + The reversed upload month, but padded. i.e. November returns "02" """ - return Variable("release_month_reversed_padded") + return self.release_date.get_string_date_metadata( + date_metadata_key="month_reversed_padded", variable_name="release_month_reversed_padded" + ) - @property - def release_month_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def release_month_padded(self: "VariableDefinitions") -> StringVariable: """ - The entry's release month padded to two digits, i.e. March returns "03" + :description: + The entry's upload month padded to two digits, i.e. March returns "03" """ - return Variable("release_month_padded") + return self.release_date.get_string_date_metadata( + date_metadata_key="month_padded", variable_name="release_month_padded" + ) - @property - def release_day_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_padded(self: "VariableDefinitions") -> StringVariable: """ - The entry's release day padded to two digits, i.e. the fifth returns "05" + :description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" """ - return Variable("release_day_padded") + return self.release_date.get_string_date_metadata( + date_metadata_key="day_padded", variable_name="release_day_padded" + ) - @property - def release_month(self: "VariableDefinitions") -> Variable: + @cached_property + def release_month(self: "VariableDefinitions") -> IntegerVariable: """ - The release month as an integer (no padding). + :description: + The upload month as an integer (no padding). """ - return Variable("release_month") + return self.release_date.get_integer_date_metadata( + date_metadata_key="month", variable_name="release_month" + ) - @property - def release_day(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day(self: "VariableDefinitions") -> IntegerVariable: """ - The release day as an integer (no padding). + :description: + The upload day as an integer (no padding). """ - return Variable("release_day") + return self.release_date.get_integer_date_metadata( + date_metadata_key="day", variable_name="release_day" + ) - @property - def release_day_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, - i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + :description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` """ - return Variable("release_day_reversed") + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_reversed", variable_name="release_day_reversed" + ) - @property - def release_day_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The reversed release day, but padded. i.e. August 30th returns "02". + :description: + The reversed upload day, but padded. i.e. August 30th returns "02". """ - return Variable("release_day_reversed_padded") + return self.release_date.get_string_date_metadata( + date_metadata_key="day_reversed_padded", variable_name="release_day_reversed_padded" + ) - @property - def release_day_of_year(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_of_year(self: "VariableDefinitions") -> IntegerVariable: """ - The day of the year, i.e. February 1st returns ``32`` + :description: + The day of the year, i.e. February 1st returns ``32`` """ - return Variable("release_day_of_year") + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_of_year", variable_name="release_day_of_year" + ) - @property - def release_day_of_year_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_of_year_padded(self: "VariableDefinitions") -> StringVariable: """ - The release day of year, but padded i.e. February 1st returns "032" + :description: + The upload day of year, but padded i.e. February 1st returns "032" """ - return Variable("release_day_of_year_padded") + return self.release_date.get_string_date_metadata( + date_metadata_key="day_of_year_padded", variable_name="release_day_of_year_padded" + ) - @property - def release_day_of_year_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_of_year_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, - i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + :description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` """ - return Variable("release_day_of_year_reversed") + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_of_year_reversed", variable_name="release_day_of_year_reversed" + ) - @property - def release_day_of_year_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def release_day_of_year_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The reversed release day of year, but padded i.e. December 31st returns "001" + :description: + The reversed upload day of year, but padded i.e. December 31st returns "001" """ - return Variable("release_day_of_year_reversed_padded") + return self.release_date.get_string_date_metadata( + date_metadata_key="day_of_year_reversed_padded", + variable_name="release_day_of_year_reversed_padded", + ) - @property - def release_date_standardized(self: "VariableDefinitions") -> Variable: + @cached_property + def release_date_standardized(self: "VariableDefinitions") -> StringVariable: """ - The release date formatted as YYYY-MM-DD + :description: + The uploaded date formatted as YYYY-MM-DD """ - return Variable("release_date_standardized") + return self.release_date.get_string_date_metadata( + date_metadata_key="date_standardized", variable_name="release_date_standardized" + ) class YtdlSubVariableDefinitions(ABC): - @property - def ytdl_sub_input_url(self: "VariableDefinitions") -> Variable: + @cached_property + def ytdl_sub_input_url(self: "VariableDefinitions") -> StringVariable: """ - The input URL used in ytdl-sub to create this entry. + :description: + The input URL used in ytdl-sub to create this entry. """ - return Variable("ytdl_sub_input_url") + return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }") - @property - def download_index(self: "VariableDefinitions") -> Variable: + @cached_property + def download_index(self: "VariableDefinitions") -> IntegerVariable: """ - The i'th entry downloaded. NOTE that this is fetched dynamically from the download - archive. + :description: + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. """ - return Variable(variable_name="download_index") + return IntegerVariable(variable_name="download_index", definition="{ %int(1) }") - @property - def download_index_padded6(self: "VariableDefinitions") -> Variable: + @cached_property + def download_index_padded6(self: "VariableDefinitions") -> StringVariable: """ - The download_index padded six digits + :description: + The download_index padded six digits """ - return Variable("download_index_padded6") + return self.download_index.to_padded_int(variable_name="download_index_padded6", pad=6) - @property - def upload_date_index(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_date_index(self: "VariableDefinitions") -> IntegerVariable: """ - The i'th entry downloaded with this upload date. + :description: + The i'th entry downloaded with this upload date. """ - return Variable(variable_name="upload_date_index") + return IntegerVariable(variable_name="upload_date_index", definition="{ %int(1) }") - @property - def upload_date_index_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_date_index_padded(self: "VariableDefinitions") -> StringVariable: """ - The upload_date_index padded two digits + :description: + The upload_date_index padded two digits """ - return Variable("upload_date_index_padded") + return self.upload_date_index.to_padded_int(variable_name="upload_date_index_padded", pad=2) - @property - def upload_date_index_reversed(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_date_index_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - 100 - upload_date_index + :description: + 100 - upload_date_index """ - return Variable("upload_date_index_reversed") + return IntegerVariable( + variable_name="upload_date_index_reversed", + definition=f"{{%sub(100, {self.upload_date_index.variable_name})}}", + ) - @property - def upload_date_index_reversed_padded(self: "VariableDefinitions") -> Variable: + @cached_property + def upload_date_index_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - The upload_date_index padded two digits + :description: + The upload_date_index padded two digits """ - return Variable("upload_date_index_reversed_padded") + return self.upload_date_index_reversed.to_padded_int( + variable_name="upload_date_index_reversed_padded", pad=2 + ) class EntryVariableDefinitions(ABC): - @property - def uid(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def uid(self: "VariableDefinitions") -> StringMetadataVariable: """ - The entry's unique ID + :description: + The entry's unique ID """ - return MetadataVariable(metadata_key="id", variable_name="uid") + return StringMetadataVariable.from_entry(metadata_key="id", variable_name="uid") - @property - def duration(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def duration(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - The duration of the entry in seconds + :description: + The duration of the entry in seconds if it exists. Defaults to zero otherwise. """ - return MetadataVariable("duration", metadata_key="duration") + return IntegerMetadataVariable.from_entry(metadata_key="duration", default=0) - @property - def uid_sanitized_plex(self: "VariableDefinitions") -> Variable: + @cached_property + def uid_sanitized_plex(self: "VariableDefinitions") -> StringVariable: """ - The sanitized uid with additional sanitizing for Plex. Replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. + :description: + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. """ - return Variable("uid_sanitized_plex") + return self.uid.to_sanitized_plex(variable_name="uid_sanitized_plex") - @property - def ie_key(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def ie_key(self: "VariableDefinitions") -> StringMetadataVariable: """ - The ie_key, used in legacy yt-dlp things as the 'info-extractor key' + :description: + The ie_key, used in legacy yt-dlp things as the 'info-extractor key'. + If it does not exist, return ``extractor_key`` """ - return MetadataVariable(metadata_key="ie_key", variable_name="ie_key") + return StringMetadataVariable.from_entry(metadata_key="ie_key", default=self.extractor_key) - @property - def extractor_key(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def extractor_key(self: "VariableDefinitions") -> StringMetadataVariable: """ - The yt-dlp extractor key + :description: + The yt-dlp extractor key """ - return MetadataVariable(metadata_key="extractor_key", variable_name="extractor_key") + return StringMetadataVariable.from_entry(metadata_key="extractor_key") - @property - def extractor(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def extractor(self: "VariableDefinitions") -> StringMetadataVariable: """ - The yt-dlp extractor name + :description: + The yt-dlp extractor name """ - return MetadataVariable(variable_name="extractor", metadata_key="extractor") + return StringMetadataVariable.from_entry(metadata_key="extractor") - @property - def epoch(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def epoch(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - The unix epoch of when the metadata was scraped by yt-dlp. + :description: + The unix epoch of when the metadata was scraped by yt-dlp. """ - return MetadataVariable(metadata_key="epoch", variable_name="epoch") + return IntegerMetadataVariable.from_entry(metadata_key="epoch") - @property - def epoch_date(self: "VariableDefinitions") -> Variable: + @cached_property + def epoch_date(self: "VariableDefinitions") -> StringVariable: """ - The epoch's date, in YYYYMMDD format. + :description: + The epoch's date, in YYYYMMDD format. """ - return Variable("epoch_date") + return StringVariable( + variable_name="epoch_date", + definition=f"{{%datetime_strftime({self.epoch.variable_name}, '%Y%m%d')}}", + ) - @property - def epoch_hour(self: "VariableDefinitions") -> Variable: + @cached_property + def epoch_hour(self: "VariableDefinitions") -> StringVariable: """ - The epoch's hour + :description: + The epoch's hour """ - return Variable("epoch_hour") + return StringVariable( + variable_name="epoch_hour", + definition=f"{{%datetime_strftime({self.epoch.variable_name}, '%H')}}", + ) - @property - def title(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def title(self: "VariableDefinitions") -> StringMetadataVariable: """ - The title of the entry. If a title does not exist, returns its unique ID. + :description: + The title of the entry. If a title does not exist, returns its unique ID. """ - return MetadataVariable(variable_name="title", metadata_key="title") + return StringMetadataVariable.from_entry(metadata_key="title", default=self.uid) - @property - def title_sanitized_plex(self: "VariableDefinitions") -> Variable: + @cached_property + def title_sanitized_plex(self: "VariableDefinitions") -> StringVariable: """ - The sanitized title with additional sanitizing for Plex. It replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. + :description: + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. """ - return Variable("title_sanitized_plex") + return self.title.to_sanitized_plex(variable_name="title_sanitized_plex") - @property - def webpage_url(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: """ - The url to the webpage. + :description: + The url to the webpage. """ - return MetadataVariable(metadata_key="webpage_url", variable_name="webpage_url") + return StringMetadataVariable.from_entry(metadata_key="webpage_url") - @property - def info_json_ext(self: "VariableDefinitions") -> Variable: + @cached_property + def info_json_ext(self: "VariableDefinitions") -> StringVariable: """ - The "info.json" extension + :description: + The "info.json" extension """ - return Variable("info_json_ext") + return StringVariable( + variable_name="info_json_ext", + definition="info.json", + ) - @property - def description(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def description(self: "VariableDefinitions") -> StringMetadataVariable: """ - The description if it exists. Otherwise, returns an emtpy string. + :description: + The description if it exists. Otherwise, returns an emtpy string. """ - return MetadataVariable(variable_name="description", metadata_key="description") + return StringMetadataVariable.from_entry( + metadata_key="description", + default="", + ) - @property - def uploader_id(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: """ - The uploader id if it exists, otherwise return the unique ID. + :description: + The uploader id if it exists, otherwise return the unique ID. """ - return MetadataVariable(variable_name="uploader_id", metadata_key="uploader_id") + return StringMetadataVariable.from_entry( + metadata_key="uploader_id", + default=self.uid, + ) - @property - def uploader(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def uploader(self: "VariableDefinitions") -> StringMetadataVariable: """ - The uploader if it exists, otherwise return the uploader ID. + :description: + The uploader if it exists, otherwise return the uploader ID. """ - return MetadataVariable(variable_name="uploader", metadata_key="uploader") + return StringMetadataVariable.from_entry( + metadata_key="uploader", + default=self.uploader_id, + ) - @property - def uploader_url(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: """ - The uploader url if it exists, otherwise returns the webpage_url. + :description: + The uploader url if it exists, otherwise returns the webpage_url. """ - return MetadataVariable("uploader_url", metadata_key="uploader_url") + return StringMetadataVariable.from_entry( + metadata_key="uploader_url", + default=self.webpage_url, + ) - @property - def creator(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def creator(self: "VariableDefinitions") -> StringMetadataVariable: """ - The creator name if it exists, otherwise returns the channel. + :description: + The creator name if it exists, otherwise returns the channel. """ - return MetadataVariable(variable_name="creator", metadata_key="creator") + return StringMetadataVariable.from_entry( + metadata_key="creator", + default=self.channel, + ) - @property - def channel(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def channel(self: "VariableDefinitions") -> StringMetadataVariable: """ - The channel name if it exists, otherwise returns the uploader. + :description: + The channel name if it exists, otherwise returns the uploader. """ - return MetadataVariable(variable_name="channel", metadata_key="channel") + return StringMetadataVariable.from_entry( + metadata_key="channel", + default=self.uploader, + ) - @property - def channel_id(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def channel_id(self: "VariableDefinitions") -> StringMetadataVariable: """ - The channel id if it exists, otherwise returns the entry uploader ID. + :description: + The channel id if it exists, otherwise returns the entry uploader ID. """ - return MetadataVariable(variable_name="channel_id", metadata_key="channel_id") + return StringMetadataVariable.from_entry( + metadata_key="channel_id", + default=self.uploader_id, + ) - @property - def ext(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def ext(self: "VariableDefinitions") -> StringMetadataVariable: """ - The downloaded entry's file extension + :description: + The downloaded entry's file extension """ - return MetadataVariable(variable_name="ext", metadata_key="ext") + return StringMetadataVariable.from_entry(metadata_key="ext") - @property - def thumbnail_ext(self: "VariableDefinitions") -> Variable: + @cached_property + def thumbnail_ext(self: "VariableDefinitions") -> StringVariable: """ - The download entry's thumbnail extension. Will always return 'jpg'. Until there is a - need to support other image types, we always convert to jpg. + :description: + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. """ - return Variable("thumbnail_ext") + return StringVariable( + variable_name="thumbnail_ext", + definition="jpg", + ) - @property - def comments(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def comments(self: "VariableDefinitions") -> ArrayMetadataVariable: """ - Comments if they are requested + :description: + Comments if they are requested """ - return MetadataVariable("comments", "comments") + return ArrayMetadataVariable( + metadata_key="comments", variable_name="comments", definition="{ [] }" + ) - @property - def chapters(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def chapters(self: "VariableDefinitions") -> ArrayMetadataVariable: """ - Chapters if they exist + :description: + Chapters if they exist """ - return MetadataVariable("chapters", "chapters") + return ArrayMetadataVariable( + metadata_key="chapters", variable_name="chapters", definition="{ [] }" + ) - @property - def sponsorblock_chapters(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def sponsorblock_chapters(self: "VariableDefinitions") -> ArrayMetadataVariable: """ - Sponsorblock Chapters if they are requested and exist + :description: + Sponsorblock Chapters if they are requested and exist """ - return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") + return ArrayMetadataVariable( + metadata_key="sponsorblock_chapters", + variable_name="sponsorblock_chapters", + definition="{ [] }", + ) - @property - def requested_subtitles(self: "VariableDefinitions") -> MetadataVariable: + @cached_property + def requested_subtitles(self: "VariableDefinitions") -> MapMetadataVariable: """ - Subtitles if they are requested and exist + :description: + Subtitles if they are requested and exist """ - return MetadataVariable("requested_subtitles", "requested_subtitles") + return MapMetadataVariable( + metadata_key="requested_subtitles", + variable_name="requested_subtitles", + definition="{ {} }", + ) class VariableDefinitions( @@ -776,8 +1071,90 @@ class VariableDefinitions( ReleaseDateVariableDefinitions, YtdlSubVariableDefinitions, ): - pass + @cache + def scripts(self) -> Dict[str, str]: + """ + Returns all variables and their scripts in dict form + """ + return { + var.variable_name: var.definition + for var in [ + getattr(self, attr) + for attr in dir(self) + if isinstance(getattr(self, attr), Variable) + ] + } + + @cache + def injected_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that get injected in the download-stage + """ + return { + self.download_index, + self.upload_date_index, + self.comments, + self.requested_subtitles, + self.chapters, + self.sponsorblock_chapters, + self.ytdl_sub_input_url, + } + + @cache + def required_entry_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that the entry requires to exist + """ + return { + self.uid, + self.extractor_key, + self.epoch, + self.webpage_url, + self.ext, + } + + @cache + def default_entry_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that reside on the entry that may or may not exist, + but have defaults + """ + return { + self.title, + self.extractor, + self.description, + self.ie_key, + self.uploader_id, + self.uploader, + self.uploader_url, + self.upload_date, + self.release_date, + self.channel, + self.creator, + self.channel_id, + self.duration, + self.playlist_index, + self.playlist_count, + self.playlist_uid, + self.playlist_title, + self.playlist_uploader_id, + } + + @cache + def unresolvable_static_variables(self) -> Set[Variable]: + """ + Returns variables that are not static (i.e. depend on runtime) + """ + return { + VARIABLES.entry_metadata, + } | self.injected_variables() -# Singleton to use externally +# Singletons to use externally VARIABLES: VariableDefinitions = VariableDefinitions() +VARIABLE_SCRIPTS: Dict[str, str] = VARIABLES.scripts() +UNRESOLVED_VARIABLES: Set[str] = { + var.variable_name for var in VARIABLES.unresolvable_static_variables() +} + +CustomFunctions.register() diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py deleted file mode 100644 index 90808c37..00000000 --- a/src/ytdl_sub/entries/script/variable_scripts.py +++ /dev/null @@ -1,296 +0,0 @@ -from typing import Dict -from typing import List -from typing import Optional -from typing import Set - -import mergedeep - -from ytdl_sub.entries.script.custom_functions import CustomFunctions -from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_definitions import Metadata -from ytdl_sub.entries.script.variable_definitions import MetadataVariable -from ytdl_sub.entries.script.variable_definitions import Variable -from ytdl_sub.entries.script.variable_definitions import VariableDefinitions - -############################################################################################### -# Helpers - -v: VariableDefinitions = VARIABLES - - -def _pad_int(key: Variable, pad: int) -> str: - return f"{{%pad_zero({key.variable_name}, {pad})}}" - - -def _sanitized_plex(key: Variable) -> str: - return f"{{%sanitize_plex_episode({key.variable_name})}}" - - -def _date_metadata(date_key: Variable, metadata_key: str) -> str: - return f"{{%map_get(%to_date_metadata({date_key.variable_name}), '{metadata_key}')}}" - - -############################################################################################### -# Metadata Getters - - -def _get( - cast: str, - metadata: Metadata, - key: MetadataVariable, - default: Optional[Variable | str | int | Dict | List], -) -> str: - if default is None: - # TODO: assert with good error message if key DNE - out = f"%map_get({metadata.variable_name}, '{key.metadata_key}')" - elif isinstance(default, Variable): - args = f"{metadata.variable_name}, '{key.metadata_key}', {default.variable_name}" - out = f"%map_get_non_empty({args})" - elif isinstance(default, str): - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', '{default}')" - elif isinstance(default, dict): - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {{}})" - elif isinstance(default, list): - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', [])" - else: - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {default})" - - return f"{{ %legacy_bracket_safety(%{cast}({out})) }}" - - -############################################################################################### -# Entry Getters - - -def _entry_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: - return _get("string", metadata=v.entry_metadata, key=key, default=default) - - -def _entry_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str: - return _get("int", metadata=v.entry_metadata, key=key, default=default) - - -def _entry_get_map(key: MetadataVariable, default: Optional[Variable | Dict] = None): - return _get("map", metadata=v.entry_metadata, key=key, default=default) - - -def _entry_get_array(key: MetadataVariable, default: Optional[Variable | List] = None): - return _get("array", metadata=v.entry_metadata, key=key, default=default) - - -############################################################################################### -# Playlist Getters - - -def _playlist_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: - return _get("string", metadata=v.playlist_metadata, key=key, default=default) - - -def _playlist_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str: - return _get("int", metadata=v.playlist_metadata, key=key, default=default) - - -############################################################################################### -# Source Getters - - -def _source_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: - return _get("string", metadata=v.source_metadata, key=key, default=default) - - -############################################################################################### -# Scripts - -ENTRY_EMPTY_METADATA: Dict[Variable, str] = {v.entry_metadata: "{ {} }"} - -ENTRY_HARDCODED_VARIABLES: Dict[Variable, str] = { - v.info_json_ext: "info.json", - v.thumbnail_ext: "jpg", -} - -ENTRY_RELATIVE_VARIABLES: Dict[MetadataVariable, str] = { - v.playlist_metadata: _entry_get_map(v.playlist_metadata, {}), - v.source_metadata: _entry_get_map(v.source_metadata, {}), - v.sibling_metadata: _entry_get_array(v.sibling_metadata, []), -} - -ENTRY_REQUIRED_VARIABLES: Dict[MetadataVariable, str] = { - v.uid: _entry_get_str(v.uid), - v.extractor_key: _entry_get_str(v.extractor_key), - v.epoch: _entry_get_int(v.epoch), - v.webpage_url: _entry_get_str(v.webpage_url), - v.ext: _entry_get_str(v.ext), -} - -ENTRY_DEFAULT_VARIABLES: Dict[MetadataVariable, str] = { - v.title: _entry_get_str(v.title, v.uid), - v.extractor: _entry_get_str(v.extractor, v.extractor_key), - v.description: _entry_get_str(v.description, ""), - v.ie_key: _entry_get_str(v.ie_key, v.extractor_key), - v.uploader_id: _entry_get_str(v.uploader_id, v.uid), - v.uploader: _entry_get_str(v.uploader, v.uploader_id), - v.uploader_url: _entry_get_str(v.uploader_url, v.webpage_url), - v.upload_date: _entry_get_str(v.upload_date, v.epoch_date), - v.release_date: _entry_get_str(v.release_date, v.upload_date), - v.channel: _entry_get_str(v.channel, v.uploader), - v.creator: _entry_get_str(v.creator, v.channel), - v.channel_id: _entry_get_str(v.channel_id, v.uploader_id), - v.duration: _entry_get_int(v.duration, 0), - v.playlist_index: _entry_get_int(v.playlist_index, 1), - v.playlist_count: _entry_get_int(v.playlist_count, 1), - v.playlist_uid: _entry_get_str(v.playlist_uid, v.uid), - v.playlist_title: _entry_get_str(v.playlist_title, v.title), - v.playlist_uploader_id: _entry_get_str(v.playlist_uploader_id, v.uploader_id), -} - -# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED IN THE DOWNLOADER -DOWNLOADER_INJECTED_VARIABLES: Dict[Variable, str] = { - v.download_index: "{%int(1)}", - v.upload_date_index: "{%int(1)}", - v.comments: "{ [] }", - v.requested_subtitles: "{ {} }", - v.chapters: "{ [] }", - v.sponsorblock_chapters: "{ [] }", - v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}", -} - -ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = { - v.uid_sanitized_plex: _sanitized_plex(v.uid), - v.title_sanitized_plex: _sanitized_plex(v.title), - v.epoch_date: f"{{%datetime_strftime({v.epoch.variable_name}, '%Y%m%d')}}", - v.epoch_hour: f"{{%datetime_strftime({v.epoch.variable_name}, '%H')}}", - v.download_index_padded6: _pad_int(v.download_index, 6), - v.upload_date_index_padded: _pad_int(v.upload_date_index, 2), - v.upload_date_index_reversed: f"{{%sub(100, {v.upload_date_index.variable_name})}}", - v.upload_date_index_reversed_padded: _pad_int(v.upload_date_index_reversed, 2), - v.playlist_index_reversed: ( - f"{{%sub({v.playlist_count.variable_name}, {v.playlist_index.variable_name}, -1)}}" - ), - v.playlist_index_padded: _pad_int(v.playlist_index, 2), - v.playlist_index_reversed_padded: _pad_int(v.playlist_index_reversed, 2), - v.playlist_index_padded6: _pad_int(v.playlist_index, 6), - v.playlist_index_reversed_padded6: _pad_int(v.playlist_index_reversed, 6), -} - -ENTRY_UPLOAD_DATE_VARIABLES: Dict[Variable, str] = { - v.upload_year: _date_metadata(v.upload_date, "year"), - v.upload_year_truncated: _date_metadata(v.upload_date, "year_truncated"), - v.upload_year_truncated_reversed: _date_metadata(v.upload_date, "year_truncated_reversed"), - v.upload_month_reversed: _date_metadata(v.upload_date, "month_reversed"), - v.upload_month_reversed_padded: _date_metadata(v.upload_date, "month_reversed_padded"), - v.upload_month_padded: _date_metadata(v.upload_date, "month_padded"), - v.upload_day_padded: _date_metadata(v.upload_date, "day_padded"), - v.upload_month: _date_metadata(v.upload_date, "month"), - v.upload_day: _date_metadata(v.upload_date, "day"), - v.upload_day_reversed: _date_metadata(v.upload_date, "day_reversed"), - v.upload_day_reversed_padded: _date_metadata(v.upload_date, "day_reversed_padded"), - v.upload_day_of_year: _date_metadata(v.upload_date, "day_of_year"), - v.upload_day_of_year_padded: _date_metadata(v.upload_date, "day_of_year_padded"), - v.upload_day_of_year_reversed: _date_metadata(v.upload_date, "day_of_year_reversed"), - v.upload_day_of_year_reversed_padded: _date_metadata( - v.upload_date, "day_of_year_reversed_padded" - ), - v.upload_date_standardized: _date_metadata(v.upload_date, "date_standardized"), -} - -ENTRY_RELEASE_DATE_VARIABLES: Dict[Variable, str] = { - v.release_year: _date_metadata(v.release_date, "year"), - v.release_year_truncated: _date_metadata(v.release_date, "year_truncated"), - v.release_year_truncated_reversed: _date_metadata(v.release_date, "year_truncated_reversed"), - v.release_month_reversed: _date_metadata(v.release_date, "month_reversed"), - v.release_month_reversed_padded: _date_metadata(v.release_date, "month_reversed_padded"), - v.release_month_padded: _date_metadata(v.release_date, "month_padded"), - v.release_day_padded: _date_metadata(v.release_date, "day_padded"), - v.release_month: _date_metadata(v.release_date, "month"), - v.release_day: _date_metadata(v.release_date, "day"), - v.release_day_reversed: _date_metadata(v.release_date, "day_reversed"), - v.release_day_reversed_padded: _date_metadata(v.release_date, "day_reversed_padded"), - v.release_day_of_year: _date_metadata(v.release_date, "day_of_year"), - v.release_day_of_year_padded: _date_metadata(v.release_date, "day_of_year_padded"), - v.release_day_of_year_reversed: _date_metadata(v.release_date, "day_of_year_reversed"), - v.release_day_of_year_reversed_padded: _date_metadata( - v.release_date, "day_of_year_reversed_padded" - ), - v.release_date_standardized: _date_metadata(v.release_date, "date_standardized"), -} - -PLAYLIST_VARIABLES: Dict[Variable, str] = { - v.playlist_webpage_url: _playlist_get_str(v.playlist_webpage_url, v.webpage_url), - v.playlist_description: _playlist_get_str(v.playlist_description, v.description), - v.playlist_uploader: _playlist_get_str(v.playlist_uploader, v.uploader), - v.playlist_uploader_url: _playlist_get_str(v.playlist_uploader_url, v.playlist_webpage_url), - v.source_index: _playlist_get_int(v.source_index, 1), - v.source_count: _playlist_get_int(v.source_count, 1), -} - - -SOURCE_VARIABLES: Dict[Variable, str] = { - v.source_uid: _source_get_str(v.source_uid, v.playlist_uid), - v.source_title: _source_get_str(v.source_title, v.playlist_title), - v.source_webpage_url: _source_get_str(v.source_webpage_url, v.playlist_webpage_url), - v.source_description: _source_get_str(v.source_description, v.playlist_description), - v.source_uploader_id: _source_get_str(v.source_uploader_id, v.playlist_uploader_id), - v.source_uploader: _source_get_str(v.source_uploader, v.playlist_uploader), - v.source_uploader_url: _source_get_str(v.source_uploader_url, v.source_webpage_url), -} - -SOURCE_DERIVED_VARIABLES: Dict[Variable, str] = { - v.source_index_padded: _pad_int(v.source_index, 2), -} - -SIBLING_VARIABLES: Dict[Variable, str] = { - v.playlist_max_upload_date: f"""{{ - %array_reduce( - %if_passthrough( - %extract_field_from_siblings('{v.upload_date.variable_name}'), - [{v.upload_date.variable_name}] - ), - %max - ) - }}""" -} - -SIBLING_DERIVED_VARIABLES: Dict[Variable, str] = { - v.playlist_max_upload_year: _date_metadata(v.playlist_max_upload_date, "year"), - v.playlist_max_upload_year_truncated: _date_metadata( - v.playlist_max_upload_date, "year_truncated" - ), -} - -_VARIABLE_SCRIPTS: Dict[Variable, str] = {} -mergedeep.merge( - _VARIABLE_SCRIPTS, - ENTRY_EMPTY_METADATA, - ENTRY_HARDCODED_VARIABLES, - ENTRY_RELATIVE_VARIABLES, - ENTRY_REQUIRED_VARIABLES, - ENTRY_DEFAULT_VARIABLES, - DOWNLOADER_INJECTED_VARIABLES, - ENTRY_DERIVED_VARIABLES, - ENTRY_UPLOAD_DATE_VARIABLES, - ENTRY_RELEASE_DATE_VARIABLES, - SIBLING_VARIABLES, - SIBLING_DERIVED_VARIABLES, - PLAYLIST_VARIABLES, - SOURCE_VARIABLES, - SOURCE_DERIVED_VARIABLES, -) - -VARIABLE_SCRIPTS: Dict[str, str] = { - var.variable_name: script for var, script in _VARIABLE_SCRIPTS.items() -} - - -def _keys(*variables: Dict[Variable, str]) -> Set[str]: - keys: Set[str] = set() - for variable_set in variables: - keys.update(set(var.variable_name for var in variable_set.keys())) - return keys - - -UNRESOLVED_VARIABLES: Set[str] = _keys( - ENTRY_EMPTY_METADATA, - DOWNLOADER_INJECTED_VARIABLES, -) - -CustomFunctions.register() diff --git a/src/ytdl_sub/entries/script/variable_types.py b/src/ytdl_sub/entries/script/variable_types.py new file mode 100644 index 00000000..7cca58cd --- /dev/null +++ b/src/ytdl_sub/entries/script/variable_types.py @@ -0,0 +1,321 @@ +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Dict +from typing import List +from typing import Optional +from typing import Type +from typing import TypeVar + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String + +ENTRY_METADATA_VARIABLE_NAME = "entry_metadata" +PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata" +SOURCE_METADATA_VARIABLE_NAME = "source_metadata" + +TMetadataVariable = TypeVar("TMetadataVariable", bound="MetadataVariable") +TVariable = TypeVar("TVariable", bound="Variable") + + +def _get( + cast: str, + metadata_variable_name: str, + metadata_key: str, + variable_name: Optional[str], + default: Optional[TVariable | str | int | Dict | List], + as_type: Type[TMetadataVariable], +) -> TMetadataVariable: + if default is None: + # TODO: assert with good error message if key DNE + out = f"%map_get({metadata_variable_name}, '{metadata_key}')" + elif isinstance(default, Variable): + args = f"{metadata_variable_name}, '{metadata_key}', {default.variable_name}" + out = f"%map_get_non_empty({args})" + elif isinstance(default, str): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', '{default}')" + elif isinstance(default, dict): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {{}})" + elif isinstance(default, list): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', [])" + else: + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {default})" + + return as_type( + variable_name=variable_name or metadata_key, + metadata_key=metadata_key, + definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}", + ) + + +@dataclass(frozen=True) +class Variable(ABC): + variable_name: str + definition: str + + @classmethod + @abstractmethod + def human_readable_type(cls) -> str: + """ + Script type of the variable, for documentation + """ + + +@dataclass(frozen=True) +class StringVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return String.__name__ + + def to_sanitized_plex(self, variable_name: str) -> "StringVariable": + """ + Converts a String variable to be plex sanitized + """ + return StringVariable( + variable_name=variable_name, + definition=f"{{%sanitize_plex_episode({self.variable_name})}}", + ) + + def as_date_variable(self) -> "StringDateVariable": + """ + Converts a String variable to a date variable (which has metadata helpers) + """ + return StringDateVariable( + variable_name=self.variable_name, + definition=self.definition, + ) + + +@dataclass(frozen=True) +class StringDateVariable(StringVariable): + def get_string_date_metadata( + self, date_metadata_key: str, variable_name: Optional[str] = None + ) -> StringVariable: + """ + Gets a string-based date metadata variable + """ + return StringVariable( + variable_name=variable_name or date_metadata_key, + definition=f"""{{ + %string( + %map_get( + %to_date_metadata({self.variable_name}), + '{date_metadata_key}' + ) + ) + }}""", + ) + + def get_integer_date_metadata( + self, date_metadata_key: str, variable_name: str + ) -> "IntegerVariable": + """ + Gets an int-based date metadata variable + """ + return IntegerVariable( + variable_name=variable_name, + definition=f"""{{ + %int( + %map_get( + %to_date_metadata({self.variable_name}), + '{date_metadata_key}' + ) + ) + }}""", + ) + + +@dataclass(frozen=True) +class IntegerVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Integer.__name__ + + def to_padded_int(self, variable_name: str, pad: int) -> StringVariable: + """ + Pads an integer + """ + return StringVariable( + variable_name=variable_name, definition=f"{{%pad_zero({self.variable_name}, {pad})}}" + ) + + +@dataclass(frozen=True) +class ArrayVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Array.__name__ + + +@dataclass(frozen=True) +class MapVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Map.__name__ + + +@dataclass(frozen=True) +class MetadataVariable(Variable, ABC): + metadata_key: str + + +@dataclass(frozen=True) +class MapMetadataVariable(MetadataVariable, MapVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional["MapMetadataVariable" | Dict] = None, + ) -> "MapMetadataVariable": + """ + Creates a map variable from entry metadata + """ + return _get( + "map", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=MapMetadataVariable, + ) + + +@dataclass(frozen=True) +class ArrayMetadataVariable(MetadataVariable, ArrayVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional["ArrayMetadataVariable" | List] = None, + ) -> "ArrayMetadataVariable": + """ + Creates an array variable from entry metadata + """ + return _get( + "array", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=ArrayMetadataVariable, + ) + + +@dataclass(frozen=True) +class StringMetadataVariable(MetadataVariable, StringVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from entry metadata + """ + return _get( + "string", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + @classmethod + def from_playlist( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from playlist metadata + """ + return _get( + "string", + metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + @classmethod + def from_source( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from source metadata + """ + return _get( + "string", + metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + def as_date_variable(self) -> "StringDateMetadataVariable": + """ + Converts a String variable to a date variable (which has metadata helpers) + """ + return StringDateMetadataVariable( + metadata_key=self.metadata_key, + variable_name=self.variable_name, + definition=self.definition, + ) + + +@dataclass(frozen=True) +class StringDateMetadataVariable(StringMetadataVariable, StringDateVariable): + pass + + +@dataclass(frozen=True) +class IntegerMetadataVariable(MetadataVariable, IntegerVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[IntegerVariable | int] = None, + ) -> "IntegerMetadataVariable": + """ + Creates an int variable from entry metadata + """ + return _get( + "int", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=IntegerMetadataVariable, + ) + + @classmethod + def from_playlist( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[IntegerVariable | int] = None, + ) -> "IntegerMetadataVariable": + """ + Creates an int variable from playlist metadata + """ + return _get( + "int", + metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=IntegerMetadataVariable, + ) diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 4e06d441..f3ee78b1 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -1,5 +1,5 @@ from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.script.functions import Functions from ytdl_sub.script.utils.name_validation import is_valid_name diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index 105bce98..aba9d1fc 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -5,9 +5,9 @@ from typing import Dict from typing import Set from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS -from ytdl_sub.entries.script.variable_definitions import Variable -from ytdl_sub.entries.script.variable_scripts import UNRESOLVED_VARIABLES -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException diff --git a/tools/docgen/entry_variables.py b/tools/docgen/entry_variables.py index 02cca731..df53b13a 100644 --- a/tools/docgen/entry_variables.py +++ b/tools/docgen/entry_variables.py @@ -4,11 +4,12 @@ from typing import Dict from typing import Type from tools.docgen.docgen import DocGen +from tools.docgen.utils import cached_properties from tools.docgen.utils import camel_case_to_human from tools.docgen.utils import get_function_docs from tools.docgen.utils import line_section -from tools.docgen.utils import properties from tools.docgen.utils import section +from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions @@ -37,10 +38,11 @@ class EntryVariablesDocGen(DocGen): docs += line_section(section_idx=idx) docs += section(name, level=1) - for variable_function_name in properties(parent_objs[name]): + for variable_function_name in cached_properties(parent_objs[name]): docs += get_function_docs( function_name=variable_function_name, obj=parent_objs[name], + pre_docstring=f":type: ``{getattr(VARIABLES, variable_function_name).human_readable_type()}``\n", level=2, ) diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py index 94cab101..81ddc3fa 100644 --- a/tools/docgen/utils.py +++ b/tools/docgen/utils.py @@ -1,4 +1,5 @@ import inspect +from functools import cached_property from typing import Any from typing import Dict from typing import List @@ -18,6 +19,10 @@ def properties(obj: Type[Any]) -> List[str]: return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property)) +def cached_properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), cached_property)) + + def static_methods(obj: Type[Any]) -> List[str]: return sorted( name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod) @@ -36,11 +41,16 @@ def camel_case_to_human(string: str) -> str: def get_function_docs( - function_name: str, obj: Any, level: int, display_function_name: Optional[str] = None + function_name: str, + obj: Any, + level: int, + display_function_name: Optional[str] = None, + pre_docstring: Optional[str] = None, ) -> str: display_function_name = display_function_name if display_function_name else function_name docs = section(display_function_name, level=level) + docs += pre_docstring or "" docs += inspect.cleandoc(getattr(obj, function_name).__doc__) docs += "\n" return docs From 735da8b647c1d8bc515c5eee8eced44834d9167a Mon Sep 17 00:00:00 2001 From: Jesse Bannon <jbann1994@gmail.com> Date: Fri, 29 Dec 2023 15:41:33 -0800 Subject: [PATCH 4/5] [BUGFIX] Actually fix `subscription_name` with periods in it (#861) Prior PR did not actually work - this actually fixes the bug (https://github.com/jmbannon/ytdl-sub/issues/844) where subscription names would drop periods from them, i.e. Mr. Beast would have the name Beast. POTENTIAL BREAKING CHANGE If you have a subscription with a period in its name prior to this fix, this change will make it so your download archive is not found since it uses the subscription name in the download archive file path. To fix, simply change the download archive JSON's name to have the actual subscription name, and not the dropped-period one. --- src/ytdl_sub/config/overrides.py | 21 ++- src/ytdl_sub/config/preset.py | 4 +- .../config/validators/variable_validation.py | 11 +- .../entries/variables/override_variables.py | 10 +- .../subscriptions/subscription_validators.py | 14 +- tests/unit/config/test_subscription.py | 147 ++++++++++-------- 6 files changed, 110 insertions(+), 97 deletions(-) diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index b576e1ad..59b787b7 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -7,8 +7,8 @@ import mergedeep from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME from ytdl_sub.entries.variables.override_variables import OverrideHelpers +from ytdl_sub.entries.variables.override_variables import OverrideVariables from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved @@ -124,14 +124,20 @@ class Overrides(DictFormatterValidator, Scriptable): initial_variables, self.dict_with_format_strings, unresolved_variables if unresolved_variables else {}, - {SUBSCRIPTION_NAME: self.subscription_name}, ) return ScriptUtils.add_sanitized_variables(initial_variables) - def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": + def initialize_script( + self, subscription_name: str, unresolved_variables: Set[str] + ) -> "Overrides": """ Initialize the override script with override variables + any unresolved variables """ + self.script.add( + ScriptUtils.add_sanitized_variables( + {OverrideVariables.subscription_name(): subscription_name} + ) + ) self.script.add( self.initial_variables( unresolved_variables={ @@ -144,15 +150,6 @@ class Overrides(DictFormatterValidator, Scriptable): self.update_script() return self - @property - def subscription_name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - return self._root_name - def apply_formatter( self, formatter: StringFormatterValidator, diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 2e5d9fb6..4fec587d 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -198,7 +198,9 @@ class Preset(_PresetShell): downloader_options=self.downloader_options, output_options=self.output_options, plugins=self.plugins, - ).initialize_overrides(overrides=self.overrides).ensure_proper_usage() + ).initialize_overrides( + subscription_name=self.name, overrides=self.overrides + ).ensure_proper_usage() @property def name(self) -> str: diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index af8a777f..1b2ea1e4 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -14,6 +14,7 @@ from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.variables.override_variables import OverrideVariables from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import validate_formatters @@ -66,7 +67,7 @@ def _get_added_and_modified_variables( def _override_variables(overrides: Overrides) -> Set[str]: - return set(list(overrides.initial_variables().keys())) + return set(list(overrides.initial_variables().keys())) | {OverrideVariables.subscription_name()} def _entry_variables() -> Set[str]: @@ -88,7 +89,9 @@ class VariableValidation: self.resolved_variables: Set[str] = set() self.unresolved_variables: Set[str] = set() - def initialize_overrides(self, overrides: Overrides) -> "VariableValidation": + def initialize_overrides( + self, subscription_name: str, overrides: Overrides + ) -> "VariableValidation": """ Do some gymnastics to initialize the Overrides script. """ @@ -127,7 +130,9 @@ class VariableValidation: # Initialize overrides with unresolved variables + modified variables to throw an error. # For modified variables, this is to prevent a resolve(update=True) to setting any # dependencies until it has been explicitly added - overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables) + overrides = overrides.initialize_script( + subscription_name=subscription_name, unresolved_variables=self.unresolved_variables + ) # copy the script and mock entry variables self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index f3ee78b1..38f7160c 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -3,9 +3,7 @@ from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.script.functions import Functions from ytdl_sub.script.utils.name_validation import is_valid_name -SUBSCRIPTION_NAME = "subscription_name" -SUBSCRIPTION_VALUE = "subscription_value" -SUBSCRIPTION_MAP = "subscription_map" +# TODO: use this SUBSCRIPTION_ARRAY = "subscription_array" @@ -15,7 +13,7 @@ class OverrideVariables: """ Name of the subscription """ - return SUBSCRIPTION_NAME + return "subscription_name" @staticmethod def subscription_value() -> str: @@ -28,7 +26,7 @@ class OverrideVariables: ``subscription_value`` gets set to ``https://...``. """ - return SUBSCRIPTION_VALUE + return "subscription_value" @staticmethod def subscription_indent_i(index: int) -> str: @@ -90,7 +88,7 @@ class OverrideVariables: ] } """ - return SUBSCRIPTION_MAP + return "subscription_map" class OverrideHelpers: diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 5dbe4d2b..62445552 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -9,9 +9,6 @@ from typing import final from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.overrides import Overrides -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_MAP -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE from ytdl_sub.entries.variables.override_variables import OverrideVariables from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator @@ -85,7 +82,6 @@ class SubscriptionPresetDictValidator(NamedSubscriptionValidator, DictValidator) output_dict["overrides"] = dict( output_dict.get("overrides", {}), **self._indent_overrides_dict(), - **{SUBSCRIPTION_NAME: self.subscription_name}, ) return {self.subscription_name: output_dict} @@ -114,7 +110,7 @@ class SubscriptionLeafValidator(NamedSubscriptionValidator, ABC): f"used as a subscription name" ) - self._overrides_to_add: Dict[str, str] = {SUBSCRIPTION_NAME: self.subscription_name} + self._overrides_to_add: Dict[str, str] = {} @final def subscription_dicts(self, global_presets_to_apply: List[str]) -> Dict[str, Dict]: @@ -147,7 +143,7 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add[SUBSCRIPTION_VALUE] = self.value + self._overrides_to_add[OverrideVariables.subscription_value()] = self.value class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator): @@ -172,7 +168,7 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid for idx, list_value in enumerate(self.list): # Write the first list value into subscription_value as well if idx == 0: - self._overrides_to_add[SUBSCRIPTION_VALUE] = list_value.value + self._overrides_to_add[OverrideVariables.subscription_value()] = list_value.value self._overrides_to_add[ OverrideVariables.subscription_value_i(index=idx) @@ -219,7 +215,9 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add = {SUBSCRIPTION_MAP: ScriptUtils.to_script(self.dict)} + self._overrides_to_add[OverrideVariables.subscription_map()] = ScriptUtils.to_script( + self.dict + ) class SubscriptionValidator(SubscriptionOutput): diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 2a3da732..bd7122c5 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -225,9 +225,9 @@ def test_subscription_file_preset_applies(config_file: ConfigFile, preset_with_f "key-4": "test_preset", } - overrides = preset_sub.overrides.dict_with_format_strings + overrides = preset_sub.overrides.script # preset overrides take precedence over __preset__ - assert overrides.get("current_override") == "test_preset" + assert overrides.get("current_override").native == "test_preset" def test_subscription_list( @@ -238,13 +238,15 @@ def test_subscription_list( subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 3 - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_value_1") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_value_2") == "is_2_1_list_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_value_1").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_value_2").native == "is_2_1_list_2" + assert ( + sub_2_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence def test_subscription_overrides_tilda( @@ -255,10 +257,10 @@ def test_subscription_overrides_tilda( subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 3 - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("current_override") == "test_2_1" # tilda sub takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("current_override").native == "test_2_1" # tilda sub takes precedence def test_subscription_overrides_map( @@ -291,7 +293,7 @@ def test_subscription_with_period_in_name( assert len(subs) == 2 assert subs[1].name == "Mr. Beast" - assert subs[1].overrides.dict_with_format_strings["subscription_name"] == "Mr. Beast" + assert subs[1].overrides.script.get("subscription_name").native == "Mr. Beast" def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables( @@ -304,26 +306,28 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 4 - sub_test_value = [sub for sub in subs if sub.name == "test_value"][ - 0 - ].overrides.dict_with_format_strings - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_test_value = [sub for sub in subs if sub.name == "test_value"][0].overrides.script + sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.script + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_test_value.get("subscription_indent_1") == "original_1" - assert sub_test_value.get("subscription_indent_2") == "original_2" + assert sub_test_value.get("subscription_indent_1").native == "original_1" + assert sub_test_value.get("subscription_indent_2").native == "original_2" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_indent_1") == "INDENT_1" - assert sub_1.get("subscription_indent_2") == "INDENT_2" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_1.get("subscription_name").native == "test_1" + assert sub_1.get("subscription_value").native == "is_1_overwritten" + assert sub_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_1.get("subscription_indent_2").native == "INDENT_2" + assert ( + sub_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_indent_1") == "INDENT_1" - assert sub_2_1.get("subscription_indent_2") == "original_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_2_1.get("subscription_indent_2").native == "original_2" + assert ( + sub_2_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence @pytest.mark.parametrize("all_same_line", [True, False]) @@ -343,28 +347,28 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 4 - sub_test_value = [sub for sub in subs if sub.name == "test_value"][ - 0 - ].overrides.dict_with_format_strings - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_test_value = [sub for sub in subs if sub.name == "test_value"][0].overrides.script + sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.script + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_test_value.get("subscription_indent_1") == "original_1" - assert sub_test_value.get("subscription_indent_2") == "original_2" + assert sub_test_value.get("subscription_indent_1").native == "original_1" + assert sub_test_value.get("subscription_indent_2").native == "original_2" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_indent_1") == "INDENT_1" - assert sub_1.get("subscription_indent_2") == "INDENT_2" - assert sub_1.get("subscription_indent_3") == "INDENT_3" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_1.get("subscription_name").native == "test_1" + assert sub_1.get("subscription_value").native == "is_1_overwritten" + assert sub_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_1.get("subscription_indent_2").native == "INDENT_2" + assert sub_1.get("subscription_indent_3").native == "INDENT_3" + # ensure __preset__ takes precedence + assert sub_1.get("current_override").native == "__preset__" - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_indent_1") == "INDENT_1" - assert sub_2_1.get("subscription_indent_2") == "original_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence - assert "subscription_indent_3" not in sub_2_1 + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_2_1.get("subscription_indent_2").native == "original_2" + # ensure __preset__ takes precedence + assert sub_2_1.get("current_override").native == "__preset__" + assert "subscription_indent_3" not in sub_2_1.variable_names def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables_same_line_old_format_errors( @@ -414,12 +418,15 @@ def test_tv_show_subscriptions(config_file: ConfigFile, tv_show_subscriptions_pa assert len(subs) == 7 assert subs[3].name == "Jake Trains" - jake_train_overrides = subs[3].overrides.dict_with_format_strings + jake_train_overrides = subs[3].overrides.script - assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" - assert jake_train_overrides["subscription_indent_1"] == "Kids" - assert jake_train_overrides["subscription_indent_2"] == "TV-Y" + assert jake_train_overrides.get("subscription_name").native == "Jake Trains" + assert ( + jake_train_overrides.get("subscription_value").native + == "https://www.youtube.com/@JakeTrains" + ) + assert jake_train_overrides.get("subscription_indent_1").native == "Kids" + assert jake_train_overrides.get("subscription_indent_2").native == "TV-Y" def test_advanced_tv_show_subscriptions( @@ -431,17 +438,20 @@ def test_advanced_tv_show_subscriptions( assert len(subs) == 9 assert subs[3].name == "Jake Trains" - jake_train_overrides = subs[3].overrides.dict_with_format_strings + jake_train_overrides = subs[3].overrides.script - assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" - assert jake_train_overrides["subscription_indent_1"] == "Kids" - assert jake_train_overrides["subscription_indent_2"] == "TV-Y" + assert jake_train_overrides.get("subscription_name").native == "Jake Trains" + assert ( + jake_train_overrides.get("subscription_value").native + == "https://www.youtube.com/@JakeTrains" + ) + assert jake_train_overrides.get("subscription_indent_1").native == "Kids" + assert jake_train_overrides.get("subscription_indent_2").native == "TV-Y" assert subs[5].name == "Gardening with Ciscoe" overrides = subs[5].overrides - assert overrides.apply_formatter(overrides.dict["subscription_name"]) == "Gardening with Ciscoe" + assert overrides.script.get("subscription_name").native == "Gardening with Ciscoe" assert ( overrides.apply_formatter(overrides.dict["url"]) == "https://www.youtube.com/@gardeningwithciscoe4430" @@ -459,11 +469,14 @@ def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_pat assert len(subs) == 14 assert subs[2].name == "Stan Getz" - monk = subs[2].overrides.dict_with_format_strings + monk = subs[2].overrides.script - assert monk["subscription_name"] == "Stan Getz" - assert monk["subscription_value"] == "https://www.youtube.com/@stangetzofficial/releases" - assert monk["subscription_indent_1"] == "Jazz" + assert monk.get("subscription_name").native == "Stan Getz" + assert ( + monk.get("subscription_value").native + == "https://www.youtube.com/@stangetzofficial/releases" + ) + assert monk.get("subscription_indent_1").native == "Jazz" def test_music_video_subscriptions(default_config: ConfigFile, music_video_subscription_path: Path): @@ -473,14 +486,14 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc assert len(subs) == 3 assert subs[1].name == "Michael Jackson" - monk = subs[1].overrides.dict_with_format_strings + monk = subs[1].overrides.script - assert monk["subscription_name"] == "Michael Jackson" + assert monk.get("subscription_name").native == "Michael Jackson" assert ( - monk["subscription_value"] + monk.get("subscription_value").native == "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" ) - assert monk["subscription_indent_1"] == "Pop" + assert monk.get("subscription_indent_1").native == "Pop" def test_default_docker_config_and_subscriptions(): From 12430ecf1edceeb3e7a32580b9a8f41661a2d672 Mon Sep 17 00:00:00 2001 From: Jesse Bannon <jbann1994@gmail.com> Date: Fri, 29 Dec 2023 22:34:36 -0800 Subject: [PATCH 5/5] [DOCS] Scripting reference start (#862) --- .../config_reference/scripting/index.rst | 164 ++++++++- .../scripting/scripting_types.rst | 319 ++++++++++++++++++ 2 files changed, 481 insertions(+), 2 deletions(-) create mode 100644 docs/source/config_reference/scripting/scripting_types.rst diff --git a/docs/source/config_reference/scripting/index.rst b/docs/source/config_reference/scripting/index.rst index c99105f8..bee9a4e8 100644 --- a/docs/source/config_reference/scripting/index.rst +++ b/docs/source/config_reference/scripting/index.rst @@ -2,11 +2,171 @@ Scripting ========= -Work in progress! Explanation of how to define scripts/variables will be added here. +``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The links below +contain reference documentation for each built-in variable and scripting function. .. toctree:: :maxdepth: 1 entry_variables override_variables - scripting_functions \ No newline at end of file + scripting_functions + scripting_types + +How it Works +------------ + +Fields in the config that support ``formatters`` mean they support scripting, and will +*format* the field using its defined script. + +In its most basic form, a script is a string comprised of variables and/or functions. + +Static String +~~~~~~~~~~~~~ + +The following example sets ``ytdl-sub``'s output directory. It is +considered *static* because it does not depend on anything from an entry. + +.. code-block:: yaml + + output_options: + output_directory: "Custom YTDL-SUB TV Show" + +Static Variables +~~~~~~~~~~~~~~~~ + +``ytdl-sub`` offers a few built-in static variables, including ``subscription_name``. +We can use this instead of hard-coding it above: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + +The syntax for variable usage is curly-braces with the variable name within it. Assuming +our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub`` +will actually write to that directory. + +Entry Variables +~~~~~~~~~~~~~~~ + +For context, an *entry* is a video or audio file downloaded from ``yt-dlp``. +*Entry variables* are variables that are derived from an entry's ``info.json`` file. This file +comes from ``yt-dlp`` and contains every piece of metadata that it scraped. + +These variables are not considered static since they change per entry download. There are a +few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For others, +we are free to use values that derive from an entry. + +Suppose we want to customize the name of an entry's output file and thumbnail to include its +title in its name. We can do that using entry variables: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + file_name: "{title}.{ext}" + thumbnail_name: "{title}.{thumbnail_ext}" + +Creating Custom Variables +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Suppose we want to include the date in our file names. This means we'd need to update +both the ``file_name`` and ``thumbnail_name`` fields to include it. + +Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method +for creating and overriding custom variables. + +These are created in the ``overrides`` section. Let's take our above example and create +a ``custom_file_name`` variable to use for the entry file and thumbnail fields: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + custom_file_name: "{upload_date_standardized} {title}" + +Sanitizing Variables +~~~~~~~~~~~~~~~~~~~~ + +For experienced ``yt-dlp`` scrapers, you may be thinking: + +- What if the title has characters that do not play nice with my operating system? + +``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic characters +with safe alternatives that can be used in file names. We can ensure our file names and directories +are safe by using: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name_sanitized}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + custom_file_name: "{upload_date_standardized} {title_sanitized}" + +Simply add a ``_sanitized`` suffix to any variable name to make it sanitized. + +.. note:: + + Make sure you do not sanitize custom variables that intentionally create directories, otherwise + they will... be sanitized and not resolve to directories! + + +Using Scripting Functions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's suppose you are an avid command-line user, and like all of your file names to be +``snake_cased_with_no_spaces``. We can use the +`replace <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_ +*scripting function* to create and use a snake-cased title. + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name_sanitized}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + snake_cased_title: >- + { + %replace( title, ' ', '_' ) + } + custom_file_name: "{upload_date_standardized}_{snake_cased_title_sanitized}" + +Scripting functions are similar to variables - they must be used within curly-braces. +It is good practice to use ``>-`` when defining variables that use functions. It is YAML's way of +saying: + +- Allow a string to be multi-lined, and do not include newlines before or after it. + +See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=canonical_yaml>`_. +Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make +scripting function usage readable. + +.. important:: + + It is important to use ``>-`` over other YAML new-line directives like ``>`` because they + add newlines before or after curly-braces, and will be included in your variable's output string. + +Advanced Scripting +------------------ + +Accessing ``info.json`` Fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +WIP + +Creating Custom Functions +~~~~~~~~~~~~~~~~~~~~~~~~~ +WIP + +Parsing Maps and Arrays +~~~~~~~~~~~~~~~~~~~~~~~ +WIP diff --git a/docs/source/config_reference/scripting/scripting_types.rst b/docs/source/config_reference/scripting/scripting_types.rst new file mode 100644 index 00000000..d389c2a9 --- /dev/null +++ b/docs/source/config_reference/scripting/scripting_types.rst @@ -0,0 +1,319 @@ + +Scripting Types +=============== + +Types +----- + +String +~~~~~~ + +Strings are a series of characters surrounded by quotes and can be defined in a few ways, including: + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + string_variable: "This is a String variable" + + .. tab-item:: In-Line + + .. code-block:: yaml + + string_variable: "{ %string('This is a String variable') }" + + .. tab-item:: Single Quote + + .. code-block:: yaml + + string_variable: >- + { + %string('This is a String variable') + } + + .. tab-item:: Double Quote + + .. code-block:: yaml + + string_variable: >- + { + %string("This is a String variable") + } + + .. tab-item:: Triple Quote + + .. code-block:: yaml + + string_variable: >- + { + %string('''This is a String variable''') + } + + .. tab-item:: Triple-Double Quote + + .. code-block:: yaml + + string_variable: >- + { + %string("""This is a String variable""") + } + +.. note:: + + For non-String types, they must be defined as parameters to scripting functions. This is because + anything in a variable definition that is not within curly-braces gets evaluated as a String. + +Integer +~~~~~~~ + +Integers are whole numbers with no decimal. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + int_variable: >- + { + %int(2022) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + int_variable: "{ %int(2022) }" + +Float +~~~~~ + +Floats are floating-point decimals numbers. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + float_variable: >- + { + %float(3.14) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + float_variable: "{ %float(3.14) }" + +Boolean +~~~~~~~ + +A type is considered boolean if it spells out ``True`` or ``False``, case-insensitive. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + bool_variable: >- + { + %bool(True) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + bool_variable: "{ %bool(FALSE) }" + +Array +~~~~~ + +An Array contains multiple types of any kind, including nested Arrays and Maps. +Arrays are defined using brackets (``[ ]``), and are accessed using zero-based indexing. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + array_variable: >- + { + [ + "element with index 0", + 1, + 2.0, + [ "Nested Array 3" ] + ] + } + element_0: >- + { + %array_at(array_variable, 0) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + array_variable: "{ ['element with index 0', 1, 2.0, ['Nested Array 3' ]] }" + element_0: "{ %array_at(array_variable, 0) }" + +Map +~~~ + +A Map is a key-value store, containing mappings between keys and values. +Maps are defined using curley-braces (``{ }``), and are accessed using their keys. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + map_variable: >- + { + { + "string_key": "string_value", + 1: "int_key", + "list_value": [ "elem0", 1, 2.0 ] + } + } + string_value: >- + { + %map_get(map_variable, "string_key") + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + map_variable: "{ {'string_key': 'string_value', 1: 'int_key', 'list_value': [ 'elem0', 1, 2.0 ]} }" + string_value: "{ %map_get(map_variable, 'string_key') }" + +Null +~~~~ +Null is represented by an empty String, and can be conveyed by spelling out ``null``, +case-insensitive. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + null_variable: "" + + .. tab-item:: In-Line + + .. code-block:: yaml + + null_variable: "{ %string(null) }" + + +Function Type-Hints +------------------- + +AnyArgument +~~~~~~~~~~~ +AnyArgument means any of the above Types are valid as input or output to a scripting function. + +.. note:: + + Strict typing is enforced. For functions that return ``AnyArgument`` need to be casted before + passing into functions that expect a particular type. + +Numeric +~~~~~~~ +Numeric refers to either an Integer or Float. + +Optional +~~~~~~~~ +Optional means a particular scripting function argument can be either provided or not included. +For example, the function +`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_ +has an optional default value. Both of these usages are valid: + +.. tab-set:: + + .. tab-item:: Map Get + + .. code-block:: yaml + + will_throw_key_does_not_exist_error: "{ %map_get( {}, 'key' ) }" + + .. tab-item:: Map Get with Optional Default Value + + .. code-block:: yaml + + will_return_default: "{ %map_get( {}, 'key', 'default value' ) }" + +Lambda +~~~~~~ +Lambda parameters are a reference to a function, and will call that lambda function +on the input. In this example, + +.. code-block:: yaml + + lambda_array_numeric_to_string: >- + { + %array_apply( [ 1, 2, 3, 4], %string ) + } + +We apply ``%string`` as a lambda function to +`array_apply <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_, +which is called on every element in the input array. The output becomes ``["1", "2", "3", "4"]``. + +This example has one input-argument being passed into the lambda. For other lambda-based functions +like `array_enumerate <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_, +it expects the lambda function to have two input arguments. These are denoted using +``LambdaTwo``, ``LambdaThree``, etc within the function spec. + +LambdaReduce +~~~~~~~~~~~~ +LambdaReduce is special type of lambda that reduces an Array to a single value by calling the +LabmdaReduce function repeatedly on two elements in the Array until it is reduced to a single value. + +In this example, + +.. code-block:: yaml + + lambda_reduce_sum: >- + { + %array_reduce( [ 1, 2, 3, 4], %add ) + } + +We call +`array_reduce <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_ +on the input array, using +`add <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_ +as the LambdaReduce function. This will reduce the Array to a single value by internally calling + +.. code-block:: + + - %add(1, 2) = 3 + - %add(3, 3) = 6 + - %add(6, 4) = 10 + +And evaluate to ``10``. + +ReturnableArguments +~~~~~~~~~~~~~~~~~~~ + +Returnable arguments are used in conditional functions like +`if <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_, +which implies the argument passed into the function is the function's output. For example, + +.. code-block:: yaml + + conditional_function: >- + { + %if( True, "Return this if True", "Return this if False" ) + } + +is going to return ``"Return this if True"`` since the condition parameter is ``True``. \ No newline at end of file