diff --git a/.gitignore b/.gitignore index 20d1b188..e18f90d3 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,5 @@ docker/testing/volumes ffmpeg.exe ffprobe.exe + +tools/docgen/out \ No newline at end of file diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/conf.py b/docs/source/conf.py index 6a48e5f0..7e551b67 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,6 +53,7 @@ html_theme_options = { } html_static_path = ["_static"] +html_css_files = ["custom.css"] # Make sure the all autosectionlabel targets are unique diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst index 521e1f0a..8e4097c3 100644 --- a/docs/source/config_reference/config_yaml.rst +++ b/docs/source/config_reference/config_yaml.rst @@ -53,51 +53,7 @@ presets ~~~~~~~ ``presets`` define a `formula` for how to format downloaded media and metadata. -download_strategy -""""""""""""""""" -Download strategies dictate what is getting downloaded from a source. Each -download strategy has its own set of parameters. - -.. _url: - -url -''' -.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions() - :members: url, playlist_thumbnails, source_thumbnails, download_reverse - :member-order: bysource - -multi_url -''''''''' -.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions() - :members: urls, variables - -------------------------------------------------------------------------------- - -output_options -"""""""""""""" - -.. autoclass:: ytdl_sub.config.preset_options.OutputOptions() - :members: - :member-order: bysource - :exclude-members: get_upload_date_range_to_keep, partial_validate - -------------------------------------------------------------------------------- - -.. _ytdl_options: - -ytdl_options -"""""""""""" -.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions() - -------------------------------------------------------------------------------- - -.. _overrides: - -overrides -""""""""" -.. autoclass:: ytdl_sub.config.overrides.Overrides() - -.. _parent preset: +This section is work-in-progress! preset """""" diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 451a0ed8..69512e8f 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -1,3 +1,4 @@ + Plugins ======= @@ -5,175 +6,315 @@ audio_extract ------------- Extracts audio from a video file. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 -codec -~~~~~ -The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, -opus, vorbis, wav, and best to grab the best possible format at runtime. +``codec`` -quality -~~~~~~~ -Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` -(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. +:expected type: String +:description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. + + +``quality`` + +:expected type: Float +:description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + + +---------------------------------------------------------------------------------------------------- chapters -------- Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False -allow_chapters_from_comments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. If chapters do not exist in the video/description itself, attempt to scrape -comments to find the chapters. Defaults to False. +``allow_chapters_from_comments`` -embed_chapters -~~~~~~~~~~~~~~ -Optional. Embed chapters into the file. Defaults to True. +:expected type: Optional[Boolean] +:description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. -force_key_frames -~~~~~~~~~~~~~~~~ -Optional. Force keyframes at cuts when removing sections. This is slow due to needing a -re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to -False. -remove_chapters_regex -~~~~~~~~~~~~~~~~~~~~~ -Optional. List of regex patterns to match chapter titles against and remove them from the -entry. +``embed_chapters`` -remove_sponsorblock_categories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. List of SponsorBlock categories to remove from the output file. Can only remove -categories that are specified in ``sponsorblock_categories`` or "all", which removes -everything specified in ``sponsorblock_categories``. +:expected type: Optional[Boolean] +:description: + Defaults to True. Embed chapters into the file. -sponsorblock_categories -~~~~~~~~~~~~~~~~~~~~~~~ -Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", -"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", -"poi_highlight", or "all" to include all categories. + +``force_key_frames`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. + + +``remove_chapters_regex`` + +:expected type: Optional[List[RegexString] +:description: + List of regex patterns to match chapter titles against and remove them from the + entry. + + +``remove_sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. + + +``sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. + + +---------------------------------------------------------------------------------------------------- date_range ---------- Only download files uploaded within the specified date range. +Dates must adhere to a yt-dlp datetime. From their docs: -Usage: +.. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + +Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. +Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest +granularity possible. + +:Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" -after -~~~~~ -Optional. Only download videos after this datetime. +``after`` -before -~~~~~~ -Optional. Only download videos before this datetime. +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +``before`` + +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +---------------------------------------------------------------------------------------------------- + +download +-------- +Sets the URL(s) to download from. Can be used in many forms, including + +:Single URL: + +.. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + +:Multi URL: + +.. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + +:Thumbnails + Variables: + +All variables must be defined for the top-most url. All subsequent URL variables can be either +overwritten or default to the top-most value. + +If an entry is returned from more than one URL, it will use the variables in the bottom-most +URL. + +.. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + +---------------------------------------------------------------------------------------------------- embed_thumbnail --------------- Whether to embed thumbnails to the audio/video file or not. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True + +---------------------------------------------------------------------------------------------------- file_convert ------------ Converts video files from one extension to another. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" -Supports custom ffmpeg conversions: +Also supports custom ffmpeg conversions: + +:Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text -convert_to -~~~~~~~~~~ -Convert to a desired file type. Supports: +``convert_to`` -* Video: avi, flv, mkv, mov, mp4, webm -* Audio: aac, flac, mp3, m4a, opus, vorbis, wav +:expected type: String +:description: + Convert to a desired file type. Supports -convert_with -~~~~~~~~~~~~ -Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within -yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified -with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav -ffmpeg_post_process_args -~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the -form of: -.. code-block:: bash +``convert_with`` - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext +:expected type: Optional[String] +:description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. -The output file will use the extension specified in ``convert_to``. Post-processing args -can still be set with ``convert_with`` set to ``yt-dlp``. + +``ffmpeg_post_process_args`` + +:expected type: Optional[OverridesFormatter] +:description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of + + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. + + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. + + +---------------------------------------------------------------------------------------------------- + +filter_exclude +-------------- +Applies a conditional OR on any number of filters comprised of either variables or scripts. +If any filter evaluates to True, the entry will be excluded. + +:Usage: + +.. code-block:: yaml + + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } + +---------------------------------------------------------------------------------------------------- + +filter_include +-------------- +Applies a conditional AND on any number of filters comprised of either variables or scripts. +If all filters evaluate to True, the entry will be included. + +:Usage: + +.. code-block:: yaml + + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + +---------------------------------------------------------------------------------------------------- format ------ @@ -184,48 +325,28 @@ Usage: .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" -format -~~~~~~ -yt-dlp format, uses same syntax as yt-dlp. +---------------------------------------------------------------------------------------------------- match_filters ------------- -Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. -Uses the same syntax as yt-dlp. +Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. +Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. +For logical AND's between match filters, use the ``&`` operator in a single match filter. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" + match_filters: + filters: + - "age_limit?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" -Supports one or multiple filters: - -.. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit`_. -Usage: +:Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" -embed_thumbnail -~~~~~~~~~~~~~~~ -Optional. Whether to embed the thumbnail into the audio file. - -tags -~~~~ -Key, values of tag names, tag values. Supports source and override variables. -Supports lists which will get written to MP3s as id3v2.4 multi-tags. +---------------------------------------------------------------------------------------------------- nfo_tags -------- Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False -kodi_safe -~~~~~~~~~ -Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some -foreign language characters. Setting this to True will replace those characters with '□'. -Defaults to False. +``kodi_safe`` -nfo_name -~~~~~~~~ -The NFO file name. +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. -nfo_root -~~~~~~~~ -The root tag of the NFO's XML. In the usage above, it would look like -.. code-block:: xml +``nfo_name`` - - - +:expected type: EntryFormatter +:description: + The NFO file name. -tags -~~~~ -Tags within the nfo_root tag. In the usage above, it would look like -.. code-block:: xml +``nfo_root`` - - - Awesome Youtube Video - 2022 - 502 - +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like -Also supports xml attributes and duplicate keys: + .. code-block:: xml -.. code-block:: yaml + + + - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" -Which translates to +``tags`` -.. code-block:: xml +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like - 2022 - Comedy - Drama + .. code-block:: xml + + + + Awesome Youtube Video + 2022 + 502 + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + 2022 + Comedy + Drama + + +---------------------------------------------------------------------------------------------------- output_directory_nfo_tags ------------------------- @@ -361,63 +484,77 @@ Usage: # optional kodi_safe: False -kodi_safe -~~~~~~~~~ -Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some -foreign language characters. Setting this to True will replace those characters with '□'. -Defaults to False. +``kodi_safe`` -nfo_name -~~~~~~~~ -The NFO file name. +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. -nfo_root -~~~~~~~~ -The root tag of the NFO's XML. In the usage above, it would look like -.. code-block:: xml +``nfo_name`` - - - +:expected type: EntryFormatter +:description: + The NFO file name. -tags -~~~~ -Tags within the nfo_root tag. In the usage above, it would look like -.. code-block:: xml +``nfo_root`` - - - Sweet youtube TV show - +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like -Also supports xml attributes and duplicate keys: + .. code-block:: xml -.. code-block:: yaml + + + - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" -Which translates to +``tags`` -.. code-block:: xml +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + .. code-block:: xml + + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <tvshow> + <title>Sweet youtube TV show + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> + + +---------------------------------------------------------------------------------------------------- output_options -------------- Defines where to output files and thumbnails after all post-processing has completed. -Usage: +:Usage: .. code-block:: yaml @@ -436,79 +573,113 @@ Usage: keep_files_before: now keep_files_after: 19000101 -download_archive_name -~~~~~~~~~~~~~~~~~~~~~ -Optional. The file name to store a subscriptions download archive placed relative to -the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` +``download_archive_name`` -file_name -~~~~~~~~~ -Required. The file name for the media file. This can include directories such as -``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. +:expected type: Optional[OverridesFormatter] +:description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` -info_json_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's info json file. This can include directories such -as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output -directory. Can be set to empty string or `null` to disable info json writes. -keep_files_after -~~~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. +``file_name`` -Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep -files after ``19000101``, which implies all files. Can be used in conjunction with -``keep_max_files``. +:expected type: EntryFormatter +:description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. -keep_files_before -~~~~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. -Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep -files before ``now``, which implies all files. Can be used in conjunction with -``keep_max_files``. +``info_json_name`` -keep_max_files -~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. -Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be -applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. -maintain_download_archive -~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Maintains a download archive file in the output directory for a subscription. -It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the -output directory. +``keep_files_after`` -The download archive contains a mapping of ytdl IDs to downloaded files. This is used to -create a ytdl download-archive file when invoking a download on a subscription. This will -prevent ytdl from redownloading media already downloaded. +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. -Defaults to False. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. -migrated_download_archive_name -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Intended to be used if you are migrating a subscription with either a new -subscription name or output directory. It will try to load the archive file using this name -first, and fallback to ``download_archive_name``. It will always save to this file -and remove the original ``download_archive_name``. -output_directory -~~~~~~~~~~~~~~~~ -Required. The output directory to store all media files downloaded. +``keep_files_before`` -thumbnail_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's thumbnail image. This can include directories such -as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output -directory. Can be set to empty string or `null` to disable thumbnail writes. +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. + + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. + + +``keep_max_files`` + +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + + +``maintain_download_archive`` + +:expected type: Optional[Boolean] +:description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. + + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. + + Defaults to False. + + +``migrated_download_archive_name`` + +:expected type: Optional[OverridesFormatter] +:description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. + + +``output_directory`` + +:expected type: OverridesFormatter +:description: + The output directory to store all media files downloaded. + + +``thumbnail_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. + + +---------------------------------------------------------------------------------------------------- overrides --------- -Optional. This section allows you to define variables that can be used in any string formatter. -For example, if you want your file and thumbnail files to match without copy-pasting a large -format string, you can define something like: +Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + +:Usage: .. code-block:: yaml @@ -531,8 +702,45 @@ In addition, any override variable defined will automatically create a ``sanitiz for use. In the example above, ``output_directory_sanitized`` will exist and perform sanitization on the value when used. +---------------------------------------------------------------------------------------------------- + regex ----- +.. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -548,82 +756,95 @@ For example, creating the override variable ``"title_and_description": "{title} and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" -skip_if_match_fails -~~~~~~~~~~~~~~~~~~~ -Defaults to True. If True, when any match fails and has no defaults, the entry will be -skipped. If False, ytdl-sub will error and all downloads will not proceed. +``skip_if_match_fails`` + +:expected type: Optional[Boolean] +:description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. + + +---------------------------------------------------------------------------------------------------- split_by_chapters ----------------- Splits a file by chapters into multiple files. Each file becomes its own entry with the -new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, -``chapter_index_padded``, ``chapter_count``. +new variables -If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is -set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" -when_no_chapters -~~~~~~~~~~~~~~~~ -Behavior to perform when no chapters are present. Supports "pass" (continue processing), -"drop" (exclude it from output), and "error" (stop processing for everything). +``when_no_chapters`` + +:expected type: String +:description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + + +---------------------------------------------------------------------------------------------------- subtitles --------- @@ -631,43 +852,60 @@ Defines how to download and store subtitles. Using this plugin creates two new v ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False -allow_auto_generated_subtitles -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Whether to allow auto generated subtitles. Defaults to False. +``allow_auto_generated_subtitles`` -embed_subtitles -~~~~~~~~~~~~~~~ -Optional. Whether to embed the subtitles into the video file. Defaults to False. -NOTE: webm files can only embed "vtt" subtitle types. +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to allow auto generated subtitles. -languages -~~~~~~~~~ -Optional. Language code(s) to download for subtitles. Supports a single or list of multiple -language codes. Defaults to "en". -subtitles_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's subtitles if they are present. This can include -directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and -will be placed in the output directory. ``lang`` is dynamic since you can download multiple -subtitles. It will set the respective language to the correct subtitle file. +``embed_subtitles`` -subtitles_type -~~~~~~~~~~~~~~ -Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. + + +``languages`` + +:expected type: Optional[List[String]] +:description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". + + +``subtitles_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. + + +``subtitles_type`` + +:expected type: Optional[String] +:description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". + + +---------------------------------------------------------------------------------------------------- throttle_protection ------------------- @@ -675,7 +913,7 @@ Provides options to make ytdl-sub look more 'human-like' to protect from throttl range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. -Usage: +:Usage: .. code-block:: yaml @@ -693,52 +931,62 @@ Usage: max: 36 subscription_download_probability: 1.0 -max_downloads_per_subscription -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Number of downloads to perform per subscription. +``max_downloads_per_subscription`` -sleep_per_download_s -~~~~~~~~~~~~~~~~~~~~ -Number in seconds to sleep between each download. Does not include time it takes for -ytdl-sub to perform post-processing. +:expected type: Optional[Range] +:description: + Number of downloads to perform per subscription. -sleep_per_subscription_s -~~~~~~~~~~~~~~~~~~~~~~~~ -Number in seconds to sleep between each subscription. -subscription_download_probability -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Probability to perform any downloads, recomputed for each subscription. This is only -recommended to set if you run ytdl-sub in a cron-job, that way you are statistically -guaranteed over time to eventually download the subscription. +``sleep_per_download_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. + + +``sleep_per_subscription_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each subscription. + + +``subscription_download_probability`` + +:expected type: Optional[Float] +:description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. + + +---------------------------------------------------------------------------------------------------- video_tags ---------- Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" -tags -~~~~ -Key/values of tag names/values. Supports source and override variables. +---------------------------------------------------------------------------------------------------- ytdl_options ------------ -Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. +Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. -ytdl_options should be formatted like: +:Usage: .. code-block:: yaml diff --git a/docs/source/config_reference/scripting/config_types.rst b/docs/source/config_reference/scripting/config_types.rst deleted file mode 100644 index 31273cde..00000000 --- a/docs/source/config_reference/scripting/config_types.rst +++ /dev/null @@ -1,17 +0,0 @@ -================== -Config Field Types -================== - -The ``config.yaml`` uses various types for its configurable fields. Below is a definition for each type. - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator() - -.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator() diff --git a/docs/source/config_reference/scripting/entry_variables.rst b/docs/source/config_reference/scripting/entry_variables.rst index cbd3c0cf..8aa85ca6 100644 --- a/docs/source/config_reference/scripting/entry_variables.rst +++ b/docs/source/config_reference/scripting/entry_variables.rst @@ -1,3 +1,4 @@ + Entry Variables =============== @@ -6,434 +7,643 @@ Entry Variables channel ~~~~~~~ -The channel name if it exists, otherwise returns the uploader. +:type: ``String`` +:description: + The channel name if it exists, otherwise returns the uploader. channel_id ~~~~~~~~~~ -The channel id if it exists, otherwise returns the entry uploader ID. +:type: ``String`` +:description: + The channel id if it exists, otherwise returns the entry uploader ID. chapters ~~~~~~~~ -Chapters if they exist +:type: ``Array`` +:description: + Chapters if they exist comments ~~~~~~~~ -Comments if they are requested +:type: ``Array`` +:description: + Comments if they are requested creator ~~~~~~~ -The creator name if it exists, otherwise returns the channel. +:type: ``String`` +:description: + The creator name if it exists, otherwise returns the channel. description ~~~~~~~~~~~ -The description if it exists. Otherwise, returns an emtpy string. +:type: ``String`` +:description: + The description if it exists. Otherwise, returns an emtpy string. duration ~~~~~~~~ -The duration of the entry in seconds +:type: ``Integer`` +:description: + The duration of the entry in seconds if it exists. Defaults to zero otherwise. epoch ~~~~~ -The unix epoch of when the metadata was scraped by yt-dlp. +:type: ``Integer`` +:description: + The unix epoch of when the metadata was scraped by yt-dlp. epoch_date ~~~~~~~~~~ -The epoch's date, in YYYYMMDD format. +:type: ``String`` +:description: + The epoch's date, in YYYYMMDD format. epoch_hour ~~~~~~~~~~ -The epoch's hour +:type: ``String`` +:description: + The epoch's hour ext ~~~ -The downloaded entry's file extension +:type: ``String`` +:description: + The downloaded entry's file extension extractor ~~~~~~~~~ -The yt-dlp extractor name +:type: ``String`` +:description: + The yt-dlp extractor name extractor_key ~~~~~~~~~~~~~ -The yt-dlp extractor key +:type: ``String`` +:description: + The yt-dlp extractor key ie_key ~~~~~~ -The ie_key, used in legacy yt-dlp things as the 'info-extractor key' +:type: ``String`` +:description: + The ie_key, used in legacy yt-dlp things as the 'info-extractor key'. + If it does not exist, return ``extractor_key`` info_json_ext ~~~~~~~~~~~~~ -The "info.json" extension +:type: ``String`` +:description: + The "info.json" extension requested_subtitles ~~~~~~~~~~~~~~~~~~~ -Subtitles if they are requested and exist +:type: ``Map`` +:description: + Subtitles if they are requested and exist sponsorblock_chapters ~~~~~~~~~~~~~~~~~~~~~ -Sponsorblock Chapters if they are requested and exist +:type: ``Array`` +:description: + Sponsorblock Chapters if they are requested and exist thumbnail_ext ~~~~~~~~~~~~~ -The download entry's thumbnail extension. Will always return 'jpg'. Until there is a -need to support other image types, we always convert to jpg. +:type: ``String`` +:description: + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. title ~~~~~ -The title of the entry. If a title does not exist, returns its unique ID. +:type: ``String`` +:description: + The title of the entry. If a title does not exist, returns its unique ID. title_sanitized_plex ~~~~~~~~~~~~~~~~~~~~ -The sanitized title with additional sanitizing for Plex. It replaces numbers with -fixed-width numbers so Plex does not recognize them as season or episode numbers. +:type: ``String`` +:description: + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. uid ~~~ -The entry's unique ID +:type: ``String`` +:description: + The entry's unique ID uid_sanitized_plex ~~~~~~~~~~~~~~~~~~ -The sanitized uid with additional sanitizing for Plex. Replaces numbers with -fixed-width numbers so Plex does not recognize them as season or episode numbers. +:type: ``String`` +:description: + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. uploader ~~~~~~~~ -The uploader if it exists, otherwise return the uploader ID. +:type: ``String`` +:description: + The uploader if it exists, otherwise return the uploader ID. uploader_id ~~~~~~~~~~~ -The uploader id if it exists, otherwise return the unique ID. +:type: ``String`` +:description: + The uploader id if it exists, otherwise return the unique ID. uploader_url ~~~~~~~~~~~~ -The uploader url if it exists, otherwise returns the webpage_url. +:type: ``String`` +:description: + The uploader url if it exists, otherwise returns the webpage_url. webpage_url ~~~~~~~~~~~ -The url to the webpage. +:type: ``String`` +:description: + The url to the webpage. + +---------------------------------------------------------------------------------------------------- Metadata Variables ------------------ entry_metadata ~~~~~~~~~~~~~~ -The entry's info.json +:type: ``Map`` +:description: + The entry's info.json playlist_metadata ~~~~~~~~~~~~~~~~~ -Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) +:type: ``Map`` +:description: + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) sibling_metadata ~~~~~~~~~~~~~~~~ -Metadata from any sibling entries that reside in the same playlist as this entry. +:type: ``Array`` +:description: + Metadata from any sibling entries that reside in the same playlist as this entry. source_metadata ~~~~~~~~~~~~~~~ -Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) +:type: ``Map`` +:description: + Metadata from the source + (i.e. the grandparent metadata, like channel -> playlist -> entry) + +---------------------------------------------------------------------------------------------------- Playlist Variables ------------------ playlist_count ~~~~~~~~~~~~~~ -Playlist count if it exists, otherwise returns ``1``. +:type: ``Integer`` +:description: + Playlist count if it exists, otherwise returns ``1``. -Note that for channels/playlists, any change (i.e. adding or removing a video) will make -this value change. Use with caution. + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. playlist_description ~~~~~~~~~~~~~~~~~~~~ -The playlist description if it exists, otherwise returns the entry's description. +:type: ``String`` +:description: + The playlist description if it exists, otherwise returns the entry's description. playlist_index ~~~~~~~~~~~~~~ -Playlist index if it exists, otherwise returns ``1``. +:type: ``Integer`` +:description: + Playlist index if it exists, otherwise returns ``1``. -Note that for channels/playlists, any change (i.e. adding or removing a video) will make -this value change. Use with caution. + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. playlist_index_padded ~~~~~~~~~~~~~~~~~~~~~ -playlist_index padded two digits +:type: ``String`` +:description: + playlist_index padded two digits playlist_index_padded6 ~~~~~~~~~~~~~~~~~~~~~~ -playlist_index padded six digits. +:type: ``String`` +:description: + playlist_index padded six digits. playlist_index_reversed ~~~~~~~~~~~~~~~~~~~~~~~ -Playlist index reversed via ``playlist_count - playlist_index + 1`` +:type: ``Integer`` +:description: + Playlist index reversed via ``playlist_count - playlist_index + 1`` playlist_index_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -playlist_index_reversed padded two digits +:type: ``String`` +:description: + playlist_index_reversed padded two digits playlist_index_reversed_padded6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -playlist_index_reversed padded six digits. +:type: ``String`` +:description: + playlist_index_reversed padded six digits. playlist_max_upload_date ~~~~~~~~~~~~~~~~~~~~~~~~ -Max upload_date for all entries in this entry's playlist if it exists, otherwise returns -``upload_date`` +:type: ``String`` +:description: + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` playlist_max_upload_year ~~~~~~~~~~~~~~~~~~~~~~~~ -Max upload_year for all entries in this entry's playlist if it exists, otherwise returns -``upload_year`` +:type: ``Integer`` +:description: + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` playlist_max_upload_year_truncated ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The max playlist truncated upload year for all entries in this entry's playlist if it -exists, otherwise returns ``upload_year_truncated``. +:type: ``Integer`` +:description: + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. playlist_title ~~~~~~~~~~~~~~ -Name of its parent playlist/channel if it exists, otherwise returns its title. +:type: ``String`` +:description: + Name of its parent playlist/channel if it exists, otherwise returns its title. playlist_uid ~~~~~~~~~~~~ -The playlist unique ID if it exists, otherwise return the entry unique ID. +:type: ``String`` +:description: + The playlist unique ID if it exists, otherwise return the entry unique ID. playlist_uploader ~~~~~~~~~~~~~~~~~ -The playlist uploader if it exists, otherwise return the entry uploader. +:type: ``String`` +:description: + The playlist uploader if it exists, otherwise return the entry uploader. playlist_uploader_id ~~~~~~~~~~~~~~~~~~~~ -The playlist uploader id if it exists, otherwise returns the entry uploader ID. +:type: ``String`` +:description: + The playlist uploader id if it exists, otherwise returns the entry uploader ID. playlist_uploader_url ~~~~~~~~~~~~~~~~~~~~~ -The playlist uploader url if it exists, otherwise returns the playlist webpage_url. +:type: ``String`` +:description: + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. playlist_webpage_url ~~~~~~~~~~~~~~~~~~~~ -The playlist webpage url if it exists. Otherwise, returns the entry webpage url. +:type: ``String`` +:description: + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + +---------------------------------------------------------------------------------------------------- Release Date Variables ---------------------- release_date ~~~~~~~~~~~~ -The entry’s release date, in YYYYMMDD format. If not present, return the upload date. +:type: ``String`` +:description: + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. release_date_standardized ~~~~~~~~~~~~~~~~~~~~~~~~~ -The release date formatted as YYYY-MM-DD +:type: ``String`` +:description: + The uploaded date formatted as YYYY-MM-DD release_day ~~~~~~~~~~~ -The release day as an integer (no padding). +:type: ``Integer`` +:description: + The upload day as an integer (no padding). release_day_of_year ~~~~~~~~~~~~~~~~~~~ -The day of the year, i.e. February 1st returns ``32`` +:type: ``Integer`` +:description: + The day of the year, i.e. February 1st returns ``32`` release_day_of_year_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The release day of year, but padded i.e. February 1st returns "032" +:type: ``String`` +:description: + The upload day of year, but padded i.e. February 1st returns "032" release_day_of_year_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, -i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` release_day_of_year_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed release day of year, but padded i.e. December 31st returns "001" +:type: ``String`` +:description: + The reversed upload day of year, but padded i.e. December 31st returns "001" release_day_padded ~~~~~~~~~~~~~~~~~~ -The entry's release day padded to two digits, i.e. the fifth returns "05" +:type: ``String`` +:description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" release_day_reversed ~~~~~~~~~~~~~~~~~~~~ -The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, -i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` release_day_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed release day, but padded. i.e. August 30th returns "02". +:type: ``String`` +:description: + The reversed upload day, but padded. i.e. August 30th returns "02". release_month ~~~~~~~~~~~~~ -The release month as an integer (no padding). +:type: ``Integer`` +:description: + The upload month as an integer (no padding). release_month_padded ~~~~~~~~~~~~~~~~~~~~ -The entry's release month padded to two digits, i.e. March returns "03" +:type: ``String`` +:description: + The entry's upload month padded to two digits, i.e. March returns "03" release_month_reversed ~~~~~~~~~~~~~~~~~~~~~~ -The release month, but reversed -using ``13 - {release_month}``, i.e. March returns ``10`` +:type: ``Integer`` +:description: + The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10`` release_month_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed release month, but padded. i.e. November returns "02" +:type: ``String`` +:description: + The reversed upload month, but padded. i.e. November returns "02" release_year ~~~~~~~~~~~~ -The entry's release year +:type: ``Integer`` +:description: + The entry's upload year release_year_truncated ~~~~~~~~~~~~~~~~~~~~~~ -The last two digits of the release year, i.e. 22 in 2022 +:type: ``Integer`` +:description: + The last two digits of the upload year, i.e. 22 in 2022 release_year_truncated_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. -2022 returns ``100 - 22`` = ``78`` +:type: ``Integer`` +:description: + The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + +---------------------------------------------------------------------------------------------------- Source Variables ---------------- source_count ~~~~~~~~~~~~ -The source count if it exists, otherwise returns the playlist count. +:type: ``Integer`` +:description: + The source count if it exists, otherwise returns ``1``. source_description ~~~~~~~~~~~~~~~~~~ -The source description if it exists, otherwise returns the playlist description. +:type: ``String`` +:description: + The source description if it exists, otherwise returns the playlist description. source_index ~~~~~~~~~~~~ -Source index if it exists, otherwise returns ``1``. +:type: ``Integer`` +:description: + Source index if it exists, otherwise returns ``1``. -It is recommended to not use this unless you know the source will never add new content -(it is easy for this value to change). + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). source_index_padded ~~~~~~~~~~~~~~~~~~~ -The source index, padded. +:type: ``String`` +:description: + The source index, padded two digits. source_title ~~~~~~~~~~~~ -Name of the source (i.e. channel with multiple playlists) if it exists, otherwise -returns its playlist_title. +:type: ``String`` +:description: + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. source_uid ~~~~~~~~~~ -The source unique id if it exists, otherwise returns the playlist unique ID. +:type: ``String`` +:description: + The source unique id if it exists, otherwise returns the playlist unique ID. source_uploader ~~~~~~~~~~~~~~~ -The source uploader if it exists, otherwise return the playlist_uploader +:type: ``String`` +:description: + The source uploader if it exists, otherwise return the playlist_uploader source_uploader_id ~~~~~~~~~~~~~~~~~~ -The source uploader id if it exists, otherwise returns the playlist_uploader_id +:type: ``String`` +:description: + The source uploader id if it exists, otherwise returns the playlist_uploader_id source_uploader_url ~~~~~~~~~~~~~~~~~~~ -The source uploader url if it exists, otherwise returns the source webpage_url. +:type: ``String`` +:description: + The source uploader url if it exists, otherwise returns the source webpage_url. source_webpage_url ~~~~~~~~~~~~~~~~~~ -The source webpage url if it exists, otherwise returns the playlist webpage url. +:type: ``String`` +:description: + The source webpage url if it exists, otherwise returns the playlist webpage url. + +---------------------------------------------------------------------------------------------------- Upload Date Variables --------------------- upload_date ~~~~~~~~~~~ -The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. +:type: ``String`` +:description: + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. upload_date_standardized ~~~~~~~~~~~~~~~~~~~~~~~~ -The uploaded date formatted as YYYY-MM-DD +:type: ``String`` +:description: + The uploaded date formatted as YYYY-MM-DD upload_day ~~~~~~~~~~ -The upload day as an integer (no padding). +:type: ``Integer`` +:description: + The upload day as an integer (no padding). upload_day_of_year ~~~~~~~~~~~~~~~~~~ -The day of the year, i.e. February 1st returns ``32`` +:type: ``Integer`` +:description: + The day of the year, i.e. February 1st returns ``32`` upload_day_of_year_padded ~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload day of year, but padded i.e. February 1st returns "032" +:type: ``String`` +:description: + The upload day of year, but padded i.e. February 1st returns "032" upload_day_of_year_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, -i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` upload_day_of_year_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed upload day of year, but padded i.e. December 31st returns "001" +:type: ``String`` +:description: + The reversed upload day of year, but padded i.e. December 31st returns "001" upload_day_padded ~~~~~~~~~~~~~~~~~ -The entry's upload day padded to two digits, i.e. the fifth returns "05" +:type: ``String`` +:description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" upload_day_reversed ~~~~~~~~~~~~~~~~~~~ -The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, -i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` +:type: ``Integer`` +:description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` upload_day_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed upload day, but padded. i.e. August 30th returns "02". +:type: ``String`` +:description: + The reversed upload day, but padded. i.e. August 30th returns "02". upload_month ~~~~~~~~~~~~ -The upload month as an integer (no padding). +:type: ``Integer`` +:description: + The upload month as an integer (no padding). upload_month_padded ~~~~~~~~~~~~~~~~~~~ -The entry's upload month padded to two digits, i.e. March returns "03" +:type: ``String`` +:description: + The entry's upload month padded to two digits, i.e. March returns "03" upload_month_reversed ~~~~~~~~~~~~~~~~~~~~~ -The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` +:type: ``Integer`` +:description: + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` upload_month_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reversed upload month, but padded. i.e. November returns "02" +:type: ``String`` +:description: + The reversed upload month, but padded. i.e. November returns "02" upload_year ~~~~~~~~~~~ -The entry's upload year +:type: ``Integer`` +:description: + The entry's upload year upload_year_truncated ~~~~~~~~~~~~~~~~~~~~~ -The last two digits of the upload year, i.e. 22 in 2022 +:type: ``Integer`` +:description: + The last two digits of the upload year, i.e. 22 in 2022 upload_year_truncated_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. -2022 returns ``100 - 22`` = ``78`` +:type: ``Integer`` +:description: + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + +---------------------------------------------------------------------------------------------------- Ytdl-Sub Variables ------------------ download_index ~~~~~~~~~~~~~~ -The i'th entry downloaded. NOTE that this is fetched dynamically from the download -archive. +:type: ``Integer`` +:description: + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. download_index_padded6 ~~~~~~~~~~~~~~~~~~~~~~ -The download_index padded six digits +:type: ``String`` +:description: + The download_index padded six digits upload_date_index ~~~~~~~~~~~~~~~~~ -The i'th entry downloaded with this upload date. +:type: ``Integer`` +:description: + The i'th entry downloaded with this upload date. upload_date_index_padded ~~~~~~~~~~~~~~~~~~~~~~~~ -The upload_date_index padded two digits +:type: ``String`` +:description: + The upload_date_index padded two digits upload_date_index_reversed ~~~~~~~~~~~~~~~~~~~~~~~~~~ -100 - upload_date_index +:type: ``Integer`` +:description: + 100 - upload_date_index upload_date_index_reversed_padded ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The upload_date_index padded two digits +:type: ``String`` +:description: + The upload_date_index padded two digits ytdl_sub_input_url ~~~~~~~~~~~~~~~~~~ -The input URL used in ytdl-sub to create this entry. \ No newline at end of file +:type: ``String`` +:description: + The input URL used in ytdl-sub to create this entry. diff --git a/docs/source/config_reference/scripting/index.rst b/docs/source/config_reference/scripting/index.rst index c6ae4532..bee9a4e8 100644 --- a/docs/source/config_reference/scripting/index.rst +++ b/docs/source/config_reference/scripting/index.rst @@ -2,8 +2,171 @@ Scripting ========= -.. toctree:: +``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The links below +contain reference documentation for each built-in variable and scripting function. + +.. toctree:: + :maxdepth: 1 + entry_variables override_variables scripting_functions - config_types \ No newline at end of file + scripting_types + +How it Works +------------ + +Fields in the config that support ``formatters`` mean they support scripting, and will +*format* the field using its defined script. + +In its most basic form, a script is a string comprised of variables and/or functions. + +Static String +~~~~~~~~~~~~~ + +The following example sets ``ytdl-sub``'s output directory. It is +considered *static* because it does not depend on anything from an entry. + +.. code-block:: yaml + + output_options: + output_directory: "Custom YTDL-SUB TV Show" + +Static Variables +~~~~~~~~~~~~~~~~ + +``ytdl-sub`` offers a few built-in static variables, including ``subscription_name``. +We can use this instead of hard-coding it above: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + +The syntax for variable usage is curly-braces with the variable name within it. Assuming +our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub`` +will actually write to that directory. + +Entry Variables +~~~~~~~~~~~~~~~ + +For context, an *entry* is a video or audio file downloaded from ``yt-dlp``. +*Entry variables* are variables that are derived from an entry's ``info.json`` file. This file +comes from ``yt-dlp`` and contains every piece of metadata that it scraped. + +These variables are not considered static since they change per entry download. There are a +few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For others, +we are free to use values that derive from an entry. + +Suppose we want to customize the name of an entry's output file and thumbnail to include its +title in its name. We can do that using entry variables: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + file_name: "{title}.{ext}" + thumbnail_name: "{title}.{thumbnail_ext}" + +Creating Custom Variables +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Suppose we want to include the date in our file names. This means we'd need to update +both the ``file_name`` and ``thumbnail_name`` fields to include it. + +Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method +for creating and overriding custom variables. + +These are created in the ``overrides`` section. Let's take our above example and create +a ``custom_file_name`` variable to use for the entry file and thumbnail fields: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + custom_file_name: "{upload_date_standardized} {title}" + +Sanitizing Variables +~~~~~~~~~~~~~~~~~~~~ + +For experienced ``yt-dlp`` scrapers, you may be thinking: + +- What if the title has characters that do not play nice with my operating system? + +``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic characters +with safe alternatives that can be used in file names. We can ensure our file names and directories +are safe by using: + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name_sanitized}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + custom_file_name: "{upload_date_standardized} {title_sanitized}" + +Simply add a ``_sanitized`` suffix to any variable name to make it sanitized. + +.. note:: + + Make sure you do not sanitize custom variables that intentionally create directories, otherwise + they will... be sanitized and not resolve to directories! + + +Using Scripting Functions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's suppose you are an avid command-line user, and like all of your file names to be +``snake_cased_with_no_spaces``. We can use the +`replace <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_ +*scripting function* to create and use a snake-cased title. + +.. code-block:: yaml + + output_options: + output_directory: "{subscription_name_sanitized}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + overrides: + snake_cased_title: >- + { + %replace( title, ' ', '_' ) + } + custom_file_name: "{upload_date_standardized}_{snake_cased_title_sanitized}" + +Scripting functions are similar to variables - they must be used within curly-braces. +It is good practice to use ``>-`` when defining variables that use functions. It is YAML's way of +saying: + +- Allow a string to be multi-lined, and do not include newlines before or after it. + +See for yourself `here <https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=canonical_yaml>`_. +Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make +scripting function usage readable. + +.. important:: + + It is important to use ``>-`` over other YAML new-line directives like ``>`` because they + add newlines before or after curly-braces, and will be included in your variable's output string. + +Advanced Scripting +------------------ + +Accessing ``info.json`` Fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +WIP + +Creating Custom Functions +~~~~~~~~~~~~~~~~~~~~~~~~~ +WIP + +Parsing Maps and Arrays +~~~~~~~~~~~~~~~~~~~~~~~ +WIP diff --git a/docs/source/config_reference/scripting/override_variables.rst b/docs/source/config_reference/scripting/override_variables.rst index 7cb23e4a..c7a9991b 100644 --- a/docs/source/config_reference/scripting/override_variables.rst +++ b/docs/source/config_reference/scripting/override_variables.rst @@ -1,3 +1,4 @@ + Override Variables ================== @@ -14,6 +15,32 @@ For subscriptions in the form of ``subscription_indent_1`` and ``subscription_indent_2`` get set to ``Indent Value 1`` and ``Indent Value 2``. +subscription_map +---------------- +For subscriptions in the form of + +.. code-block:: yaml + + + Subscription Name: + Music Videos: + - "https://url1.com/..." + Concerts: + - "https://url2.com/..." + +Stores all the contents under the subscription name into the override variable +``subscription_map`` as a Map value. The above example is stored as: + +.. code-block:: python + + { + "Music Videos": [ + "https://url1.com/..." + ], + "Concerts: [ + "https://url2.com/..." + ] + } + subscription_name ----------------- Name of the subscription @@ -41,4 +68,3 @@ For subscriptions in the form of ``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...`` and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to ``subscription_value``. - diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index b462acf0..c271669e 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -7,414 +7,571 @@ Array Functions array ~~~~~ -``array(maybe_array: AnyArgument) -> Array`` +:spec: ``array(maybe_array: AnyArgument) -> Array`` -Tries to cast an unknown variable type to an Array. +:description: + Tries to cast an unknown variable type to an Array. array_apply ~~~~~~~~~~~ -``array_apply(array: Array, lambda_function: Lambda) -> Array`` +:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array`` -Apply a lambda function on every element in the Array. +:description: + Apply a lambda function on every element in the Array. +:usage: + +.. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] + +array_apply_fixed +~~~~~~~~~~~~~~~~~ +:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array`` + +:description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. array_at ~~~~~~~~ -``array_at(array: Array, idx: Integer) -> AnyArgument`` +:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` -Return the element in the Array at index ``idx``. +:description: + Return the element in the Array at index ``idx``. array_contains ~~~~~~~~~~~~~~ -``array_contains(array: Array, value: AnyArgument) -> Boolean`` +:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean`` -Return True if the value exists in the Array. False otherwise. +:description: + Return True if the value exists in the Array. False otherwise. array_enumerate ~~~~~~~~~~~~~~~ -``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` +:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` -Apply a lambda function on every element in the Array, where each arg -passed to the lambda function is ``idx, element`` as two separate args. +:description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. array_extend ~~~~~~~~~~~~ -``array_extend(arrays: Array, ...) -> Array`` +:spec: ``array_extend(arrays: Array, ...) -> Array`` -Combine multiple Arrays into a single Array. +:description: + Combine multiple Arrays into a single Array. + +array_first +~~~~~~~~~~~ +:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument`` + +:description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. array_flatten ~~~~~~~~~~~~~ -``array_flatten(array: Array) -> Array`` +:spec: ``array_flatten(array: Array) -> Array`` -Flatten any nested Arrays into a single-dimensional Array. +:description: + Flatten any nested Arrays into a single-dimensional Array. array_index ~~~~~~~~~~~ -``array_index(array: Array, value: AnyArgument) -> Integer`` +:spec: ``array_index(array: Array, value: AnyArgument) -> Integer`` -Return the index of the value within the Array if it exists. If it does not, it will -throw an error. +:description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. + +array_overlay +~~~~~~~~~~~~~ +:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array`` + +:description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. array_product ~~~~~~~~~~~~~ -``array_product(arrays: Array, ...) -> Array`` +:spec: ``array_product(arrays: Array, ...) -> Array`` -Returns the Cartesian product of elements from different arrays +:description: + Returns the Cartesian product of elements from different arrays array_reduce ~~~~~~~~~~~~ -``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` +:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` -Apply a reduce function on pairs of elements in the Array, until one element remains. -Executes using the left-most and reduces in the right direction. +:description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. array_reverse ~~~~~~~~~~~~~ -``array_reverse(array: Array) -> Array`` +:spec: ``array_reverse(array: Array) -> Array`` -Reverse an Array. +:description: + Reverse an Array. array_size ~~~~~~~~~~ -``array_size(array: Array) -> Integer`` +:spec: ``array_size(array: Array) -> Integer`` -Returns the size of an Array. +:description: + Returns the size of an Array. array_slice ~~~~~~~~~~~ -``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` +:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` -Returns the slice of the Array. +:description: + Returns the slice of the Array. + +---------------------------------------------------------------------------------------------------- Boolean Functions ----------------- and ~~~ -``and(values: AnyArgument, ...) -> Boolean`` +:spec: ``and(values: AnyArgument, ...) -> Boolean`` -``and`` operator. Returns True if all values evaluate to True. False otherwise. +:description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. bool ~~~~ -``bool(value: AnyArgument) -> Boolean`` +:spec: ``bool(value: AnyArgument) -> Boolean`` -Cast any type to a Boolean. +:description: + Cast any type to a Boolean. eq ~~ -``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` -``==`` operator. Returns True if left == right. False otherwise. +:description: + ``==`` operator. Returns True if left == right. False otherwise. gt ~~ -``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` -``>`` operator. Returns True if left > right. False otherwise. +:description: + ``>`` operator. Returns True if left > right. False otherwise. gte ~~~ -``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` -``>=`` operator. Returns True if left >= right. False otherwise. +:description: + ``>=`` operator. Returns True if left >= right. False otherwise. + +is_null +~~~~~~~ +:spec: ``is_null(value: AnyArgument) -> Boolean`` + +:description: + Returns True if a value is null (i.e. an empty string). False otherwise. lt ~~ -``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` -``<`` operator. Returns True if left < right. False otherwise. +:description: + ``<`` operator. Returns True if left < right. False otherwise. lte ~~~ -``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` -``<=`` operator. Returns True if left <= right. False otherwise. +:description: + ``<=`` operator. Returns True if left <= right. False otherwise. ne ~~ -``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` -``!=`` operator. Returns True if left != right. False otherwise. +:description: + ``!=`` operator. Returns True if left != right. False otherwise. not ~~~ -``not(value: Boolean) -> Boolean`` +:spec: ``not(value: Boolean) -> Boolean`` -``not`` operator. Returns the opposite of value. +:description: + ``not`` operator. Returns the opposite of value. or ~~ -``or(values: AnyArgument, ...) -> Boolean`` +:spec: ``or(values: AnyArgument, ...) -> Boolean`` -``or`` operator. Returns True if any value evaluates to True. False otherwise. +:description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. xor ~~~ -``xor(values: AnyArgument, ...) -> Boolean`` +:spec: ``xor(values: AnyArgument, ...) -> Boolean`` -``^`` operator. Returns True if exactly one value is set to True. False otherwise. +:description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + +---------------------------------------------------------------------------------------------------- Conditional Functions --------------------- if ~~ -``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` +:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` -Conditional ``if`` statement that returns the ``true`` or ``false`` parameter -depending on the ``condition`` value. +:description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. if_passthrough ~~~~~~~~~~~~~~ -``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` +:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` -Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, -otherwise returns ``else_arg``. +:description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. + +---------------------------------------------------------------------------------------------------- Date Functions -------------- datetime_strftime ~~~~~~~~~~~~~~~~~ -``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` +:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` -Converts a posix timestamp to a date using strftime formatting. +:description: + Converts a posix timestamp to a date using strftime formatting. + +---------------------------------------------------------------------------------------------------- Error Functions --------------- assert ~~~~~~ -``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` +:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` -Explicitly throw an error with the provided assert message if ``value`` evaluates to False. -If it evaluates to True, it will return ``value``. +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. + +assert_eq +~~~~~~~~~ +:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. + +assert_ne +~~~~~~~~~ +:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. + +assert_then +~~~~~~~~~~~ +:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. throw ~~~~~ -``throw(error_message: String) -> AnyArgument`` +:spec: ``throw(error_message: String) -> AnyArgument`` -Explicitly throw an error with the provided error message. +:description: + Explicitly throw an error with the provided error message. + +---------------------------------------------------------------------------------------------------- Json Functions -------------- from_json ~~~~~~~~~ -``from_json(argument: String) -> AnyArgument`` +:spec: ``from_json(argument: String) -> AnyArgument`` -Converts a JSON string into an actual type. +:description: + Converts a JSON string into an actual type. + +---------------------------------------------------------------------------------------------------- Map Functions ------------- map ~~~ -``map(maybe_mapping: AnyArgument) -> Map`` +:spec: ``map(maybe_mapping: AnyArgument) -> Map`` -Tries to cast an unknown variable type to a Map. +:description: + Tries to cast an unknown variable type to a Map. map_apply ~~~~~~~~~ -``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` +:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` -Apply a lambda function on the Map, where each arg -passed to the lambda function is ``key, value`` as two separate args. +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. map_contains ~~~~~~~~~~~~ -``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` +:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` -Returns True if the key is in the Map. False otherwise. +:description: + Returns True if the key is in the Map. False otherwise. map_enumerate ~~~~~~~~~~~~~ -``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` +:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` -Apply a lambda function on the Map, where each arg -passed to the lambda function is ``idx, key, value`` as three separate args. +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. map_get ~~~~~~~ -``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` +:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` -Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is -provided, it will return ``default``. Otherwise, will error. +:description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. map_get_non_empty ~~~~~~~~~~~~~~~~~ -``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` +:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` -Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, -return ``default``. Otherwise, will error. +:description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. map_size ~~~~~~~~ -``map_size(mapping: Map) -> Integer`` +:spec: ``map_size(mapping: Map) -> Integer`` -Returns the size of a Map. +:description: + Returns the size of a Map. + +---------------------------------------------------------------------------------------------------- Numeric Functions ----------------- add ~~~ -``add(values: Numeric, ...) -> Numeric`` +:spec: ``add(values: Numeric, ...) -> Numeric`` -``+`` operator. Returns the sum of all values. +:description: + ``+`` operator. Returns the sum of all values. div ~~~ -``div(left: Numeric, right: Numeric) -> Numeric`` +:spec: ``div(left: Numeric, right: Numeric) -> Numeric`` -``/`` operator. Returns ``left / right``. +:description: + ``/`` operator. Returns ``left / right``. float ~~~~~ -``float(value: AnyArgument) -> Float`` +:spec: ``float(value: AnyArgument) -> Float`` -Cast to Float. +:description: + Cast to Float. int ~~~ -``int(value: AnyArgument) -> Integer`` +:spec: ``int(value: AnyArgument) -> Integer`` -Cast to Integer. +:description: + Cast to Integer. max ~~~ -``max(values: Numeric, ...) -> Numeric`` +:spec: ``max(values: Numeric, ...) -> Numeric`` -Returns max of all values. +:description: + Returns max of all values. min ~~~ -``min(values: Numeric, ...) -> Numeric`` +:spec: ``min(values: Numeric, ...) -> Numeric`` -Returns min of all values. +:description: + Returns min of all values. mod ~~~ -``mod(left: Numeric, right: Numeric) -> Numeric`` +:spec: ``mod(left: Numeric, right: Numeric) -> Numeric`` -``%`` operator. Returns ``left % right``. +:description: + ``%`` operator. Returns ``left % right``. mul ~~~ -``mul(values: Numeric, ...) -> Numeric`` +:spec: ``mul(values: Numeric, ...) -> Numeric`` -``*`` operator. Returns the product of all values. +:description: + ``*`` operator. Returns the product of all values. pow ~~~ -``pow(base: Numeric, exponent: Numeric) -> Numeric`` +:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric`` -``**`` operator. Returns the exponential of the base and exponent value. +:description: + ``**`` operator. Returns the exponential of the base and exponent value. sub ~~~ -``sub(values: Numeric, ...) -> Numeric`` +:spec: ``sub(values: Numeric, ...) -> Numeric`` -``-`` operator. Subtracts all values from left to right. +:description: + ``-`` operator. Subtracts all values from left to right. + +---------------------------------------------------------------------------------------------------- Regex Functions --------------- +regex_capture_groups +~~~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_groups(regex: String) -> Integer`` + +:description: + Returns number of capture groups in regex + regex_fullmatch ~~~~~~~~~~~~~~~ -``regex_fullmatch(regex: String, string: String) -> Array`` +:spec: ``regex_fullmatch(regex: String, string: String) -> Array`` -Checks for entire string to be a match. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. regex_match ~~~~~~~~~~~ -``regex_match(regex: String, string: String) -> Array`` +:spec: ``regex_match(regex: String, string: String) -> Array`` -Checks for a match only at the beginning of the string. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. regex_search ~~~~~~~~~~~~ -``regex_search(regex: String, string: String) -> Array`` +:spec: ``regex_search(regex: String, string: String) -> Array`` -Checks for a match anywhere in the string. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + +---------------------------------------------------------------------------------------------------- String Functions ---------------- capitalize ~~~~~~~~~~ -``capitalize(string: String) -> String`` +:spec: ``capitalize(string: String) -> String`` -Capitalize the first character in the string. +:description: + Capitalize the first character in the string. concat ~~~~~~ -``concat(values: String, ...) -> String`` +:spec: ``concat(values: String, ...) -> String`` -Concatenate multiple Strings into a single String. +:description: + Concatenate multiple Strings into a single String. + +contains +~~~~~~~~ +:spec: ``contains(string: String, contains: String) -> Boolean`` + +:description: + Returns True if ``contains`` is in ``string``. False otherwise. lower ~~~~~ -``lower(string: String) -> String`` +:spec: ``lower(string: String) -> String`` -Lower-case the entire String. +:description: + Lower-case the entire String. pad ~~~ -``pad(string: String, length: Integer, char: String) -> String`` +:spec: ``pad(string: String, length: Integer, char: String) -> String`` -Pads the string to the given length +:description: + Pads the string to the given length pad_zero ~~~~~~~~ -``pad_zero(numeric: Numeric, length: Integer) -> String`` +:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String`` -Pads a numeric with zeros to the given length +:description: + Pads a numeric with zeros to the given length replace ~~~~~~~ -``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` +:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` -Replace the ``old`` part of the String with the ``new``. Optionally only replace it -``count`` number of times. +:description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. slice ~~~~~ -``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` +:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` -Returns the slice of the Array. +:description: + Returns the slice of the Array. string ~~~~~~ -``string(value: AnyArgument) -> String`` +:spec: ``string(value: AnyArgument) -> String`` -Cast to String. +:description: + Cast to String. titlecase ~~~~~~~~~ -``titlecase(string: String) -> String`` +:spec: ``titlecase(string: String) -> String`` -Capitalize each word in the string. +:description: + Capitalize each word in the string. upper ~~~~~ -``upper(string: String) -> String`` +:spec: ``upper(string: String) -> String`` -Upper-case the entire String. +:description: + Upper-case the entire String. + +---------------------------------------------------------------------------------------------------- Ytdl-Sub Functions ------------------ legacy_bracket_safety ~~~~~~~~~~~~~~~~~~~~~ -``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` +:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not interfere with its legacy variable scripting system. This function replicates that @@ -422,14 +579,14 @@ behavior. sanitize ~~~~~~~~ -``sanitize(value: AnyArgument) -> String`` +:spec: ``sanitize(value: AnyArgument) -> String`` Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use for file/directory names on any OS. sanitize_plex_episode ~~~~~~~~~~~~~~~~~~~~~ -``sanitize_plex_episode(string: String) -> String`` +:spec: ``sanitize_plex_episode(string: String) -> String`` Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the @@ -437,7 +594,7 @@ season and/or episode. to_date_metadata ~~~~~~~~~~~~~~~~ -``to_date_metadata(yyyymmdd: String) -> Map`` +:spec: ``to_date_metadata(yyyymmdd: String) -> Map`` Takes a date in the form of YYYYMMDD and returns a Map containing: @@ -446,7 +603,7 @@ Takes a date in the form of YYYYMMDD and returns a Map containing: - year (Integer) - month (Integer) - day (Integer) -- year_truncated (String, YY from YY[YY]) +- year_truncated (Integer, YY from YY[YY]) - month_padded (String) - day_padded (String) - year_truncated_reversed (Integer, 100 - year_truncated) @@ -461,14 +618,14 @@ Takes a date in the form of YYYYMMDD and returns a Map containing: to_native_filepath ~~~~~~~~~~~~~~~~~~ -``to_native_filepath(filepath: String) -> String`` +:spec: ``to_native_filepath(filepath: String) -> String`` Convert any unix-based path separators ('/') with the OS's native separator. truncate_filepath_if_too_long ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``truncate_filepath_if_too_long(filepath: String) -> String`` +:spec: ``truncate_filepath_if_too_long(filepath: String) -> String`` If a file-path is too long for the OS, this function will truncate it while preserving the extension. diff --git a/docs/source/config_reference/scripting/scripting_types.rst b/docs/source/config_reference/scripting/scripting_types.rst new file mode 100644 index 00000000..d389c2a9 --- /dev/null +++ b/docs/source/config_reference/scripting/scripting_types.rst @@ -0,0 +1,319 @@ + +Scripting Types +=============== + +Types +----- + +String +~~~~~~ + +Strings are a series of characters surrounded by quotes and can be defined in a few ways, including: + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + string_variable: "This is a String variable" + + .. tab-item:: In-Line + + .. code-block:: yaml + + string_variable: "{ %string('This is a String variable') }" + + .. tab-item:: Single Quote + + .. code-block:: yaml + + string_variable: >- + { + %string('This is a String variable') + } + + .. tab-item:: Double Quote + + .. code-block:: yaml + + string_variable: >- + { + %string("This is a String variable") + } + + .. tab-item:: Triple Quote + + .. code-block:: yaml + + string_variable: >- + { + %string('''This is a String variable''') + } + + .. tab-item:: Triple-Double Quote + + .. code-block:: yaml + + string_variable: >- + { + %string("""This is a String variable""") + } + +.. note:: + + For non-String types, they must be defined as parameters to scripting functions. This is because + anything in a variable definition that is not within curly-braces gets evaluated as a String. + +Integer +~~~~~~~ + +Integers are whole numbers with no decimal. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + int_variable: >- + { + %int(2022) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + int_variable: "{ %int(2022) }" + +Float +~~~~~ + +Floats are floating-point decimals numbers. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + float_variable: >- + { + %float(3.14) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + float_variable: "{ %float(3.14) }" + +Boolean +~~~~~~~ + +A type is considered boolean if it spells out ``True`` or ``False``, case-insensitive. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + bool_variable: >- + { + %bool(True) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + bool_variable: "{ %bool(FALSE) }" + +Array +~~~~~ + +An Array contains multiple types of any kind, including nested Arrays and Maps. +Arrays are defined using brackets (``[ ]``), and are accessed using zero-based indexing. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + array_variable: >- + { + [ + "element with index 0", + 1, + 2.0, + [ "Nested Array 3" ] + ] + } + element_0: >- + { + %array_at(array_variable, 0) + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + array_variable: "{ ['element with index 0', 1, 2.0, ['Nested Array 3' ]] }" + element_0: "{ %array_at(array_variable, 0) }" + +Map +~~~ + +A Map is a key-value store, containing mappings between keys and values. +Maps are defined using curley-braces (``{ }``), and are accessed using their keys. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + map_variable: >- + { + { + "string_key": "string_value", + 1: "int_key", + "list_value": [ "elem0", 1, 2.0 ] + } + } + string_value: >- + { + %map_get(map_variable, "string_key") + } + + .. tab-item:: In-Line + + .. code-block:: yaml + + map_variable: "{ {'string_key': 'string_value', 1: 'int_key', 'list_value': [ 'elem0', 1, 2.0 ]} }" + string_value: "{ %map_get(map_variable, 'string_key') }" + +Null +~~~~ +Null is represented by an empty String, and can be conveyed by spelling out ``null``, +case-insensitive. + +.. tab-set:: + + .. tab-item:: Literal + + .. code-block:: yaml + + null_variable: "" + + .. tab-item:: In-Line + + .. code-block:: yaml + + null_variable: "{ %string(null) }" + + +Function Type-Hints +------------------- + +AnyArgument +~~~~~~~~~~~ +AnyArgument means any of the above Types are valid as input or output to a scripting function. + +.. note:: + + Strict typing is enforced. For functions that return ``AnyArgument`` need to be casted before + passing into functions that expect a particular type. + +Numeric +~~~~~~~ +Numeric refers to either an Integer or Float. + +Optional +~~~~~~~~ +Optional means a particular scripting function argument can be either provided or not included. +For example, the function +`map_get <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_ +has an optional default value. Both of these usages are valid: + +.. tab-set:: + + .. tab-item:: Map Get + + .. code-block:: yaml + + will_throw_key_does_not_exist_error: "{ %map_get( {}, 'key' ) }" + + .. tab-item:: Map Get with Optional Default Value + + .. code-block:: yaml + + will_return_default: "{ %map_get( {}, 'key', 'default value' ) }" + +Lambda +~~~~~~ +Lambda parameters are a reference to a function, and will call that lambda function +on the input. In this example, + +.. code-block:: yaml + + lambda_array_numeric_to_string: >- + { + %array_apply( [ 1, 2, 3, 4], %string ) + } + +We apply ``%string`` as a lambda function to +`array_apply <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_, +which is called on every element in the input array. The output becomes ``["1", "2", "3", "4"]``. + +This example has one input-argument being passed into the lambda. For other lambda-based functions +like `array_enumerate <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_, +it expects the lambda function to have two input arguments. These are denoted using +``LambdaTwo``, ``LambdaThree``, etc within the function spec. + +LambdaReduce +~~~~~~~~~~~~ +LambdaReduce is special type of lambda that reduces an Array to a single value by calling the +LabmdaReduce function repeatedly on two elements in the Array until it is reduced to a single value. + +In this example, + +.. code-block:: yaml + + lambda_reduce_sum: >- + { + %array_reduce( [ 1, 2, 3, 4], %add ) + } + +We call +`array_reduce <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_ +on the input array, using +`add <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_ +as the LambdaReduce function. This will reduce the Array to a single value by internally calling + +.. code-block:: + + - %add(1, 2) = 3 + - %add(3, 3) = 6 + - %add(6, 4) = 10 + +And evaluate to ``10``. + +ReturnableArguments +~~~~~~~~~~~~~~~~~~~ + +Returnable arguments are used in conditional functions like +`if <https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_, +which implies the argument passed into the function is the function's output. For example, + +.. code-block:: yaml + + conditional_function: >- + { + %if( True, "Return this if True", "Return this if False" ) + } + +is going to return ``"Return this if True"`` since the condition parameter is ``True``. \ No newline at end of file diff --git a/docs/source/prebuilt_presets/tv_shows.rst b/docs/source/prebuilt_presets/tv_shows.rst index cded4940..f2b1da1d 100644 --- a/docs/source/prebuilt_presets/tv_shows.rst +++ b/docs/source/prebuilt_presets/tv_shows.rst @@ -18,7 +18,7 @@ Jellyfin Kodi ~~~~ * Everything that the Jellyfin version does -* Turns on :ref:`config_reference/plugins:kodi_safe`, replacing characters that would break kodi with safer characters +* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□`` Plex ~~~~ diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 8f6903d2..59b787b7 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -7,7 +7,7 @@ import mergedeep from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME +from ytdl_sub.entries.variables.override_variables import OverrideHelpers from ytdl_sub.entries.variables.override_variables import OverrideVariables from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script @@ -23,9 +23,9 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid class Overrides(DictFormatterValidator, Scriptable): """ - Optional. This section allows you to define variables that can be used in any string formatter. - For example, if you want your file and thumbnail files to match without copy-pasting a large - format string, you can define something like: + Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + + :Usage: .. code-block:: yaml @@ -89,7 +89,7 @@ class Overrides(DictFormatterValidator, Scriptable): """ Ensures the variable name does not collide with any entry variables or built-in functions. """ - if not OverrideVariables.is_valid_name(name): + if not OverrideHelpers.is_valid_name(name): override_type = "function" if name.startswith("%") else "variable" raise self._validation_exception( f"Override {override_type} with name {name} is invalid. Names must be" @@ -97,14 +97,14 @@ class Overrides(DictFormatterValidator, Scriptable): exception_class=InvalidVariableNameException, ) - if OverrideVariables.is_entry_variable_name(name): + if OverrideHelpers.is_entry_variable_name(name): raise self._validation_exception( f"Override variable with name {name} cannot be used since it is a" " built-in ytdl-sub entry variable name.", exception_class=InvalidVariableNameException, ) - if OverrideVariables.is_function_name(name): + if OverrideHelpers.is_function_name(name): raise self._validation_exception( f"Override function definition with name {name} cannot be used since it is" " a built-in ytdl-sub function name.", @@ -124,14 +124,20 @@ class Overrides(DictFormatterValidator, Scriptable): initial_variables, self.dict_with_format_strings, unresolved_variables if unresolved_variables else {}, - {SUBSCRIPTION_NAME: self.subscription_name}, ) return ScriptUtils.add_sanitized_variables(initial_variables) - def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": + def initialize_script( + self, subscription_name: str, unresolved_variables: Set[str] + ) -> "Overrides": """ Initialize the override script with override variables + any unresolved variables """ + self.script.add( + ScriptUtils.add_sanitized_variables( + {OverrideVariables.subscription_name(): subscription_name} + ) + ) self.script.add( self.initial_variables( unresolved_variables={ @@ -144,15 +150,6 @@ class Overrides(DictFormatterValidator, Scriptable): self.update_script() return self - @property - def subscription_name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - return self._root_name - def apply_formatter( self, formatter: StringFormatterValidator, diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 2e5d9fb6..4fec587d 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -198,7 +198,9 @@ class Preset(_PresetShell): downloader_options=self.downloader_options, output_options=self.output_options, plugins=self.plugins, - ).initialize_overrides(overrides=self.overrides).ensure_proper_usage() + ).initialize_overrides( + subscription_name=self.name, overrides=self.overrides + ).ensure_proper_usage() @property def name(self) -> str: diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index d6b2bc2e..8cc4697c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -15,12 +15,12 @@ from ytdl_sub.validators.validators import LiteralDictValidator class YTDLOptions(LiteralDictValidator): """ - Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. + Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. - ytdl_options should be formatted like: + :Usage: .. code-block:: yaml @@ -58,7 +58,7 @@ class OutputOptions(StrictDictValidator): """ Defines where to output files and thumbnails after all post-processing has completed. - Usage: + :Usage: .. code-block:: yaml @@ -156,97 +156,119 @@ class OutputOptions(StrictDictValidator): @property def output_directory(self) -> OverridesStringFormatterValidator: """ - Required. The output directory to store all media files downloaded. + :expected type: OverridesFormatter + :description: + The output directory to store all media files downloaded. """ return self._output_directory @property def file_name(self) -> StringFormatterValidator: """ - Required. The file name for the media file. This can include directories such as - ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. + :expected type: EntryFormatter + :description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. """ return self._file_name @property def thumbnail_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's thumbnail image. This can include directories such - as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable thumbnail writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. """ return self._thumbnail_name @property def info_json_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's info json file. This can include directories such - as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable info json writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. """ return self._info_json_name @property def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. The file name to store a subscriptions download archive placed relative to - the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` + :expected type: Optional[OverridesFormatter] + :description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` """ return self._download_archive_name @property def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. Intended to be used if you are migrating a subscription with either a new - subscription name or output directory. It will try to load the archive file using this name - first, and fallback to ``download_archive_name``. It will always save to this file - and remove the original ``download_archive_name``. + :expected type: Optional[OverridesFormatter] + :description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. """ return self._migrated_download_archive_name @property def maintain_download_archive(self) -> bool: """ - Optional. Maintains a download archive file in the output directory for a subscription. - It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the - output directory. + :expected type: Optional[Boolean] + :description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. - The download archive contains a mapping of ytdl IDs to downloaded files. This is used to - create a ytdl download-archive file when invoking a download on a subscription. This will - prevent ytdl from redownloading media already downloaded. + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. - Defaults to False. + Defaults to False. """ return self._maintain_download_archive.value @property def keep_files_before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep - files before ``now``, which implies all files. Can be used in conjunction with - ``keep_max_files``. + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_before @property def keep_files_after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep - files after ``19000101``, which implies all files. Can be used in conjunction with - ``keep_max_files``. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_after @property def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. - Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be - applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. """ return self._keep_max_files diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index d52489a7..1b2ea1e4 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -13,7 +13,8 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.variables.override_variables import OverrideVariables from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import validate_formatters @@ -66,7 +67,7 @@ def _get_added_and_modified_variables( def _override_variables(overrides: Overrides) -> Set[str]: - return set(list(overrides.initial_variables().keys())) + return set(list(overrides.initial_variables().keys())) | {OverrideVariables.subscription_name()} def _entry_variables() -> Set[str]: @@ -88,7 +89,9 @@ class VariableValidation: self.resolved_variables: Set[str] = set() self.unresolved_variables: Set[str] = set() - def initialize_overrides(self, overrides: Overrides) -> "VariableValidation": + def initialize_overrides( + self, subscription_name: str, overrides: Overrides + ) -> "VariableValidation": """ Do some gymnastics to initialize the Overrides script. """ @@ -127,7 +130,9 @@ class VariableValidation: # Initialize overrides with unresolved variables + modified variables to throw an error. # For modified variables, this is to prevent a resolve(update=True) to setting any # dependencies until it has been explicitly added - overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables) + overrides = overrides.initialize_script( + subscription_name=subscription_name, unresolved_variables=self.unresolved_variables + ) # copy the script and mock entry variables self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index f1b29aa4..509a4f8d 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -11,10 +11,9 @@ from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions -from ytdl_sub.entries.script.variable_scripts import DOWNLOADER_INJECTED_VARIABLES -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import get_file_extension @@ -114,7 +113,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): inj.variable_name, VARIABLE_SCRIPTS[inj.variable_name], ) - for inj in DOWNLOADER_INJECTED_VARIABLES + for inj in v.injected_variables() } ) entries.append(entry) diff --git a/src/ytdl_sub/downloaders/url/multi_url.py b/src/ytdl_sub/downloaders/url/multi_url.py deleted file mode 100644 index b55b8eed..00000000 --- a/src/ytdl_sub/downloaders/url/multi_url.py +++ /dev/null @@ -1,37 +0,0 @@ -from ytdl_sub.downloaders.url.validators import MultiUrlValidator - - -# TODO: Remove later - keep for docstring -class MultiUrlDownloadOptions(MultiUrlValidator): - """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - urls: - - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "1" - season_name: "Uploads" - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "2" - season_name: "Playlist as Season" - playlist_thumbnails: - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - """ diff --git a/src/ytdl_sub/downloaders/url/url.py b/src/ytdl_sub/downloaders/url/url.py deleted file mode 100644 index e5589645..00000000 --- a/src/ytdl_sub/downloaders/url/url.py +++ /dev/null @@ -1,25 +0,0 @@ -from ytdl_sub.downloaders.url.validators import UrlValidator - - -# TODO: Remove later - keep for docstring -class UrlDownloadOptions(UrlValidator): - """ - Downloads from a single URL supported by yt-dlp. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - # optional - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - download_reverse: True - """ diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index eb999b15..631ee979 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -215,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]): class MultiUrlValidator(OptionsValidator): """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. + Sets the URL(s) to download from. Can be used in many forms, including + + :Single URL: + + .. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + + :Multi URL: + + .. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + + :Thumbnails + Variables: + + All variables must be defined for the top-most url. All subsequent URL variables can be either + overwritten or default to the top-most value. + + If an entry is returned from more than one URL, it will use the variables in the bottom-most + URL. + + .. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" """ @classmethod diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index b996c20b..c7442992 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -12,8 +12,10 @@ from typing import final from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_definitions import Variable from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.entries.script.variable_types import ArrayVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.scriptable import Scriptable @@ -23,8 +25,12 @@ from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS v: VariableDefinitions = VARIABLES _YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables" -ytdl_sub_chapters_from_comments = Variable("ytdl_sub_chapters_from_comments") -ytdl_sub_split_by_chapters_parent_uid = Variable("ytdl_sub_split_by_chapters_parent_uid") +ytdl_sub_chapters_from_comments = ArrayVariable( + "ytdl_sub_chapters_from_comments", definition="{ [] }" +) +ytdl_sub_split_by_chapters_parent_uid = StringVariable( + "ytdl_sub_split_by_chapters_parent_uid", definition="{ %string('') }" +) TypeT = TypeVar("TypeT") diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 21ff7d18..6cf7dcb1 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -3,15 +3,14 @@ from typing import Any from typing import Dict from typing import List from typing import Optional +from typing import Set from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import TBaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_definitions import MetadataVariable from ytdl_sub.entries.script.variable_definitions import VariableDefinitions -from ytdl_sub.entries.script.variable_scripts import ENTRY_DEFAULT_VARIABLES -from ytdl_sub.entries.script.variable_scripts import ENTRY_REQUIRED_VARIABLES +from ytdl_sub.entries.script.variable_types import MetadataVariable v: VariableDefinitions = VARIABLES @@ -53,8 +52,8 @@ class EntryParent(BaseEntry): def _sibling_entry_metadata(self) -> List[Dict[str, Any]]: sibling_entry_metadata: List[Dict[str, Any]] = [] - variable_filter: List[MetadataVariable] = list(ENTRY_REQUIRED_VARIABLES.keys()) + list( - ENTRY_DEFAULT_VARIABLES.keys() + variable_filter: Set[MetadataVariable] = ( + v.required_entry_variables() | v.default_entry_variables() ) for entry in self.entry_children(): sibling_entry_metadata.append( diff --git a/src/ytdl_sub/entries/script/custom_functions.py b/src/ytdl_sub/entries/script/custom_functions.py index a7440b9d..690095de 100644 --- a/src/ytdl_sub/entries/script/custom_functions.py +++ b/src/ytdl_sub/entries/script/custom_functions.py @@ -101,7 +101,7 @@ class CustomFunctions: - year (Integer) - month (Integer) - day (Integer) - - year_truncated (String, YY from YY[YY]) + - year_truncated (Integer, YY from YY[YY]) - month_padded (String) - day_padded (String) - year_truncated_reversed (Integer, 100 - year_truncated) @@ -149,7 +149,7 @@ class CustomFunctions: String("year"): Integer(year), String("month"): Integer(month), String("day"): Integer(day), - String("year_truncated"): String(year_truncated), + String("year_truncated"): Integer(year_truncated), String("month_padded"): String(month_padded), String("day_padded"): String(day_padded), String("year_truncated_reversed"): Integer(100 - year_truncated), diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 6c576db4..d06b015f 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1,991 +1,1160 @@ -from dataclasses import dataclass +from abc import ABC +from functools import cache +from functools import cached_property +from typing import Dict +from typing import Set + +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.entries.script.variable_types import ArrayMetadataVariable +from ytdl_sub.entries.script.variable_types import IntegerMetadataVariable +from ytdl_sub.entries.script.variable_types import IntegerVariable +from ytdl_sub.entries.script.variable_types import MapMetadataVariable +from ytdl_sub.entries.script.variable_types import MapVariable +from ytdl_sub.entries.script.variable_types import MetadataVariable +from ytdl_sub.entries.script.variable_types import StringDateMetadataVariable +from ytdl_sub.entries.script.variable_types import StringDateVariable +from ytdl_sub.entries.script.variable_types import StringMetadataVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable # This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion # pylint: disable=no-member # pylint: disable=too-many-public-methods +# pylint: disable=too-many-lines -@dataclass(frozen=True) -class Variable: - variable_name: str +class MetadataVariableDefinitions(ABC): + @cached_property + def entry_metadata(self: "VariableDefinitions") -> MapVariable: + """ + :description: + The entry's info.json + """ + return MapVariable(variable_name="entry_metadata", definition="{ {} }") - -@dataclass(frozen=True) -class InternalVariable(Variable): - pass - - -@dataclass(frozen=True) -class Metadata(Variable): - pass - - -@dataclass(frozen=True) -class MetadataVariable(Variable): - metadata_key: str - - -@dataclass(frozen=True) -class RelativeMetadata(MetadataVariable, Metadata): - pass - - -@dataclass(frozen=True) -class SiblingMetadata(MetadataVariable): - pass - - -class VariableDefinitions: - @property - def entry_metadata(self) -> Metadata: + @cached_property + def playlist_metadata(self: "VariableDefinitions") -> MapMetadataVariable: """ - The entry's info.json + :description: + Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) """ - return Metadata("entry_metadata") - - @property - def playlist_metadata(self) -> RelativeMetadata: - """ - Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) - """ - return RelativeMetadata("playlist_metadata", metadata_key="playlist_metadata") - - @property - def source_metadata(self) -> RelativeMetadata: - """ - Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) - """ - return RelativeMetadata("source_metadata", metadata_key="source_metadata") - - @property - def sibling_metadata(self) -> SiblingMetadata: - """ - Metadata from any sibling entries that reside in the same playlist as this entry. - """ - return SiblingMetadata("sibling_metadata", metadata_key="sibling_metadata") - - @property - def uid(self) -> MetadataVariable: - """ - The entry's unique ID - """ - return MetadataVariable(metadata_key="id", variable_name="uid") - - @property - def duration(self) -> MetadataVariable: - """ - The duration of the entry in seconds - """ - return MetadataVariable("duration", metadata_key="duration") - - @property - def uid_sanitized_plex(self) -> Variable: - """ - The sanitized uid with additional sanitizing for Plex. Replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. - """ - return Variable("uid_sanitized_plex") - - @property - def ie_key(self) -> MetadataVariable: - """ - The ie_key, used in legacy yt-dlp things as the 'info-extractor key' - """ - return MetadataVariable(metadata_key="ie_key", variable_name="ie_key") - - @property - def extractor_key(self) -> MetadataVariable: - """ - The yt-dlp extractor key - """ - return MetadataVariable(metadata_key="extractor_key", variable_name="extractor_key") - - @property - def extractor(self) -> MetadataVariable: - """ - The yt-dlp extractor name - """ - return MetadataVariable(variable_name="extractor", metadata_key="extractor") - - @property - def epoch(self) -> MetadataVariable: - """ - The unix epoch of when the metadata was scraped by yt-dlp. - """ - return MetadataVariable(metadata_key="epoch", variable_name="epoch") - - @property - def epoch_date(self) -> Variable: - """ - The epoch's date, in YYYYMMDD format. - """ - return Variable("epoch_date") - - @property - def epoch_hour(self) -> Variable: - """ - The epoch's hour - """ - return Variable("epoch_hour") - - @property - def title(self) -> MetadataVariable: - """ - The title of the entry. If a title does not exist, returns its unique ID. - """ - return MetadataVariable(variable_name="title", metadata_key="title") - - @property - def title_sanitized_plex(self) -> Variable: - """ - The sanitized title with additional sanitizing for Plex. It replaces numbers with - fixed-width numbers so Plex does not recognize them as season or episode numbers. - """ - return Variable("title_sanitized_plex") - - @property - def webpage_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The url to the webpage. - """ - return MetadataVariable(metadata_key="webpage_url", variable_name="webpage_url") - - @property - def info_json_ext(self) -> Variable: - """ - Returns - ------- - str - The "info.json" extension - """ - return Variable("info_json_ext") - - @property - def description(self) -> MetadataVariable: - """ - Returns - ------- - str - The description if it exists. Otherwise, returns an emtpy string. - """ - return MetadataVariable(variable_name="description", metadata_key="description") - - @property - def uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The uploader id if it exists, otherwise return the unique ID. - """ - return MetadataVariable(variable_name="uploader_id", metadata_key="uploader_id") - - @property - def uploader(self) -> MetadataVariable: - """ - Returns - ------- - str - The uploader if it exists, otherwise return the uploader ID. - """ - return MetadataVariable(variable_name="uploader", metadata_key="uploader") - - @property - def uploader_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The uploader url if it exists, otherwise returns the webpage_url. - """ - return MetadataVariable("uploader_url", metadata_key="uploader_url") - - @property - def source_title(self) -> MetadataVariable: - """ - Returns - ------- - str - Name of the source (i.e. channel with multiple playlists) if it exists, otherwise - returns its playlist_title. - """ - return MetadataVariable("source_title", metadata_key=self.title.metadata_key) - - @property - def source_uid(self) -> MetadataVariable: - """ - Returns - ------- - str - The source unique id if it exists, otherwise returns the playlist unique ID. - """ - return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) - - @property - def source_index(self) -> MetadataVariable: - """ - Returns - ------- - int - Source index if it exists, otherwise returns ``1``. - - It is recommended to not use this unless you know the source will never add new content - (it is easy for this value to change). - """ - return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) - - @property - def source_index_padded(self) -> Variable: - """ - Returns - ------- - int - The source index, padded. - """ - return Variable("source_index_padded") - - @property - def source_count(self) -> MetadataVariable: - """ - Returns - ------- - int - The source count if it exists, otherwise returns the playlist count. - """ - return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) - - @property - def source_webpage_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The source webpage url if it exists, otherwise returns the playlist webpage url. - """ - return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) - - @property - def source_description(self) -> MetadataVariable: - """ - Returns - ------- - str - The source description if it exists, otherwise returns the playlist description. - """ - return MetadataVariable("source_description", metadata_key=self.description.metadata_key) - - @property - def playlist_uid(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist unique ID if it exists, otherwise return the entry unique ID. - """ - return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") - - @property - def playlist_title(self) -> MetadataVariable: - """ - Returns - ------- - str - Name of its parent playlist/channel if it exists, otherwise returns its title. - """ - return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") - - @property - def playlist_index(self) -> MetadataVariable: - """ - Returns - ------- - int - Playlist index if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") - - @property - def playlist_index_reversed(self) -> Variable: - """ - Returns - ------- - int - Playlist index reversed via ``playlist_count - playlist_index + 1`` - """ - return Variable("playlist_index_reversed") - - @property - def playlist_index_padded(self) -> Variable: - """ - Returns - ------- - str - playlist_index padded two digits - """ - return Variable("playlist_index_padded") - - @property - def playlist_index_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - playlist_index_reversed padded two digits - """ - return Variable("playlist_index_reversed_padded") - - @property - def playlist_index_padded6(self) -> Variable: - """ - Returns - ------- - str - playlist_index padded six digits. - """ - return Variable("playlist_index_padded6") - - @property - def playlist_index_reversed_padded6(self) -> Variable: - """ - Returns - ------- - str - playlist_index_reversed padded six digits. - """ - return Variable("playlist_index_reversed_padded6") - - @property - def playlist_count(self) -> MetadataVariable: - """ - Returns - ------- - int - Playlist count if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") - - @property - def playlist_description(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist description if it exists, otherwise returns the entry's description. - """ - return MetadataVariable( - variable_name="playlist_description", metadata_key=self.description.metadata_key + return MapMetadataVariable.from_entry( + metadata_key="playlist_metadata", + default={}, ) - @property - def playlist_webpage_url(self) -> MetadataVariable: + @cached_property + def source_metadata(self: "VariableDefinitions") -> MapMetadataVariable: """ - Returns - ------- - str - The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + :description: + Metadata from the source + (i.e. the grandparent metadata, like channel -> playlist -> entry) """ - return MetadataVariable( - variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key + return MapMetadataVariable.from_entry( + metadata_key="source_metadata", + default={}, ) - @property - def playlist_max_upload_date(self) -> Variable: + @cached_property + def sibling_metadata(self: "VariableDefinitions") -> ArrayMetadataVariable: """ - Returns - ------- - Max upload_date for all entries in this entry's playlist if it exists, otherwise returns - ``upload_date`` + :description: + Metadata from any sibling entries that reside in the same playlist as this entry. """ - return Variable("playlist_max_upload_date") - - @property - def playlist_max_upload_year(self) -> Variable: - """ - Returns - ------- - int - Max upload_year for all entries in this entry's playlist if it exists, otherwise returns - ``upload_year`` - """ - # override in EntryParent - return Variable("playlist_max_upload_year") - - @property - def playlist_max_upload_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The max playlist truncated upload year for all entries in this entry's playlist if it - exists, otherwise returns ``upload_year_truncated``. - """ - return Variable("playlist_max_upload_year_truncated") - - @property - def playlist_uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader id if it exists, otherwise returns the entry uploader ID. - """ - return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") - - @property - def playlist_uploader(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader if it exists, otherwise return the entry uploader. - """ - return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) - - @property - def playlist_uploader_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader url if it exists, otherwise returns the playlist webpage_url. - """ - return MetadataVariable( - "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key + return ArrayMetadataVariable.from_entry( + metadata_key="sibling_metadata", + default=[], ) - @property - def source_uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader id if it exists, otherwise returns the playlist_uploader_id - """ - return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) - @property - def source_uploader(self) -> MetadataVariable: +class PlaylistVariableDefinitions(ABC): + @cached_property + def playlist_uid(self: "VariableDefinitions") -> StringMetadataVariable: """ - Returns - ------- - str - The source uploader if it exists, otherwise return the playlist_uploader + :description: + The playlist unique ID if it exists, otherwise return the entry unique ID. """ - return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) + return StringMetadataVariable.from_playlist( + metadata_key="playlist_id", variable_name="playlist_uid", default=self.uid + ) - @property - def source_uploader_url(self) -> MetadataVariable: + @cached_property + def playlist_title(self: "VariableDefinitions") -> StringMetadataVariable: """ - Returns - ------- - str - The source uploader url if it exists, otherwise returns the source webpage_url. + :description: + Name of its parent playlist/channel if it exists, otherwise returns its title. """ - return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) + return StringMetadataVariable.from_entry(metadata_key="playlist_title", default=self.title) - @property - def creator(self) -> MetadataVariable: + @cached_property + def playlist_index(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - Returns - ------- - str - The creator name if it exists, otherwise returns the channel. - """ - return MetadataVariable(variable_name="creator", metadata_key="creator") + :description: + Playlist index if it exists, otherwise returns ``1``. - @property - def channel(self) -> MetadataVariable: + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. """ - Returns - ------- - str - The channel name if it exists, otherwise returns the uploader. - """ - return MetadataVariable(variable_name="channel", metadata_key="channel") + return IntegerMetadataVariable.from_entry(metadata_key="playlist_index", default=1) - @property - def channel_id(self) -> MetadataVariable: + @cached_property + def playlist_index_reversed(self: "VariableDefinitions") -> IntegerVariable: """ - Returns - ------- - str - The channel id if it exists, otherwise returns the entry uploader ID. + :description: + Playlist index reversed via ``playlist_count - playlist_index + 1`` """ - return MetadataVariable(variable_name="channel_id", metadata_key="channel_id") + return IntegerVariable( + variable_name="playlist_index_reversed", + definition=f"""{{ + %sub( + {self.playlist_count.variable_name}, + {self.playlist_index.variable_name}, + -1 + ) + }}""", + ) - @property - def ext(self) -> MetadataVariable: + @cached_property + def playlist_index_padded(self: "VariableDefinitions") -> StringVariable: """ - Returns - ------- - str - The downloaded entry's file extension + :description: + playlist_index padded two digits """ - return MetadataVariable(variable_name="ext", metadata_key="ext") + return self.playlist_index.to_padded_int(variable_name="playlist_index_padded", pad=2) - @property - def thumbnail_ext(self) -> Variable: + @cached_property + def playlist_index_reversed_padded(self: "VariableDefinitions") -> StringVariable: """ - Returns - ------- - str - The download entry's thumbnail extension. Will always return 'jpg'. Until there is a - need to support other image types, we always convert to jpg. + :description: + playlist_index_reversed padded two digits """ - return Variable("thumbnail_ext") + return self.playlist_index_reversed.to_padded_int( + variable_name="playlist_index_reversed_padded", + pad=2, + ) - @property - def comments(self) -> MetadataVariable: + @cached_property + def playlist_index_padded6(self: "VariableDefinitions") -> StringVariable: """ - Comments if they are requested + :description: + playlist_index padded six digits. """ - return MetadataVariable("comments", "comments") + return self.playlist_index.to_padded_int(variable_name="playlist_index_padded6", pad=6) - @property - def chapters(self) -> MetadataVariable: + @cached_property + def playlist_index_reversed_padded6(self: "VariableDefinitions") -> StringVariable: """ - Chapters if they exist + :description: + playlist_index_reversed padded six digits. """ - return MetadataVariable("chapters", "chapters") + return self.playlist_index_reversed.to_padded_int( + variable_name="playlist_index_reversed_padded6", pad=6 + ) - @property - def sponsorblock_chapters(self) -> MetadataVariable: + @cached_property + def playlist_count(self: "VariableDefinitions") -> IntegerMetadataVariable: """ - Sponsorblock Chapters if they are requested and exist - """ - return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") + :description: + Playlist count if it exists, otherwise returns ``1``. - @property - def requested_subtitles(self) -> MetadataVariable: + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. """ - Subtitles if they are requested and exist - """ - return MetadataVariable("requested_subtitles", "requested_subtitles") + return IntegerMetadataVariable.from_entry(metadata_key="playlist_count", default=1) - @property - def ytdl_sub_input_url(self) -> Variable: + @cached_property + def playlist_description(self: "VariableDefinitions") -> StringMetadataVariable: """ - The input URL used in ytdl-sub to create this entry. + :description: + The playlist description if it exists, otherwise returns the entry's description. """ - return Variable("ytdl_sub_input_url") + return StringMetadataVariable.from_playlist( + metadata_key=self.description.metadata_key, + variable_name="playlist_description", + default=self.description, + ) - @property - def download_index(self) -> Variable: + @cached_property + def playlist_webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: """ - Returns - ------- - int - The i'th entry downloaded. NOTE that this is fetched dynamically from the download - archive. + :description: + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. """ - return Variable(variable_name="download_index") + return StringMetadataVariable.from_playlist( + metadata_key=self.webpage_url.metadata_key, + variable_name="playlist_webpage_url", + default=self.webpage_url, + ) - @property - def download_index_padded6(self) -> Variable: + @cached_property + def playlist_max_upload_date(self: "VariableDefinitions") -> StringDateVariable: """ - Returns - ------- - str - The download_index padded six digits + :description: + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` """ - return Variable("download_index_padded6") + return StringVariable( + variable_name="playlist_max_upload_date", + definition=f"""{{ + %array_reduce( + %if_passthrough( + %extract_field_from_siblings('{self.upload_date.variable_name}'), + [{self.upload_date.variable_name}] + ), + %max + ) + }}""", + ).as_date_variable() - @property - def upload_date_index(self) -> Variable: + @cached_property + def playlist_max_upload_year(self: "VariableDefinitions") -> IntegerVariable: """ - Returns - ------- - int - The i'th entry downloaded with this upload date. + :description: + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` """ - return Variable(variable_name="upload_date_index") + return self.playlist_max_upload_date.get_integer_date_metadata( + date_metadata_key="year", + variable_name="playlist_max_upload_year", + ) - @property - def upload_date_index_padded(self) -> Variable: + @cached_property + def playlist_max_upload_year_truncated(self: "VariableDefinitions") -> IntegerVariable: """ - Returns - ------- - int - The upload_date_index padded two digits + :description: + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. """ - return Variable("upload_date_index_padded") + return self.playlist_max_upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="playlist_max_upload_year_truncated" + ) - @property - def upload_date_index_reversed(self) -> Variable: + @cached_property + def playlist_uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: """ - Returns - ------- - int - 100 - upload_date_index + :description: + The playlist uploader id if it exists, otherwise returns the entry uploader ID. """ - return Variable("upload_date_index_reversed") + return StringMetadataVariable.from_entry( + metadata_key="playlist_uploader_id", + default=self.uploader_id, + ) - @property - def upload_date_index_reversed_padded(self) -> Variable: + @cached_property + def playlist_uploader(self: "VariableDefinitions") -> StringMetadataVariable: """ - Returns - ------- - int - The upload_date_index padded two digits + :description: + The playlist uploader if it exists, otherwise return the entry uploader. """ - return Variable("upload_date_index_reversed_padded") + return StringMetadataVariable.from_playlist( + metadata_key=self.uploader.metadata_key, + variable_name="playlist_uploader", + default=self.uploader, + ) - @property - def upload_date(self) -> MetadataVariable: + @cached_property + def playlist_uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: """ - Returns - ------- - str - The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + :description: + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. """ - return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") - - @property - def upload_year(self) -> Variable: - """ - Returns - ------- - int - The entry's upload year - """ - return Variable("upload_year") - - @property - def upload_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The last two digits of the upload year, i.e. 22 in 2022 - """ - return Variable("upload_year_truncated") - - @property - def upload_year_truncated_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return Variable("upload_year_truncated_reversed") - - @property - def upload_month_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` - """ - return Variable("upload_month_reversed") - - @property - def upload_month_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload month, but padded. i.e. November returns "02" - """ - return Variable("upload_month_reversed_padded") - - @property - def upload_month_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's upload month padded to two digits, i.e. March returns "03" - """ - return Variable("upload_month_padded") - - @property - def upload_day_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's upload day padded to two digits, i.e. the fifth returns "05" - """ - return Variable("upload_day_padded") - - @property - def upload_month(self) -> Variable: - """ - Returns - ------- - int - The upload month as an integer (no padding). - """ - return Variable("upload_month") - - @property - def upload_day(self) -> Variable: - """ - Returns - ------- - int - The upload day as an integer (no padding). - """ - return Variable("upload_day") - - @property - def upload_day_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, - i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - return Variable("upload_day_reversed") - - @property - def upload_day_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload day, but padded. i.e. August 30th returns "02". - """ - return Variable("upload_day_reversed_padded") - - @property - def upload_day_of_year(self) -> Variable: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - return Variable("upload_day_of_year") - - @property - def upload_day_of_year_padded(self) -> Variable: - """ - Returns - ------- - str - The upload day of year, but padded i.e. February 1st returns "032" - """ - return Variable("upload_day_of_year_padded") - - @property - def upload_day_of_year_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, - i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - return Variable("upload_day_of_year_reversed") - - @property - def upload_day_of_year_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload day of year, but padded i.e. December 31st returns "001" - """ - return Variable("upload_day_of_year_reversed_padded") - - @property - def upload_date_standardized(self) -> Variable: - """ - Returns - ------- - str - The uploaded date formatted as YYYY-MM-DD - """ - return Variable("upload_date_standardized") - - @property - def release_date(self) -> MetadataVariable: - """ - Returns - ------- - str - The entry’s release date, in YYYYMMDD format. If not present, return the upload date. - """ - return MetadataVariable(variable_name="release_date", metadata_key="release_date") - - @property - def release_year(self) -> Variable: - """ - Returns - ------- - int - The entry's release year - """ - return Variable("release_year") - - @property - def release_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The last two digits of the release year, i.e. 22 in 2022 - """ - return Variable("release_year_truncated") - - @property - def release_year_truncated_reversed(self) -> Variable: - """ - Returns - ------- - int - The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return Variable("release_year_truncated_reversed") - - @property - def release_month_reversed(self) -> Variable: - """ - Returns - ------- - int - The release month, but reversed - using ``13 - {release_month}``, i.e. March returns ``10`` - """ - return Variable("release_month_reversed") - - @property - def release_month_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release month, but padded. i.e. November returns "02" - """ - return Variable("release_month_reversed_padded") - - @property - def release_month_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's release month padded to two digits, i.e. March returns "03" - """ - return Variable("release_month_padded") - - @property - def release_day_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's release day padded to two digits, i.e. the fifth returns "05" - """ - return Variable("release_day_padded") - - @property - def release_month(self) -> Variable: - """ - Returns - ------- - int - The release month as an integer (no padding). - """ - return Variable("release_month") - - @property - def release_day(self) -> Variable: - """ - Returns - ------- - int - The release day as an integer (no padding). - """ - return Variable("release_day") - - @property - def release_day_reversed(self) -> Variable: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, - i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - return Variable("release_day_reversed") - - @property - def release_day_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release day, but padded. i.e. August 30th returns "02". - """ - return Variable("release_day_reversed_padded") - - @property - def release_day_of_year(self) -> Variable: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - return Variable("release_day_of_year") - - @property - def release_day_of_year_padded(self) -> Variable: - """ - Returns - ------- - str - The release day of year, but padded i.e. February 1st returns "032" - """ - return Variable("release_day_of_year_padded") - - @property - def release_day_of_year_reversed(self) -> Variable: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, - i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - return Variable("release_day_of_year_reversed") - - @property - def release_day_of_year_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release day of year, but padded i.e. December 31st returns "001" - """ - return Variable("release_day_of_year_reversed_padded") - - @property - def release_date_standardized(self) -> Variable: - """ - Returns - ------- - str - The release date formatted as YYYY-MM-DD - """ - return Variable("release_date_standardized") + return StringMetadataVariable.from_playlist( + metadata_key=self.uploader_url.metadata_key, + variable_name="playlist_uploader_url", + default=self.webpage_url, + ) -# Singleton to use externally +class SourceVariableDefinitions(ABC): + @cached_property + def source_title(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + """ + return StringMetadataVariable.from_source( + metadata_key=self.title.metadata_key, + variable_name="source_title", + default=self.playlist_title, + ) + + @cached_property + def source_uid(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source unique id if it exists, otherwise returns the playlist unique ID. + """ + return StringMetadataVariable.from_source( + metadata_key=self.uid.metadata_key, + variable_name="source_uid", + default=self.playlist_uid, + ) + + @cached_property + def source_index(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + """ + return IntegerMetadataVariable.from_playlist( + metadata_key=self.playlist_index.metadata_key, + variable_name="source_index", + default=1, + ) + + @cached_property + def source_index_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The source index, padded two digits. + """ + return self.source_index.to_padded_int(variable_name="source_index_padded", pad=2) + + @cached_property + def source_count(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + The source count if it exists, otherwise returns ``1``. + """ + return IntegerMetadataVariable.from_playlist( + metadata_key=self.playlist_count.metadata_key, + variable_name="source_count", + default=1, + ) + + @cached_property + def source_webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source webpage url if it exists, otherwise returns the playlist webpage url. + """ + return StringMetadataVariable.from_source( + metadata_key=self.webpage_url.metadata_key, + variable_name="source_webpage_url", + default=self.playlist_webpage_url, + ) + + @cached_property + def source_description(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source description if it exists, otherwise returns the playlist description. + """ + return StringMetadataVariable.from_source( + metadata_key=self.description.metadata_key, + variable_name="source_description", + default=self.playlist_description, + ) + + @cached_property + def source_uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source uploader id if it exists, otherwise returns the playlist_uploader_id + """ + return StringMetadataVariable.from_source( + metadata_key=self.uploader_id.metadata_key, + variable_name="source_uploader_id", + default=self.playlist_uploader_id, + ) + + @cached_property + def source_uploader(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source uploader if it exists, otherwise return the playlist_uploader + """ + return StringMetadataVariable.from_source( + metadata_key=self.uploader.metadata_key, + variable_name="source_uploader", + default=self.playlist_uploader, + ) + + @cached_property + def source_uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The source uploader url if it exists, otherwise returns the source webpage_url. + """ + return StringMetadataVariable.from_source( + metadata_key=self.uploader_url.metadata_key, + variable_name="source_uploader_url", + default=self.source_webpage_url, + ) + + +class UploadDateVariableDefinitions(ABC): + @cached_property + def upload_date(self: "VariableDefinitions") -> StringDateMetadataVariable: + """ + :description: + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + """ + return StringDateMetadataVariable.from_entry(metadata_key="upload_date").as_date_variable() + + @cached_property + def upload_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The entry's upload year + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year", variable_name="upload_year" + ) + + @cached_property + def upload_year_truncated(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The last two digits of the upload year, i.e. 22 in 2022 + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="upload_year_truncated" + ) + + @cached_property + def upload_year_truncated_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="year_truncated_reversed", + variable_name="upload_year_truncated_reversed", + ) + + @cached_property + def upload_month_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="month_reversed", variable_name="upload_month_reversed" + ) + + @cached_property + def upload_month_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload month, but padded. i.e. November returns "02" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="month_reversed_padded", variable_name="upload_month_reversed_padded" + ) + + @cached_property + def upload_month_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="month_padded", variable_name="upload_month_padded" + ) + + @cached_property + def upload_day_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_padded", variable_name="upload_day_padded" + ) + + @cached_property + def upload_month(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month as an integer (no padding). + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="month", variable_name="upload_month" + ) + + @cached_property + def upload_day(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day as an integer (no padding). + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day", variable_name="upload_day" + ) + + @cached_property + def upload_day_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_reversed", variable_name="upload_day_reversed" + ) + + @cached_property + def upload_day_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_reversed_padded", variable_name="upload_day_reversed_padded" + ) + + @cached_property + def upload_day_of_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The day of the year, i.e. February 1st returns ``32`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_of_year", variable_name="upload_day_of_year" + ) + + @cached_property + def upload_day_of_year_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload day of year, but padded i.e. February 1st returns "032" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_of_year_padded", variable_name="upload_day_of_year_padded" + ) + + @cached_property + def upload_day_of_year_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return self.upload_date.get_integer_date_metadata( + date_metadata_key="day_of_year_reversed", variable_name="upload_day_of_year_reversed" + ) + + @cached_property + def upload_day_of_year_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="day_of_year_reversed_padded", + variable_name="upload_day_of_year_reversed_padded", + ) + + @cached_property + def upload_date_standardized(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The uploaded date formatted as YYYY-MM-DD + """ + return self.upload_date.get_string_date_metadata( + date_metadata_key="date_standardized", variable_name="upload_date_standardized" + ) + + +class ReleaseDateVariableDefinitions(ABC): + @cached_property + def release_date(self: "VariableDefinitions") -> StringDateMetadataVariable: + """ + :description: + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + """ + return StringMetadataVariable.from_entry( + metadata_key="release_date", default=self.upload_date + ).as_date_variable() + + @cached_property + def release_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The entry's upload year + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="year", variable_name="release_year" + ) + + @cached_property + def release_year_truncated(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The last two digits of the upload year, i.e. 22 in 2022 + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="year_truncated", variable_name="release_year_truncated" + ) + + @cached_property + def release_year_truncated_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="year_truncated_reversed", + variable_name="release_year_truncated_reversed", + ) + + @cached_property + def release_month_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="month_reversed", variable_name="release_month_reversed" + ) + + @cached_property + def release_month_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload month, but padded. i.e. November returns "02" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="month_reversed_padded", variable_name="release_month_reversed_padded" + ) + + @cached_property + def release_month_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="month_padded", variable_name="release_month_padded" + ) + + @cached_property + def release_day_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_padded", variable_name="release_day_padded" + ) + + @cached_property + def release_month(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload month as an integer (no padding). + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="month", variable_name="release_month" + ) + + @cached_property + def release_day(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day as an integer (no padding). + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day", variable_name="release_day" + ) + + @cached_property + def release_day_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_reversed", variable_name="release_day_reversed" + ) + + @cached_property + def release_day_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_reversed_padded", variable_name="release_day_reversed_padded" + ) + + @cached_property + def release_day_of_year(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The day of the year, i.e. February 1st returns ``32`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_of_year", variable_name="release_day_of_year" + ) + + @cached_property + def release_day_of_year_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload day of year, but padded i.e. February 1st returns "032" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_of_year_padded", variable_name="release_day_of_year_padded" + ) + + @cached_property + def release_day_of_year_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return self.release_date.get_integer_date_metadata( + date_metadata_key="day_of_year_reversed", variable_name="release_day_of_year_reversed" + ) + + @cached_property + def release_day_of_year_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="day_of_year_reversed_padded", + variable_name="release_day_of_year_reversed_padded", + ) + + @cached_property + def release_date_standardized(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The uploaded date formatted as YYYY-MM-DD + """ + return self.release_date.get_string_date_metadata( + date_metadata_key="date_standardized", variable_name="release_date_standardized" + ) + + +class YtdlSubVariableDefinitions(ABC): + @cached_property + def ytdl_sub_input_url(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The input URL used in ytdl-sub to create this entry. + """ + return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }") + + @cached_property + def download_index(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. + """ + return IntegerVariable(variable_name="download_index", definition="{ %int(1) }") + + @cached_property + def download_index_padded6(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The download_index padded six digits + """ + return self.download_index.to_padded_int(variable_name="download_index_padded6", pad=6) + + @cached_property + def upload_date_index(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + The i'th entry downloaded with this upload date. + """ + return IntegerVariable(variable_name="upload_date_index", definition="{ %int(1) }") + + @cached_property + def upload_date_index_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload_date_index padded two digits + """ + return self.upload_date_index.to_padded_int(variable_name="upload_date_index_padded", pad=2) + + @cached_property + def upload_date_index_reversed(self: "VariableDefinitions") -> IntegerVariable: + """ + :description: + 100 - upload_date_index + """ + return IntegerVariable( + variable_name="upload_date_index_reversed", + definition=f"{{%sub(100, {self.upload_date_index.variable_name})}}", + ) + + @cached_property + def upload_date_index_reversed_padded(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The upload_date_index padded two digits + """ + return self.upload_date_index_reversed.to_padded_int( + variable_name="upload_date_index_reversed_padded", pad=2 + ) + + +class EntryVariableDefinitions(ABC): + @cached_property + def uid(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The entry's unique ID + """ + return StringMetadataVariable.from_entry(metadata_key="id", variable_name="uid") + + @cached_property + def duration(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + The duration of the entry in seconds if it exists. Defaults to zero otherwise. + """ + return IntegerMetadataVariable.from_entry(metadata_key="duration", default=0) + + @cached_property + def uid_sanitized_plex(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The sanitized uid with additional sanitizing for Plex. Replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + """ + return self.uid.to_sanitized_plex(variable_name="uid_sanitized_plex") + + @cached_property + def ie_key(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The ie_key, used in legacy yt-dlp things as the 'info-extractor key'. + If it does not exist, return ``extractor_key`` + """ + return StringMetadataVariable.from_entry(metadata_key="ie_key", default=self.extractor_key) + + @cached_property + def extractor_key(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The yt-dlp extractor key + """ + return StringMetadataVariable.from_entry(metadata_key="extractor_key") + + @cached_property + def extractor(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The yt-dlp extractor name + """ + return StringMetadataVariable.from_entry(metadata_key="extractor") + + @cached_property + def epoch(self: "VariableDefinitions") -> IntegerMetadataVariable: + """ + :description: + The unix epoch of when the metadata was scraped by yt-dlp. + """ + return IntegerMetadataVariable.from_entry(metadata_key="epoch") + + @cached_property + def epoch_date(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The epoch's date, in YYYYMMDD format. + """ + return StringVariable( + variable_name="epoch_date", + definition=f"{{%datetime_strftime({self.epoch.variable_name}, '%Y%m%d')}}", + ) + + @cached_property + def epoch_hour(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The epoch's hour + """ + return StringVariable( + variable_name="epoch_hour", + definition=f"{{%datetime_strftime({self.epoch.variable_name}, '%H')}}", + ) + + @cached_property + def title(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The title of the entry. If a title does not exist, returns its unique ID. + """ + return StringMetadataVariable.from_entry(metadata_key="title", default=self.uid) + + @cached_property + def title_sanitized_plex(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The sanitized title with additional sanitizing for Plex. It replaces numbers with + fixed-width numbers so Plex does not recognize them as season or episode numbers. + """ + return self.title.to_sanitized_plex(variable_name="title_sanitized_plex") + + @cached_property + def webpage_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The url to the webpage. + """ + return StringMetadataVariable.from_entry(metadata_key="webpage_url") + + @cached_property + def info_json_ext(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The "info.json" extension + """ + return StringVariable( + variable_name="info_json_ext", + definition="info.json", + ) + + @cached_property + def description(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The description if it exists. Otherwise, returns an emtpy string. + """ + return StringMetadataVariable.from_entry( + metadata_key="description", + default="", + ) + + @cached_property + def uploader_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The uploader id if it exists, otherwise return the unique ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="uploader_id", + default=self.uid, + ) + + @cached_property + def uploader(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The uploader if it exists, otherwise return the uploader ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="uploader", + default=self.uploader_id, + ) + + @cached_property + def uploader_url(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The uploader url if it exists, otherwise returns the webpage_url. + """ + return StringMetadataVariable.from_entry( + metadata_key="uploader_url", + default=self.webpage_url, + ) + + @cached_property + def creator(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The creator name if it exists, otherwise returns the channel. + """ + return StringMetadataVariable.from_entry( + metadata_key="creator", + default=self.channel, + ) + + @cached_property + def channel(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The channel name if it exists, otherwise returns the uploader. + """ + return StringMetadataVariable.from_entry( + metadata_key="channel", + default=self.uploader, + ) + + @cached_property + def channel_id(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The channel id if it exists, otherwise returns the entry uploader ID. + """ + return StringMetadataVariable.from_entry( + metadata_key="channel_id", + default=self.uploader_id, + ) + + @cached_property + def ext(self: "VariableDefinitions") -> StringMetadataVariable: + """ + :description: + The downloaded entry's file extension + """ + return StringMetadataVariable.from_entry(metadata_key="ext") + + @cached_property + def thumbnail_ext(self: "VariableDefinitions") -> StringVariable: + """ + :description: + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. + """ + return StringVariable( + variable_name="thumbnail_ext", + definition="jpg", + ) + + @cached_property + def comments(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Comments if they are requested + """ + return ArrayMetadataVariable( + metadata_key="comments", variable_name="comments", definition="{ [] }" + ) + + @cached_property + def chapters(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Chapters if they exist + """ + return ArrayMetadataVariable( + metadata_key="chapters", variable_name="chapters", definition="{ [] }" + ) + + @cached_property + def sponsorblock_chapters(self: "VariableDefinitions") -> ArrayMetadataVariable: + """ + :description: + Sponsorblock Chapters if they are requested and exist + """ + return ArrayMetadataVariable( + metadata_key="sponsorblock_chapters", + variable_name="sponsorblock_chapters", + definition="{ [] }", + ) + + @cached_property + def requested_subtitles(self: "VariableDefinitions") -> MapMetadataVariable: + """ + :description: + Subtitles if they are requested and exist + """ + return MapMetadataVariable( + metadata_key="requested_subtitles", + variable_name="requested_subtitles", + definition="{ {} }", + ) + + +class VariableDefinitions( + EntryVariableDefinitions, + MetadataVariableDefinitions, + PlaylistVariableDefinitions, + SourceVariableDefinitions, + UploadDateVariableDefinitions, + ReleaseDateVariableDefinitions, + YtdlSubVariableDefinitions, +): + @cache + def scripts(self) -> Dict[str, str]: + """ + Returns all variables and their scripts in dict form + """ + return { + var.variable_name: var.definition + for var in [ + getattr(self, attr) + for attr in dir(self) + if isinstance(getattr(self, attr), Variable) + ] + } + + @cache + def injected_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that get injected in the download-stage + """ + return { + self.download_index, + self.upload_date_index, + self.comments, + self.requested_subtitles, + self.chapters, + self.sponsorblock_chapters, + self.ytdl_sub_input_url, + } + + @cache + def required_entry_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that the entry requires to exist + """ + return { + self.uid, + self.extractor_key, + self.epoch, + self.webpage_url, + self.ext, + } + + @cache + def default_entry_variables(self) -> Set[MetadataVariable]: + """ + Returns variables that reside on the entry that may or may not exist, + but have defaults + """ + return { + self.title, + self.extractor, + self.description, + self.ie_key, + self.uploader_id, + self.uploader, + self.uploader_url, + self.upload_date, + self.release_date, + self.channel, + self.creator, + self.channel_id, + self.duration, + self.playlist_index, + self.playlist_count, + self.playlist_uid, + self.playlist_title, + self.playlist_uploader_id, + } + + @cache + def unresolvable_static_variables(self) -> Set[Variable]: + """ + Returns variables that are not static (i.e. depend on runtime) + """ + return { + VARIABLES.entry_metadata, + } | self.injected_variables() + + +# Singletons to use externally VARIABLES: VariableDefinitions = VariableDefinitions() +VARIABLE_SCRIPTS: Dict[str, str] = VARIABLES.scripts() +UNRESOLVED_VARIABLES: Set[str] = { + var.variable_name for var in VARIABLES.unresolvable_static_variables() +} + +CustomFunctions.register() diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py deleted file mode 100644 index 90808c37..00000000 --- a/src/ytdl_sub/entries/script/variable_scripts.py +++ /dev/null @@ -1,296 +0,0 @@ -from typing import Dict -from typing import List -from typing import Optional -from typing import Set - -import mergedeep - -from ytdl_sub.entries.script.custom_functions import CustomFunctions -from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_definitions import Metadata -from ytdl_sub.entries.script.variable_definitions import MetadataVariable -from ytdl_sub.entries.script.variable_definitions import Variable -from ytdl_sub.entries.script.variable_definitions import VariableDefinitions - -############################################################################################### -# Helpers - -v: VariableDefinitions = VARIABLES - - -def _pad_int(key: Variable, pad: int) -> str: - return f"{{%pad_zero({key.variable_name}, {pad})}}" - - -def _sanitized_plex(key: Variable) -> str: - return f"{{%sanitize_plex_episode({key.variable_name})}}" - - -def _date_metadata(date_key: Variable, metadata_key: str) -> str: - return f"{{%map_get(%to_date_metadata({date_key.variable_name}), '{metadata_key}')}}" - - -############################################################################################### -# Metadata Getters - - -def _get( - cast: str, - metadata: Metadata, - key: MetadataVariable, - default: Optional[Variable | str | int | Dict | List], -) -> str: - if default is None: - # TODO: assert with good error message if key DNE - out = f"%map_get({metadata.variable_name}, '{key.metadata_key}')" - elif isinstance(default, Variable): - args = f"{metadata.variable_name}, '{key.metadata_key}', {default.variable_name}" - out = f"%map_get_non_empty({args})" - elif isinstance(default, str): - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', '{default}')" - elif isinstance(default, dict): - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {{}})" - elif isinstance(default, list): - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', [])" - else: - out = f"%map_get_non_empty({metadata.variable_name}, '{key.metadata_key}', {default})" - - return f"{{ %legacy_bracket_safety(%{cast}({out})) }}" - - -############################################################################################### -# Entry Getters - - -def _entry_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: - return _get("string", metadata=v.entry_metadata, key=key, default=default) - - -def _entry_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str: - return _get("int", metadata=v.entry_metadata, key=key, default=default) - - -def _entry_get_map(key: MetadataVariable, default: Optional[Variable | Dict] = None): - return _get("map", metadata=v.entry_metadata, key=key, default=default) - - -def _entry_get_array(key: MetadataVariable, default: Optional[Variable | List] = None): - return _get("array", metadata=v.entry_metadata, key=key, default=default) - - -############################################################################################### -# Playlist Getters - - -def _playlist_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: - return _get("string", metadata=v.playlist_metadata, key=key, default=default) - - -def _playlist_get_int(key: MetadataVariable, default: Optional[Variable | int] = None) -> str: - return _get("int", metadata=v.playlist_metadata, key=key, default=default) - - -############################################################################################### -# Source Getters - - -def _source_get_str(key: MetadataVariable, default: Optional[Variable | str] = None) -> str: - return _get("string", metadata=v.source_metadata, key=key, default=default) - - -############################################################################################### -# Scripts - -ENTRY_EMPTY_METADATA: Dict[Variable, str] = {v.entry_metadata: "{ {} }"} - -ENTRY_HARDCODED_VARIABLES: Dict[Variable, str] = { - v.info_json_ext: "info.json", - v.thumbnail_ext: "jpg", -} - -ENTRY_RELATIVE_VARIABLES: Dict[MetadataVariable, str] = { - v.playlist_metadata: _entry_get_map(v.playlist_metadata, {}), - v.source_metadata: _entry_get_map(v.source_metadata, {}), - v.sibling_metadata: _entry_get_array(v.sibling_metadata, []), -} - -ENTRY_REQUIRED_VARIABLES: Dict[MetadataVariable, str] = { - v.uid: _entry_get_str(v.uid), - v.extractor_key: _entry_get_str(v.extractor_key), - v.epoch: _entry_get_int(v.epoch), - v.webpage_url: _entry_get_str(v.webpage_url), - v.ext: _entry_get_str(v.ext), -} - -ENTRY_DEFAULT_VARIABLES: Dict[MetadataVariable, str] = { - v.title: _entry_get_str(v.title, v.uid), - v.extractor: _entry_get_str(v.extractor, v.extractor_key), - v.description: _entry_get_str(v.description, ""), - v.ie_key: _entry_get_str(v.ie_key, v.extractor_key), - v.uploader_id: _entry_get_str(v.uploader_id, v.uid), - v.uploader: _entry_get_str(v.uploader, v.uploader_id), - v.uploader_url: _entry_get_str(v.uploader_url, v.webpage_url), - v.upload_date: _entry_get_str(v.upload_date, v.epoch_date), - v.release_date: _entry_get_str(v.release_date, v.upload_date), - v.channel: _entry_get_str(v.channel, v.uploader), - v.creator: _entry_get_str(v.creator, v.channel), - v.channel_id: _entry_get_str(v.channel_id, v.uploader_id), - v.duration: _entry_get_int(v.duration, 0), - v.playlist_index: _entry_get_int(v.playlist_index, 1), - v.playlist_count: _entry_get_int(v.playlist_count, 1), - v.playlist_uid: _entry_get_str(v.playlist_uid, v.uid), - v.playlist_title: _entry_get_str(v.playlist_title, v.title), - v.playlist_uploader_id: _entry_get_str(v.playlist_uploader_id, v.uploader_id), -} - -# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED IN THE DOWNLOADER -DOWNLOADER_INJECTED_VARIABLES: Dict[Variable, str] = { - v.download_index: "{%int(1)}", - v.upload_date_index: "{%int(1)}", - v.comments: "{ [] }", - v.requested_subtitles: "{ {} }", - v.chapters: "{ [] }", - v.sponsorblock_chapters: "{ [] }", - v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}", -} - -ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = { - v.uid_sanitized_plex: _sanitized_plex(v.uid), - v.title_sanitized_plex: _sanitized_plex(v.title), - v.epoch_date: f"{{%datetime_strftime({v.epoch.variable_name}, '%Y%m%d')}}", - v.epoch_hour: f"{{%datetime_strftime({v.epoch.variable_name}, '%H')}}", - v.download_index_padded6: _pad_int(v.download_index, 6), - v.upload_date_index_padded: _pad_int(v.upload_date_index, 2), - v.upload_date_index_reversed: f"{{%sub(100, {v.upload_date_index.variable_name})}}", - v.upload_date_index_reversed_padded: _pad_int(v.upload_date_index_reversed, 2), - v.playlist_index_reversed: ( - f"{{%sub({v.playlist_count.variable_name}, {v.playlist_index.variable_name}, -1)}}" - ), - v.playlist_index_padded: _pad_int(v.playlist_index, 2), - v.playlist_index_reversed_padded: _pad_int(v.playlist_index_reversed, 2), - v.playlist_index_padded6: _pad_int(v.playlist_index, 6), - v.playlist_index_reversed_padded6: _pad_int(v.playlist_index_reversed, 6), -} - -ENTRY_UPLOAD_DATE_VARIABLES: Dict[Variable, str] = { - v.upload_year: _date_metadata(v.upload_date, "year"), - v.upload_year_truncated: _date_metadata(v.upload_date, "year_truncated"), - v.upload_year_truncated_reversed: _date_metadata(v.upload_date, "year_truncated_reversed"), - v.upload_month_reversed: _date_metadata(v.upload_date, "month_reversed"), - v.upload_month_reversed_padded: _date_metadata(v.upload_date, "month_reversed_padded"), - v.upload_month_padded: _date_metadata(v.upload_date, "month_padded"), - v.upload_day_padded: _date_metadata(v.upload_date, "day_padded"), - v.upload_month: _date_metadata(v.upload_date, "month"), - v.upload_day: _date_metadata(v.upload_date, "day"), - v.upload_day_reversed: _date_metadata(v.upload_date, "day_reversed"), - v.upload_day_reversed_padded: _date_metadata(v.upload_date, "day_reversed_padded"), - v.upload_day_of_year: _date_metadata(v.upload_date, "day_of_year"), - v.upload_day_of_year_padded: _date_metadata(v.upload_date, "day_of_year_padded"), - v.upload_day_of_year_reversed: _date_metadata(v.upload_date, "day_of_year_reversed"), - v.upload_day_of_year_reversed_padded: _date_metadata( - v.upload_date, "day_of_year_reversed_padded" - ), - v.upload_date_standardized: _date_metadata(v.upload_date, "date_standardized"), -} - -ENTRY_RELEASE_DATE_VARIABLES: Dict[Variable, str] = { - v.release_year: _date_metadata(v.release_date, "year"), - v.release_year_truncated: _date_metadata(v.release_date, "year_truncated"), - v.release_year_truncated_reversed: _date_metadata(v.release_date, "year_truncated_reversed"), - v.release_month_reversed: _date_metadata(v.release_date, "month_reversed"), - v.release_month_reversed_padded: _date_metadata(v.release_date, "month_reversed_padded"), - v.release_month_padded: _date_metadata(v.release_date, "month_padded"), - v.release_day_padded: _date_metadata(v.release_date, "day_padded"), - v.release_month: _date_metadata(v.release_date, "month"), - v.release_day: _date_metadata(v.release_date, "day"), - v.release_day_reversed: _date_metadata(v.release_date, "day_reversed"), - v.release_day_reversed_padded: _date_metadata(v.release_date, "day_reversed_padded"), - v.release_day_of_year: _date_metadata(v.release_date, "day_of_year"), - v.release_day_of_year_padded: _date_metadata(v.release_date, "day_of_year_padded"), - v.release_day_of_year_reversed: _date_metadata(v.release_date, "day_of_year_reversed"), - v.release_day_of_year_reversed_padded: _date_metadata( - v.release_date, "day_of_year_reversed_padded" - ), - v.release_date_standardized: _date_metadata(v.release_date, "date_standardized"), -} - -PLAYLIST_VARIABLES: Dict[Variable, str] = { - v.playlist_webpage_url: _playlist_get_str(v.playlist_webpage_url, v.webpage_url), - v.playlist_description: _playlist_get_str(v.playlist_description, v.description), - v.playlist_uploader: _playlist_get_str(v.playlist_uploader, v.uploader), - v.playlist_uploader_url: _playlist_get_str(v.playlist_uploader_url, v.playlist_webpage_url), - v.source_index: _playlist_get_int(v.source_index, 1), - v.source_count: _playlist_get_int(v.source_count, 1), -} - - -SOURCE_VARIABLES: Dict[Variable, str] = { - v.source_uid: _source_get_str(v.source_uid, v.playlist_uid), - v.source_title: _source_get_str(v.source_title, v.playlist_title), - v.source_webpage_url: _source_get_str(v.source_webpage_url, v.playlist_webpage_url), - v.source_description: _source_get_str(v.source_description, v.playlist_description), - v.source_uploader_id: _source_get_str(v.source_uploader_id, v.playlist_uploader_id), - v.source_uploader: _source_get_str(v.source_uploader, v.playlist_uploader), - v.source_uploader_url: _source_get_str(v.source_uploader_url, v.source_webpage_url), -} - -SOURCE_DERIVED_VARIABLES: Dict[Variable, str] = { - v.source_index_padded: _pad_int(v.source_index, 2), -} - -SIBLING_VARIABLES: Dict[Variable, str] = { - v.playlist_max_upload_date: f"""{{ - %array_reduce( - %if_passthrough( - %extract_field_from_siblings('{v.upload_date.variable_name}'), - [{v.upload_date.variable_name}] - ), - %max - ) - }}""" -} - -SIBLING_DERIVED_VARIABLES: Dict[Variable, str] = { - v.playlist_max_upload_year: _date_metadata(v.playlist_max_upload_date, "year"), - v.playlist_max_upload_year_truncated: _date_metadata( - v.playlist_max_upload_date, "year_truncated" - ), -} - -_VARIABLE_SCRIPTS: Dict[Variable, str] = {} -mergedeep.merge( - _VARIABLE_SCRIPTS, - ENTRY_EMPTY_METADATA, - ENTRY_HARDCODED_VARIABLES, - ENTRY_RELATIVE_VARIABLES, - ENTRY_REQUIRED_VARIABLES, - ENTRY_DEFAULT_VARIABLES, - DOWNLOADER_INJECTED_VARIABLES, - ENTRY_DERIVED_VARIABLES, - ENTRY_UPLOAD_DATE_VARIABLES, - ENTRY_RELEASE_DATE_VARIABLES, - SIBLING_VARIABLES, - SIBLING_DERIVED_VARIABLES, - PLAYLIST_VARIABLES, - SOURCE_VARIABLES, - SOURCE_DERIVED_VARIABLES, -) - -VARIABLE_SCRIPTS: Dict[str, str] = { - var.variable_name: script for var, script in _VARIABLE_SCRIPTS.items() -} - - -def _keys(*variables: Dict[Variable, str]) -> Set[str]: - keys: Set[str] = set() - for variable_set in variables: - keys.update(set(var.variable_name for var in variable_set.keys())) - return keys - - -UNRESOLVED_VARIABLES: Set[str] = _keys( - ENTRY_EMPTY_METADATA, - DOWNLOADER_INJECTED_VARIABLES, -) - -CustomFunctions.register() diff --git a/src/ytdl_sub/entries/script/variable_types.py b/src/ytdl_sub/entries/script/variable_types.py new file mode 100644 index 00000000..7cca58cd --- /dev/null +++ b/src/ytdl_sub/entries/script/variable_types.py @@ -0,0 +1,321 @@ +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from typing import Dict +from typing import List +from typing import Optional +from typing import Type +from typing import TypeVar + +from ytdl_sub.script.types.array import Array +from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import Integer +from ytdl_sub.script.types.resolvable import String + +ENTRY_METADATA_VARIABLE_NAME = "entry_metadata" +PLAYLIST_METADATA_VARIABLE_NAME = "playlist_metadata" +SOURCE_METADATA_VARIABLE_NAME = "source_metadata" + +TMetadataVariable = TypeVar("TMetadataVariable", bound="MetadataVariable") +TVariable = TypeVar("TVariable", bound="Variable") + + +def _get( + cast: str, + metadata_variable_name: str, + metadata_key: str, + variable_name: Optional[str], + default: Optional[TVariable | str | int | Dict | List], + as_type: Type[TMetadataVariable], +) -> TMetadataVariable: + if default is None: + # TODO: assert with good error message if key DNE + out = f"%map_get({metadata_variable_name}, '{metadata_key}')" + elif isinstance(default, Variable): + args = f"{metadata_variable_name}, '{metadata_key}', {default.variable_name}" + out = f"%map_get_non_empty({args})" + elif isinstance(default, str): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', '{default}')" + elif isinstance(default, dict): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {{}})" + elif isinstance(default, list): + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', [])" + else: + out = f"%map_get_non_empty({metadata_variable_name}, '{metadata_key}', {default})" + + return as_type( + variable_name=variable_name or metadata_key, + metadata_key=metadata_key, + definition=f"{{ %legacy_bracket_safety(%{cast}({out})) }}", + ) + + +@dataclass(frozen=True) +class Variable(ABC): + variable_name: str + definition: str + + @classmethod + @abstractmethod + def human_readable_type(cls) -> str: + """ + Script type of the variable, for documentation + """ + + +@dataclass(frozen=True) +class StringVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return String.__name__ + + def to_sanitized_plex(self, variable_name: str) -> "StringVariable": + """ + Converts a String variable to be plex sanitized + """ + return StringVariable( + variable_name=variable_name, + definition=f"{{%sanitize_plex_episode({self.variable_name})}}", + ) + + def as_date_variable(self) -> "StringDateVariable": + """ + Converts a String variable to a date variable (which has metadata helpers) + """ + return StringDateVariable( + variable_name=self.variable_name, + definition=self.definition, + ) + + +@dataclass(frozen=True) +class StringDateVariable(StringVariable): + def get_string_date_metadata( + self, date_metadata_key: str, variable_name: Optional[str] = None + ) -> StringVariable: + """ + Gets a string-based date metadata variable + """ + return StringVariable( + variable_name=variable_name or date_metadata_key, + definition=f"""{{ + %string( + %map_get( + %to_date_metadata({self.variable_name}), + '{date_metadata_key}' + ) + ) + }}""", + ) + + def get_integer_date_metadata( + self, date_metadata_key: str, variable_name: str + ) -> "IntegerVariable": + """ + Gets an int-based date metadata variable + """ + return IntegerVariable( + variable_name=variable_name, + definition=f"""{{ + %int( + %map_get( + %to_date_metadata({self.variable_name}), + '{date_metadata_key}' + ) + ) + }}""", + ) + + +@dataclass(frozen=True) +class IntegerVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Integer.__name__ + + def to_padded_int(self, variable_name: str, pad: int) -> StringVariable: + """ + Pads an integer + """ + return StringVariable( + variable_name=variable_name, definition=f"{{%pad_zero({self.variable_name}, {pad})}}" + ) + + +@dataclass(frozen=True) +class ArrayVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Array.__name__ + + +@dataclass(frozen=True) +class MapVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Map.__name__ + + +@dataclass(frozen=True) +class MetadataVariable(Variable, ABC): + metadata_key: str + + +@dataclass(frozen=True) +class MapMetadataVariable(MetadataVariable, MapVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional["MapMetadataVariable" | Dict] = None, + ) -> "MapMetadataVariable": + """ + Creates a map variable from entry metadata + """ + return _get( + "map", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=MapMetadataVariable, + ) + + +@dataclass(frozen=True) +class ArrayMetadataVariable(MetadataVariable, ArrayVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional["ArrayMetadataVariable" | List] = None, + ) -> "ArrayMetadataVariable": + """ + Creates an array variable from entry metadata + """ + return _get( + "array", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=ArrayMetadataVariable, + ) + + +@dataclass(frozen=True) +class StringMetadataVariable(MetadataVariable, StringVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from entry metadata + """ + return _get( + "string", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + @classmethod + def from_playlist( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from playlist metadata + """ + return _get( + "string", + metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + @classmethod + def from_source( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[StringVariable | str] = None, + ) -> "StringMetadataVariable": + """ + Creates a string variable from source metadata + """ + return _get( + "string", + metadata_variable_name=SOURCE_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=StringMetadataVariable, + ) + + def as_date_variable(self) -> "StringDateMetadataVariable": + """ + Converts a String variable to a date variable (which has metadata helpers) + """ + return StringDateMetadataVariable( + metadata_key=self.metadata_key, + variable_name=self.variable_name, + definition=self.definition, + ) + + +@dataclass(frozen=True) +class StringDateMetadataVariable(StringMetadataVariable, StringDateVariable): + pass + + +@dataclass(frozen=True) +class IntegerMetadataVariable(MetadataVariable, IntegerVariable): + @classmethod + def from_entry( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[IntegerVariable | int] = None, + ) -> "IntegerMetadataVariable": + """ + Creates an int variable from entry metadata + """ + return _get( + "int", + metadata_variable_name=ENTRY_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=IntegerMetadataVariable, + ) + + @classmethod + def from_playlist( + cls, + metadata_key: str, + variable_name: Optional[str] = None, + default: Optional[IntegerVariable | int] = None, + ) -> "IntegerMetadataVariable": + """ + Creates an int variable from playlist metadata + """ + return _get( + "int", + metadata_variable_name=PLAYLIST_METADATA_VARIABLE_NAME, + metadata_key=metadata_key, + variable_name=variable_name, + default=default, + as_type=IntegerMetadataVariable, + ) diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index a798dd18..38f7160c 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -1,24 +1,22 @@ from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.script.functions import Functions from ytdl_sub.script.utils.name_validation import is_valid_name -SUBSCRIPTION_NAME = "subscription_name" -SUBSCRIPTION_VALUE = "subscription_value" -SUBSCRIPTION_MAP = "subscription_map" +# TODO: use this SUBSCRIPTION_ARRAY = "subscription_array" class OverrideVariables: - @classmethod - def subscription_name(cls) -> str: + @staticmethod + def subscription_name() -> str: """ Name of the subscription """ - return SUBSCRIPTION_NAME + return "subscription_name" - @classmethod - def subscription_value(cls) -> str: + @staticmethod + def subscription_value() -> str: """ For subscriptions in the form of @@ -28,10 +26,10 @@ class OverrideVariables: ``subscription_value`` gets set to ``https://...``. """ - return SUBSCRIPTION_VALUE + return "subscription_value" - @classmethod - def subscription_indent_i(cls, index: int) -> str: + @staticmethod + def subscription_indent_i(index: int) -> str: """ For subscriptions in the form of @@ -46,8 +44,8 @@ class OverrideVariables: """ return f"subscription_indent_{index + 1}" - @classmethod - def subscription_value_i(cls, index: int) -> str: + @staticmethod + def subscription_value_i(index: int) -> str: """ For subscriptions in the form of @@ -63,6 +61,37 @@ class OverrideVariables: """ return f"subscription_value_{index + 1}" + @staticmethod + def subscription_map() -> str: + """ + For subscriptions in the form of + + .. code-block:: yaml + + + Subscription Name: + Music Videos: + - "https://url1.com/..." + Concerts: + - "https://url2.com/..." + + Stores all the contents under the subscription name into the override variable + ``subscription_map`` as a Map value. The above example is stored as: + + .. code-block:: python + + { + "Music Videos": [ + "https://url1.com/..." + ], + "Concerts: [ + "https://url2.com/..." + ] + } + """ + return "subscription_map" + + +class OverrideHelpers: @classmethod def is_entry_variable_name(cls, name: str) -> bool: """ diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index e728f392..644fa1c1 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -25,15 +25,13 @@ class AudioExtractOptions(OptionsDictValidator): """ Extracts audio from a video file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 """ _required_keys = {"codec"} @@ -56,16 +54,20 @@ class AudioExtractOptions(OptionsDictValidator): @property def codec(self) -> str: """ - The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, - opus, vorbis, wav, and best to grab the best possible format at runtime. + :expected type: String + :description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. """ return self._codec @property def quality(self) -> Optional[float]: """ - Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` - (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + :expected type: Float + :description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. """ if self._quality is not None: return self._quality.value diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index bbb10ddd..fbd11c09 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -64,32 +64,29 @@ class ChaptersOptions(OptionsDictValidator): Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" - - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False """ _optional_keys = { @@ -135,23 +132,29 @@ class ChaptersOptions(OptionsDictValidator): @property def embed_chapters(self) -> Optional[bool]: """ - Optional. Embed chapters into the file. Defaults to True. + :expected type: Optional[Boolean] + :description: + Defaults to True. Embed chapters into the file. """ return self._embed_chapters @property def allow_chapters_from_comments(self) -> bool: """ - Optional. If chapters do not exist in the video/description itself, attempt to scrape - comments to find the chapters. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. """ return self._allow_chapters_from_comments @property def remove_chapters_regex(self) -> Optional[List[re.Pattern]]: """ - Optional. List of regex patterns to match chapter titles against and remove them from the - entry. + :expected type: Optional[List[RegexString] + :description: + List of regex patterns to match chapter titles against and remove them from the + entry. """ if self._remove_chapters_regex: return [validator.compiled_regex for validator in self._remove_chapters_regex.list] @@ -160,9 +163,11 @@ class ChaptersOptions(OptionsDictValidator): @property def sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", - "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", - "poi_highlight", or "all" to include all categories. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. """ if self._sponsorblock_categories: category_list = [validator.value for validator in self._sponsorblock_categories.list] @@ -174,9 +179,11 @@ class ChaptersOptions(OptionsDictValidator): @property def remove_sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to remove from the output file. Can only remove - categories that are specified in ``sponsorblock_categories`` or "all", which removes - everything specified in ``sponsorblock_categories``. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. """ if self._remove_sponsorblock_categories: category_list = [ @@ -190,9 +197,10 @@ class ChaptersOptions(OptionsDictValidator): @property def force_key_frames(self) -> bool: """ - Optional. Force keyframes at cuts when removing sections. This is slow due to needing a - re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to - False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. """ return self._force_key_frames diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 092d1054..3948cb1e 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -11,16 +11,24 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator class DateRangeOptions(OptionsDictValidator): """ Only download files uploaded within the specified date range. + Dates must adhere to a yt-dlp datetime. From their docs: - Usage: + .. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + + Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. + Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest + granularity possible. + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" """ _optional_keys = {"before", "after"} @@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator): @property def before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos before this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._before @property def after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos after this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._after diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 64703870..b3ef63e2 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -20,13 +20,11 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator): """ Whether to embed thumbnails to the audio/video file or not. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True """ diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 3b56e1c1..8fccaedb 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -32,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator): """ Converts video files from one extension to another. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" - Supports custom ffmpeg conversions: + Also supports custom ffmpeg conversions: + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text """ _required_keys = {"convert_to"} @@ -89,35 +87,38 @@ class FileConvertOptions(OptionsDictValidator): @property def convert_to(self) -> str: """ - Convert to a desired file type. Supports: + :expected type: String + :description: + Convert to a desired file type. Supports - * Video: avi, flv, mkv, mov, mp4, webm - * Audio: aac, flac, mp3, m4a, opus, vorbis, wav + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav """ return self._convert_to @property def convert_with(self) -> Optional[str]: """ - Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within - yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified - with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - + :expected type: Optional[String] + :description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. """ return self._convert_with @property def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the - form of: + :expected type: Optional[OverridesFormatter] + :description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of - .. code-block:: bash + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext - - The output file will use the extension specified in ``convert_to``. Post-processing args - can still be set with ``convert_with`` set to ``yt-dlp``. + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. """ return self._ffmpeg_post_process_args diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index cceb4700..7f381db1 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -19,15 +19,15 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): Applies a conditional OR on any number of filters comprised of either variables or scripts. If any filter evaluates to True, the entry will be excluded. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - filter_exclude: - - { %contains( %lower(title), '#short' ) } - - { %contains( %lower(description), '#short' ) } + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } """ diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 988426fe..41a98f78 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -19,24 +19,23 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): Applies a conditional AND on any number of filters comprised of either variables or scripts. If all filters evaluate to True, the entry will be included. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - filter_include: - - {description} - - >- - { - %regex_search_any( - title, - [ - "Full Episode", - "FULL", - ] - ) - } + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } """ diff --git a/src/ytdl_sub/plugins/format.py b/src/ytdl_sub/plugins/format.py index 3b5c334f..6e2d545d 100644 --- a/src/ytdl_sub/plugins/format.py +++ b/src/ytdl_sub/plugins/format.py @@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator): .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" """ def __init__(self, name, value): diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 4e1ad498..fe10a69b 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -58,32 +58,20 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: class MatchFiltersOptions(OptionsDictValidator): """ - Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. - Uses the same syntax as yt-dlp. + Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. + Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. + For logical AND's between match filters, use the ``&`` operator in a single match filter. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" - - Supports one or multiple filters: - - .. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit<?18" - - "like_count>?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit<?18" - # - "availability=?public" + match_filters: + filters: + - "age_limit<?18 & like_count>?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" """ _optional_keys = {"filters"} diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index ec99d50f..f6098517 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -75,23 +75,22 @@ class MusicTagsOptions(OptionsDictValidator): a full list of tags for various file types in MediaFile's `source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_. - Usage: + :Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" """ _optional_keys = {"tags", "embed_thumbnail"} diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 92e2b968..728259e1 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -60,7 +60,9 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def nfo_name(self) -> StringFormatterFileNameValidator: """ - The NFO file name. + :expected type: EntryFormatter + :description: + The NFO file name. """ return self._nfo_name @@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def kodi_safe(self) -> Optional[bool]: """ - Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some - foreign language characters. Setting this to True will replace those characters with '□'. - Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. """ return self._kodi_safe @@ -190,22 +194,18 @@ class NfoTagsOptions(SharedNfoTagsOptions): Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False """ _formatter_validator = StringFormatterValidator @@ -215,50 +215,54 @@ class NfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - </episodedetails> + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <episodedetails> + </episodedetails> """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - <title>Awesome Youtube Video - 2022 - 502 - + + + Awesome Youtube Video + 2022 + 502 + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - 2022 - Comedy - Drama + 2022 + Comedy + Drama """ return self._tags diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 72f1828d..f754ec44 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -37,48 +37,52 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - + + + """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - Sweet youtube TV show - + + + Sweet youtube TV show + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + <title year="2022">Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> """ return self._tags diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 325649ea..47b34262 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -122,6 +122,41 @@ class FromSourceVariablesRegex(DictValidator): class RegexOptions(OptionsDictValidator): r""" + .. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -137,51 +172,49 @@ class RegexOptions(OptionsDictValidator): and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" """ _required_keys = {"from"} @@ -209,8 +242,10 @@ class RegexOptions(OptionsDictValidator): @property def skip_if_match_fails(self) -> Optional[bool]: """ - Defaults to True. If True, when any match fails and has no defaults, the entry will be - skipped. If False, ytdl-sub will error and all downloads will not proceed. + :expected type: Optional[Boolean] + :description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. """ return self._skip_if_match_fails diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index fc83a07c..b812c0b9 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -48,23 +48,22 @@ class WhenNoChaptersValidator(StringSelectValidator): class SplitByChaptersOptions(OptionsDictValidator): """ Splits a file by chapters into multiple files. Each file becomes its own entry with the - new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, - ``chapter_index_padded``, ``chapter_count``. + new variables - If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is - set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" """ _required_keys = {"when_no_chapters"} @@ -100,8 +99,16 @@ class SplitByChaptersOptions(OptionsDictValidator): @property def when_no_chapters(self) -> str: """ - Behavior to perform when no chapters are present. Supports "pass" (continue processing), - "drop" (exclude it from output), and "error" (stop processing for everything). + :expected type: String + :description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. """ return self._when_no_chapters diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 445a00a9..9c53a9bd 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -37,18 +37,18 @@ class SubtitleOptions(OptionsDictValidator): ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False """ _optional_keys = { @@ -82,40 +82,50 @@ class SubtitleOptions(OptionsDictValidator): @property def subtitles_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's subtitles if they are present. This can include - directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and - will be placed in the output directory. ``lang`` is dynamic since you can download multiple - subtitles. It will set the respective language to the correct subtitle file. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. """ return self._subtitles_name @property def subtitles_type(self) -> Optional[str]: """ - Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" + :expected type: Optional[String] + :description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". """ return self._subtitles_type @property def embed_subtitles(self) -> Optional[bool]: """ - Optional. Whether to embed the subtitles into the video file. Defaults to False. - NOTE: webm files can only embed "vtt" subtitle types. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. """ return self._embed_subtitles @property def languages(self) -> Optional[List[str]]: """ - Optional. Language code(s) to download for subtitles. Supports a single or list of multiple - language codes. Defaults to "en". + :expected type: Optional[List[String]] + :description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". """ return [lang.value for lang in self._languages] @property def allow_auto_generated_subtitles(self) -> Optional[bool]: """ - Optional. Whether to allow auto generated subtitles. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to allow auto generated subtitles. """ return self._allow_auto_generated_subtitles diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 5bab172c..5295c4f2 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator): range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. - Usage: + :Usage: .. code-block:: yaml @@ -110,31 +110,39 @@ class ThrottleProtectionOptions(OptionsDictValidator): @property def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each download. Does not include time it takes for - ytdl-sub to perform post-processing. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. """ return self._sleep_per_download_s @property def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each subscription. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each subscription. """ return self._sleep_per_subscription_s @property def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]: """ - Number of downloads to perform per subscription. + :expected type: Optional[Range] + :description: + Number of downloads to perform per subscription. """ return self._max_downloads_per_subscription @property def subscription_download_probability(self) -> Optional[ProbabilityValidator]: """ - Probability to perform any downloads, recomputed for each subscription. This is only - recommended to set if you run ytdl-sub in a cron-job, that way you are statistically - guaranteed over time to eventually download the subscription. + :expected type: Optional[Float] + :description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. """ return self._subscription_download_probability diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 9ce3d8fd..c3b2e6ec 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -17,16 +17,14 @@ class VideoTagsOptions(OptionsDictValidator): """ Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" """ _optional_keys = {"tags"} diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py index 5dc0e653..4af6f63c 100644 --- a/src/ytdl_sub/script/functions/array_functions.py +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -19,12 +19,8 @@ class ArrayFunctions: @staticmethod def array(maybe_array: AnyArgument) -> Array: """ - Tries to cast an unknown variable type to an Array. - - Raises - ------ - FunctionRuntimeException - If the input type is not actually an Array. + :description: + Tries to cast an unknown variable type to an Array. """ if not isinstance(maybe_array, Array): raise FunctionRuntimeException( @@ -35,14 +31,16 @@ class ArrayFunctions: @staticmethod def array_size(array: Array) -> Integer: """ - Returns the size of an Array. + :description: + Returns the size of an Array. """ return Integer(len(array.value)) @staticmethod def array_extend(*arrays: Array) -> Array: """ - Combine multiple Arrays into a single Array. + :description: + Combine multiple Arrays into a single Array. """ output: List[Resolvable] = [] for array in arrays: @@ -55,7 +53,8 @@ class ArrayFunctions: array: Array, overlap: Array, only_missing: Optional[Boolean] = None ) -> Array: """ - Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. + :description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. """ output: List[Resolvable] = [] output.extend(array.value) @@ -76,15 +75,17 @@ class ArrayFunctions: @staticmethod def array_at(array: Array, idx: Integer) -> AnyArgument: """ - Return the element in the Array at index ``idx``. + :description: + Return the element in the Array at index ``idx``. """ return array.value[idx.value] @staticmethod def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: """ - Returns the first element whose boolean conversion is True. Returns fallback - if all elements evaluate to False. + :description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. """ for val in array.value: if bool(val.value): @@ -95,15 +96,17 @@ class ArrayFunctions: @staticmethod def array_contains(array: Array, value: AnyArgument) -> Boolean: """ - Return True if the value exists in the Array. False otherwise. + :description: + Return True if the value exists in the Array. False otherwise. """ return Boolean(value in array.value) @staticmethod def array_index(array: Array, value: AnyArgument) -> Integer: """ - Return the index of the value within the Array if it exists. If it does not, it will - throw an error. + :description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. """ if not ArrayFunctions.array_contains(array=array, value=value): raise ArrayValueDoesNotExist( @@ -118,7 +121,8 @@ class ArrayFunctions: @staticmethod def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array: """ - Returns the slice of the Array. + :description: + Returns the slice of the Array. """ if end is not None: return Array(array.value[start.value : end.value]) @@ -127,7 +131,8 @@ class ArrayFunctions: @staticmethod def array_flatten(array: Array) -> Array: """ - Flatten any nested Arrays into a single-dimensional Array. + :description: + Flatten any nested Arrays into a single-dimensional Array. """ output: List[Resolvable] = [] for elem in array.value: @@ -141,14 +146,16 @@ class ArrayFunctions: @staticmethod def array_reverse(array: Array) -> Array: """ - Reverse an Array. + :description: + Reverse an Array. """ return Array(list(reversed(array.value))) @staticmethod def array_product(*arrays: Array) -> Array: """ - Returns the Cartesian product of elements from different arrays + :description: + Returns the Cartesian product of elements from different arrays """ out: List[Resolvable] = [] for combo in itertools.product(*[arr.value for arr in arrays]): @@ -161,7 +168,17 @@ class ArrayFunctions: @staticmethod def array_apply(array: Array, lambda_function: Lambda) -> Array: """ - Apply a lambda function on every element in the Array. + :description: + Apply a lambda function on every element in the Array. + :usage: + + .. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] """ return Array([Array([val]) for val in array.value]) @@ -173,8 +190,9 @@ class ArrayFunctions: reverse_args: Optional[Boolean] = None, ) -> Array: """ - Apply a lambda function on every element in the Array, with ``fixed_argument`` - passed as a second argument to every invocation. + :description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. """ if reverse_args and reverse_args.value: return Array([Array([fixed_argument, val]) for val in array.value]) @@ -184,15 +202,17 @@ class ArrayFunctions: @staticmethod def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array: """ - Apply a lambda function on every element in the Array, where each arg - passed to the lambda function is ``idx, element`` as two separate args. + :description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. """ return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)]) @staticmethod def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument: """ - Apply a reduce function on pairs of elements in the Array, until one element remains. - Executes using the left-most and reduces in the right direction. + :description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. """ return array diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py index f1bb9a35..bfb7416f 100644 --- a/src/ytdl_sub/script/functions/boolean_functions.py +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -13,70 +13,80 @@ class BooleanFunctions: @staticmethod def bool(value: AnyArgument) -> Boolean: """ - Cast any type to a Boolean. + :description: + Cast any type to a Boolean. """ return Boolean(bool(value.value)) @staticmethod def eq(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``==`` operator. Returns True if left == right. False otherwise. + :description: + ``==`` operator. Returns True if left == right. False otherwise. """ return Boolean(left.value == right.value) @staticmethod def ne(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``!=`` operator. Returns True if left != right. False otherwise. + :description: + ``!=`` operator. Returns True if left != right. False otherwise. """ return Boolean(left.value != right.value) @staticmethod def lt(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``<`` operator. Returns True if left < right. False otherwise. + :description: + ``<`` operator. Returns True if left < right. False otherwise. """ return Boolean(left.value < right.value) @staticmethod def lte(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``<=`` operator. Returns True if left <= right. False otherwise. + :description: + ``<=`` operator. Returns True if left <= right. False otherwise. """ return Boolean(left.value <= right.value) @staticmethod def gt(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``>`` operator. Returns True if left > right. False otherwise. + :description: + ``>`` operator. Returns True if left > right. False otherwise. """ return Boolean(left.value > right.value) @staticmethod def gte(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``>=`` operator. Returns True if left >= right. False otherwise. + :description: + ``>=`` operator. Returns True if left >= right. False otherwise. """ return Boolean(left.value >= right.value) @staticmethod def and_(*values: AnyArgument) -> Boolean: """ - ``and`` operator. Returns True if all values evaluate to True. False otherwise. + :description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. """ return Boolean(all(bool(val.value) for val in values)) @staticmethod def or_(*values: AnyArgument) -> Boolean: """ - ``or`` operator. Returns True if any value evaluates to True. False otherwise. + :description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. """ return Boolean(any(bool(val.value) for val in values)) @staticmethod def xor(*values: AnyArgument) -> Boolean: """ - ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + :description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. """ bit_array = [bool(val.value) for val in values] @@ -85,13 +95,15 @@ class BooleanFunctions: @staticmethod def not_(value: Boolean) -> Boolean: """ - ``not`` operator. Returns the opposite of value. + :description: + ``not`` operator. Returns the opposite of value. """ return Boolean(not value.value) @staticmethod def is_null(value: AnyArgument) -> Boolean: """ - Returns True if a value is null (i.e. an empty string). False otherwise. + :description: + Returns True if a value is null (i.e. an empty string). False otherwise. """ return Boolean(isinstance(value, String) and value.value == "") diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py index 8caaae02..e532031b 100644 --- a/src/ytdl_sub/script/functions/conditional_functions.py +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -11,8 +11,9 @@ class ConditionalFunctions: condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: """ - Conditional ``if`` statement that returns the ``true`` or ``false`` parameter - depending on the ``condition`` value. + :description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. """ if condition.value: return true @@ -23,8 +24,9 @@ class ConditionalFunctions: maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: """ - Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, - otherwise returns ``else_arg``. + :description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. """ if bool(maybe_true_arg.value): return maybe_true_arg diff --git a/src/ytdl_sub/script/functions/date_functions.py b/src/ytdl_sub/script/functions/date_functions.py index 8d68f052..fea1095c 100644 --- a/src/ytdl_sub/script/functions/date_functions.py +++ b/src/ytdl_sub/script/functions/date_functions.py @@ -8,6 +8,7 @@ class DateFunctions: @staticmethod def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String: """ - Converts a posix timestamp to a date using strftime formatting. + :description: + Converts a posix timestamp to a date using strftime formatting. """ return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value)) diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py index 517c6141..3a1be9f5 100644 --- a/src/ytdl_sub/script/functions/error_functions.py +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -8,15 +8,17 @@ class ErrorFunctions: @staticmethod def throw(error_message: String) -> AnyArgument: """ - Explicitly throw an error with the provided error message. + :description: + Explicitly throw an error with the provided error message. """ raise UserThrownRuntimeError(error_message) @staticmethod def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` evaluates to False. - If it evaluates to True, it will return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) @@ -27,8 +29,9 @@ class ErrorFunctions: value: AnyArgument, ret: ReturnableArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` evaluates to False. - If it evaluates to True, it will return ``ret``. + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) @@ -39,8 +42,9 @@ class ErrorFunctions: value: ReturnableArgument, equals: AnyArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` does not equal - ``equals``. If they do equal, then return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. """ if not value.value == equals.value: raise UserThrownRuntimeError(assert_message) @@ -51,8 +55,9 @@ class ErrorFunctions: value: ReturnableArgument, equals: AnyArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` equals - ``equals``. If they do equal, then return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. """ if value.value == equals.value: raise UserThrownRuntimeError(assert_message) diff --git a/src/ytdl_sub/script/functions/json_functions.py b/src/ytdl_sub/script/functions/json_functions.py index 915fedf1..c706a86f 100644 --- a/src/ytdl_sub/script/functions/json_functions.py +++ b/src/ytdl_sub/script/functions/json_functions.py @@ -35,6 +35,7 @@ class JsonFunctions: @staticmethod def from_json(argument: String) -> AnyArgument: """ - Converts a JSON string into an actual type. + :description: + Converts a JSON string into an actual type. """ return _from_json(json.loads(argument.value)) diff --git a/src/ytdl_sub/script/functions/map_functions.py b/src/ytdl_sub/script/functions/map_functions.py index 45ca13f1..b34fa611 100644 --- a/src/ytdl_sub/script/functions/map_functions.py +++ b/src/ytdl_sub/script/functions/map_functions.py @@ -18,12 +18,8 @@ class MapFunctions: @staticmethod def map(maybe_mapping: AnyArgument) -> Map: """ - Tries to cast an unknown variable type to a Map. - - Raises - ------ - FunctionRuntimeException - If the input type is not actually a Map. + :description: + Tries to cast an unknown variable type to a Map. """ if not isinstance(maybe_mapping, Map): raise FunctionRuntimeException( @@ -34,14 +30,16 @@ class MapFunctions: @staticmethod def map_size(mapping: Map) -> Integer: """ - Returns the size of a Map. + :description: + Returns the size of a Map. """ return Integer(len(mapping.value)) @staticmethod def map_contains(mapping: Map, key: AnyArgument) -> Boolean: """ - Returns True if the key is in the Map. False otherwise. + :description: + Returns True if the key is in the Map. False otherwise. """ if not isinstance(key, Hashable): raise KeyNotHashableRuntimeException( @@ -55,8 +53,9 @@ class MapFunctions: mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None ) -> AnyArgument: """ - Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is - provided, it will return ``default``. Otherwise, will error. + :description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. """ if not MapFunctions.map_contains(mapping=mapping, key=key).value: if default is not None: @@ -70,8 +69,9 @@ class MapFunctions: @staticmethod def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument: """ - Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, - return ``default``. Otherwise, will error. + :description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. """ output = MapFunctions.map_get(mapping, key, default) if isinstance(output, String) and output.value == "": @@ -83,16 +83,18 @@ class MapFunctions: @staticmethod def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array: """ - Apply a lambda function on the Map, where each arg - passed to the lambda function is ``key, value`` as two separate args. + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. """ return Array([Array([key, value]) for key, value in mapping.value.items()]) @staticmethod def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array: """ - Apply a lambda function on the Map, where each arg - passed to the lambda function is ``idx, key, value`` as three separate args. + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. """ return Array( [ diff --git a/src/ytdl_sub/script/functions/numeric_functions.py b/src/ytdl_sub/script/functions/numeric_functions.py index 5211c690..b25374f9 100644 --- a/src/ytdl_sub/script/functions/numeric_functions.py +++ b/src/ytdl_sub/script/functions/numeric_functions.py @@ -16,28 +16,32 @@ class NumericFunctions: @staticmethod def float(value: AnyArgument) -> Float: """ - Cast to Float. + :description: + Cast to Float. """ return Float(value=float(value.value)) @staticmethod def int(value: AnyArgument) -> Integer: """ - Cast to Integer. + :description: + Cast to Integer. """ return Integer(value=int(value.value)) @staticmethod def add(*values: Numeric) -> Numeric: """ - ``+`` operator. Returns the sum of all values. + :description: + ``+`` operator. Returns the sum of all values. """ return _to_numeric(sum(val.value for val in values)) @staticmethod def sub(*values: Numeric) -> Numeric: """ - ``-`` operator. Subtracts all values from left to right. + :description: + ``-`` operator. Subtracts all values from left to right. """ output = values[0].value for val in values[1:]: @@ -48,41 +52,47 @@ class NumericFunctions: @staticmethod def mul(*values: Numeric) -> Numeric: """ - ``*`` operator. Returns the product of all values. + :description: + ``*`` operator. Returns the product of all values. """ return _to_numeric(math.prod([val.value for val in values])) @staticmethod def pow(base: Numeric, exponent: Numeric) -> Numeric: """ - ``**`` operator. Returns the exponential of the base and exponent value. + :description: + ``**`` operator. Returns the exponential of the base and exponent value. """ return _to_numeric(math.pow(base.value, exponent.value)) @staticmethod def div(left: Numeric, right: Numeric) -> Numeric: """ - ``/`` operator. Returns ``left / right``. + :description: + ``/`` operator. Returns ``left / right``. """ return _to_numeric(left.value / right.value) @staticmethod def mod(left: Numeric, right: Numeric) -> Numeric: """ - ``%`` operator. Returns ``left % right``. + :description: + ``%`` operator. Returns ``left % right``. """ return _to_numeric(value=left.value % right.value) @staticmethod def max(*values: Numeric) -> Numeric: """ - Returns max of all values. + :description: + Returns max of all values. """ return _to_numeric(max(val.value for val in values)) @staticmethod def min(*values: Numeric) -> Numeric: """ - Returns min of all values. + :description: + Returns min of all values. """ return _to_numeric(min(val.value for val in values)) diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py index e38b3139..f837be32 100644 --- a/src/ytdl_sub/script/functions/regex_functions.py +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -18,33 +18,37 @@ class RegexFunctions: @staticmethod def regex_match(regex: String, string: String) -> Array: """ - Checks for a match only at the beginning of the string. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.match(regex.value, string.value)) @staticmethod def regex_search(regex: String, string: String) -> Array: """ - Checks for a match anywhere in the string. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.search(regex.value, string.value)) @staticmethod def regex_fullmatch(regex: String, string: String) -> Array: """ - Checks for entire string to be a match. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.fullmatch(regex.value, string.value)) @staticmethod def regex_capture_groups(regex: String) -> Integer: """ - Returns number of capture groups in regex + :description: + Returns number of capture groups in regex """ return Integer(re.compile(regex.value).groups) diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index 807a5dd4..1f0280b5 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -11,21 +11,24 @@ class StringFunctions: @staticmethod def string(value: AnyArgument) -> String: """ - Cast to String. + :description: + Cast to String. """ return String(value=str(value.value)) @staticmethod def contains(string: String, contains: String) -> Boolean: """ - Returns True if ``contains`` is in ``string``. False otherwise. + :description: + Returns True if ``contains`` is in ``string``. False otherwise. """ return Boolean(contains.value in string.value) @staticmethod def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: """ - Returns the slice of the Array. + :description: + Returns the slice of the Array. """ if end is not None: return String(string.value[start.value : end.value]) @@ -34,28 +37,32 @@ class StringFunctions: @staticmethod def lower(string: String) -> String: """ - Lower-case the entire String. + :description: + Lower-case the entire String. """ return String(string.value.lower()) @staticmethod def upper(string: String) -> String: """ - Upper-case the entire String. + :description: + Upper-case the entire String. """ return String(string.value.upper()) @staticmethod def capitalize(string: String) -> String: """ - Capitalize the first character in the string. + :description: + Capitalize the first character in the string. """ return String(string.value.capitalize()) @staticmethod def titlecase(string: String) -> String: """ - Capitalize each word in the string. + :description: + Capitalize each word in the string. """ return String(string.value.title()) @@ -64,8 +71,9 @@ class StringFunctions: string: String, old: String, new: String, count: Optional[Integer] = None ) -> String: """ - Replace the ``old`` part of the String with the ``new``. Optionally only replace it - ``count`` number of times. + :description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. """ if count: return String(string.value.replace(old.value, new.value, count.value)) @@ -75,14 +83,16 @@ class StringFunctions: @staticmethod def concat(*values: String) -> String: """ - Concatenate multiple Strings into a single String. + :description: + Concatenate multiple Strings into a single String. """ return String("".join(val.value for val in values)) @staticmethod def pad(string: String, length: Integer, char: String) -> String: """ - Pads the string to the given length + :description: + Pads the string to the given length """ output = string.value while len(output) < length.value: @@ -93,7 +103,8 @@ class StringFunctions: @staticmethod def pad_zero(numeric: Numeric, length: Integer) -> String: """ - Pads a numeric with zeros to the given length + :description: + Pads a numeric with zeros to the given length """ return StringFunctions.pad( string=String(str(numeric.value)), diff --git a/src/ytdl_sub/script/utils/exception_formatters.py b/src/ytdl_sub/script/utils/exception_formatters.py index fb3b5e6e..6e14354e 100644 --- a/src/ytdl_sub/script/utils/exception_formatters.py +++ b/src/ytdl_sub/script/utils/exception_formatters.py @@ -1,16 +1,11 @@ import sys from typing import List -from typing import Type from typing import TypeVar -from typing import Union from ytdl_sub.script.types.resolvable import BuiltInFunctionType -from ytdl_sub.script.types.resolvable import NamedType from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.type_checking import FunctionSpec -from ytdl_sub.script.utils.type_checking import get_optional_type -from ytdl_sub.script.utils.type_checking import is_optional from ytdl_sub.script.utils.type_checking import is_union TUserException = TypeVar("TUserException", bound=UserException) @@ -103,28 +98,10 @@ class FunctionArgumentsExceptionFormatter: input_spec: FunctionSpec, function_instance: BuiltInFunctionType, ): - self._args = input_spec.args - self._varargs = input_spec.varargs + self._input_spec = input_spec self._name = function_instance.name self._input_args = function_instance.args - @classmethod - def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: - if is_optional(python_type): - return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" - if is_union(python_type): - return ", ".join( - sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) - ) - return python_type.type_name() - - def _expected_args_str(self) -> str: - if self._args is not None: - return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})" - if self._varargs is not None: - return f"({self._to_human_readable_name(self._varargs)}, ...)" - return "()" - def _received_args_str(self) -> str: received_type_names: List[str] = [] for arg in self._input_args: @@ -149,5 +126,6 @@ class FunctionArgumentsExceptionFormatter: """ return IncompatibleFunctionArguments( f"Incompatible arguments passed to function {self._name}.\n" - f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}" + f"Expected {self._input_spec.human_readable_input_args()}\n" + f"Received {self._received_args_str()}" ) diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py index cc7d87ff..2170c3e0 100644 --- a/src/ytdl_sub/script/utils/type_checking.py +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -119,6 +119,7 @@ def is_type_compatible( @dataclass(frozen=True) class FunctionSpec: return_type: Type[Resolvable] + arg_names: List[str] args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None varargs: Optional[Type[Resolvable]] = None @@ -223,6 +224,42 @@ class FunctionSpec: return l_type return None + @classmethod + def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: + if is_optional(python_type): + return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" + if is_union(python_type): + args = ", ".join( + sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) + ) + return f"Union[{args}]" + return python_type.type_name() + + def human_readable_input_args(self) -> str: + """ + Returns + ------- + input arg string in human-readable format + """ + if self.args is not None: + args = ", ".join( + f"{name}: {self._to_human_readable_name(type_)}" + for name, type_ in zip(self.arg_names, self.args) + ) + elif self.varargs is not None: + args = f"{self.arg_names[0]}: {self._to_human_readable_name(self.varargs)}, ..." + else: + args = "" + return f"({args})" + + def human_readable_output_type(self) -> str: + """ + Returns + ------- + output type string in human-readable format + """ + return self._to_human_readable_name(self.return_type) + @classmethod def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec": """ @@ -234,10 +271,12 @@ class FunctionSpec: if arg_spec.varargs: return FunctionSpec( return_type=arg_spec.annotations["return"], + arg_names=[arg_spec.varargs], varargs=arg_spec.annotations[arg_spec.varargs], ) return FunctionSpec( return_type=arg_spec.annotations["return"], + arg_names=arg_spec.args, args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args], ) diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 55e1ea25..62445552 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -9,11 +9,11 @@ from typing import final from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.overrides import Overrides -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.validators import DictValidator +from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator @@ -82,7 +82,6 @@ class SubscriptionPresetDictValidator(NamedSubscriptionValidator, DictValidator) output_dict["overrides"] = dict( output_dict.get("overrides", {}), **self._indent_overrides_dict(), - **{SUBSCRIPTION_NAME: self.subscription_name}, ) return {self.subscription_name: output_dict} @@ -111,7 +110,7 @@ class SubscriptionLeafValidator(NamedSubscriptionValidator, ABC): f"used as a subscription name" ) - self._overrides_to_add: Dict[str, str] = {SUBSCRIPTION_NAME: self.subscription_name} + self._overrides_to_add: Dict[str, str] = {} @final def subscription_dicts(self, global_presets_to_apply: List[str]) -> Dict[str, Dict]: @@ -144,7 +143,7 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add[SUBSCRIPTION_VALUE] = self.value + self._overrides_to_add[OverrideVariables.subscription_value()] = self.value class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator): @@ -169,7 +168,7 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid for idx, list_value in enumerate(self.list): # Write the first list value into subscription_value as well if idx == 0: - self._overrides_to_add[SUBSCRIPTION_VALUE] = list_value.value + self._overrides_to_add[OverrideVariables.subscription_value()] = list_value.value self._overrides_to_add[ OverrideVariables.subscription_value_i(index=idx) @@ -198,6 +197,29 @@ class SubscriptionWithOverridesValidator(SubscriptionLeafValidator, DictFormatte self._overrides_to_add = dict(self.dict_with_format_strings, **self._overrides_to_add) +class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator): + def __init__( + self, + name, + value, + subscription_name: str, + config: ConfigFile, + presets: List[str], + indent_overrides: List[str], + ): + super().__init__( + name=name, + value=value, + subscription_name=subscription_name, + config=config, + presets=presets, + indent_overrides=indent_overrides, + ) + self._overrides_to_add[OverrideVariables.subscription_map()] = ScriptUtils.to_script( + self.dict + ) + + class SubscriptionValidator(SubscriptionOutput): """ Top-level subscription validator @@ -284,7 +306,21 @@ class SubscriptionValidator(SubscriptionOutput): SubscriptionWithOverridesValidator( name=obj_name, value=obj, - subscription_name=key[1:], + subscription_name=key[1:].lstrip(), + config=config, + presets=presets, + indent_overrides=indent_overrides, + ) + ) + # Subscription defined as + # "\Sub Name": + # custom_key: "value" + elif key.startswith("+"): + self._children.append( + SubscriptionMapValidator( + name=obj_name, + value=obj, + subscription_name=key[1:].lstrip(), config=config, presets=presets, indent_overrides=indent_overrides, diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index 7cf5ef2b..646d44e1 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -33,6 +33,13 @@ def get_file_extension(file_name: Path | str) -> str: return file_name.rsplit(".", maxsplit=1)[-1] +def get_md5_hash(contents: str) -> str: + """ + Helper function to compute md5 hash + """ + return hashlib.md5(contents.encode()).hexdigest() + + def get_file_md5_hash(full_file_path: Path | str) -> str: """ Parameters diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index 105bce98..aba9d1fc 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -5,9 +5,9 @@ from typing import Dict from typing import Set from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS -from ytdl_sub.entries.script.variable_definitions import Variable -from ytdl_sub.entries.script.variable_scripts import UNRESOLVED_VARIABLES -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES +from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt index 77140703..4b7fb2b5 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt @@ -1,5 +1,5 @@ { ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475", - "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c", + "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "d53121df33ac8c4a4699ec8919196552", "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json index 890aca91..f077d34c 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json @@ -2,6 +2,6 @@ ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75", "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8", "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b", - "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "b2d6388b4ddf8e3fbca042cb123672c3", + "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975", "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 30f23eb5..74e55e90 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -2,7 +2,7 @@ "Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "6fb0ce965b75035079f82c84ad341e85", + "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "246fa05b6443337785575987904848df", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "INFO_JSON", @@ -10,7 +10,7 @@ "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "43f271ef8d3a19877f0dc9bc5040b42a", + "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "ae1d5e2e3979cea3c96e6a4cfcae8073", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index 29dfe4cf..c4fb6cca 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json index 143b2310..c5490aa6 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-JMC-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video.json b/tests/resources/expected_downloads_summaries/youtube/test_video.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json index 04e7c7d9..b425d91d 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json @@ -1,4 +1,4 @@ { - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "718c187e6196c85eea73d16ebd489c91", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d9d2d12feee44ee97729b39ba981c542", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 730ad6eb..bd7122c5 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -156,7 +156,7 @@ def preset_with_subscription_overrides_tilda( preset_with_subscription_value, **{ "parent_preset_2 | parent_preset_1": { - "~test_2_1": { + "~ test_2_1": { "current_override": "test_2_1", } }, @@ -164,6 +164,27 @@ def preset_with_subscription_overrides_tilda( ) +@pytest.fixture +def preset_with_subscription_overrides_map( + preset_with_subscription_value: Dict, +): + return dict( + preset_with_subscription_value, + **{ + "parent_preset_2 | parent_preset_1": { + "+ test_2_1": { + "custom_key": "custom_value", + "custom_list": [ + "elem1", + "elem2", + "elem3", + ], + } + }, + }, + ) + + @pytest.fixture def preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors( preset_with_subscription_value: Dict, @@ -204,9 +225,9 @@ def test_subscription_file_preset_applies(config_file: ConfigFile, preset_with_f "key-4": "test_preset", } - overrides = preset_sub.overrides.dict_with_format_strings + overrides = preset_sub.overrides.script # preset overrides take precedence over __preset__ - assert overrides.get("current_override") == "test_preset" + assert overrides.get("current_override").native == "test_preset" def test_subscription_list( @@ -217,13 +238,15 @@ def test_subscription_list( subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 3 - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_value_1") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_value_2") == "is_2_1_list_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_value_1").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_value_2").native == "is_2_1_list_2" + assert ( + sub_2_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence def test_subscription_overrides_tilda( @@ -234,10 +257,31 @@ def test_subscription_overrides_tilda( subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 3 - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("current_override") == "test_2_1" # tilda sub takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("current_override").native == "test_2_1" # tilda sub takes precedence + + +def test_subscription_overrides_map( + config_file: ConfigFile, + preset_with_subscription_overrides_map: Dict, +): + with mock_load_yaml(preset_dict=preset_with_subscription_overrides_map): + subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") + assert len(subs) == 3 + + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script + + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_map").native == { + "custom_key": "custom_value", + "custom_list": [ + "elem1", + "elem2", + "elem3", + ], + } def test_subscription_with_period_in_name( @@ -249,7 +293,7 @@ def test_subscription_with_period_in_name( assert len(subs) == 2 assert subs[1].name == "Mr. Beast" - assert subs[1].overrides.dict_with_format_strings["subscription_name"] == "Mr. Beast" + assert subs[1].overrides.script.get("subscription_name").native == "Mr. Beast" def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables( @@ -262,26 +306,28 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 4 - sub_test_value = [sub for sub in subs if sub.name == "test_value"][ - 0 - ].overrides.dict_with_format_strings - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_test_value = [sub for sub in subs if sub.name == "test_value"][0].overrides.script + sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.script + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_test_value.get("subscription_indent_1") == "original_1" - assert sub_test_value.get("subscription_indent_2") == "original_2" + assert sub_test_value.get("subscription_indent_1").native == "original_1" + assert sub_test_value.get("subscription_indent_2").native == "original_2" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_indent_1") == "INDENT_1" - assert sub_1.get("subscription_indent_2") == "INDENT_2" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_1.get("subscription_name").native == "test_1" + assert sub_1.get("subscription_value").native == "is_1_overwritten" + assert sub_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_1.get("subscription_indent_2").native == "INDENT_2" + assert ( + sub_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_indent_1") == "INDENT_1" - assert sub_2_1.get("subscription_indent_2") == "original_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_2_1.get("subscription_indent_2").native == "original_2" + assert ( + sub_2_1.get("current_override").native == "__preset__" + ) # ensure __preset__ takes precedence @pytest.mark.parametrize("all_same_line", [True, False]) @@ -301,28 +347,28 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia subs = Subscription.from_file_path(config=config_file, subscription_path="mocked") assert len(subs) == 4 - sub_test_value = [sub for sub in subs if sub.name == "test_value"][ - 0 - ].overrides.dict_with_format_strings - sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.dict_with_format_strings - sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.dict_with_format_strings + sub_test_value = [sub for sub in subs if sub.name == "test_value"][0].overrides.script + sub_1 = [sub for sub in subs if sub.name == "test_1"][0].overrides.script + sub_2_1 = [sub for sub in subs if sub.name == "test_2_1"][0].overrides.script - assert sub_test_value.get("subscription_indent_1") == "original_1" - assert sub_test_value.get("subscription_indent_2") == "original_2" + assert sub_test_value.get("subscription_indent_1").native == "original_1" + assert sub_test_value.get("subscription_indent_2").native == "original_2" - assert sub_1.get("subscription_name") == "test_1" - assert sub_1.get("subscription_value") == "is_1_overwritten" - assert sub_1.get("subscription_indent_1") == "INDENT_1" - assert sub_1.get("subscription_indent_2") == "INDENT_2" - assert sub_1.get("subscription_indent_3") == "INDENT_3" - assert sub_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence + assert sub_1.get("subscription_name").native == "test_1" + assert sub_1.get("subscription_value").native == "is_1_overwritten" + assert sub_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_1.get("subscription_indent_2").native == "INDENT_2" + assert sub_1.get("subscription_indent_3").native == "INDENT_3" + # ensure __preset__ takes precedence + assert sub_1.get("current_override").native == "__preset__" - assert sub_2_1.get("subscription_name") == "test_2_1" - assert sub_2_1.get("subscription_value") == "is_2_1_overwritten" - assert sub_2_1.get("subscription_indent_1") == "INDENT_1" - assert sub_2_1.get("subscription_indent_2") == "original_2" - assert sub_2_1.get("current_override") == "__preset__" # ensure __preset__ takes precedence - assert "subscription_indent_3" not in sub_2_1 + assert sub_2_1.get("subscription_name").native == "test_2_1" + assert sub_2_1.get("subscription_value").native == "is_2_1_overwritten" + assert sub_2_1.get("subscription_indent_1").native == "INDENT_1" + assert sub_2_1.get("subscription_indent_2").native == "original_2" + # ensure __preset__ takes precedence + assert sub_2_1.get("current_override").native == "__preset__" + assert "subscription_indent_3" not in sub_2_1.variable_names def test_subscription_file_value_applies_from_config_and_nested_and_indent_variables_same_line_old_format_errors( @@ -372,12 +418,15 @@ def test_tv_show_subscriptions(config_file: ConfigFile, tv_show_subscriptions_pa assert len(subs) == 7 assert subs[3].name == "Jake Trains" - jake_train_overrides = subs[3].overrides.dict_with_format_strings + jake_train_overrides = subs[3].overrides.script - assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" - assert jake_train_overrides["subscription_indent_1"] == "Kids" - assert jake_train_overrides["subscription_indent_2"] == "TV-Y" + assert jake_train_overrides.get("subscription_name").native == "Jake Trains" + assert ( + jake_train_overrides.get("subscription_value").native + == "https://www.youtube.com/@JakeTrains" + ) + assert jake_train_overrides.get("subscription_indent_1").native == "Kids" + assert jake_train_overrides.get("subscription_indent_2").native == "TV-Y" def test_advanced_tv_show_subscriptions( @@ -389,17 +438,20 @@ def test_advanced_tv_show_subscriptions( assert len(subs) == 9 assert subs[3].name == "Jake Trains" - jake_train_overrides = subs[3].overrides.dict_with_format_strings + jake_train_overrides = subs[3].overrides.script - assert jake_train_overrides["subscription_name"] == "Jake Trains" - assert jake_train_overrides["subscription_value"] == "https://www.youtube.com/@JakeTrains" - assert jake_train_overrides["subscription_indent_1"] == "Kids" - assert jake_train_overrides["subscription_indent_2"] == "TV-Y" + assert jake_train_overrides.get("subscription_name").native == "Jake Trains" + assert ( + jake_train_overrides.get("subscription_value").native + == "https://www.youtube.com/@JakeTrains" + ) + assert jake_train_overrides.get("subscription_indent_1").native == "Kids" + assert jake_train_overrides.get("subscription_indent_2").native == "TV-Y" assert subs[5].name == "Gardening with Ciscoe" overrides = subs[5].overrides - assert overrides.apply_formatter(overrides.dict["subscription_name"]) == "Gardening with Ciscoe" + assert overrides.script.get("subscription_name").native == "Gardening with Ciscoe" assert ( overrides.apply_formatter(overrides.dict["url"]) == "https://www.youtube.com/@gardeningwithciscoe4430" @@ -417,11 +469,14 @@ def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_pat assert len(subs) == 14 assert subs[2].name == "Stan Getz" - monk = subs[2].overrides.dict_with_format_strings + monk = subs[2].overrides.script - assert monk["subscription_name"] == "Stan Getz" - assert monk["subscription_value"] == "https://www.youtube.com/@stangetzofficial/releases" - assert monk["subscription_indent_1"] == "Jazz" + assert monk.get("subscription_name").native == "Stan Getz" + assert ( + monk.get("subscription_value").native + == "https://www.youtube.com/@stangetzofficial/releases" + ) + assert monk.get("subscription_indent_1").native == "Jazz" def test_music_video_subscriptions(default_config: ConfigFile, music_video_subscription_path: Path): @@ -431,14 +486,14 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc assert len(subs) == 3 assert subs[1].name == "Michael Jackson" - monk = subs[1].overrides.dict_with_format_strings + monk = subs[1].overrides.script - assert monk["subscription_name"] == "Michael Jackson" + assert monk.get("subscription_name").native == "Michael Jackson" assert ( - monk["subscription_value"] + monk.get("subscription_value").native == "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" ) - assert monk["subscription_indent_1"] == "Pop" + assert monk.get("subscription_indent_1").native == "Pop" def test_default_docker_config_and_subscriptions(): diff --git a/tests/unit/docgen/__init__.py b/tests/unit/docgen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py new file mode 100644 index 00000000..82768347 --- /dev/null +++ b/tests/unit/docgen/test_docgen.py @@ -0,0 +1,30 @@ +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.entry_variables import EntryVariablesDocGen +from tools.docgen.override_variables import OverrideVariablesDocGen +from tools.docgen.plugins import PluginsDocGen +from tools.docgen.scripting_functions import ScriptingFunctionsDocGen +from ytdl_sub.utils.file_handler import get_md5_hash + + +def _test_doc_gen(doc_gen: Type[DocGen]) -> None: + expected_md5_hash = get_md5_hash(doc_gen.generate_and_maybe_write_to_file()) + with open(doc_gen.LOCATION, "r", encoding="utf-8") as file_doc: + md5_hash = get_md5_hash(file_doc.read()) + + assert md5_hash == expected_md5_hash + + +class TestDocGen: + def test_entry_variables_generated(self): + _test_doc_gen(EntryVariablesDocGen) + + def test_override_variables_generated(self): + _test_doc_gen(OverrideVariablesDocGen) + + def test_scripting_functions_generated(self): + _test_doc_gen(ScriptingFunctionsDocGen) + + def test_plugins_generated(self): + _test_doc_gen(PluginsDocGen) diff --git a/tests/unit/docgen/test_docgen_regenerate_disabled.py b/tests/unit/docgen/test_docgen_regenerate_disabled.py new file mode 100644 index 00000000..836b13ea --- /dev/null +++ b/tests/unit/docgen/test_docgen_regenerate_disabled.py @@ -0,0 +1,5 @@ +from tools.docgen.docgen import REGENERATE_DOCS + + +def test_docgen_regenerate_disabled(): + assert REGENERATE_DOCS is False diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py index 1331d485..014e4a85 100644 --- a/tests/unit/script/types/test_function.py +++ b/tests/unit/script/types/test_function.py @@ -40,7 +40,7 @@ class TestFunction: with pytest.raises( IncompatibleFunctionArguments, match=_incompatible_arguments_match( - expected="Map, AnyArgument, Optional[AnyArgument]", + expected="mapping: Map, key: AnyArgument, default: Optional[AnyArgument]", recieved="%if(...)->Union[Array, Map], String", ), ): @@ -49,11 +49,11 @@ class TestFunction: @pytest.mark.parametrize( "function_str, expected_types, received_types", [ - ("{%array_at({'a': 'dict?'}, 1)}", "Array, Integer", "Map, Integer"), - ("{%array_extend('not', 'array')}", "Array, ...", "String, String"), + ("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), + ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), ( "{%replace('hi mom', 'mom', 'dad', 1, 0)}", - "String, String, String, Optional[Integer]", + "string: String, old: String, new: String, count: Optional[Integer]", "String, String, String, Integer, Integer", ), ], diff --git a/tools/docgen/docgen.py b/tools/docgen/docgen.py new file mode 100644 index 00000000..f1307b29 --- /dev/null +++ b/tools/docgen/docgen.py @@ -0,0 +1,32 @@ +from abc import abstractmethod +from pathlib import Path + +REGENERATE_DOCS: bool = False + + +class DocGen: + """ + Home-made auto doc generation + """ + + LOCATION: Path + + @classmethod + @abstractmethod + def generate(cls) -> str: + """ + Generate the docs as a single string + """ + + @classmethod + def generate_and_maybe_write_to_file(cls) -> str: + """ + Maybe writes the docs to their file if the global is set to True, and returns + the generated docs + """ + contents = cls.generate() + if REGENERATE_DOCS: + with open(cls.LOCATION, "w", encoding="utf-8") as out: + out.write(contents) + + return contents diff --git a/tools/docgen/entry_variables.py b/tools/docgen/entry_variables.py new file mode 100644 index 00000000..df53b13a --- /dev/null +++ b/tools/docgen/entry_variables.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import cached_properties +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import line_section +from tools.docgen.utils import section +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + + +def _variable_class_to_name(obj: Type[Any]) -> str: + assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions" + return ( + camel_case_to_human(obj.__name__) + .replace("Variable Definitions", "Variables") + .replace("Ytdl Sub", "Ytdl-Sub") + ) + + +class EntryVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/entry_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Entry Variables", level=0) + + parent_objs: Dict[str, Type[Any]] = { + _variable_class_to_name(obj): obj for obj in VariableDefinitions.__bases__ + } + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for variable_function_name in cached_properties(parent_objs[name]): + docs += get_function_docs( + function_name=variable_function_name, + obj=parent_objs[name], + pre_docstring=f":type: ``{getattr(VARIABLES, variable_function_name).human_readable_type()}``\n", + level=2, + ) + + return docs diff --git a/tools/docgen/override_variables.py b/tools/docgen/override_variables.py new file mode 100644 index 00000000..e919877c --- /dev/null +++ b/tools/docgen/override_variables.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.variables.override_variables import OverrideVariables + + +class OverrideVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Override Variables", level=0) + + for name in static_methods(OverrideVariables): + docs += get_function_docs( + function_name=name, + obj=OverrideVariables, + level=1, + ) + + return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py new file mode 100644 index 00000000..10323066 --- /dev/null +++ b/tools/docgen/plugins.py @@ -0,0 +1,91 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import line_section +from tools.docgen.utils import properties +from tools.docgen.utils import section +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator + + +def should_filter_all_properties(plugin_name: str) -> bool: + return plugin_name in ( + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "video_tags", + "download", + ) + + +def should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + ) + + +def get_function_docs(function_name: str, obj: Any, level: int) -> str: + docs = f"\n``{function_name}``\n\n" + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n\n" + return docs + + +def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: + docs = "" + docs += section(name, level=offset + 0) + + docs += inspect.cleandoc(options.__doc__) + docs += "\n" + + if should_filter_all_properties(name): + return docs + + property_names = [prop for prop in properties(options) if not should_filter_property(prop)] + for property_name in sorted(property_names): + docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) + + return docs + + +class PluginsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/plugins.rst") + + @classmethod + def generate(cls): + options_dict: Dict[str, Type[OptionsValidator]] = { + "output_options": OutputOptions, + "ytdl_options": YTDLOptions, + "overrides": Overrides, + "download": MultiUrlValidator, + } + for plugin_name, plugin_type in PluginMapping._MAPPING.items(): + if plugin_name.startswith("_"): + continue + options_dict[plugin_name] = plugin_type.plugin_options_type + + docs = section("Plugins", level=0) + for idx, name in enumerate(sorted(options_dict.keys())): + docs += line_section(section_idx=idx) + docs += generate_plugin_docs(name, options_dict[name], offset=1) + + return docs diff --git a/tools/docgen/scripting_functions.py b/tools/docgen/scripting_functions.py new file mode 100644 index 00000000..c4663477 --- /dev/null +++ b/tools/docgen/scripting_functions.py @@ -0,0 +1,82 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import line_section +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.type_checking import FunctionSpec + + +def maybe_get_function_name(function_name: str) -> Optional[str]: + if function_name in ["register"]: + return None + + if function_name.endswith("_"): + return function_name[:-1] + return function_name + + +def function_class_to_name(obj: Type[Any]) -> str: + assert "Functions" in obj.__name__ + return camel_case_to_human(obj.__name__) + + +def function_type_hinting(display_function_name: str, function: Any) -> str: + spec = FunctionSpec.from_callable(function) + out = ":spec: ``" + out += display_function_name + out += spec.human_readable_input_args() + out += " -> " + out += spec.human_readable_output_type() + out += "``\n\n" + return out + + +def get_function_docstring( + function_name: str, function: Any, level: int, display_function_name: Optional[str] = None +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + + docs += function_type_hinting(display_function_name=display_function_name, function=function) + docs += inspect.cleandoc(function.__doc__) + docs += "\n" + return docs + + +class ScriptingFunctionsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/scripting_functions.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Scripting Functions", level=0) + + parent_objs: Dict[str, Type[Any]] = { + function_class_to_name(obj): obj for obj in Functions.__bases__ + } + parent_objs["Ytdl-Sub Functions"] = CustomFunctions + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for function_name in static_methods(parent_objs[name]): + if display_function_name := maybe_get_function_name(function_name): + docs += get_function_docstring( + function_name=function_name, + display_function_name=display_function_name, + function=getattr(parent_objs[name], function_name), + level=2, + ) + + return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py new file mode 100644 index 00000000..81ddc3fa --- /dev/null +++ b/tools/docgen/utils.py @@ -0,0 +1,66 @@ +import inspect +from functools import cached_property +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Type + +LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} + + +def section(name: str, level: int, as_code: bool = False) -> str: + if as_code: + name = f"``{name}``" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" + + +def properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property)) + + +def cached_properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), cached_property)) + + +def static_methods(obj: Type[Any]) -> List[str]: + return sorted( + name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod) + ) + + +def camel_case_to_human(string: str) -> str: + output_str = string[0] + for char in string[1:]: + if char.islower(): + output_str += char + else: + output_str += f" {char}" + + return output_str + + +def get_function_docs( + function_name: str, + obj: Any, + level: int, + display_function_name: Optional[str] = None, + pre_docstring: Optional[str] = None, +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + docs += pre_docstring or "" + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n" + return docs + + +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return ""