diff --git a/.gitignore b/.gitignore index 20d1b188..e18f90d3 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,5 @@ docker/testing/volumes ffmpeg.exe ffprobe.exe + +tools/docgen/out \ No newline at end of file diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/conf.py b/docs/source/conf.py index 6a48e5f0..7e551b67 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,6 +53,7 @@ html_theme_options = { } html_static_path = ["_static"] +html_css_files = ["custom.css"] # Make sure the all autosectionlabel targets are unique diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst index 521e1f0a..8e4097c3 100644 --- a/docs/source/config_reference/config_yaml.rst +++ b/docs/source/config_reference/config_yaml.rst @@ -53,51 +53,7 @@ presets ~~~~~~~ ``presets`` define a `formula` for how to format downloaded media and metadata. -download_strategy -""""""""""""""""" -Download strategies dictate what is getting downloaded from a source. Each -download strategy has its own set of parameters. - -.. _url: - -url -''' -.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions() - :members: url, playlist_thumbnails, source_thumbnails, download_reverse - :member-order: bysource - -multi_url -''''''''' -.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions() - :members: urls, variables - -------------------------------------------------------------------------------- - -output_options -"""""""""""""" - -.. autoclass:: ytdl_sub.config.preset_options.OutputOptions() - :members: - :member-order: bysource - :exclude-members: get_upload_date_range_to_keep, partial_validate - -------------------------------------------------------------------------------- - -.. _ytdl_options: - -ytdl_options -"""""""""""" -.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions() - -------------------------------------------------------------------------------- - -.. _overrides: - -overrides -""""""""" -.. autoclass:: ytdl_sub.config.overrides.Overrides() - -.. _parent preset: +This section is work-in-progress! preset """""" diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 451a0ed8..69512e8f 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -1,3 +1,4 @@ + Plugins ======= @@ -5,175 +6,315 @@ audio_extract ------------- Extracts audio from a video file. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 -codec -~~~~~ -The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, -opus, vorbis, wav, and best to grab the best possible format at runtime. +``codec`` -quality -~~~~~~~ -Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` -(worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. +:expected type: String +:description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. + + +``quality`` + +:expected type: Float +:description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + + +---------------------------------------------------------------------------------------------------- chapters -------- Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False -allow_chapters_from_comments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. If chapters do not exist in the video/description itself, attempt to scrape -comments to find the chapters. Defaults to False. +``allow_chapters_from_comments`` -embed_chapters -~~~~~~~~~~~~~~ -Optional. Embed chapters into the file. Defaults to True. +:expected type: Optional[Boolean] +:description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. -force_key_frames -~~~~~~~~~~~~~~~~ -Optional. Force keyframes at cuts when removing sections. This is slow due to needing a -re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to -False. -remove_chapters_regex -~~~~~~~~~~~~~~~~~~~~~ -Optional. List of regex patterns to match chapter titles against and remove them from the -entry. +``embed_chapters`` -remove_sponsorblock_categories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. List of SponsorBlock categories to remove from the output file. Can only remove -categories that are specified in ``sponsorblock_categories`` or "all", which removes -everything specified in ``sponsorblock_categories``. +:expected type: Optional[Boolean] +:description: + Defaults to True. Embed chapters into the file. -sponsorblock_categories -~~~~~~~~~~~~~~~~~~~~~~~ -Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", -"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", -"poi_highlight", or "all" to include all categories. + +``force_key_frames`` + +:expected type: Optional[Boolean] +:description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. + + +``remove_chapters_regex`` + +:expected type: Optional[List[RegexString] +:description: + List of regex patterns to match chapter titles against and remove them from the + entry. + + +``remove_sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. + + +``sponsorblock_categories`` + +:expected type: Optional[List[String]] +:description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. + + +---------------------------------------------------------------------------------------------------- date_range ---------- Only download files uploaded within the specified date range. +Dates must adhere to a yt-dlp datetime. From their docs: -Usage: +.. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + +Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. +Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest +granularity possible. + +:Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" -after -~~~~~ -Optional. Only download videos after this datetime. +``after`` -before -~~~~~~ -Optional. Only download videos before this datetime. +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +``before`` + +:expected type: Optional[OverridesFormatter] +:description: + Only download videos before this datetime. + + +---------------------------------------------------------------------------------------------------- + +download +-------- +Sets the URL(s) to download from. Can be used in many forms, including + +:Single URL: + +.. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + +:Multi URL: + +.. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + +:Thumbnails + Variables: + +All variables must be defined for the top-most url. All subsequent URL variables can be either +overwritten or default to the top-most value. + +If an entry is returned from more than one URL, it will use the variables in the bottom-most +URL. + +.. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + +---------------------------------------------------------------------------------------------------- embed_thumbnail --------------- Whether to embed thumbnails to the audio/video file or not. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True + +---------------------------------------------------------------------------------------------------- file_convert ------------ Converts video files from one extension to another. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" -Supports custom ffmpeg conversions: +Also supports custom ffmpeg conversions: + +:Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text -convert_to -~~~~~~~~~~ -Convert to a desired file type. Supports: +``convert_to`` -* Video: avi, flv, mkv, mov, mp4, webm -* Audio: aac, flac, mp3, m4a, opus, vorbis, wav +:expected type: String +:description: + Convert to a desired file type. Supports -convert_with -~~~~~~~~~~~~ -Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within -yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified -with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav -ffmpeg_post_process_args -~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the -form of: -.. code-block:: bash +``convert_with`` - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext +:expected type: Optional[String] +:description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. -The output file will use the extension specified in ``convert_to``. Post-processing args -can still be set with ``convert_with`` set to ``yt-dlp``. + +``ffmpeg_post_process_args`` + +:expected type: Optional[OverridesFormatter] +:description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of + + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. + + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. + + +---------------------------------------------------------------------------------------------------- + +filter_exclude +-------------- +Applies a conditional OR on any number of filters comprised of either variables or scripts. +If any filter evaluates to True, the entry will be excluded. + +:Usage: + +.. code-block:: yaml + + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } + +---------------------------------------------------------------------------------------------------- + +filter_include +-------------- +Applies a conditional AND on any number of filters comprised of either variables or scripts. +If all filters evaluate to True, the entry will be included. + +:Usage: + +.. code-block:: yaml + + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } + +---------------------------------------------------------------------------------------------------- format ------ @@ -184,48 +325,28 @@ Usage: .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" -format -~~~~~~ -yt-dlp format, uses same syntax as yt-dlp. +---------------------------------------------------------------------------------------------------- match_filters ------------- -Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. -Uses the same syntax as yt-dlp. +Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. +Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. +For logical AND's between match filters, use the ``&`` operator in a single match filter. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" + match_filters: + filters: + - "age_limit?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" -Supports one or multiple filters: - -.. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit`_. -Usage: +:Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" -embed_thumbnail -~~~~~~~~~~~~~~~ -Optional. Whether to embed the thumbnail into the audio file. - -tags -~~~~ -Key, values of tag names, tag values. Supports source and override variables. -Supports lists which will get written to MP3s as id3v2.4 multi-tags. +---------------------------------------------------------------------------------------------------- nfo_tags -------- Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False -kodi_safe -~~~~~~~~~ -Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some -foreign language characters. Setting this to True will replace those characters with '□'. -Defaults to False. +``kodi_safe`` -nfo_name -~~~~~~~~ -The NFO file name. +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. -nfo_root -~~~~~~~~ -The root tag of the NFO's XML. In the usage above, it would look like -.. code-block:: xml +``nfo_name`` - - - +:expected type: EntryFormatter +:description: + The NFO file name. -tags -~~~~ -Tags within the nfo_root tag. In the usage above, it would look like -.. code-block:: xml +``nfo_root`` - - - Awesome Youtube Video - 2022 - 502 - +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like -Also supports xml attributes and duplicate keys: + .. code-block:: xml -.. code-block:: yaml + + + - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" -Which translates to +``tags`` -.. code-block:: xml +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like - 2022 - Comedy - Drama + .. code-block:: xml + + + + Awesome Youtube Video + 2022 + 502 + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + 2022 + Comedy + Drama + + +---------------------------------------------------------------------------------------------------- output_directory_nfo_tags ------------------------- @@ -361,63 +484,77 @@ Usage: # optional kodi_safe: False -kodi_safe -~~~~~~~~~ -Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some -foreign language characters. Setting this to True will replace those characters with '□'. -Defaults to False. +``kodi_safe`` -nfo_name -~~~~~~~~ -The NFO file name. +:expected type: Optional[Boolean] +:description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. -nfo_root -~~~~~~~~ -The root tag of the NFO's XML. In the usage above, it would look like -.. code-block:: xml +``nfo_name`` - - - +:expected type: EntryFormatter +:description: + The NFO file name. -tags -~~~~ -Tags within the nfo_root tag. In the usage above, it would look like -.. code-block:: xml +``nfo_root`` - - - Sweet youtube TV show - +:expected type: EntryFormatter +:description: + The root tag of the NFO's XML. In the usage above, it would look like -Also supports xml attributes and duplicate keys: + .. code-block:: xml -.. code-block:: yaml + + + - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" -Which translates to +``tags`` -.. code-block:: xml +:expected type: NfoTags +:description: + Tags within the nfo_root tag. In the usage above, it would look like - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + .. code-block:: xml + + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <tvshow> + <title>Sweet youtube TV show + + + Also supports xml attributes and duplicate keys: + + .. code-block:: yaml + + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" + + Which translates to + + .. code-block:: xml + + Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> + + +---------------------------------------------------------------------------------------------------- output_options -------------- Defines where to output files and thumbnails after all post-processing has completed. -Usage: +:Usage: .. code-block:: yaml @@ -436,79 +573,113 @@ Usage: keep_files_before: now keep_files_after: 19000101 -download_archive_name -~~~~~~~~~~~~~~~~~~~~~ -Optional. The file name to store a subscriptions download archive placed relative to -the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` +``download_archive_name`` -file_name -~~~~~~~~~ -Required. The file name for the media file. This can include directories such as -``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. +:expected type: Optional[OverridesFormatter] +:description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` -info_json_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's info json file. This can include directories such -as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output -directory. Can be set to empty string or `null` to disable info json writes. -keep_files_after -~~~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. +``file_name`` -Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep -files after ``19000101``, which implies all files. Can be used in conjunction with -``keep_max_files``. +:expected type: EntryFormatter +:description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. -keep_files_before -~~~~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. -Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep -files before ``now``, which implies all files. Can be used in conjunction with -``keep_max_files``. +``info_json_name`` -keep_max_files -~~~~~~~~~~~~~~ -Optional. Requires ``maintain_download_archive`` set to True. +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. -Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be -applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. -maintain_download_archive -~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Maintains a download archive file in the output directory for a subscription. -It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the -output directory. +``keep_files_after`` -The download archive contains a mapping of ytdl IDs to downloaded files. This is used to -create a ytdl download-archive file when invoking a download on a subscription. This will -prevent ytdl from redownloading media already downloaded. +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. -Defaults to False. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. -migrated_download_archive_name -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Intended to be used if you are migrating a subscription with either a new -subscription name or output directory. It will try to load the archive file using this name -first, and fallback to ``download_archive_name``. It will always save to this file -and remove the original ``download_archive_name``. -output_directory -~~~~~~~~~~~~~~~~ -Required. The output directory to store all media files downloaded. +``keep_files_before`` -thumbnail_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's thumbnail image. This can include directories such -as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output -directory. Can be set to empty string or `null` to disable thumbnail writes. +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. + + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. + + +``keep_max_files`` + +:expected type: Optional[OverridesFormatter] +:description: + Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + + +``maintain_download_archive`` + +:expected type: Optional[Boolean] +:description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. + + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. + + Defaults to False. + + +``migrated_download_archive_name`` + +:expected type: Optional[OverridesFormatter] +:description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. + + +``output_directory`` + +:expected type: OverridesFormatter +:description: + The output directory to store all media files downloaded. + + +``thumbnail_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. + + +---------------------------------------------------------------------------------------------------- overrides --------- -Optional. This section allows you to define variables that can be used in any string formatter. -For example, if you want your file and thumbnail files to match without copy-pasting a large -format string, you can define something like: +Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + +:Usage: .. code-block:: yaml @@ -531,8 +702,45 @@ In addition, any override variable defined will automatically create a ``sanitiz for use. In the example above, ``output_directory_sanitized`` will exist and perform sanitization on the value when used. +---------------------------------------------------------------------------------------------------- + regex ----- +.. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -548,82 +756,95 @@ For example, creating the override variable ``"title_and_description": "{title} and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" -skip_if_match_fails -~~~~~~~~~~~~~~~~~~~ -Defaults to True. If True, when any match fails and has no defaults, the entry will be -skipped. If False, ytdl-sub will error and all downloads will not proceed. +``skip_if_match_fails`` + +:expected type: Optional[Boolean] +:description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. + + +---------------------------------------------------------------------------------------------------- split_by_chapters ----------------- Splits a file by chapters into multiple files. Each file becomes its own entry with the -new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, -``chapter_index_padded``, ``chapter_count``. +new variables -If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is -set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" -when_no_chapters -~~~~~~~~~~~~~~~~ -Behavior to perform when no chapters are present. Supports "pass" (continue processing), -"drop" (exclude it from output), and "error" (stop processing for everything). +``when_no_chapters`` + +:expected type: String +:description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + + +---------------------------------------------------------------------------------------------------- subtitles --------- @@ -631,43 +852,60 @@ Defines how to download and store subtitles. Using this plugin creates two new v ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False -allow_auto_generated_subtitles -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Optional. Whether to allow auto generated subtitles. Defaults to False. +``allow_auto_generated_subtitles`` -embed_subtitles -~~~~~~~~~~~~~~~ -Optional. Whether to embed the subtitles into the video file. Defaults to False. -NOTE: webm files can only embed "vtt" subtitle types. +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to allow auto generated subtitles. -languages -~~~~~~~~~ -Optional. Language code(s) to download for subtitles. Supports a single or list of multiple -language codes. Defaults to "en". -subtitles_name -~~~~~~~~~~~~~~ -Optional. The file name for the media's subtitles if they are present. This can include -directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and -will be placed in the output directory. ``lang`` is dynamic since you can download multiple -subtitles. It will set the respective language to the correct subtitle file. +``embed_subtitles`` -subtitles_type -~~~~~~~~~~~~~~ -Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" +:expected type: Optional[Boolean] +:description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. + + +``languages`` + +:expected type: Optional[List[String]] +:description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". + + +``subtitles_name`` + +:expected type: Optional[EntryFormatter] +:description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. + + +``subtitles_type`` + +:expected type: Optional[String] +:description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". + + +---------------------------------------------------------------------------------------------------- throttle_protection ------------------- @@ -675,7 +913,7 @@ Provides options to make ytdl-sub look more 'human-like' to protect from throttl range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. -Usage: +:Usage: .. code-block:: yaml @@ -693,52 +931,62 @@ Usage: max: 36 subscription_download_probability: 1.0 -max_downloads_per_subscription -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Number of downloads to perform per subscription. +``max_downloads_per_subscription`` -sleep_per_download_s -~~~~~~~~~~~~~~~~~~~~ -Number in seconds to sleep between each download. Does not include time it takes for -ytdl-sub to perform post-processing. +:expected type: Optional[Range] +:description: + Number of downloads to perform per subscription. -sleep_per_subscription_s -~~~~~~~~~~~~~~~~~~~~~~~~ -Number in seconds to sleep between each subscription. -subscription_download_probability -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Probability to perform any downloads, recomputed for each subscription. This is only -recommended to set if you run ytdl-sub in a cron-job, that way you are statistically -guaranteed over time to eventually download the subscription. +``sleep_per_download_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. + + +``sleep_per_subscription_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each subscription. + + +``subscription_download_probability`` + +:expected type: Optional[Float] +:description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. + + +---------------------------------------------------------------------------------------------------- video_tags ---------- Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. -Usage: +:Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" -tags -~~~~ -Key/values of tag names/values. Supports source and override variables. +---------------------------------------------------------------------------------------------------- ytdl_options ------------ -Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. +Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. -ytdl_options should be formatted like: +:Usage: .. code-block:: yaml diff --git a/docs/source/config_reference/scripting/config_types.rst b/docs/source/config_reference/scripting/config_types.rst deleted file mode 100644 index 31273cde..00000000 --- a/docs/source/config_reference/scripting/config_types.rst +++ /dev/null @@ -1,17 +0,0 @@ -================== -Config Field Types -================== - -The ``config.yaml`` uses various types for its configurable fields. Below is a definition for each type. - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator() - -.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator() - -.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator() - -.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator() diff --git a/docs/source/config_reference/scripting/entry_variables.rst b/docs/source/config_reference/scripting/entry_variables.rst index cbd3c0cf..c4467330 100644 --- a/docs/source/config_reference/scripting/entry_variables.rst +++ b/docs/source/config_reference/scripting/entry_variables.rst @@ -1,3 +1,4 @@ + Entry Variables =============== @@ -111,6 +112,8 @@ webpage_url ~~~~~~~~~~~ The url to the webpage. +---------------------------------------------------------------------------------------------------- + Metadata Variables ------------------ @@ -130,6 +133,8 @@ source_metadata ~~~~~~~~~~~~~~~ Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) +---------------------------------------------------------------------------------------------------- + Playlist Variables ------------------ @@ -210,6 +215,8 @@ playlist_webpage_url ~~~~~~~~~~~~~~~~~~~~ The playlist webpage url if it exists. Otherwise, returns the entry webpage url. +---------------------------------------------------------------------------------------------------- + Release Date Variables ---------------------- @@ -285,6 +292,8 @@ release_year_truncated_reversed The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. 2022 returns ``100 - 22`` = ``78`` +---------------------------------------------------------------------------------------------------- + Source Variables ---------------- @@ -332,6 +341,8 @@ source_webpage_url ~~~~~~~~~~~~~~~~~~ The source webpage url if it exists, otherwise returns the playlist webpage url. +---------------------------------------------------------------------------------------------------- + Upload Date Variables --------------------- @@ -406,6 +417,8 @@ upload_year_truncated_reversed The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. 2022 returns ``100 - 22`` = ``78`` +---------------------------------------------------------------------------------------------------- + Ytdl-Sub Variables ------------------ @@ -436,4 +449,4 @@ The upload_date_index padded two digits ytdl_sub_input_url ~~~~~~~~~~~~~~~~~~ -The input URL used in ytdl-sub to create this entry. \ No newline at end of file +The input URL used in ytdl-sub to create this entry. diff --git a/docs/source/config_reference/scripting/index.rst b/docs/source/config_reference/scripting/index.rst index c6ae4532..c99105f8 100644 --- a/docs/source/config_reference/scripting/index.rst +++ b/docs/source/config_reference/scripting/index.rst @@ -2,8 +2,11 @@ Scripting ========= -.. toctree:: +Work in progress! Explanation of how to define scripts/variables will be added here. + +.. toctree:: + :maxdepth: 1 + entry_variables override_variables - scripting_functions - config_types \ No newline at end of file + scripting_functions \ No newline at end of file diff --git a/docs/source/config_reference/scripting/override_variables.rst b/docs/source/config_reference/scripting/override_variables.rst index 7cb23e4a..3b02c4c7 100644 --- a/docs/source/config_reference/scripting/override_variables.rst +++ b/docs/source/config_reference/scripting/override_variables.rst @@ -1,3 +1,4 @@ + Override Variables ================== @@ -41,4 +42,3 @@ For subscriptions in the form of ``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...`` and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to ``subscription_value``. - diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index b462acf0..d757f07c 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -7,414 +7,571 @@ Array Functions array ~~~~~ -``array(maybe_array: AnyArgument) -> Array`` +:spec: ``array(maybe_array: AnyArgument) -> Array`` -Tries to cast an unknown variable type to an Array. +:description: + Tries to cast an unknown variable type to an Array. array_apply ~~~~~~~~~~~ -``array_apply(array: Array, lambda_function: Lambda) -> Array`` +:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array`` -Apply a lambda function on every element in the Array. +:description: + Apply a lambda function on every element in the Array. +:usage: + +.. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] + +array_apply_fixed +~~~~~~~~~~~~~~~~~ +:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array`` + +:description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. array_at ~~~~~~~~ -``array_at(array: Array, idx: Integer) -> AnyArgument`` +:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` -Return the element in the Array at index ``idx``. +:description: + Return the element in the Array at index ``idx``. array_contains ~~~~~~~~~~~~~~ -``array_contains(array: Array, value: AnyArgument) -> Boolean`` +:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean`` -Return True if the value exists in the Array. False otherwise. +:description: + Return True if the value exists in the Array. False otherwise. array_enumerate ~~~~~~~~~~~~~~~ -``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` +:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array`` -Apply a lambda function on every element in the Array, where each arg -passed to the lambda function is ``idx, element`` as two separate args. +:description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. array_extend ~~~~~~~~~~~~ -``array_extend(arrays: Array, ...) -> Array`` +:spec: ``array_extend(arrays: Array, ...) -> Array`` -Combine multiple Arrays into a single Array. +:description: + Combine multiple Arrays into a single Array. + +array_first +~~~~~~~~~~~ +:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument`` + +:description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. array_flatten ~~~~~~~~~~~~~ -``array_flatten(array: Array) -> Array`` +:spec: ``array_flatten(array: Array) -> Array`` -Flatten any nested Arrays into a single-dimensional Array. +:description: + Flatten any nested Arrays into a single-dimensional Array. array_index ~~~~~~~~~~~ -``array_index(array: Array, value: AnyArgument) -> Integer`` +:spec: ``array_index(array: Array, value: AnyArgument) -> Integer`` -Return the index of the value within the Array if it exists. If it does not, it will -throw an error. +:description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. + +array_overlay +~~~~~~~~~~~~~ +:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array`` + +:description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. array_product ~~~~~~~~~~~~~ -``array_product(arrays: Array, ...) -> Array`` +:spec: ``array_product(arrays: Array, ...) -> Array`` -Returns the Cartesian product of elements from different arrays +:description: + Returns the Cartesian product of elements from different arrays array_reduce ~~~~~~~~~~~~ -``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` +:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument`` -Apply a reduce function on pairs of elements in the Array, until one element remains. -Executes using the left-most and reduces in the right direction. +:description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. array_reverse ~~~~~~~~~~~~~ -``array_reverse(array: Array) -> Array`` +:spec: ``array_reverse(array: Array) -> Array`` -Reverse an Array. +:description: + Reverse an Array. array_size ~~~~~~~~~~ -``array_size(array: Array) -> Integer`` +:spec: ``array_size(array: Array) -> Integer`` -Returns the size of an Array. +:description: + Returns the size of an Array. array_slice ~~~~~~~~~~~ -``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` +:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array`` -Returns the slice of the Array. +:description: + Returns the slice of the Array. + +---------------------------------------------------------------------------------------------------- Boolean Functions ----------------- and ~~~ -``and(values: AnyArgument, ...) -> Boolean`` +:spec: ``and(values: AnyArgument, ...) -> Boolean`` -``and`` operator. Returns True if all values evaluate to True. False otherwise. +:description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. bool ~~~~ -``bool(value: AnyArgument) -> Boolean`` +:spec: ``bool(value: AnyArgument) -> Boolean`` -Cast any type to a Boolean. +:description: + Cast any type to a Boolean. eq ~~ -``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean`` -``==`` operator. Returns True if left == right. False otherwise. +:description: + ``==`` operator. Returns True if left == right. False otherwise. gt ~~ -``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean`` -``>`` operator. Returns True if left > right. False otherwise. +:description: + ``>`` operator. Returns True if left > right. False otherwise. gte ~~~ -``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean`` -``>=`` operator. Returns True if left >= right. False otherwise. +:description: + ``>=`` operator. Returns True if left >= right. False otherwise. + +is_null +~~~~~~~ +:spec: ``is_null(value: AnyArgument) -> Boolean`` + +:description: + Returns True if a value is null (i.e. an empty string). False otherwise. lt ~~ -``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean`` -``<`` operator. Returns True if left < right. False otherwise. +:description: + ``<`` operator. Returns True if left < right. False otherwise. lte ~~~ -``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean`` -``<=`` operator. Returns True if left <= right. False otherwise. +:description: + ``<=`` operator. Returns True if left <= right. False otherwise. ne ~~ -``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` +:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean`` -``!=`` operator. Returns True if left != right. False otherwise. +:description: + ``!=`` operator. Returns True if left != right. False otherwise. not ~~~ -``not(value: Boolean) -> Boolean`` +:spec: ``not(value: Boolean) -> Boolean`` -``not`` operator. Returns the opposite of value. +:description: + ``not`` operator. Returns the opposite of value. or ~~ -``or(values: AnyArgument, ...) -> Boolean`` +:spec: ``or(values: AnyArgument, ...) -> Boolean`` -``or`` operator. Returns True if any value evaluates to True. False otherwise. +:description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. xor ~~~ -``xor(values: AnyArgument, ...) -> Boolean`` +:spec: ``xor(values: AnyArgument, ...) -> Boolean`` -``^`` operator. Returns True if exactly one value is set to True. False otherwise. +:description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + +---------------------------------------------------------------------------------------------------- Conditional Functions --------------------- if ~~ -``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` +:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` -Conditional ``if`` statement that returns the ``true`` or ``false`` parameter -depending on the ``condition`` value. +:description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. if_passthrough ~~~~~~~~~~~~~~ -``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` +:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` -Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, -otherwise returns ``else_arg``. +:description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. + +---------------------------------------------------------------------------------------------------- Date Functions -------------- datetime_strftime ~~~~~~~~~~~~~~~~~ -``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` +:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String`` -Converts a posix timestamp to a date using strftime formatting. +:description: + Converts a posix timestamp to a date using strftime formatting. + +---------------------------------------------------------------------------------------------------- Error Functions --------------- assert ~~~~~~ -``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` +:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument`` -Explicitly throw an error with the provided assert message if ``value`` evaluates to False. -If it evaluates to True, it will return ``value``. +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. + +assert_eq +~~~~~~~~~ +:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. + +assert_ne +~~~~~~~~~ +:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. + +assert_then +~~~~~~~~~~~ +:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument`` + +:description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. throw ~~~~~ -``throw(error_message: String) -> AnyArgument`` +:spec: ``throw(error_message: String) -> AnyArgument`` -Explicitly throw an error with the provided error message. +:description: + Explicitly throw an error with the provided error message. + +---------------------------------------------------------------------------------------------------- Json Functions -------------- from_json ~~~~~~~~~ -``from_json(argument: String) -> AnyArgument`` +:spec: ``from_json(argument: String) -> AnyArgument`` -Converts a JSON string into an actual type. +:description: + Converts a JSON string into an actual type. + +---------------------------------------------------------------------------------------------------- Map Functions ------------- map ~~~ -``map(maybe_mapping: AnyArgument) -> Map`` +:spec: ``map(maybe_mapping: AnyArgument) -> Map`` -Tries to cast an unknown variable type to a Map. +:description: + Tries to cast an unknown variable type to a Map. map_apply ~~~~~~~~~ -``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` +:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array`` -Apply a lambda function on the Map, where each arg -passed to the lambda function is ``key, value`` as two separate args. +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. map_contains ~~~~~~~~~~~~ -``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` +:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean`` -Returns True if the key is in the Map. False otherwise. +:description: + Returns True if the key is in the Map. False otherwise. map_enumerate ~~~~~~~~~~~~~ -``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` +:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array`` -Apply a lambda function on the Map, where each arg -passed to the lambda function is ``idx, key, value`` as three separate args. +:description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. map_get ~~~~~~~ -``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` +:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument`` -Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is -provided, it will return ``default``. Otherwise, will error. +:description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. map_get_non_empty ~~~~~~~~~~~~~~~~~ -``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` +:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument`` -Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, -return ``default``. Otherwise, will error. +:description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. map_size ~~~~~~~~ -``map_size(mapping: Map) -> Integer`` +:spec: ``map_size(mapping: Map) -> Integer`` -Returns the size of a Map. +:description: + Returns the size of a Map. + +---------------------------------------------------------------------------------------------------- Numeric Functions ----------------- add ~~~ -``add(values: Numeric, ...) -> Numeric`` +:spec: ``add(values: Numeric, ...) -> Numeric`` -``+`` operator. Returns the sum of all values. +:description: + ``+`` operator. Returns the sum of all values. div ~~~ -``div(left: Numeric, right: Numeric) -> Numeric`` +:spec: ``div(left: Numeric, right: Numeric) -> Numeric`` -``/`` operator. Returns ``left / right``. +:description: + ``/`` operator. Returns ``left / right``. float ~~~~~ -``float(value: AnyArgument) -> Float`` +:spec: ``float(value: AnyArgument) -> Float`` -Cast to Float. +:description: + Cast to Float. int ~~~ -``int(value: AnyArgument) -> Integer`` +:spec: ``int(value: AnyArgument) -> Integer`` -Cast to Integer. +:description: + Cast to Integer. max ~~~ -``max(values: Numeric, ...) -> Numeric`` +:spec: ``max(values: Numeric, ...) -> Numeric`` -Returns max of all values. +:description: + Returns max of all values. min ~~~ -``min(values: Numeric, ...) -> Numeric`` +:spec: ``min(values: Numeric, ...) -> Numeric`` -Returns min of all values. +:description: + Returns min of all values. mod ~~~ -``mod(left: Numeric, right: Numeric) -> Numeric`` +:spec: ``mod(left: Numeric, right: Numeric) -> Numeric`` -``%`` operator. Returns ``left % right``. +:description: + ``%`` operator. Returns ``left % right``. mul ~~~ -``mul(values: Numeric, ...) -> Numeric`` +:spec: ``mul(values: Numeric, ...) -> Numeric`` -``*`` operator. Returns the product of all values. +:description: + ``*`` operator. Returns the product of all values. pow ~~~ -``pow(base: Numeric, exponent: Numeric) -> Numeric`` +:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric`` -``**`` operator. Returns the exponential of the base and exponent value. +:description: + ``**`` operator. Returns the exponential of the base and exponent value. sub ~~~ -``sub(values: Numeric, ...) -> Numeric`` +:spec: ``sub(values: Numeric, ...) -> Numeric`` -``-`` operator. Subtracts all values from left to right. +:description: + ``-`` operator. Subtracts all values from left to right. + +---------------------------------------------------------------------------------------------------- Regex Functions --------------- +regex_capture_groups +~~~~~~~~~~~~~~~~~~~~ +:spec: ``regex_capture_groups(regex: String) -> Integer`` + +:description: + Returns number of capture groups in regex + regex_fullmatch ~~~~~~~~~~~~~~~ -``regex_fullmatch(regex: String, string: String) -> Array`` +:spec: ``regex_fullmatch(regex: String, string: String) -> Array`` -Checks for entire string to be a match. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. regex_match ~~~~~~~~~~~ -``regex_match(regex: String, string: String) -> Array`` +:spec: ``regex_match(regex: String, string: String) -> Array`` -Checks for a match only at the beginning of the string. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. regex_search ~~~~~~~~~~~~ -``regex_search(regex: String, string: String) -> Array`` +:spec: ``regex_search(regex: String, string: String) -> Array`` -Checks for a match anywhere in the string. If a match exists, returns -the string as the first element of the Array. If there are capture groups, returns each -group as a subsequent element in the Array. +:description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. + +---------------------------------------------------------------------------------------------------- String Functions ---------------- capitalize ~~~~~~~~~~ -``capitalize(string: String) -> String`` +:spec: ``capitalize(string: String) -> String`` -Capitalize the first character in the string. +:description: + Capitalize the first character in the string. concat ~~~~~~ -``concat(values: String, ...) -> String`` +:spec: ``concat(values: String, ...) -> String`` -Concatenate multiple Strings into a single String. +:description: + Concatenate multiple Strings into a single String. + +contains +~~~~~~~~ +:spec: ``contains(string: String, contains: String) -> Boolean`` + +:description: + Returns True if ``contains`` is in ``string``. False otherwise. lower ~~~~~ -``lower(string: String) -> String`` +:spec: ``lower(string: String) -> String`` -Lower-case the entire String. +:description: + Lower-case the entire String. pad ~~~ -``pad(string: String, length: Integer, char: String) -> String`` +:spec: ``pad(string: String, length: Integer, char: String) -> String`` -Pads the string to the given length +:description: + Pads the string to the given length pad_zero ~~~~~~~~ -``pad_zero(numeric: Numeric, length: Integer) -> String`` +:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String`` -Pads a numeric with zeros to the given length +:description: + Pads a numeric with zeros to the given length replace ~~~~~~~ -``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` +:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String`` -Replace the ``old`` part of the String with the ``new``. Optionally only replace it -``count`` number of times. +:description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. slice ~~~~~ -``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` +:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String`` -Returns the slice of the Array. +:description: + Returns the slice of the Array. string ~~~~~~ -``string(value: AnyArgument) -> String`` +:spec: ``string(value: AnyArgument) -> String`` -Cast to String. +:description: + Cast to String. titlecase ~~~~~~~~~ -``titlecase(string: String) -> String`` +:spec: ``titlecase(string: String) -> String`` -Capitalize each word in the string. +:description: + Capitalize each word in the string. upper ~~~~~ -``upper(string: String) -> String`` +:spec: ``upper(string: String) -> String`` -Upper-case the entire String. +:description: + Upper-case the entire String. + +---------------------------------------------------------------------------------------------------- Ytdl-Sub Functions ------------------ legacy_bracket_safety ~~~~~~~~~~~~~~~~~~~~~ -``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` +:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument`` ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not interfere with its legacy variable scripting system. This function replicates that @@ -422,14 +579,14 @@ behavior. sanitize ~~~~~~~~ -``sanitize(value: AnyArgument) -> String`` +:spec: ``sanitize(value: AnyArgument) -> String`` Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use for file/directory names on any OS. sanitize_plex_episode ~~~~~~~~~~~~~~~~~~~~~ -``sanitize_plex_episode(string: String) -> String`` +:spec: ``sanitize_plex_episode(string: String) -> String`` Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the @@ -437,7 +594,7 @@ season and/or episode. to_date_metadata ~~~~~~~~~~~~~~~~ -``to_date_metadata(yyyymmdd: String) -> Map`` +:spec: ``to_date_metadata(yyyymmdd: String) -> Map`` Takes a date in the form of YYYYMMDD and returns a Map containing: @@ -461,14 +618,14 @@ Takes a date in the form of YYYYMMDD and returns a Map containing: to_native_filepath ~~~~~~~~~~~~~~~~~~ -``to_native_filepath(filepath: String) -> String`` +:spec: ``to_native_filepath(filepath: String) -> String`` Convert any unix-based path separators ('/') with the OS's native separator. truncate_filepath_if_too_long ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``truncate_filepath_if_too_long(filepath: String) -> String`` +:spec: ``truncate_filepath_if_too_long(filepath: String) -> String`` If a file-path is too long for the OS, this function will truncate it while preserving the extension. diff --git a/docs/source/prebuilt_presets/tv_shows.rst b/docs/source/prebuilt_presets/tv_shows.rst index cded4940..f2b1da1d 100644 --- a/docs/source/prebuilt_presets/tv_shows.rst +++ b/docs/source/prebuilt_presets/tv_shows.rst @@ -18,7 +18,7 @@ Jellyfin Kodi ~~~~ * Everything that the Jellyfin version does -* Turns on :ref:`config_reference/plugins:kodi_safe`, replacing characters that would break kodi with safer characters +* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with ``□`` Plex ~~~~ diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 8f6903d2..b576e1ad 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -8,7 +8,7 @@ import mergedeep from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME -from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.entries.variables.override_variables import OverrideHelpers from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved @@ -23,9 +23,9 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid class Overrides(DictFormatterValidator, Scriptable): """ - Optional. This section allows you to define variables that can be used in any string formatter. - For example, if you want your file and thumbnail files to match without copy-pasting a large - format string, you can define something like: + Allows you to define variables that can be used in any EntryFormatter or OverridesFormatter. + + :Usage: .. code-block:: yaml @@ -89,7 +89,7 @@ class Overrides(DictFormatterValidator, Scriptable): """ Ensures the variable name does not collide with any entry variables or built-in functions. """ - if not OverrideVariables.is_valid_name(name): + if not OverrideHelpers.is_valid_name(name): override_type = "function" if name.startswith("%") else "variable" raise self._validation_exception( f"Override {override_type} with name {name} is invalid. Names must be" @@ -97,14 +97,14 @@ class Overrides(DictFormatterValidator, Scriptable): exception_class=InvalidVariableNameException, ) - if OverrideVariables.is_entry_variable_name(name): + if OverrideHelpers.is_entry_variable_name(name): raise self._validation_exception( f"Override variable with name {name} cannot be used since it is a" " built-in ytdl-sub entry variable name.", exception_class=InvalidVariableNameException, ) - if OverrideVariables.is_function_name(name): + if OverrideHelpers.is_function_name(name): raise self._validation_exception( f"Override function definition with name {name} cannot be used since it is" " a built-in ytdl-sub function name.", diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index d6b2bc2e..8cc4697c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -15,12 +15,12 @@ from ytdl_sub.validators.validators import LiteralDictValidator class YTDLOptions(LiteralDictValidator): """ - Optional. This section allows you to add any ytdl argument to ytdl-sub's downloader. + Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_ for more details. - ytdl_options should be formatted like: + :Usage: .. code-block:: yaml @@ -58,7 +58,7 @@ class OutputOptions(StrictDictValidator): """ Defines where to output files and thumbnails after all post-processing has completed. - Usage: + :Usage: .. code-block:: yaml @@ -156,97 +156,119 @@ class OutputOptions(StrictDictValidator): @property def output_directory(self) -> OverridesStringFormatterValidator: """ - Required. The output directory to store all media files downloaded. + :expected type: OverridesFormatter + :description: + The output directory to store all media files downloaded. """ return self._output_directory @property def file_name(self) -> StringFormatterValidator: """ - Required. The file name for the media file. This can include directories such as - ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. + :expected type: EntryFormatter + :description: + The file name for the media file. This can include directories such as + ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. """ return self._file_name @property def thumbnail_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's thumbnail image. This can include directories such - as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable thumbnail writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's thumbnail image. This can include directories such + as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable thumbnail writes. """ return self._thumbnail_name @property def info_json_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's info json file. This can include directories such - as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output - directory. Can be set to empty string or `null` to disable info json writes. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's info json file. This can include directories such + as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output + directory. Can be set to empty string or `null` to disable info json writes. """ return self._info_json_name @property def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. The file name to store a subscriptions download archive placed relative to - the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` + :expected type: Optional[OverridesFormatter] + :description: + The file name to store a subscriptions download archive placed relative to + the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` """ return self._download_archive_name @property def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. Intended to be used if you are migrating a subscription with either a new - subscription name or output directory. It will try to load the archive file using this name - first, and fallback to ``download_archive_name``. It will always save to this file - and remove the original ``download_archive_name``. + :expected type: Optional[OverridesFormatter] + :description: + Intended to be used if you are migrating a subscription with either a new + subscription name or output directory. It will try to load the archive file using this + name first, and fallback to ``download_archive_name``. It will always save to this file + and remove the original ``download_archive_name``. """ return self._migrated_download_archive_name @property def maintain_download_archive(self) -> bool: """ - Optional. Maintains a download archive file in the output directory for a subscription. - It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the - output directory. + :expected type: Optional[Boolean] + :description: + Maintains a download archive file in the output directory for a subscription. + It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the + output directory. - The download archive contains a mapping of ytdl IDs to downloaded files. This is used to - create a ytdl download-archive file when invoking a download on a subscription. This will - prevent ytdl from redownloading media already downloaded. + The download archive contains a mapping of ytdl IDs to downloaded files. This is used to + create a ytdl download-archive file when invoking a download on a subscription. This will + prevent ytdl from redownloading media already downloaded. - Defaults to False. + Defaults to False. """ return self._maintain_download_archive.value @property def keep_files_before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep - files before ``now``, which implies all files. Can be used in conjunction with - ``keep_max_files``. + Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_before @property def keep_files_after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. Uses the same syntax as the + ``date_range`` plugin. - Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep - files after ``19000101``, which implies all files. Can be used in conjunction with - ``keep_max_files``. + Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_after @property def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: """ - Optional. Requires ``maintain_download_archive`` set to True. + :expected type: Optional[OverridesFormatter] + :description: + Requires ``maintain_download_archive`` set to True. - Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be - applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. """ return self._keep_max_files diff --git a/src/ytdl_sub/downloaders/url/multi_url.py b/src/ytdl_sub/downloaders/url/multi_url.py deleted file mode 100644 index b55b8eed..00000000 --- a/src/ytdl_sub/downloaders/url/multi_url.py +++ /dev/null @@ -1,37 +0,0 @@ -from ytdl_sub.downloaders.url.validators import MultiUrlValidator - - -# TODO: Remove later - keep for docstring -class MultiUrlDownloadOptions(MultiUrlValidator): - """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - urls: - - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "1" - season_name: "Uploads" - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - variables: - season_index: "2" - season_name: "Playlist as Season" - playlist_thumbnails: - - name: "season{season_index}-poster.jpg" - uid: "latest_entry" - """ diff --git a/src/ytdl_sub/downloaders/url/url.py b/src/ytdl_sub/downloaders/url/url.py deleted file mode 100644 index e5589645..00000000 --- a/src/ytdl_sub/downloaders/url/url.py +++ /dev/null @@ -1,25 +0,0 @@ -from ytdl_sub.downloaders.url.validators import UrlValidator - - -# TODO: Remove later - keep for docstring -class UrlDownloadOptions(UrlValidator): - """ - Downloads from a single URL supported by yt-dlp. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - download: - # required - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" - # optional - playlist_thumbnails: - - name: "poster.jpg" - uid: "avatar_uncropped" - - name: "fanart.jpg" - uid: "banner_uncropped" - download_reverse: True - """ diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index eb999b15..631ee979 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -215,8 +215,53 @@ class UrlListValidator(ListValidator[UrlStringOrDictValidator]): class MultiUrlValidator(OptionsValidator): """ - Downloads from multiple URLs. If an entry is returned from more than one URL, it will - resolve to the bottom-most URL settings. + Sets the URL(s) to download from. Can be used in many forms, including + + :Single URL: + + .. code-block:: yaml + + download: "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + + :Multi URL: + + .. code-block:: yaml + + download: + - "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + - "https://www.youtube.com/watch?v=3BFTio5296w" + + :Thumbnails + Variables: + + All variables must be defined for the top-most url. All subsequent URL variables can be either + overwritten or default to the top-most value. + + If an entry is returned from more than one URL, it will use the variables in the bottom-most + URL. + + .. code-block:: yaml + + download: + # required + urls: + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" """ @classmethod diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 6c576db4..e711a214 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1,3 +1,4 @@ +from abc import ABC from dataclasses import dataclass # This file contains mixins to a BaseEntry subclass. Ignore pylint's "no kwargs member" suggestion @@ -35,51 +36,567 @@ class SiblingMetadata(MetadataVariable): pass -class VariableDefinitions: +class MetadataVariableDefinitions(ABC): @property - def entry_metadata(self) -> Metadata: + def entry_metadata(self: "VariableDefinitions") -> Metadata: """ The entry's info.json """ return Metadata("entry_metadata") @property - def playlist_metadata(self) -> RelativeMetadata: + def playlist_metadata(self: "VariableDefinitions") -> RelativeMetadata: """ Metadata from the playlist (i.e. the parent metadata, like playlist -> entry) """ return RelativeMetadata("playlist_metadata", metadata_key="playlist_metadata") @property - def source_metadata(self) -> RelativeMetadata: + def source_metadata(self: "VariableDefinitions") -> RelativeMetadata: """ Metadata from the source (i.e. the grandparent metadata, like channel -> playlist -> entry) """ return RelativeMetadata("source_metadata", metadata_key="source_metadata") @property - def sibling_metadata(self) -> SiblingMetadata: + def sibling_metadata(self: "VariableDefinitions") -> SiblingMetadata: """ Metadata from any sibling entries that reside in the same playlist as this entry. """ return SiblingMetadata("sibling_metadata", metadata_key="sibling_metadata") + +class PlaylistVariableDefinitions(ABC): @property - def uid(self) -> MetadataVariable: + def playlist_uid(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist unique ID if it exists, otherwise return the entry unique ID. + """ + return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") + + @property + def playlist_title(self: "VariableDefinitions") -> MetadataVariable: + """ + Name of its parent playlist/channel if it exists, otherwise returns its title. + """ + return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") + + @property + def playlist_index(self: "VariableDefinitions") -> MetadataVariable: + """ + Playlist index if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") + + @property + def playlist_index_reversed(self: "VariableDefinitions") -> Variable: + """ + Playlist index reversed via ``playlist_count - playlist_index + 1`` + """ + return Variable("playlist_index_reversed") + + @property + def playlist_index_padded(self: "VariableDefinitions") -> Variable: + """ + playlist_index padded two digits + """ + return Variable("playlist_index_padded") + + @property + def playlist_index_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + playlist_index_reversed padded two digits + """ + return Variable("playlist_index_reversed_padded") + + @property + def playlist_index_padded6(self: "VariableDefinitions") -> Variable: + """ + playlist_index padded six digits. + """ + return Variable("playlist_index_padded6") + + @property + def playlist_index_reversed_padded6(self: "VariableDefinitions") -> Variable: + """ + playlist_index_reversed padded six digits. + """ + return Variable("playlist_index_reversed_padded6") + + @property + def playlist_count(self: "VariableDefinitions") -> MetadataVariable: + """ + Playlist count if it exists, otherwise returns ``1``. + + Note that for channels/playlists, any change (i.e. adding or removing a video) will make + this value change. Use with caution. + """ + return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") + + @property + def playlist_description(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist description if it exists, otherwise returns the entry's description. + """ + return MetadataVariable( + variable_name="playlist_description", metadata_key=self.description.metadata_key + ) + + @property + def playlist_webpage_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist webpage url if it exists. Otherwise, returns the entry webpage url. + """ + return MetadataVariable( + variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key + ) + + @property + def playlist_max_upload_date(self: "VariableDefinitions") -> Variable: + """ + Max upload_date for all entries in this entry's playlist if it exists, otherwise returns + ``upload_date`` + """ + return Variable("playlist_max_upload_date") + + @property + def playlist_max_upload_year(self: "VariableDefinitions") -> Variable: + """ + Max upload_year for all entries in this entry's playlist if it exists, otherwise returns + ``upload_year`` + """ + # override in EntryParent + return Variable("playlist_max_upload_year") + + @property + def playlist_max_upload_year_truncated(self: "VariableDefinitions") -> Variable: + """ + The max playlist truncated upload year for all entries in this entry's playlist if it + exists, otherwise returns ``upload_year_truncated``. + """ + return Variable("playlist_max_upload_year_truncated") + + @property + def playlist_uploader_id(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist uploader id if it exists, otherwise returns the entry uploader ID. + """ + return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") + + @property + def playlist_uploader(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist uploader if it exists, otherwise return the entry uploader. + """ + return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) + + @property + def playlist_uploader_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The playlist uploader url if it exists, otherwise returns the playlist webpage_url. + """ + return MetadataVariable( + "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key + ) + + +class SourceVariableDefinitions(ABC): + @property + def source_title(self: "VariableDefinitions") -> MetadataVariable: + """ + Name of the source (i.e. channel with multiple playlists) if it exists, otherwise + returns its playlist_title. + """ + return MetadataVariable("source_title", metadata_key=self.title.metadata_key) + + @property + def source_uid(self: "VariableDefinitions") -> MetadataVariable: + """ + The source unique id if it exists, otherwise returns the playlist unique ID. + """ + return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) + + @property + def source_index(self: "VariableDefinitions") -> MetadataVariable: + """ + Source index if it exists, otherwise returns ``1``. + + It is recommended to not use this unless you know the source will never add new content + (it is easy for this value to change). + """ + return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) + + @property + def source_index_padded(self: "VariableDefinitions") -> Variable: + """ + The source index, padded. + """ + return Variable("source_index_padded") + + @property + def source_count(self: "VariableDefinitions") -> MetadataVariable: + """ + The source count if it exists, otherwise returns the playlist count. + """ + return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) + + @property + def source_webpage_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The source webpage url if it exists, otherwise returns the playlist webpage url. + """ + return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) + + @property + def source_description(self: "VariableDefinitions") -> MetadataVariable: + """ + The source description if it exists, otherwise returns the playlist description. + """ + return MetadataVariable("source_description", metadata_key=self.description.metadata_key) + + @property + def source_uploader_id(self: "VariableDefinitions") -> MetadataVariable: + """ + The source uploader id if it exists, otherwise returns the playlist_uploader_id + """ + return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) + + @property + def source_uploader(self: "VariableDefinitions") -> MetadataVariable: + """ + The source uploader if it exists, otherwise return the playlist_uploader + """ + return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) + + @property + def source_uploader_url(self: "VariableDefinitions") -> MetadataVariable: + """ + The source uploader url if it exists, otherwise returns the source webpage_url. + """ + return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) + + +class UploadDateVariableDefinitions(ABC): + @property + def upload_date(self: "VariableDefinitions") -> MetadataVariable: + """ + The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. + """ + return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") + + @property + def upload_year(self: "VariableDefinitions") -> Variable: + """ + The entry's upload year + """ + return Variable("upload_year") + + @property + def upload_year_truncated(self: "VariableDefinitions") -> Variable: + """ + The last two digits of the upload year, i.e. 22 in 2022 + """ + return Variable("upload_year_truncated") + + @property + def upload_year_truncated_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return Variable("upload_year_truncated_reversed") + + @property + def upload_month_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` + """ + return Variable("upload_month_reversed") + + @property + def upload_month_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed upload month, but padded. i.e. November returns "02" + """ + return Variable("upload_month_reversed_padded") + + @property + def upload_month_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's upload month padded to two digits, i.e. March returns "03" + """ + return Variable("upload_month_padded") + + @property + def upload_day_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's upload day padded to two digits, i.e. the fifth returns "05" + """ + return Variable("upload_day_padded") + + @property + def upload_month(self: "VariableDefinitions") -> Variable: + """ + The upload month as an integer (no padding). + """ + return Variable("upload_month") + + @property + def upload_day(self: "VariableDefinitions") -> Variable: + """ + The upload day as an integer (no padding). + """ + return Variable("upload_day") + + @property + def upload_day_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, + i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return Variable("upload_day_reversed") + + @property + def upload_day_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed upload day, but padded. i.e. August 30th returns "02". + """ + return Variable("upload_day_reversed_padded") + + @property + def upload_day_of_year(self: "VariableDefinitions") -> Variable: + """ + The day of the year, i.e. February 1st returns ``32`` + """ + return Variable("upload_day_of_year") + + @property + def upload_day_of_year_padded(self: "VariableDefinitions") -> Variable: + """ + The upload day of year, but padded i.e. February 1st returns "032" + """ + return Variable("upload_day_of_year_padded") + + @property + def upload_day_of_year_reversed(self: "VariableDefinitions") -> Variable: + """ + The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, + i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return Variable("upload_day_of_year_reversed") + + @property + def upload_day_of_year_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed upload day of year, but padded i.e. December 31st returns "001" + """ + return Variable("upload_day_of_year_reversed_padded") + + @property + def upload_date_standardized(self: "VariableDefinitions") -> Variable: + """ + The uploaded date formatted as YYYY-MM-DD + """ + return Variable("upload_date_standardized") + + +class ReleaseDateVariableDefinitions(ABC): + @property + def release_date(self: "VariableDefinitions") -> MetadataVariable: + """ + The entry’s release date, in YYYYMMDD format. If not present, return the upload date. + """ + return MetadataVariable(variable_name="release_date", metadata_key="release_date") + + @property + def release_year(self: "VariableDefinitions") -> Variable: + """ + The entry's release year + """ + return Variable("release_year") + + @property + def release_year_truncated(self: "VariableDefinitions") -> Variable: + """ + The last two digits of the release year, i.e. 22 in 2022 + """ + return Variable("release_year_truncated") + + @property + def release_year_truncated_reversed(self: "VariableDefinitions") -> Variable: + """ + The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. + 2022 returns ``100 - 22`` = ``78`` + """ + return Variable("release_year_truncated_reversed") + + @property + def release_month_reversed(self: "VariableDefinitions") -> Variable: + """ + The release month, but reversed + using ``13 - {release_month}``, i.e. March returns ``10`` + """ + return Variable("release_month_reversed") + + @property + def release_month_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed release month, but padded. i.e. November returns "02" + """ + return Variable("release_month_reversed_padded") + + @property + def release_month_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's release month padded to two digits, i.e. March returns "03" + """ + return Variable("release_month_padded") + + @property + def release_day_padded(self: "VariableDefinitions") -> Variable: + """ + The entry's release day padded to two digits, i.e. the fifth returns "05" + """ + return Variable("release_day_padded") + + @property + def release_month(self: "VariableDefinitions") -> Variable: + """ + The release month as an integer (no padding). + """ + return Variable("release_month") + + @property + def release_day(self: "VariableDefinitions") -> Variable: + """ + The release day as an integer (no padding). + """ + return Variable("release_day") + + @property + def release_day_reversed(self: "VariableDefinitions") -> Variable: + """ + The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, + i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` + """ + return Variable("release_day_reversed") + + @property + def release_day_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed release day, but padded. i.e. August 30th returns "02". + """ + return Variable("release_day_reversed_padded") + + @property + def release_day_of_year(self: "VariableDefinitions") -> Variable: + """ + The day of the year, i.e. February 1st returns ``32`` + """ + return Variable("release_day_of_year") + + @property + def release_day_of_year_padded(self: "VariableDefinitions") -> Variable: + """ + The release day of year, but padded i.e. February 1st returns "032" + """ + return Variable("release_day_of_year_padded") + + @property + def release_day_of_year_reversed(self: "VariableDefinitions") -> Variable: + """ + The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, + i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` + """ + return Variable("release_day_of_year_reversed") + + @property + def release_day_of_year_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The reversed release day of year, but padded i.e. December 31st returns "001" + """ + return Variable("release_day_of_year_reversed_padded") + + @property + def release_date_standardized(self: "VariableDefinitions") -> Variable: + """ + The release date formatted as YYYY-MM-DD + """ + return Variable("release_date_standardized") + + +class YtdlSubVariableDefinitions(ABC): + @property + def ytdl_sub_input_url(self: "VariableDefinitions") -> Variable: + """ + The input URL used in ytdl-sub to create this entry. + """ + return Variable("ytdl_sub_input_url") + + @property + def download_index(self: "VariableDefinitions") -> Variable: + """ + The i'th entry downloaded. NOTE that this is fetched dynamically from the download + archive. + """ + return Variable(variable_name="download_index") + + @property + def download_index_padded6(self: "VariableDefinitions") -> Variable: + """ + The download_index padded six digits + """ + return Variable("download_index_padded6") + + @property + def upload_date_index(self: "VariableDefinitions") -> Variable: + """ + The i'th entry downloaded with this upload date. + """ + return Variable(variable_name="upload_date_index") + + @property + def upload_date_index_padded(self: "VariableDefinitions") -> Variable: + """ + The upload_date_index padded two digits + """ + return Variable("upload_date_index_padded") + + @property + def upload_date_index_reversed(self: "VariableDefinitions") -> Variable: + """ + 100 - upload_date_index + """ + return Variable("upload_date_index_reversed") + + @property + def upload_date_index_reversed_padded(self: "VariableDefinitions") -> Variable: + """ + The upload_date_index padded two digits + """ + return Variable("upload_date_index_reversed_padded") + + +class EntryVariableDefinitions(ABC): + @property + def uid(self: "VariableDefinitions") -> MetadataVariable: """ The entry's unique ID """ return MetadataVariable(metadata_key="id", variable_name="uid") @property - def duration(self) -> MetadataVariable: + def duration(self: "VariableDefinitions") -> MetadataVariable: """ The duration of the entry in seconds """ return MetadataVariable("duration", metadata_key="duration") @property - def uid_sanitized_plex(self) -> Variable: + def uid_sanitized_plex(self: "VariableDefinitions") -> Variable: """ The sanitized uid with additional sanitizing for Plex. Replaces numbers with fixed-width numbers so Plex does not recognize them as season or episode numbers. @@ -87,56 +604,56 @@ class VariableDefinitions: return Variable("uid_sanitized_plex") @property - def ie_key(self) -> MetadataVariable: + def ie_key(self: "VariableDefinitions") -> MetadataVariable: """ The ie_key, used in legacy yt-dlp things as the 'info-extractor key' """ return MetadataVariable(metadata_key="ie_key", variable_name="ie_key") @property - def extractor_key(self) -> MetadataVariable: + def extractor_key(self: "VariableDefinitions") -> MetadataVariable: """ The yt-dlp extractor key """ return MetadataVariable(metadata_key="extractor_key", variable_name="extractor_key") @property - def extractor(self) -> MetadataVariable: + def extractor(self: "VariableDefinitions") -> MetadataVariable: """ The yt-dlp extractor name """ return MetadataVariable(variable_name="extractor", metadata_key="extractor") @property - def epoch(self) -> MetadataVariable: + def epoch(self: "VariableDefinitions") -> MetadataVariable: """ The unix epoch of when the metadata was scraped by yt-dlp. """ return MetadataVariable(metadata_key="epoch", variable_name="epoch") @property - def epoch_date(self) -> Variable: + def epoch_date(self: "VariableDefinitions") -> Variable: """ The epoch's date, in YYYYMMDD format. """ return Variable("epoch_date") @property - def epoch_hour(self) -> Variable: + def epoch_hour(self: "VariableDefinitions") -> Variable: """ The epoch's hour """ return Variable("epoch_hour") @property - def title(self) -> MetadataVariable: + def title(self: "VariableDefinitions") -> MetadataVariable: """ The title of the entry. If a title does not exist, returns its unique ID. """ return MetadataVariable(variable_name="title", metadata_key="title") @property - def title_sanitized_plex(self) -> Variable: + def title_sanitized_plex(self: "VariableDefinitions") -> Variable: """ The sanitized title with additional sanitizing for Plex. It replaces numbers with fixed-width numbers so Plex does not recognize them as season or episode numbers. @@ -144,847 +661,122 @@ class VariableDefinitions: return Variable("title_sanitized_plex") @property - def webpage_url(self) -> MetadataVariable: + def webpage_url(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The url to the webpage. + The url to the webpage. """ return MetadataVariable(metadata_key="webpage_url", variable_name="webpage_url") @property - def info_json_ext(self) -> Variable: + def info_json_ext(self: "VariableDefinitions") -> Variable: """ - Returns - ------- - str - The "info.json" extension + The "info.json" extension """ return Variable("info_json_ext") @property - def description(self) -> MetadataVariable: + def description(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The description if it exists. Otherwise, returns an emtpy string. + The description if it exists. Otherwise, returns an emtpy string. """ return MetadataVariable(variable_name="description", metadata_key="description") @property - def uploader_id(self) -> MetadataVariable: + def uploader_id(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The uploader id if it exists, otherwise return the unique ID. + The uploader id if it exists, otherwise return the unique ID. """ return MetadataVariable(variable_name="uploader_id", metadata_key="uploader_id") @property - def uploader(self) -> MetadataVariable: + def uploader(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The uploader if it exists, otherwise return the uploader ID. + The uploader if it exists, otherwise return the uploader ID. """ return MetadataVariable(variable_name="uploader", metadata_key="uploader") @property - def uploader_url(self) -> MetadataVariable: + def uploader_url(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The uploader url if it exists, otherwise returns the webpage_url. + The uploader url if it exists, otherwise returns the webpage_url. """ return MetadataVariable("uploader_url", metadata_key="uploader_url") @property - def source_title(self) -> MetadataVariable: + def creator(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - Name of the source (i.e. channel with multiple playlists) if it exists, otherwise - returns its playlist_title. - """ - return MetadataVariable("source_title", metadata_key=self.title.metadata_key) - - @property - def source_uid(self) -> MetadataVariable: - """ - Returns - ------- - str - The source unique id if it exists, otherwise returns the playlist unique ID. - """ - return MetadataVariable("source_uid", metadata_key=self.uid.metadata_key) - - @property - def source_index(self) -> MetadataVariable: - """ - Returns - ------- - int - Source index if it exists, otherwise returns ``1``. - - It is recommended to not use this unless you know the source will never add new content - (it is easy for this value to change). - """ - return MetadataVariable("source_index", metadata_key=self.playlist_index.metadata_key) - - @property - def source_index_padded(self) -> Variable: - """ - Returns - ------- - int - The source index, padded. - """ - return Variable("source_index_padded") - - @property - def source_count(self) -> MetadataVariable: - """ - Returns - ------- - int - The source count if it exists, otherwise returns the playlist count. - """ - return MetadataVariable("source_count", metadata_key=self.playlist_count.metadata_key) - - @property - def source_webpage_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The source webpage url if it exists, otherwise returns the playlist webpage url. - """ - return MetadataVariable("source_webpage_url", metadata_key=self.webpage_url.metadata_key) - - @property - def source_description(self) -> MetadataVariable: - """ - Returns - ------- - str - The source description if it exists, otherwise returns the playlist description. - """ - return MetadataVariable("source_description", metadata_key=self.description.metadata_key) - - @property - def playlist_uid(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist unique ID if it exists, otherwise return the entry unique ID. - """ - return MetadataVariable(variable_name="playlist_uid", metadata_key="playlist_id") - - @property - def playlist_title(self) -> MetadataVariable: - """ - Returns - ------- - str - Name of its parent playlist/channel if it exists, otherwise returns its title. - """ - return MetadataVariable(variable_name="playlist_title", metadata_key="playlist_title") - - @property - def playlist_index(self) -> MetadataVariable: - """ - Returns - ------- - int - Playlist index if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(metadata_key="playlist_index", variable_name="playlist_index") - - @property - def playlist_index_reversed(self) -> Variable: - """ - Returns - ------- - int - Playlist index reversed via ``playlist_count - playlist_index + 1`` - """ - return Variable("playlist_index_reversed") - - @property - def playlist_index_padded(self) -> Variable: - """ - Returns - ------- - str - playlist_index padded two digits - """ - return Variable("playlist_index_padded") - - @property - def playlist_index_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - playlist_index_reversed padded two digits - """ - return Variable("playlist_index_reversed_padded") - - @property - def playlist_index_padded6(self) -> Variable: - """ - Returns - ------- - str - playlist_index padded six digits. - """ - return Variable("playlist_index_padded6") - - @property - def playlist_index_reversed_padded6(self) -> Variable: - """ - Returns - ------- - str - playlist_index_reversed padded six digits. - """ - return Variable("playlist_index_reversed_padded6") - - @property - def playlist_count(self) -> MetadataVariable: - """ - Returns - ------- - int - Playlist count if it exists, otherwise returns ``1``. - - Note that for channels/playlists, any change (i.e. adding or removing a video) will make - this value change. Use with caution. - """ - return MetadataVariable(variable_name="playlist_count", metadata_key="playlist_count") - - @property - def playlist_description(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist description if it exists, otherwise returns the entry's description. - """ - return MetadataVariable( - variable_name="playlist_description", metadata_key=self.description.metadata_key - ) - - @property - def playlist_webpage_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist webpage url if it exists. Otherwise, returns the entry webpage url. - """ - return MetadataVariable( - variable_name="playlist_webpage_url", metadata_key=self.webpage_url.metadata_key - ) - - @property - def playlist_max_upload_date(self) -> Variable: - """ - Returns - ------- - Max upload_date for all entries in this entry's playlist if it exists, otherwise returns - ``upload_date`` - """ - return Variable("playlist_max_upload_date") - - @property - def playlist_max_upload_year(self) -> Variable: - """ - Returns - ------- - int - Max upload_year for all entries in this entry's playlist if it exists, otherwise returns - ``upload_year`` - """ - # override in EntryParent - return Variable("playlist_max_upload_year") - - @property - def playlist_max_upload_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The max playlist truncated upload year for all entries in this entry's playlist if it - exists, otherwise returns ``upload_year_truncated``. - """ - return Variable("playlist_max_upload_year_truncated") - - @property - def playlist_uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader id if it exists, otherwise returns the entry uploader ID. - """ - return MetadataVariable("playlist_uploader_id", metadata_key="playlist_uploader_id") - - @property - def playlist_uploader(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader if it exists, otherwise return the entry uploader. - """ - return MetadataVariable("playlist_uploader", metadata_key=self.uploader.metadata_key) - - @property - def playlist_uploader_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The playlist uploader url if it exists, otherwise returns the playlist webpage_url. - """ - return MetadataVariable( - "playlist_uploader_url", metadata_key=self.uploader_url.metadata_key - ) - - @property - def source_uploader_id(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader id if it exists, otherwise returns the playlist_uploader_id - """ - return MetadataVariable("source_uploader_id", metadata_key=self.uploader_id.metadata_key) - - @property - def source_uploader(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader if it exists, otherwise return the playlist_uploader - """ - return MetadataVariable("source_uploader", metadata_key=self.uploader.metadata_key) - - @property - def source_uploader_url(self) -> MetadataVariable: - """ - Returns - ------- - str - The source uploader url if it exists, otherwise returns the source webpage_url. - """ - return MetadataVariable("source_uploader_url", metadata_key=self.uploader_url.metadata_key) - - @property - def creator(self) -> MetadataVariable: - """ - Returns - ------- - str - The creator name if it exists, otherwise returns the channel. + The creator name if it exists, otherwise returns the channel. """ return MetadataVariable(variable_name="creator", metadata_key="creator") @property - def channel(self) -> MetadataVariable: + def channel(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The channel name if it exists, otherwise returns the uploader. + The channel name if it exists, otherwise returns the uploader. """ return MetadataVariable(variable_name="channel", metadata_key="channel") @property - def channel_id(self) -> MetadataVariable: + def channel_id(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The channel id if it exists, otherwise returns the entry uploader ID. + The channel id if it exists, otherwise returns the entry uploader ID. """ return MetadataVariable(variable_name="channel_id", metadata_key="channel_id") @property - def ext(self) -> MetadataVariable: + def ext(self: "VariableDefinitions") -> MetadataVariable: """ - Returns - ------- - str - The downloaded entry's file extension + The downloaded entry's file extension """ return MetadataVariable(variable_name="ext", metadata_key="ext") @property - def thumbnail_ext(self) -> Variable: + def thumbnail_ext(self: "VariableDefinitions") -> Variable: """ - Returns - ------- - str - The download entry's thumbnail extension. Will always return 'jpg'. Until there is a - need to support other image types, we always convert to jpg. + The download entry's thumbnail extension. Will always return 'jpg'. Until there is a + need to support other image types, we always convert to jpg. """ return Variable("thumbnail_ext") @property - def comments(self) -> MetadataVariable: + def comments(self: "VariableDefinitions") -> MetadataVariable: """ Comments if they are requested """ return MetadataVariable("comments", "comments") @property - def chapters(self) -> MetadataVariable: + def chapters(self: "VariableDefinitions") -> MetadataVariable: """ Chapters if they exist """ return MetadataVariable("chapters", "chapters") @property - def sponsorblock_chapters(self) -> MetadataVariable: + def sponsorblock_chapters(self: "VariableDefinitions") -> MetadataVariable: """ Sponsorblock Chapters if they are requested and exist """ return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") @property - def requested_subtitles(self) -> MetadataVariable: + def requested_subtitles(self: "VariableDefinitions") -> MetadataVariable: """ Subtitles if they are requested and exist """ return MetadataVariable("requested_subtitles", "requested_subtitles") - @property - def ytdl_sub_input_url(self) -> Variable: - """ - The input URL used in ytdl-sub to create this entry. - """ - return Variable("ytdl_sub_input_url") - @property - def download_index(self) -> Variable: - """ - Returns - ------- - int - The i'th entry downloaded. NOTE that this is fetched dynamically from the download - archive. - """ - return Variable(variable_name="download_index") - - @property - def download_index_padded6(self) -> Variable: - """ - Returns - ------- - str - The download_index padded six digits - """ - return Variable("download_index_padded6") - - @property - def upload_date_index(self) -> Variable: - """ - Returns - ------- - int - The i'th entry downloaded with this upload date. - """ - return Variable(variable_name="upload_date_index") - - @property - def upload_date_index_padded(self) -> Variable: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return Variable("upload_date_index_padded") - - @property - def upload_date_index_reversed(self) -> Variable: - """ - Returns - ------- - int - 100 - upload_date_index - """ - return Variable("upload_date_index_reversed") - - @property - def upload_date_index_reversed_padded(self) -> Variable: - """ - Returns - ------- - int - The upload_date_index padded two digits - """ - return Variable("upload_date_index_reversed_padded") - - @property - def upload_date(self) -> MetadataVariable: - """ - Returns - ------- - str - The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date. - """ - return MetadataVariable(variable_name="upload_date", metadata_key="upload_date") - - @property - def upload_year(self) -> Variable: - """ - Returns - ------- - int - The entry's upload year - """ - return Variable("upload_year") - - @property - def upload_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The last two digits of the upload year, i.e. 22 in 2022 - """ - return Variable("upload_year_truncated") - - @property - def upload_year_truncated_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return Variable("upload_year_truncated_reversed") - - @property - def upload_month_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10`` - """ - return Variable("upload_month_reversed") - - @property - def upload_month_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload month, but padded. i.e. November returns "02" - """ - return Variable("upload_month_reversed_padded") - - @property - def upload_month_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's upload month padded to two digits, i.e. March returns "03" - """ - return Variable("upload_month_padded") - - @property - def upload_day_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's upload day padded to two digits, i.e. the fifth returns "05" - """ - return Variable("upload_day_padded") - - @property - def upload_month(self) -> Variable: - """ - Returns - ------- - int - The upload month as an integer (no padding). - """ - return Variable("upload_month") - - @property - def upload_day(self) -> Variable: - """ - Returns - ------- - int - The upload day as an integer (no padding). - """ - return Variable("upload_day") - - @property - def upload_day_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``, - i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - return Variable("upload_day_reversed") - - @property - def upload_day_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload day, but padded. i.e. August 30th returns "02". - """ - return Variable("upload_day_reversed_padded") - - @property - def upload_day_of_year(self) -> Variable: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - return Variable("upload_day_of_year") - - @property - def upload_day_of_year_padded(self) -> Variable: - """ - Returns - ------- - str - The upload day of year, but padded i.e. February 1st returns "032" - """ - return Variable("upload_day_of_year_padded") - - @property - def upload_day_of_year_reversed(self) -> Variable: - """ - Returns - ------- - int - The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``, - i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - return Variable("upload_day_of_year_reversed") - - @property - def upload_day_of_year_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed upload day of year, but padded i.e. December 31st returns "001" - """ - return Variable("upload_day_of_year_reversed_padded") - - @property - def upload_date_standardized(self) -> Variable: - """ - Returns - ------- - str - The uploaded date formatted as YYYY-MM-DD - """ - return Variable("upload_date_standardized") - - @property - def release_date(self) -> MetadataVariable: - """ - Returns - ------- - str - The entry’s release date, in YYYYMMDD format. If not present, return the upload date. - """ - return MetadataVariable(variable_name="release_date", metadata_key="release_date") - - @property - def release_year(self) -> Variable: - """ - Returns - ------- - int - The entry's release year - """ - return Variable("release_year") - - @property - def release_year_truncated(self) -> Variable: - """ - Returns - ------- - int - The last two digits of the release year, i.e. 22 in 2022 - """ - return Variable("release_year_truncated") - - @property - def release_year_truncated_reversed(self) -> Variable: - """ - Returns - ------- - int - The release year truncated, but reversed using ``100 - {release_year_truncated}``, i.e. - 2022 returns ``100 - 22`` = ``78`` - """ - return Variable("release_year_truncated_reversed") - - @property - def release_month_reversed(self) -> Variable: - """ - Returns - ------- - int - The release month, but reversed - using ``13 - {release_month}``, i.e. March returns ``10`` - """ - return Variable("release_month_reversed") - - @property - def release_month_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release month, but padded. i.e. November returns "02" - """ - return Variable("release_month_reversed_padded") - - @property - def release_month_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's release month padded to two digits, i.e. March returns "03" - """ - return Variable("release_month_padded") - - @property - def release_day_padded(self) -> Variable: - """ - Returns - ------- - str - The entry's release day padded to two digits, i.e. the fifth returns "05" - """ - return Variable("release_day_padded") - - @property - def release_month(self) -> Variable: - """ - Returns - ------- - int - The release month as an integer (no padding). - """ - return Variable("release_month") - - @property - def release_day(self) -> Variable: - """ - Returns - ------- - int - The release day as an integer (no padding). - """ - return Variable("release_day") - - @property - def release_day_reversed(self) -> Variable: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_month} + 1 - {release_day}``, - i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24`` - """ - return Variable("release_day_reversed") - - @property - def release_day_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release day, but padded. i.e. August 30th returns "02". - """ - return Variable("release_day_reversed_padded") - - @property - def release_day_of_year(self) -> Variable: - """ - Returns - ------- - int - The day of the year, i.e. February 1st returns ``32`` - """ - return Variable("release_day_of_year") - - @property - def release_day_of_year_padded(self) -> Variable: - """ - Returns - ------- - str - The release day of year, but padded i.e. February 1st returns "032" - """ - return Variable("release_day_of_year_padded") - - @property - def release_day_of_year_reversed(self) -> Variable: - """ - Returns - ------- - int - The release day, but reversed using ``{total_days_in_year} + 1 - {release_day}``, - i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334`` - """ - return Variable("release_day_of_year_reversed") - - @property - def release_day_of_year_reversed_padded(self) -> Variable: - """ - Returns - ------- - str - The reversed release day of year, but padded i.e. December 31st returns "001" - """ - return Variable("release_day_of_year_reversed_padded") - - @property - def release_date_standardized(self) -> Variable: - """ - Returns - ------- - str - The release date formatted as YYYY-MM-DD - """ - return Variable("release_date_standardized") +class VariableDefinitions( + EntryVariableDefinitions, + MetadataVariableDefinitions, + PlaylistVariableDefinitions, + SourceVariableDefinitions, + UploadDateVariableDefinitions, + ReleaseDateVariableDefinitions, + YtdlSubVariableDefinitions, +): + pass # Singleton to use externally diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index a798dd18..04af57b0 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -10,15 +10,15 @@ SUBSCRIPTION_ARRAY = "subscription_array" class OverrideVariables: - @classmethod - def subscription_name(cls) -> str: + @staticmethod + def subscription_name() -> str: """ Name of the subscription """ return SUBSCRIPTION_NAME - @classmethod - def subscription_value(cls) -> str: + @staticmethod + def subscription_value() -> str: """ For subscriptions in the form of @@ -30,8 +30,8 @@ class OverrideVariables: """ return SUBSCRIPTION_VALUE - @classmethod - def subscription_indent_i(cls, index: int) -> str: + @staticmethod + def subscription_indent_i(index: int) -> str: """ For subscriptions in the form of @@ -46,8 +46,8 @@ class OverrideVariables: """ return f"subscription_indent_{index + 1}" - @classmethod - def subscription_value_i(cls, index: int) -> str: + @staticmethod + def subscription_value_i(index: int) -> str: """ For subscriptions in the form of @@ -63,6 +63,8 @@ class OverrideVariables: """ return f"subscription_value_{index + 1}" + +class OverrideHelpers: @classmethod def is_entry_variable_name(cls, name: str) -> bool: """ diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index e728f392..644fa1c1 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -25,15 +25,13 @@ class AudioExtractOptions(OptionsDictValidator): """ Extracts audio from a video file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - audio_extract: - codec: "mp3" - quality: 128 + audio_extract: + codec: "mp3" + quality: 128 """ _required_keys = {"codec"} @@ -56,16 +54,20 @@ class AudioExtractOptions(OptionsDictValidator): @property def codec(self) -> str: """ - The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, - opus, vorbis, wav, and best to grab the best possible format at runtime. + :expected type: String + :description: + The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, + opus, vorbis, wav, and best to grab the best possible format at runtime. """ return self._codec @property def quality(self) -> Optional[float]: """ - Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` - (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. + :expected type: Float + :description: + Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` + (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. """ if self._quality is not None: return self._quality.value diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index bbb10ddd..fbd11c09 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -64,32 +64,29 @@ class ChaptersOptions(OptionsDictValidator): Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - chapters: - # Embedded Chapter Fields - embed_chapters: True - allow_chapters_from_comments: False - remove_chapters_regex: - - "Intro" - - "Outro" - - # Sponsorblock Fields - sponsorblock_categories: - - "outro" - - "selfpromo" - - "preview" - - "interaction" - - "sponsor" - - "music_offtopic" - - "intro" - remove_sponsorblock_categories: "all" - force_key_frames: False + chapters: + # Embedded Chapter Fields + embed_chapters: True + allow_chapters_from_comments: False + remove_chapters_regex: + - "Intro" + - "Outro" + # Sponsorblock Fields + sponsorblock_categories: + - "outro" + - "selfpromo" + - "preview" + - "interaction" + - "sponsor" + - "music_offtopic" + - "intro" + remove_sponsorblock_categories: "all" + force_key_frames: False """ _optional_keys = { @@ -135,23 +132,29 @@ class ChaptersOptions(OptionsDictValidator): @property def embed_chapters(self) -> Optional[bool]: """ - Optional. Embed chapters into the file. Defaults to True. + :expected type: Optional[Boolean] + :description: + Defaults to True. Embed chapters into the file. """ return self._embed_chapters @property def allow_chapters_from_comments(self) -> bool: """ - Optional. If chapters do not exist in the video/description itself, attempt to scrape - comments to find the chapters. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. If chapters do not exist in the video/description itself, attempt to + scrape comments to find the chapters. """ return self._allow_chapters_from_comments @property def remove_chapters_regex(self) -> Optional[List[re.Pattern]]: """ - Optional. List of regex patterns to match chapter titles against and remove them from the - entry. + :expected type: Optional[List[RegexString] + :description: + List of regex patterns to match chapter titles against and remove them from the + entry. """ if self._remove_chapters_regex: return [validator.compiled_regex for validator in self._remove_chapters_regex.list] @@ -160,9 +163,11 @@ class ChaptersOptions(OptionsDictValidator): @property def sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", - "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", - "poi_highlight", or "all" to include all categories. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. """ if self._sponsorblock_categories: category_list = [validator.value for validator in self._sponsorblock_categories.list] @@ -174,9 +179,11 @@ class ChaptersOptions(OptionsDictValidator): @property def remove_sponsorblock_categories(self) -> Optional[List[str]]: """ - Optional. List of SponsorBlock categories to remove from the output file. Can only remove - categories that are specified in ``sponsorblock_categories`` or "all", which removes - everything specified in ``sponsorblock_categories``. + :expected type: Optional[List[String]] + :description: + List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. """ if self._remove_sponsorblock_categories: category_list = [ @@ -190,9 +197,10 @@ class ChaptersOptions(OptionsDictValidator): @property def force_key_frames(self) -> bool: """ - Optional. Force keyframes at cuts when removing sections. This is slow due to needing a - re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to - False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Force keyframes at cuts when removing sections. This is slow due to + needing a re-encode, but the resulting video may have fewer artifacts around the cuts. """ return self._force_key_frames diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 092d1054..3948cb1e 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -11,16 +11,24 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator class DateRangeOptions(OptionsDictValidator): """ Only download files uploaded within the specified date range. + Dates must adhere to a yt-dlp datetime. From their docs: - Usage: + .. code-block:: Markdown + + A string in the format YYYYMMDD or + (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s) + + Valid examples are ``now-2weeks`` or ``20200101``. Can use override variables in this. + Note that yt-dlp will round times to the closest day, meaning that `day` is the lowest + granularity possible. + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - date_range: - before: "now" - after: "today-2weeks" + date_range: + before: "now" + after: "today-2weeks" """ _optional_keys = {"before", "after"} @@ -33,14 +41,18 @@ class DateRangeOptions(OptionsDictValidator): @property def before(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos before this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._before @property def after(self) -> Optional[StringDatetimeValidator]: """ - Optional. Only download videos after this datetime. + :expected type: Optional[OverridesFormatter] + :description: + Only download videos before this datetime. """ return self._after diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 64703870..b3ef63e2 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -20,13 +20,11 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator): """ Whether to embed thumbnails to the audio/video file or not. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - embed_thumbnail: True + embed_thumbnail: True """ diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 3b56e1c1..8fccaedb 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -32,29 +32,27 @@ class FileConvertOptions(OptionsDictValidator): """ Converts video files from one extension to another. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mp4" + file_convert: + convert_to: "mp4" - Supports custom ffmpeg conversions: + Also supports custom ffmpeg conversions: + + :Usage: .. code-block:: yaml - presets: - my_example_preset: - file_convert: - convert_to: "mkv" - convert_with: "ffmpeg" - ffmpeg_post_process_args: > - -bitexact - -vcodec copy - -acodec copy - -scodec mov_text + file_convert: + convert_to: "mkv" + convert_with: "ffmpeg" + ffmpeg_post_process_args: > + -bitexact + -vcodec copy + -acodec copy + -scodec mov_text """ _required_keys = {"convert_to"} @@ -89,35 +87,38 @@ class FileConvertOptions(OptionsDictValidator): @property def convert_to(self) -> str: """ - Convert to a desired file type. Supports: + :expected type: String + :description: + Convert to a desired file type. Supports - * Video: avi, flv, mkv, mov, mp4, webm - * Audio: aac, flac, mp3, m4a, opus, vorbis, wav + - Video: avi, flv, mkv, mov, mp4, webm + - Audio: aac, flac, mp3, m4a, opus, vorbis, wav """ return self._convert_to @property def convert_with(self) -> Optional[str]: """ - Optional. Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within - yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified - with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - + :expected type: Optional[String] + :description: + Supports ``yt-dlp`` and ``ffmpeg``. ``yt-dlp`` will convert files within + yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified + with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. """ return self._convert_with @property def ffmpeg_post_process_args(self) -> Optional[OverridesStringFormatterValidator]: """ - Optional. ffmpeg args to post-process an entry file with. The args will be inserted in the - form of: + :expected type: Optional[OverridesFormatter] + :description: + ffmpeg args to post-process an entry file with. The args will be inserted in the + form of - .. code-block:: bash + ``ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext``. - ffmpeg -i input_file.ext {ffmpeg_post_process_args) output_file.output_ext - - The output file will use the extension specified in ``convert_to``. Post-processing args - can still be set with ``convert_with`` set to ``yt-dlp``. + The output file will use the extension specified in ``convert_to``. Post-processing args + can still be set with ``convert_with`` set to ``yt-dlp``. """ return self._ffmpeg_post_process_args diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index cceb4700..7f381db1 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -19,15 +19,15 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): Applies a conditional OR on any number of filters comprised of either variables or scripts. If any filter evaluates to True, the entry will be excluded. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - filter_exclude: - - { %contains( %lower(title), '#short' ) } - - { %contains( %lower(description), '#short' ) } + filter_exclude: + - >- + { %contains( %lower(title), '#short' ) } + - >- + { %contains( %lower(description), '#short' ) } """ diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 988426fe..41a98f78 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -19,24 +19,23 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): Applies a conditional AND on any number of filters comprised of either variables or scripts. If all filters evaluate to True, the entry will be included. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - filter_include: - - {description} - - >- - { - %regex_search_any( - title, - [ - "Full Episode", - "FULL", - ] - ) - } + filter_include: + - >- + {description} + - >- + { + %regex_search_any( + title, + [ + "Full Episode", + "FULL", + ] + ) + } """ diff --git a/src/ytdl_sub/plugins/format.py b/src/ytdl_sub/plugins/format.py index 3b5c334f..6e2d545d 100644 --- a/src/ytdl_sub/plugins/format.py +++ b/src/ytdl_sub/plugins/format.py @@ -15,9 +15,7 @@ class FormatOptions(OptionsValidator): .. code-block:: yaml - presets: - my_example_preset: - format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" + format: "(bv*[height<=1080]+bestaudio/best[height<=1080])" """ def __init__(self, name, value): diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 4e1ad498..fe10a69b 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -58,32 +58,20 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: class MatchFiltersOptions(OptionsDictValidator): """ - Set ``--match-filters``` to pass into yt-dlp to filter entries from being downloaded. - Uses the same syntax as yt-dlp. + Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. + Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. + For logical AND's between match filters, use the ``&`` operator in a single match filter. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - match_filters: - filters: "original_url!*=/shorts/" - - Supports one or multiple filters: - - .. code-block:: yaml - - presets: - my_example_preset: - match_filters: - filters: - - "age_limit<?18" - - "like_count>?100" - # Other common match-filters - # - "original_url!*=/shorts/ & !is_live" - # - "age_limit<?18" - # - "availability=?public" + match_filters: + filters: + - "age_limit<?18 & like_count>?100" + # Other common match-filters + # - "original_url!*=/shorts/ & !is_live" + # - "availability=?public" """ _optional_keys = {"filters"} diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index ec99d50f..f6098517 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -75,23 +75,22 @@ class MusicTagsOptions(OptionsDictValidator): a full list of tags for various file types in MediaFile's `source code <https://github.com/beetbox/mediafile/blob/v0.9.0/mediafile.py#L1770>`_. - Usage: + :Usage: .. code-block:: yaml presets: my_example_preset: music_tags: - tags: - artist: "{artist}" - album: "{album}" - # Supports id3v2.4 multi-tags - genres: - - "{genre}" - - "ytdl-sub" - albumartists: - - "{artist}" - - "ytdl-sub" + artist: "{artist}" + album: "{album}" + # Supports id3v2.4 multi-tags + genres: + - "{genre}" + - "ytdl-sub" + albumartists: + - "{artist}" + - "ytdl-sub" """ _optional_keys = {"tags", "embed_thumbnail"} diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 92e2b968..728259e1 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -60,7 +60,9 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def nfo_name(self) -> StringFormatterFileNameValidator: """ - The NFO file name. + :expected type: EntryFormatter + :description: + The NFO file name. """ return self._nfo_name @@ -81,9 +83,11 @@ class SharedNfoTagsOptions(OptionsDictValidator): @property def kodi_safe(self) -> Optional[bool]: """ - Optional. Kodi does not support > 3-byte unicode characters, which include emojis and some - foreign language characters. Setting this to True will replace those characters with '□'. - Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Kodi does not support > 3-byte unicode characters, which include + emojis and some foreign language characters. Setting this to True will replace those + characters with '□'. """ return self._kodi_safe @@ -190,22 +194,18 @@ class NfoTagsOptions(SharedNfoTagsOptions): Adds an NFO file for every download file. An NFO file is simply an XML file with a ``.nfo`` extension. You can add any values into the NFO. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - nfo_tags: - # required - nfo_name: "{title_sanitized}.nfo" - nfo_root: "episodedetails" - tags: - title: "{title}" - season: "{upload_year}" - episode: "{upload_month}{upload_day_padded}" - # optional - kodi_safe: False + nfo_tags: + nfo_name: "{title_sanitized}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + kodi_safe: False """ _formatter_validator = StringFormatterValidator @@ -215,50 +215,54 @@ class NfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - </episodedetails> + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <episodedetails> + </episodedetails> """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <episodedetails> - <title>Awesome Youtube Video - 2022 - 502 - + + + Awesome Youtube Video + 2022 + 502 + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - season: - attributes: - name: "Best Year" - tag: "{upload_year}" - genre: - - "Comedy" - - "Drama" + tags: + season: + attributes: + name: "Best Year" + tag: "{upload_year}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - 2022 - Comedy - Drama + 2022 + Comedy + Drama """ return self._tags diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 72f1828d..f754ec44 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -37,48 +37,52 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): @property def nfo_root(self) -> StringFormatterValidator: """ - The root tag of the NFO's XML. In the usage above, it would look like + :expected type: EntryFormatter + :description: + The root tag of the NFO's XML. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - + + + """ return self._nfo_root @property def tags(self) -> NfoTagsValidator: """ - Tags within the nfo_root tag. In the usage above, it would look like + :expected type: NfoTags + :description: + Tags within the nfo_root tag. In the usage above, it would look like - .. code-block:: xml + .. code-block:: xml - - - Sweet youtube TV show - + + + Sweet youtube TV show + - Also supports xml attributes and duplicate keys: + Also supports xml attributes and duplicate keys: - .. code-block:: yaml + .. code-block:: yaml - tags: - named_season: - - tag: "{source_title}" - attributes: - number: "{collection_index}" - genre: - - "Comedy" - - "Drama" + tags: + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + genre: + - "Comedy" + - "Drama" - Which translates to + Which translates to - .. code-block:: xml + .. code-block:: xml - Sweet youtube TV show</season> - <genre>Comedy</genre> - <genre>Drama</genre> + <title year="2022">Sweet youtube TV show</season> + <genre>Comedy</genre> + <genre>Drama</genre> """ return self._tags diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 325649ea..47b34262 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -122,6 +122,41 @@ class FromSourceVariablesRegex(DictValidator): class RegexOptions(OptionsDictValidator): r""" + .. attention:: + + This plugin will eventually be deprecated and replaced by scripting functions. + You can replicate the example below using the following. + + .. code-block:: yaml + + # Only includes videos with 'Official Video' + filter_include: + - >- + { %contains( %lower(title), "official video" ) } + + # Excludes videos with '#short' in its description + filter_exclude: + - >- + { %contains( %lower(description), '#short' ) } + + # Creates a capture array with defaults, and assigns + # each capture group to its own variable + overrides: + description_date_capture: >- + { + %regex_capture_many_with_defaults( + description, + [ "([0-9]{4})-([0-9]{2})-([0-9]{2})" ], + [ upload_year, upload_month, upload_day ] + ) + } + captured_upload_year: >- + { %array_at(description_date_capture, 1) } + captured_upload_month: >- + { %array_at(description_date_capture, 2) } + captured_upload_day: >- + { %array_at(description_date_capture, 3) } + Performs regex matching on an entry's source or override variables. Regex can be used to filter entries from proceeding with download or capture groups to create new source variables. @@ -137,51 +172,49 @@ class RegexOptions(OptionsDictValidator): and using ``title_and_description`` can regex match/exclude from either ``title`` or ``description``. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - regex: - # By default, if any match fails and has no defaults, the entry will - # be skipped. If False, ytdl-sub will error and stop all downloads - # from proceeding. - skip_if_match_fails: True + regex: + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. + skip_if_match_fails: True - from: - # For each entry's `title` value... - title: - # Perform this regex match on it to act as a filter. - # This will only download videos with "[Official Video]" in it. Note that we - # double backslash to make YAML happy - match: - - '\\[Official Video\\]' + from: + # For each entry's `title` value... + title: + # Perform this regex match on it to act as a filter. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy + match: + - '\\[Official Video\\]' - # For each entry's `description` value... - description: - # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new - # source variables - match: - - '([0-9]{4})-([0-9]{2})-([0-9]{2})' - # Exclude any entry where the description contains #short - exclude: - - '#short' + # For each entry's `description` value... + description: + # Match with capture groups and defaults. + # This tries to scrape a date from the description and produce new + # source variables + match: + - '([0-9]{4})-([0-9]{2})-([0-9]{2})' + # Exclude any entry where the description contains #short + exclude: + - '#short' - # Each capture group creates these new source variables, respectively, - # as well a sanitized version, i.e. `captured_upload_year_sanitized` - capture_group_names: - - "captured_upload_year" - - "captured_upload_month" - - "captured_upload_day" + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` + capture_group_names: + - "captured_upload_year" + - "captured_upload_month" + - "captured_upload_day" - # And if the string does not match, use these as respective default - # values for the new source variables. - capture_group_defaults: - - "{upload_year}" - - "{upload_month}" - - "{upload_day}" + # And if the string does not match, use these as respective default + # values for the new source variables. + capture_group_defaults: + - "{upload_year}" + - "{upload_month}" + - "{upload_day}" """ _required_keys = {"from"} @@ -209,8 +242,10 @@ class RegexOptions(OptionsDictValidator): @property def skip_if_match_fails(self) -> Optional[bool]: """ - Defaults to True. If True, when any match fails and has no defaults, the entry will be - skipped. If False, ytdl-sub will error and all downloads will not proceed. + :expected type: Optional[Boolean] + :description: + Defaults to True. If True, when any match fails and has no defaults, the entry will be + skipped. If False, ytdl-sub will error and all downloads will not proceed. """ return self._skip_if_match_fails diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index fc83a07c..b812c0b9 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -48,23 +48,22 @@ class WhenNoChaptersValidator(StringSelectValidator): class SplitByChaptersOptions(OptionsDictValidator): """ Splits a file by chapters into multiple files. Each file becomes its own entry with the - new source variables ``chapter_title``, ``chapter_title_sanitized``, ``chapter_index``, - ``chapter_index_padded``, ``chapter_count``. + new variables - If a file has no chapters, and ``when_no_chapters`` is set to "pass", then ``chapter_title`` is - set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. + - ``chapter_title`` + - ``chapter_index`` + - ``chapter_index_padded`` + - ``chapter_count`` Note that when using this plugin and performing dry-run, it assumes embedded chapters are being used with no modifications. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - split_by_chapters: - when_no_chapters: "pass" + split_by_chapters: + when_no_chapters: "pass" """ _required_keys = {"when_no_chapters"} @@ -100,8 +99,16 @@ class SplitByChaptersOptions(OptionsDictValidator): @property def when_no_chapters(self) -> str: """ - Behavior to perform when no chapters are present. Supports "pass" (continue processing), - "drop" (exclude it from output), and "error" (stop processing for everything). + :expected type: String + :description: + Behavior to perform when no chapters are present. Supports + + - "pass" (continue processing), + - "drop" (exclude it from output) + - "error" (stop processing for everything). + + If a file has no chapters and is set to "pass", then ``chapter_title`` is + set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. """ return self._when_no_chapters diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 445a00a9..9c53a9bd 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -37,18 +37,18 @@ class SubtitleOptions(OptionsDictValidator): ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - subtitles: - subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" - subtitles_type: "srt" - embed_subtitles: False - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: False + subtitles: + subtitles_name: "{title_sanitized}.{lang}.{subtitles_ext}" + subtitles_type: "srt" + embed_subtitles: False + languages: + - "en" # supports multiple languages + - "de" + allow_auto_generated_subtitles: False """ _optional_keys = { @@ -82,40 +82,50 @@ class SubtitleOptions(OptionsDictValidator): @property def subtitles_name(self) -> Optional[StringFormatterValidator]: """ - Optional. The file name for the media's subtitles if they are present. This can include - directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, and - will be placed in the output directory. ``lang`` is dynamic since you can download multiple - subtitles. It will set the respective language to the correct subtitle file. + :expected type: Optional[EntryFormatter] + :description: + The file name for the media's subtitles if they are present. This can include + directories such as ``"Season {upload_year}/{title_sanitized}.{lang}.{subtitles_ext}"``, + and will be placed in the output directory. ``lang`` is dynamic since you can download + multiple subtitles. It will set the respective language to the correct subtitle file. """ return self._subtitles_name @property def subtitles_type(self) -> Optional[str]: """ - Optional. One of the subtitle file types "srt", "vtt", "ass", "lrc". Defaults to "srt" + :expected type: Optional[String] + :description: + Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". """ return self._subtitles_type @property def embed_subtitles(self) -> Optional[bool]: """ - Optional. Whether to embed the subtitles into the video file. Defaults to False. - NOTE: webm files can only embed "vtt" subtitle types. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to embed the subtitles into the video file. Note that + webm files can only embed "vtt" subtitle types. """ return self._embed_subtitles @property def languages(self) -> Optional[List[str]]: """ - Optional. Language code(s) to download for subtitles. Supports a single or list of multiple - language codes. Defaults to "en". + :expected type: Optional[List[String]] + :description: + Language code(s) to download for subtitles. Supports a single or list of multiple + language codes. Defaults to only "en". """ return [lang.value for lang in self._languages] @property def allow_auto_generated_subtitles(self) -> Optional[bool]: """ - Optional. Whether to allow auto generated subtitles. Defaults to False. + :expected type: Optional[Boolean] + :description: + Defaults to False. Whether to allow auto generated subtitles. """ return self._allow_auto_generated_subtitles diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 5bab172c..5295c4f2 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -65,7 +65,7 @@ class ThrottleProtectionOptions(OptionsDictValidator): range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. - Usage: + :Usage: .. code-block:: yaml @@ -110,31 +110,39 @@ class ThrottleProtectionOptions(OptionsDictValidator): @property def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each download. Does not include time it takes for - ytdl-sub to perform post-processing. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each download. Does not include time it takes for + ytdl-sub to perform post-processing. """ return self._sleep_per_download_s @property def sleep_per_subscription_s(self) -> Optional[RandomizedRangeValidator]: """ - Number in seconds to sleep between each subscription. + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each subscription. """ return self._sleep_per_subscription_s @property def max_downloads_per_subscription(self) -> Optional[RandomizedRangeValidator]: """ - Number of downloads to perform per subscription. + :expected type: Optional[Range] + :description: + Number of downloads to perform per subscription. """ return self._max_downloads_per_subscription @property def subscription_download_probability(self) -> Optional[ProbabilityValidator]: """ - Probability to perform any downloads, recomputed for each subscription. This is only - recommended to set if you run ytdl-sub in a cron-job, that way you are statistically - guaranteed over time to eventually download the subscription. + :expected type: Optional[Float] + :description: + Probability to perform any downloads, recomputed for each subscription. This is only + recommended to set if you run ytdl-sub in a cron-job, that way you are statistically + guaranteed over time to eventually download the subscription. """ return self._subscription_download_probability diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 9ce3d8fd..c3b2e6ec 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -17,16 +17,14 @@ class VideoTagsOptions(OptionsDictValidator): """ Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. - Usage: + :Usage: .. code-block:: yaml - presets: - my_example_preset: - video_tags: - title: "{title}" - date: "{upload_date}" - description: "{description}" + video_tags: + title: "{title}" + date: "{upload_date}" + description: "{description}" """ _optional_keys = {"tags"} diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py index 5dc0e653..4af6f63c 100644 --- a/src/ytdl_sub/script/functions/array_functions.py +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -19,12 +19,8 @@ class ArrayFunctions: @staticmethod def array(maybe_array: AnyArgument) -> Array: """ - Tries to cast an unknown variable type to an Array. - - Raises - ------ - FunctionRuntimeException - If the input type is not actually an Array. + :description: + Tries to cast an unknown variable type to an Array. """ if not isinstance(maybe_array, Array): raise FunctionRuntimeException( @@ -35,14 +31,16 @@ class ArrayFunctions: @staticmethod def array_size(array: Array) -> Integer: """ - Returns the size of an Array. + :description: + Returns the size of an Array. """ return Integer(len(array.value)) @staticmethod def array_extend(*arrays: Array) -> Array: """ - Combine multiple Arrays into a single Array. + :description: + Combine multiple Arrays into a single Array. """ output: List[Resolvable] = [] for array in arrays: @@ -55,7 +53,8 @@ class ArrayFunctions: array: Array, overlap: Array, only_missing: Optional[Boolean] = None ) -> Array: """ - Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. + :description: + Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices. """ output: List[Resolvable] = [] output.extend(array.value) @@ -76,15 +75,17 @@ class ArrayFunctions: @staticmethod def array_at(array: Array, idx: Integer) -> AnyArgument: """ - Return the element in the Array at index ``idx``. + :description: + Return the element in the Array at index ``idx``. """ return array.value[idx.value] @staticmethod def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: """ - Returns the first element whose boolean conversion is True. Returns fallback - if all elements evaluate to False. + :description: + Returns the first element whose boolean conversion is True. Returns fallback + if all elements evaluate to False. """ for val in array.value: if bool(val.value): @@ -95,15 +96,17 @@ class ArrayFunctions: @staticmethod def array_contains(array: Array, value: AnyArgument) -> Boolean: """ - Return True if the value exists in the Array. False otherwise. + :description: + Return True if the value exists in the Array. False otherwise. """ return Boolean(value in array.value) @staticmethod def array_index(array: Array, value: AnyArgument) -> Integer: """ - Return the index of the value within the Array if it exists. If it does not, it will - throw an error. + :description: + Return the index of the value within the Array if it exists. If it does not, it will + throw an error. """ if not ArrayFunctions.array_contains(array=array, value=value): raise ArrayValueDoesNotExist( @@ -118,7 +121,8 @@ class ArrayFunctions: @staticmethod def array_slice(array: Array, start: Integer, end: Optional[Integer] = None) -> Array: """ - Returns the slice of the Array. + :description: + Returns the slice of the Array. """ if end is not None: return Array(array.value[start.value : end.value]) @@ -127,7 +131,8 @@ class ArrayFunctions: @staticmethod def array_flatten(array: Array) -> Array: """ - Flatten any nested Arrays into a single-dimensional Array. + :description: + Flatten any nested Arrays into a single-dimensional Array. """ output: List[Resolvable] = [] for elem in array.value: @@ -141,14 +146,16 @@ class ArrayFunctions: @staticmethod def array_reverse(array: Array) -> Array: """ - Reverse an Array. + :description: + Reverse an Array. """ return Array(list(reversed(array.value))) @staticmethod def array_product(*arrays: Array) -> Array: """ - Returns the Cartesian product of elements from different arrays + :description: + Returns the Cartesian product of elements from different arrays """ out: List[Resolvable] = [] for combo in itertools.product(*[arr.value for arr in arrays]): @@ -161,7 +168,17 @@ class ArrayFunctions: @staticmethod def array_apply(array: Array, lambda_function: Lambda) -> Array: """ - Apply a lambda function on every element in the Array. + :description: + Apply a lambda function on every element in the Array. + :usage: + + .. code-block:: python + + { + %array_apply( [1, 2, 3] , %string ) + } + + # ["1", "2", "3"] """ return Array([Array([val]) for val in array.value]) @@ -173,8 +190,9 @@ class ArrayFunctions: reverse_args: Optional[Boolean] = None, ) -> Array: """ - Apply a lambda function on every element in the Array, with ``fixed_argument`` - passed as a second argument to every invocation. + :description: + Apply a lambda function on every element in the Array, with ``fixed_argument`` + passed as a second argument to every invocation. """ if reverse_args and reverse_args.value: return Array([Array([fixed_argument, val]) for val in array.value]) @@ -184,15 +202,17 @@ class ArrayFunctions: @staticmethod def array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array: """ - Apply a lambda function on every element in the Array, where each arg - passed to the lambda function is ``idx, element`` as two separate args. + :description: + Apply a lambda function on every element in the Array, where each arg + passed to the lambda function is ``idx, element`` as two separate args. """ return Array([Array([Integer(idx), val]) for idx, val in enumerate(array.value)]) @staticmethod def array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument: """ - Apply a reduce function on pairs of elements in the Array, until one element remains. - Executes using the left-most and reduces in the right direction. + :description: + Apply a reduce function on pairs of elements in the Array, until one element remains. + Executes using the left-most and reduces in the right direction. """ return array diff --git a/src/ytdl_sub/script/functions/boolean_functions.py b/src/ytdl_sub/script/functions/boolean_functions.py index f1bb9a35..bfb7416f 100644 --- a/src/ytdl_sub/script/functions/boolean_functions.py +++ b/src/ytdl_sub/script/functions/boolean_functions.py @@ -13,70 +13,80 @@ class BooleanFunctions: @staticmethod def bool(value: AnyArgument) -> Boolean: """ - Cast any type to a Boolean. + :description: + Cast any type to a Boolean. """ return Boolean(bool(value.value)) @staticmethod def eq(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``==`` operator. Returns True if left == right. False otherwise. + :description: + ``==`` operator. Returns True if left == right. False otherwise. """ return Boolean(left.value == right.value) @staticmethod def ne(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``!=`` operator. Returns True if left != right. False otherwise. + :description: + ``!=`` operator. Returns True if left != right. False otherwise. """ return Boolean(left.value != right.value) @staticmethod def lt(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``<`` operator. Returns True if left < right. False otherwise. + :description: + ``<`` operator. Returns True if left < right. False otherwise. """ return Boolean(left.value < right.value) @staticmethod def lte(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``<=`` operator. Returns True if left <= right. False otherwise. + :description: + ``<=`` operator. Returns True if left <= right. False otherwise. """ return Boolean(left.value <= right.value) @staticmethod def gt(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``>`` operator. Returns True if left > right. False otherwise. + :description: + ``>`` operator. Returns True if left > right. False otherwise. """ return Boolean(left.value > right.value) @staticmethod def gte(left: AnyArgument, right: AnyArgument) -> Boolean: """ - ``>=`` operator. Returns True if left >= right. False otherwise. + :description: + ``>=`` operator. Returns True if left >= right. False otherwise. """ return Boolean(left.value >= right.value) @staticmethod def and_(*values: AnyArgument) -> Boolean: """ - ``and`` operator. Returns True if all values evaluate to True. False otherwise. + :description: + ``and`` operator. Returns True if all values evaluate to True. False otherwise. """ return Boolean(all(bool(val.value) for val in values)) @staticmethod def or_(*values: AnyArgument) -> Boolean: """ - ``or`` operator. Returns True if any value evaluates to True. False otherwise. + :description: + ``or`` operator. Returns True if any value evaluates to True. False otherwise. """ return Boolean(any(bool(val.value) for val in values)) @staticmethod def xor(*values: AnyArgument) -> Boolean: """ - ``^`` operator. Returns True if exactly one value is set to True. False otherwise. + :description: + ``^`` operator. Returns True if exactly one value is set to True. False otherwise. """ bit_array = [bool(val.value) for val in values] @@ -85,13 +95,15 @@ class BooleanFunctions: @staticmethod def not_(value: Boolean) -> Boolean: """ - ``not`` operator. Returns the opposite of value. + :description: + ``not`` operator. Returns the opposite of value. """ return Boolean(not value.value) @staticmethod def is_null(value: AnyArgument) -> Boolean: """ - Returns True if a value is null (i.e. an empty string). False otherwise. + :description: + Returns True if a value is null (i.e. an empty string). False otherwise. """ return Boolean(isinstance(value, String) and value.value == "") diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py index 8caaae02..e532031b 100644 --- a/src/ytdl_sub/script/functions/conditional_functions.py +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -11,8 +11,9 @@ class ConditionalFunctions: condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: """ - Conditional ``if`` statement that returns the ``true`` or ``false`` parameter - depending on the ``condition`` value. + :description: + Conditional ``if`` statement that returns the ``true`` or ``false`` parameter + depending on the ``condition`` value. """ if condition.value: return true @@ -23,8 +24,9 @@ class ConditionalFunctions: maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB ) -> Union[ReturnableArgumentA, ReturnableArgumentB]: """ - Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, - otherwise returns ``else_arg``. + :description: + Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True, + otherwise returns ``else_arg``. """ if bool(maybe_true_arg.value): return maybe_true_arg diff --git a/src/ytdl_sub/script/functions/date_functions.py b/src/ytdl_sub/script/functions/date_functions.py index 8d68f052..fea1095c 100644 --- a/src/ytdl_sub/script/functions/date_functions.py +++ b/src/ytdl_sub/script/functions/date_functions.py @@ -8,6 +8,7 @@ class DateFunctions: @staticmethod def datetime_strftime(posix_timestamp: Integer, date_format: String) -> String: """ - Converts a posix timestamp to a date using strftime formatting. + :description: + Converts a posix timestamp to a date using strftime formatting. """ return String(datetime.utcfromtimestamp(posix_timestamp.value).strftime(date_format.value)) diff --git a/src/ytdl_sub/script/functions/error_functions.py b/src/ytdl_sub/script/functions/error_functions.py index 517c6141..3a1be9f5 100644 --- a/src/ytdl_sub/script/functions/error_functions.py +++ b/src/ytdl_sub/script/functions/error_functions.py @@ -8,15 +8,17 @@ class ErrorFunctions: @staticmethod def throw(error_message: String) -> AnyArgument: """ - Explicitly throw an error with the provided error message. + :description: + Explicitly throw an error with the provided error message. """ raise UserThrownRuntimeError(error_message) @staticmethod def assert_(value: ReturnableArgument, assert_message: String) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` evaluates to False. - If it evaluates to True, it will return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``value``. """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) @@ -27,8 +29,9 @@ class ErrorFunctions: value: AnyArgument, ret: ReturnableArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` evaluates to False. - If it evaluates to True, it will return ``ret``. + :description: + Explicitly throw an error with the provided assert message if ``value`` evaluates to + False. If it evaluates to True, it will return ``ret``. """ if not bool(value.value): raise UserThrownRuntimeError(assert_message) @@ -39,8 +42,9 @@ class ErrorFunctions: value: ReturnableArgument, equals: AnyArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` does not equal - ``equals``. If they do equal, then return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` does not equal + ``equals``. If they do equal, then return ``value``. """ if not value.value == equals.value: raise UserThrownRuntimeError(assert_message) @@ -51,8 +55,9 @@ class ErrorFunctions: value: ReturnableArgument, equals: AnyArgument, assert_message: String ) -> ReturnableArgument: """ - Explicitly throw an error with the provided assert message if ``value`` equals - ``equals``. If they do equal, then return ``value``. + :description: + Explicitly throw an error with the provided assert message if ``value`` equals + ``equals``. If they do equal, then return ``value``. """ if value.value == equals.value: raise UserThrownRuntimeError(assert_message) diff --git a/src/ytdl_sub/script/functions/json_functions.py b/src/ytdl_sub/script/functions/json_functions.py index 915fedf1..c706a86f 100644 --- a/src/ytdl_sub/script/functions/json_functions.py +++ b/src/ytdl_sub/script/functions/json_functions.py @@ -35,6 +35,7 @@ class JsonFunctions: @staticmethod def from_json(argument: String) -> AnyArgument: """ - Converts a JSON string into an actual type. + :description: + Converts a JSON string into an actual type. """ return _from_json(json.loads(argument.value)) diff --git a/src/ytdl_sub/script/functions/map_functions.py b/src/ytdl_sub/script/functions/map_functions.py index 45ca13f1..b34fa611 100644 --- a/src/ytdl_sub/script/functions/map_functions.py +++ b/src/ytdl_sub/script/functions/map_functions.py @@ -18,12 +18,8 @@ class MapFunctions: @staticmethod def map(maybe_mapping: AnyArgument) -> Map: """ - Tries to cast an unknown variable type to a Map. - - Raises - ------ - FunctionRuntimeException - If the input type is not actually a Map. + :description: + Tries to cast an unknown variable type to a Map. """ if not isinstance(maybe_mapping, Map): raise FunctionRuntimeException( @@ -34,14 +30,16 @@ class MapFunctions: @staticmethod def map_size(mapping: Map) -> Integer: """ - Returns the size of a Map. + :description: + Returns the size of a Map. """ return Integer(len(mapping.value)) @staticmethod def map_contains(mapping: Map, key: AnyArgument) -> Boolean: """ - Returns True if the key is in the Map. False otherwise. + :description: + Returns True if the key is in the Map. False otherwise. """ if not isinstance(key, Hashable): raise KeyNotHashableRuntimeException( @@ -55,8 +53,9 @@ class MapFunctions: mapping: Map, key: AnyArgument, default: Optional[AnyArgument] = None ) -> AnyArgument: """ - Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is - provided, it will return ``default``. Otherwise, will error. + :description: + Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is + provided, it will return ``default``. Otherwise, will error. """ if not MapFunctions.map_contains(mapping=mapping, key=key).value: if default is not None: @@ -70,8 +69,9 @@ class MapFunctions: @staticmethod def map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument: """ - Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, - return ``default``. Otherwise, will error. + :description: + Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string, + return ``default``. Otherwise, will error. """ output = MapFunctions.map_get(mapping, key, default) if isinstance(output, String) and output.value == "": @@ -83,16 +83,18 @@ class MapFunctions: @staticmethod def map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array: """ - Apply a lambda function on the Map, where each arg - passed to the lambda function is ``key, value`` as two separate args. + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``key, value`` as two separate args. """ return Array([Array([key, value]) for key, value in mapping.value.items()]) @staticmethod def map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array: """ - Apply a lambda function on the Map, where each arg - passed to the lambda function is ``idx, key, value`` as three separate args. + :description: + Apply a lambda function on the Map, where each arg + passed to the lambda function is ``idx, key, value`` as three separate args. """ return Array( [ diff --git a/src/ytdl_sub/script/functions/numeric_functions.py b/src/ytdl_sub/script/functions/numeric_functions.py index 5211c690..b25374f9 100644 --- a/src/ytdl_sub/script/functions/numeric_functions.py +++ b/src/ytdl_sub/script/functions/numeric_functions.py @@ -16,28 +16,32 @@ class NumericFunctions: @staticmethod def float(value: AnyArgument) -> Float: """ - Cast to Float. + :description: + Cast to Float. """ return Float(value=float(value.value)) @staticmethod def int(value: AnyArgument) -> Integer: """ - Cast to Integer. + :description: + Cast to Integer. """ return Integer(value=int(value.value)) @staticmethod def add(*values: Numeric) -> Numeric: """ - ``+`` operator. Returns the sum of all values. + :description: + ``+`` operator. Returns the sum of all values. """ return _to_numeric(sum(val.value for val in values)) @staticmethod def sub(*values: Numeric) -> Numeric: """ - ``-`` operator. Subtracts all values from left to right. + :description: + ``-`` operator. Subtracts all values from left to right. """ output = values[0].value for val in values[1:]: @@ -48,41 +52,47 @@ class NumericFunctions: @staticmethod def mul(*values: Numeric) -> Numeric: """ - ``*`` operator. Returns the product of all values. + :description: + ``*`` operator. Returns the product of all values. """ return _to_numeric(math.prod([val.value for val in values])) @staticmethod def pow(base: Numeric, exponent: Numeric) -> Numeric: """ - ``**`` operator. Returns the exponential of the base and exponent value. + :description: + ``**`` operator. Returns the exponential of the base and exponent value. """ return _to_numeric(math.pow(base.value, exponent.value)) @staticmethod def div(left: Numeric, right: Numeric) -> Numeric: """ - ``/`` operator. Returns ``left / right``. + :description: + ``/`` operator. Returns ``left / right``. """ return _to_numeric(left.value / right.value) @staticmethod def mod(left: Numeric, right: Numeric) -> Numeric: """ - ``%`` operator. Returns ``left % right``. + :description: + ``%`` operator. Returns ``left % right``. """ return _to_numeric(value=left.value % right.value) @staticmethod def max(*values: Numeric) -> Numeric: """ - Returns max of all values. + :description: + Returns max of all values. """ return _to_numeric(max(val.value for val in values)) @staticmethod def min(*values: Numeric) -> Numeric: """ - Returns min of all values. + :description: + Returns min of all values. """ return _to_numeric(min(val.value for val in values)) diff --git a/src/ytdl_sub/script/functions/regex_functions.py b/src/ytdl_sub/script/functions/regex_functions.py index e38b3139..f837be32 100644 --- a/src/ytdl_sub/script/functions/regex_functions.py +++ b/src/ytdl_sub/script/functions/regex_functions.py @@ -18,33 +18,37 @@ class RegexFunctions: @staticmethod def regex_match(regex: String, string: String) -> Array: """ - Checks for a match only at the beginning of the string. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for a match only at the beginning of the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.match(regex.value, string.value)) @staticmethod def regex_search(regex: String, string: String) -> Array: """ - Checks for a match anywhere in the string. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for a match anywhere in the string. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.search(regex.value, string.value)) @staticmethod def regex_fullmatch(regex: String, string: String) -> Array: """ - Checks for entire string to be a match. If a match exists, returns - the string as the first element of the Array. If there are capture groups, returns each - group as a subsequent element in the Array. + :description: + Checks for entire string to be a match. If a match exists, returns + the string as the first element of the Array. If there are capture groups, returns each + group as a subsequent element in the Array. """ return _re_output_to_array(re.fullmatch(regex.value, string.value)) @staticmethod def regex_capture_groups(regex: String) -> Integer: """ - Returns number of capture groups in regex + :description: + Returns number of capture groups in regex """ return Integer(re.compile(regex.value).groups) diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index 807a5dd4..1f0280b5 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -11,21 +11,24 @@ class StringFunctions: @staticmethod def string(value: AnyArgument) -> String: """ - Cast to String. + :description: + Cast to String. """ return String(value=str(value.value)) @staticmethod def contains(string: String, contains: String) -> Boolean: """ - Returns True if ``contains`` is in ``string``. False otherwise. + :description: + Returns True if ``contains`` is in ``string``. False otherwise. """ return Boolean(contains.value in string.value) @staticmethod def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: """ - Returns the slice of the Array. + :description: + Returns the slice of the Array. """ if end is not None: return String(string.value[start.value : end.value]) @@ -34,28 +37,32 @@ class StringFunctions: @staticmethod def lower(string: String) -> String: """ - Lower-case the entire String. + :description: + Lower-case the entire String. """ return String(string.value.lower()) @staticmethod def upper(string: String) -> String: """ - Upper-case the entire String. + :description: + Upper-case the entire String. """ return String(string.value.upper()) @staticmethod def capitalize(string: String) -> String: """ - Capitalize the first character in the string. + :description: + Capitalize the first character in the string. """ return String(string.value.capitalize()) @staticmethod def titlecase(string: String) -> String: """ - Capitalize each word in the string. + :description: + Capitalize each word in the string. """ return String(string.value.title()) @@ -64,8 +71,9 @@ class StringFunctions: string: String, old: String, new: String, count: Optional[Integer] = None ) -> String: """ - Replace the ``old`` part of the String with the ``new``. Optionally only replace it - ``count`` number of times. + :description: + Replace the ``old`` part of the String with the ``new``. Optionally only replace it + ``count`` number of times. """ if count: return String(string.value.replace(old.value, new.value, count.value)) @@ -75,14 +83,16 @@ class StringFunctions: @staticmethod def concat(*values: String) -> String: """ - Concatenate multiple Strings into a single String. + :description: + Concatenate multiple Strings into a single String. """ return String("".join(val.value for val in values)) @staticmethod def pad(string: String, length: Integer, char: String) -> String: """ - Pads the string to the given length + :description: + Pads the string to the given length """ output = string.value while len(output) < length.value: @@ -93,7 +103,8 @@ class StringFunctions: @staticmethod def pad_zero(numeric: Numeric, length: Integer) -> String: """ - Pads a numeric with zeros to the given length + :description: + Pads a numeric with zeros to the given length """ return StringFunctions.pad( string=String(str(numeric.value)), diff --git a/src/ytdl_sub/script/utils/exception_formatters.py b/src/ytdl_sub/script/utils/exception_formatters.py index fb3b5e6e..6e14354e 100644 --- a/src/ytdl_sub/script/utils/exception_formatters.py +++ b/src/ytdl_sub/script/utils/exception_formatters.py @@ -1,16 +1,11 @@ import sys from typing import List -from typing import Type from typing import TypeVar -from typing import Union from ytdl_sub.script.types.resolvable import BuiltInFunctionType -from ytdl_sub.script.types.resolvable import NamedType from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.type_checking import FunctionSpec -from ytdl_sub.script.utils.type_checking import get_optional_type -from ytdl_sub.script.utils.type_checking import is_optional from ytdl_sub.script.utils.type_checking import is_union TUserException = TypeVar("TUserException", bound=UserException) @@ -103,28 +98,10 @@ class FunctionArgumentsExceptionFormatter: input_spec: FunctionSpec, function_instance: BuiltInFunctionType, ): - self._args = input_spec.args - self._varargs = input_spec.varargs + self._input_spec = input_spec self._name = function_instance.name self._input_args = function_instance.args - @classmethod - def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: - if is_optional(python_type): - return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" - if is_union(python_type): - return ", ".join( - sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) - ) - return python_type.type_name() - - def _expected_args_str(self) -> str: - if self._args is not None: - return f"({', '.join([self._to_human_readable_name(type_) for type_ in self._args])})" - if self._varargs is not None: - return f"({self._to_human_readable_name(self._varargs)}, ...)" - return "()" - def _received_args_str(self) -> str: received_type_names: List[str] = [] for arg in self._input_args: @@ -149,5 +126,6 @@ class FunctionArgumentsExceptionFormatter: """ return IncompatibleFunctionArguments( f"Incompatible arguments passed to function {self._name}.\n" - f"Expected {self._expected_args_str()}\nReceived {self._received_args_str()}" + f"Expected {self._input_spec.human_readable_input_args()}\n" + f"Received {self._received_args_str()}" ) diff --git a/src/ytdl_sub/script/utils/type_checking.py b/src/ytdl_sub/script/utils/type_checking.py index cc7d87ff..2170c3e0 100644 --- a/src/ytdl_sub/script/utils/type_checking.py +++ b/src/ytdl_sub/script/utils/type_checking.py @@ -119,6 +119,7 @@ def is_type_compatible( @dataclass(frozen=True) class FunctionSpec: return_type: Type[Resolvable] + arg_names: List[str] args: Optional[List[Type[Resolvable | Optional[Resolvable]]]] = None varargs: Optional[Type[Resolvable]] = None @@ -223,6 +224,42 @@ class FunctionSpec: return l_type return None + @classmethod + def _to_human_readable_name(cls, python_type: Type[NamedType] | Type[Union[NamedType]]) -> str: + if is_optional(python_type): + return f"Optional[{cls._to_human_readable_name(get_optional_type(python_type))}]" + if is_union(python_type): + args = ", ".join( + sorted(cls._to_human_readable_name(arg) for arg in python_type.__args__) + ) + return f"Union[{args}]" + return python_type.type_name() + + def human_readable_input_args(self) -> str: + """ + Returns + ------- + input arg string in human-readable format + """ + if self.args is not None: + args = ", ".join( + f"{name}: {self._to_human_readable_name(type_)}" + for name, type_ in zip(self.arg_names, self.args) + ) + elif self.varargs is not None: + args = f"{self.arg_names[0]}: {self._to_human_readable_name(self.varargs)}, ..." + else: + args = "" + return f"({args})" + + def human_readable_output_type(self) -> str: + """ + Returns + ------- + output type string in human-readable format + """ + return self._to_human_readable_name(self.return_type) + @classmethod def from_callable(cls, callable_ref: Callable[..., Resolvable]) -> "FunctionSpec": """ @@ -234,10 +271,12 @@ class FunctionSpec: if arg_spec.varargs: return FunctionSpec( return_type=arg_spec.annotations["return"], + arg_names=[arg_spec.varargs], varargs=arg_spec.annotations[arg_spec.varargs], ) return FunctionSpec( return_type=arg_spec.annotations["return"], + arg_names=arg_spec.args, args=[arg_spec.annotations[arg_name] for arg_name in arg_spec.args], ) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index 7cf5ef2b..646d44e1 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -33,6 +33,13 @@ def get_file_extension(file_name: Path | str) -> str: return file_name.rsplit(".", maxsplit=1)[-1] +def get_md5_hash(contents: str) -> str: + """ + Helper function to compute md5 hash + """ + return hashlib.md5(contents.encode()).hexdigest() + + def get_file_md5_hash(full_file_path: Path | str) -> str: """ Parameters diff --git a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt index 77140703..4b7fb2b5 100644 --- a/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt +++ b/tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt @@ -1,5 +1,5 @@ { ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475", - "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c", + "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "d53121df33ac8c4a4699ec8919196552", "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json index 890aca91..f077d34c 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json @@ -2,6 +2,6 @@ ".ytdl-sub-multiple_songs_test-download-archive.json": "54237df5e00d1598dfd39f341ee03d75", "Project Zombie/[2011] Jesse's Minecraft Server/01 - Jesse's Minecraft Server [Trailer - Mar.21].ogg": "5657c5b92f8980b20d8bbee0fdc7e5d8", "Project Zombie/[2011] Jesse's Minecraft Server/02 - Jesse's Minecraft Server [Trailer - Feb.27].ogg": "a2a3a34e02e26a6c0265530d4499473b", - "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "b2d6388b4ddf8e3fbca042cb123672c3", + "Project Zombie/[2011] Jesse's Minecraft Server/03 - Jesse's Minecraft Server [Trailer - Feb.1].ogg": "0a385da3aa06b994a69b8ab812b44975", "Project Zombie/[2011] Jesse's Minecraft Server/folder.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 30f23eb5..74e55e90 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -2,7 +2,7 @@ "Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "6fb0ce965b75035079f82c84ad341e85", + "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "246fa05b6443337785575987904848df", "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1", "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "INFO_JSON", @@ -10,7 +10,7 @@ "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo": "4ad498ce223454a4baa7d64bb4a837d6", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "43f271ef8d3a19877f0dc9bc5040b42a", + "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "ae1d5e2e3979cea3c96e6a4cfcae8073", "Project ⧸ Zombie/Season 2011/s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "073eefa6e5c6d76edde80258ddf452ee", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "Project ⧸ Zombie/Season 2011/s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index 29dfe4cf..c4fb6cca 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json index 143b2310..c5490aa6 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json @@ -2,7 +2,7 @@ "JMC/.ytdl-sub-JMC-download-archive.json": "3fdab8d103e51aa70430b6da0ceb07e2", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "INFO_JSON", - "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "2b9b7968e0db88c53d820868e542a31c", + "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "82bcd97a13f2ba361e66ad631aeac32f", "JMC/Season 01/s01.e11020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "2a2997cbf16fb6b943d9933ad267331e", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", "JMC/Season 01/s01.e11022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "INFO_JSON", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video.json b/tests/resources/expected_downloads_summaries/youtube/test_video.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json index 4cbb7eca..78aeb87a 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json @@ -1,5 +1,5 @@ { "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "0448c9fd3eeaba4eca7f650fb93fe21b", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json index 04e7c7d9..b425d91d 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json @@ -1,4 +1,4 @@ { - "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "718c187e6196c85eea73d16ebd489c91", + "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d9d2d12feee44ee97729b39ba981c542", "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb" } \ No newline at end of file diff --git a/tests/unit/docgen/__init__.py b/tests/unit/docgen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py new file mode 100644 index 00000000..82768347 --- /dev/null +++ b/tests/unit/docgen/test_docgen.py @@ -0,0 +1,30 @@ +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.entry_variables import EntryVariablesDocGen +from tools.docgen.override_variables import OverrideVariablesDocGen +from tools.docgen.plugins import PluginsDocGen +from tools.docgen.scripting_functions import ScriptingFunctionsDocGen +from ytdl_sub.utils.file_handler import get_md5_hash + + +def _test_doc_gen(doc_gen: Type[DocGen]) -> None: + expected_md5_hash = get_md5_hash(doc_gen.generate_and_maybe_write_to_file()) + with open(doc_gen.LOCATION, "r", encoding="utf-8") as file_doc: + md5_hash = get_md5_hash(file_doc.read()) + + assert md5_hash == expected_md5_hash + + +class TestDocGen: + def test_entry_variables_generated(self): + _test_doc_gen(EntryVariablesDocGen) + + def test_override_variables_generated(self): + _test_doc_gen(OverrideVariablesDocGen) + + def test_scripting_functions_generated(self): + _test_doc_gen(ScriptingFunctionsDocGen) + + def test_plugins_generated(self): + _test_doc_gen(PluginsDocGen) diff --git a/tests/unit/docgen/test_docgen_regenerate_disabled.py b/tests/unit/docgen/test_docgen_regenerate_disabled.py new file mode 100644 index 00000000..836b13ea --- /dev/null +++ b/tests/unit/docgen/test_docgen_regenerate_disabled.py @@ -0,0 +1,5 @@ +from tools.docgen.docgen import REGENERATE_DOCS + + +def test_docgen_regenerate_disabled(): + assert REGENERATE_DOCS is False diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py index 1331d485..014e4a85 100644 --- a/tests/unit/script/types/test_function.py +++ b/tests/unit/script/types/test_function.py @@ -40,7 +40,7 @@ class TestFunction: with pytest.raises( IncompatibleFunctionArguments, match=_incompatible_arguments_match( - expected="Map, AnyArgument, Optional[AnyArgument]", + expected="mapping: Map, key: AnyArgument, default: Optional[AnyArgument]", recieved="%if(...)->Union[Array, Map], String", ), ): @@ -49,11 +49,11 @@ class TestFunction: @pytest.mark.parametrize( "function_str, expected_types, received_types", [ - ("{%array_at({'a': 'dict?'}, 1)}", "Array, Integer", "Map, Integer"), - ("{%array_extend('not', 'array')}", "Array, ...", "String, String"), + ("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), + ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), ( "{%replace('hi mom', 'mom', 'dad', 1, 0)}", - "String, String, String, Optional[Integer]", + "string: String, old: String, new: String, count: Optional[Integer]", "String, String, String, Integer, Integer", ), ], diff --git a/tools/docgen/docgen.py b/tools/docgen/docgen.py new file mode 100644 index 00000000..f1307b29 --- /dev/null +++ b/tools/docgen/docgen.py @@ -0,0 +1,32 @@ +from abc import abstractmethod +from pathlib import Path + +REGENERATE_DOCS: bool = False + + +class DocGen: + """ + Home-made auto doc generation + """ + + LOCATION: Path + + @classmethod + @abstractmethod + def generate(cls) -> str: + """ + Generate the docs as a single string + """ + + @classmethod + def generate_and_maybe_write_to_file(cls) -> str: + """ + Maybe writes the docs to their file if the global is set to True, and returns + the generated docs + """ + contents = cls.generate() + if REGENERATE_DOCS: + with open(cls.LOCATION, "w", encoding="utf-8") as out: + out.write(contents) + + return contents diff --git a/tools/docgen/entry_variables.py b/tools/docgen/entry_variables.py new file mode 100644 index 00000000..02cca731 --- /dev/null +++ b/tools/docgen/entry_variables.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import line_section +from tools.docgen.utils import properties +from tools.docgen.utils import section +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions + + +def _variable_class_to_name(obj: Type[Any]) -> str: + assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions" + return ( + camel_case_to_human(obj.__name__) + .replace("Variable Definitions", "Variables") + .replace("Ytdl Sub", "Ytdl-Sub") + ) + + +class EntryVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/entry_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Entry Variables", level=0) + + parent_objs: Dict[str, Type[Any]] = { + _variable_class_to_name(obj): obj for obj in VariableDefinitions.__bases__ + } + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for variable_function_name in properties(parent_objs[name]): + docs += get_function_docs( + function_name=variable_function_name, + obj=parent_objs[name], + level=2, + ) + + return docs diff --git a/tools/docgen/override_variables.py b/tools/docgen/override_variables.py new file mode 100644 index 00000000..e919877c --- /dev/null +++ b/tools/docgen/override_variables.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.variables.override_variables import OverrideVariables + + +class OverrideVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Override Variables", level=0) + + for name in static_methods(OverrideVariables): + docs += get_function_docs( + function_name=name, + obj=OverrideVariables, + level=1, + ) + + return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py new file mode 100644 index 00000000..10323066 --- /dev/null +++ b/tools/docgen/plugins.py @@ -0,0 +1,91 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import line_section +from tools.docgen.utils import properties +from tools.docgen.utils import section +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator + + +def should_filter_all_properties(plugin_name: str) -> bool: + return plugin_name in ( + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "video_tags", + "download", + ) + + +def should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + ) + + +def get_function_docs(function_name: str, obj: Any, level: int) -> str: + docs = f"\n``{function_name}``\n\n" + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n\n" + return docs + + +def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: + docs = "" + docs += section(name, level=offset + 0) + + docs += inspect.cleandoc(options.__doc__) + docs += "\n" + + if should_filter_all_properties(name): + return docs + + property_names = [prop for prop in properties(options) if not should_filter_property(prop)] + for property_name in sorted(property_names): + docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) + + return docs + + +class PluginsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/plugins.rst") + + @classmethod + def generate(cls): + options_dict: Dict[str, Type[OptionsValidator]] = { + "output_options": OutputOptions, + "ytdl_options": YTDLOptions, + "overrides": Overrides, + "download": MultiUrlValidator, + } + for plugin_name, plugin_type in PluginMapping._MAPPING.items(): + if plugin_name.startswith("_"): + continue + options_dict[plugin_name] = plugin_type.plugin_options_type + + docs = section("Plugins", level=0) + for idx, name in enumerate(sorted(options_dict.keys())): + docs += line_section(section_idx=idx) + docs += generate_plugin_docs(name, options_dict[name], offset=1) + + return docs diff --git a/tools/docgen/scripting_functions.py b/tools/docgen/scripting_functions.py new file mode 100644 index 00000000..c4663477 --- /dev/null +++ b/tools/docgen/scripting_functions.py @@ -0,0 +1,82 @@ +import inspect +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional +from typing import Type + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import camel_case_to_human +from tools.docgen.utils import line_section +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.script.custom_functions import CustomFunctions +from ytdl_sub.script.functions import Functions +from ytdl_sub.script.utils.type_checking import FunctionSpec + + +def maybe_get_function_name(function_name: str) -> Optional[str]: + if function_name in ["register"]: + return None + + if function_name.endswith("_"): + return function_name[:-1] + return function_name + + +def function_class_to_name(obj: Type[Any]) -> str: + assert "Functions" in obj.__name__ + return camel_case_to_human(obj.__name__) + + +def function_type_hinting(display_function_name: str, function: Any) -> str: + spec = FunctionSpec.from_callable(function) + out = ":spec: ``" + out += display_function_name + out += spec.human_readable_input_args() + out += " -> " + out += spec.human_readable_output_type() + out += "``\n\n" + return out + + +def get_function_docstring( + function_name: str, function: Any, level: int, display_function_name: Optional[str] = None +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + + docs += function_type_hinting(display_function_name=display_function_name, function=function) + docs += inspect.cleandoc(function.__doc__) + docs += "\n" + return docs + + +class ScriptingFunctionsDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/scripting_functions.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Scripting Functions", level=0) + + parent_objs: Dict[str, Type[Any]] = { + function_class_to_name(obj): obj for obj in Functions.__bases__ + } + parent_objs["Ytdl-Sub Functions"] = CustomFunctions + + for idx, name in enumerate(sorted(parent_objs.keys())): + docs += line_section(section_idx=idx) + docs += section(name, level=1) + + for function_name in static_methods(parent_objs[name]): + if display_function_name := maybe_get_function_name(function_name): + docs += get_function_docstring( + function_name=function_name, + display_function_name=display_function_name, + function=getattr(parent_objs[name], function_name), + level=2, + ) + + return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py new file mode 100644 index 00000000..94cab101 --- /dev/null +++ b/tools/docgen/utils.py @@ -0,0 +1,56 @@ +import inspect +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Type + +LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} + + +def section(name: str, level: int, as_code: bool = False) -> str: + if as_code: + name = f"``{name}``" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" + + +def properties(obj: Type[Any]) -> List[str]: + return sorted(prop for prop in dir(obj) if isinstance(getattr(obj, prop), property)) + + +def static_methods(obj: Type[Any]) -> List[str]: + return sorted( + name for name in dir(obj) if isinstance(inspect.getattr_static(obj, name), staticmethod) + ) + + +def camel_case_to_human(string: str) -> str: + output_str = string[0] + for char in string[1:]: + if char.islower(): + output_str += char + else: + output_str += f" {char}" + + return output_str + + +def get_function_docs( + function_name: str, obj: Any, level: int, display_function_name: Optional[str] = None +) -> str: + display_function_name = display_function_name if display_function_name else function_name + + docs = section(display_function_name, level=level) + docs += inspect.cleandoc(getattr(obj, function_name).__doc__) + docs += "\n" + return docs + + +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return ""