From 9af03eea9ba3476f11705dc0c5829bffb75abdef Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 5 Feb 2025 09:48:57 -0800 Subject: [PATCH] plugin modularize for config docs --- docs/source/config_reference/plugins.rst | 50 ----------------- tools/docgen/configuration.py | 3 +- tools/docgen/plugins.py | 68 +++++++----------------- tools/docgen/utils.py | 57 ++++++++++++++++---- 4 files changed, 67 insertions(+), 111 deletions(-) diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index c3161a27..b48fe78c 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -21,7 +21,6 @@ Extracts audio from a video file. The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, opus, vorbis, wav, and best to grab the best possible format at runtime. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -30,7 +29,6 @@ Extracts audio from a video file. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``quality`` :expected type: Float @@ -38,7 +36,6 @@ Extracts audio from a video file. Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. - ---------------------------------------------------------------------------------------------------- chapters @@ -77,14 +74,12 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to False. If chapters do not exist in the video/description itself, attempt to scrape comments to find the chapters. - ``embed_chapters`` :expected type: Optional[Boolean] :description: Defaults to True. Embed chapters into the file. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -93,7 +88,6 @@ chapters and remove specific ones. Can also remove chapters using regex. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``force_key_frames`` :expected type: Optional[Boolean] @@ -101,7 +95,6 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to False. Force keyframes at cuts when removing sections. This is slow due to needing a re-encode, but the resulting video may have fewer artifacts around the cuts. - ``remove_chapters_regex`` :expected type: Optional[List[RegexString] @@ -109,7 +102,6 @@ chapters and remove specific ones. Can also remove chapters using regex. List of regex patterns to match chapter titles against and remove them from the entry. - ``remove_sponsorblock_categories`` :expected type: Optional[List[String]] @@ -118,7 +110,6 @@ chapters and remove specific ones. Can also remove chapters using regex. categories that are specified in ``sponsorblock_categories`` or "all", which removes everything specified in ``sponsorblock_categories``. - ``sponsorblock_categories`` :expected type: Optional[List[String]] @@ -127,7 +118,6 @@ chapters and remove specific ones. Can also remove chapters using regex. "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", "poi_highlight", or "all" to include all categories. - ---------------------------------------------------------------------------------------------------- date_range @@ -158,14 +148,12 @@ granularity possible. :description: Only download videos after this datetime. - ``before`` :expected type: Optional[OverridesFormatter] :description: Only download videos before this datetime. - ``breaks`` :expected type: Optional[OverridesFormatter] @@ -173,7 +161,6 @@ granularity possible. Toggle to enable breaking subsequent metadata downloads if an entry's upload date is out of range. Defaults to True. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -182,7 +169,6 @@ granularity possible. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ---------------------------------------------------------------------------------------------------- download @@ -286,7 +272,6 @@ Also supports custom ffmpeg conversions: - Video: avi, flv, mkv, mov, mp4, webm - Audio: aac, flac, mp3, m4a, opus, vorbis, wav - ``convert_with`` :expected type: Optional[String] @@ -295,7 +280,6 @@ Also supports custom ffmpeg conversions: yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -304,7 +288,6 @@ Also supports custom ffmpeg conversions: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``ffmpeg_post_process_args`` :expected type: Optional[OverridesFormatter] @@ -317,7 +300,6 @@ Also supports custom ffmpeg conversions: The output file will use the extension specified in ``convert_to``. Post-processing args can still be set with ``convert_with`` set to ``yt-dlp``. - ---------------------------------------------------------------------------------------------------- filter_exclude @@ -453,7 +435,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: Optional[Boolean] @@ -462,14 +443,12 @@ with a ``.nfo`` extension. You can add any values into the NFO. emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -482,7 +461,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. - ``tags`` :expected type: NfoTags @@ -519,7 +497,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. Comedy Drama - ---------------------------------------------------------------------------------------------------- output_directory_nfo_tags @@ -551,7 +528,6 @@ Usage: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: Optional[Boolean] @@ -560,14 +536,12 @@ Usage: emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -580,7 +554,6 @@ Usage: - ``tags`` :expected type: NfoTags @@ -615,7 +588,6 @@ Usage: Comedy Drama - ---------------------------------------------------------------------------------------------------- output_options @@ -648,7 +620,6 @@ Defines where to output files and thumbnails after all post-processing has compl The file name to store a subscriptions download archive placed relative to the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` - ``file_name`` :expected type: EntryFormatter @@ -656,7 +627,6 @@ Defines where to output files and thumbnails after all post-processing has compl The file name for the media file. This can include directories such as ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. - ``info_json_name`` :expected type: Optional[EntryFormatter] @@ -665,7 +635,6 @@ Defines where to output files and thumbnails after all post-processing has compl as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output directory. Can be set to empty string or `null` to disable info json writes. - ``keep_files_after`` :expected type: Optional[OverridesFormatter] @@ -677,7 +646,6 @@ Defines where to output files and thumbnails after all post-processing has compl files after ``19000101``, which implies all files. Can be used in conjunction with ``keep_max_files``. - ``keep_files_before`` :expected type: Optional[OverridesFormatter] @@ -689,7 +657,6 @@ Defines where to output files and thumbnails after all post-processing has compl files before ``now``, which implies all files. Can be used in conjunction with ``keep_max_files``. - ``keep_max_files`` :expected type: Optional[OverridesFormatter] @@ -699,7 +666,6 @@ Defines where to output files and thumbnails after all post-processing has compl Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. - ``maintain_download_archive`` :expected type: Optional[Boolean] @@ -714,7 +680,6 @@ Defines where to output files and thumbnails after all post-processing has compl Defaults to False. - ``migrated_download_archive_name`` :expected type: Optional[OverridesFormatter] @@ -724,14 +689,12 @@ Defines where to output files and thumbnails after all post-processing has compl name first, and fallback to ``download_archive_name``. It will always save to this file and remove the original ``download_archive_name``. - ``output_directory`` :expected type: OverridesFormatter :description: The output directory to store all media files downloaded. - ``thumbnail_name`` :expected type: Optional[EntryFormatter] @@ -740,7 +703,6 @@ Defines where to output files and thumbnails after all post-processing has compl as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output directory. Can be set to empty string or `null` to disable thumbnail writes. - ---------------------------------------------------------------------------------------------------- overrides @@ -805,7 +767,6 @@ used with no modifications. If a file has no chapters and is set to "pass", then ``chapter_title`` is set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. - ---------------------------------------------------------------------------------------------------- subtitles @@ -833,7 +794,6 @@ It will set the respective language to the correct subtitle file. :description: Defaults to False. Whether to allow auto generated subtitles. - ``embed_subtitles`` :expected type: Optional[Boolean] @@ -841,7 +801,6 @@ It will set the respective language to the correct subtitle file. Defaults to False. Whether to embed the subtitles into the video file. Note that webm files can only embed "vtt" subtitle types. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -850,7 +809,6 @@ It will set the respective language to the correct subtitle file. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``languages`` :expected type: Optional[List[String]] @@ -858,7 +816,6 @@ It will set the respective language to the correct subtitle file. Language code(s) to download for subtitles. Supports a single or list of multiple language codes. Defaults to only "en". - ``subtitles_name`` :expected type: Optional[EntryFormatter] @@ -868,14 +825,12 @@ It will set the respective language to the correct subtitle file. and will be placed in the output directory. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - ``subtitles_type`` :expected type: Optional[String] :description: Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". - ---------------------------------------------------------------------------------------------------- throttle_protection @@ -910,14 +865,12 @@ scripted. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``max_downloads_per_subscription`` :expected type: Optional[Range] :description: Number of downloads to perform per subscription. - ``sleep_per_download_s`` :expected type: Optional[Range] @@ -925,14 +878,12 @@ scripted. Number in seconds to sleep between each download. Does not include time it takes for ytdl-sub to perform post-processing. - ``sleep_per_subscription_s`` :expected type: Optional[Range] :description: Number in seconds to sleep between each subscription. - ``subscription_download_probability`` :expected type: Optional[Float] @@ -941,7 +892,6 @@ scripted. recommended to set if you run ytdl-sub in a cron-job, that way you are statistically guaranteed over time to eventually download the subscription. - ---------------------------------------------------------------------------------------------------- video_tags diff --git a/tools/docgen/configuration.py b/tools/docgen/configuration.py index 95b3eef2..721efcec 100644 --- a/tools/docgen/configuration.py +++ b/tools/docgen/configuration.py @@ -9,11 +9,10 @@ from tools.docgen.utils import camel_case_to_human from tools.docgen.utils import get_function_docs from tools.docgen.utils import line_section from tools.docgen.utils import section +from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions -from ytdl_sub.config.config_validator import ConfigOptions - def _variable_class_to_name(obj: Type[Any]) -> str: assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions" diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py index 1483ab38..5b33a39c 100644 --- a/tools/docgen/plugins.py +++ b/tools/docgen/plugins.py @@ -1,13 +1,11 @@ -import inspect from pathlib import Path -from typing import Any from typing import Dict -from typing import Optional +from typing import Set from typing import Type from tools.docgen.docgen import DocGen -from tools.docgen.utils import line_section, get_function_docs -from tools.docgen.utils import properties +from tools.docgen.utils import generate_options_validator_docs +from tools.docgen.utils import line_section from tools.docgen.utils import section from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_mapping import PluginMapping @@ -16,49 +14,16 @@ from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator - -def should_filter_all_properties(plugin_name: str) -> bool: - return plugin_name in ( - "format", - "match_filters", - "music_tags", - "filter_include", - "filter_exclude", - "embed_thumbnail", - "video_tags", - "download", - ) - - -def should_filter_property(property_name: str) -> bool: - return property_name.startswith("_") or property_name in ( - "value", - "source_variable_capture_dict", - "dict", - "keys", - "dict_with_format_strings", - "subscription_name", - "list", - "script", - "unresolvable", - ) - - -def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: - docs = "" - docs += section(name, level=offset + 0) - - docs += inspect.cleandoc(options.__doc__) - docs += "\n" - - if should_filter_all_properties(name): - return docs - - property_names = [prop for prop in properties(options) if not should_filter_property(prop)] - for property_name in sorted(property_names): - docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) - - return docs +PLUGIN_NAMES_TO_SKIP_PROPERTIES: Set[str] = { + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "video_tags", + "download", +} class PluginsDocGen(DocGen): @@ -81,6 +46,11 @@ class PluginsDocGen(DocGen): docs = section("Plugins", level=0) for idx, name in enumerate(sorted(options_dict.keys())): docs += line_section(section_idx=idx) - docs += generate_plugin_docs(name, options_dict[name], offset=1) + docs += generate_options_validator_docs( + name=name, + options=options_dict[name], + offset=1, + skip_properties=name in PLUGIN_NAMES_TO_SKIP_PROPERTIES, + ) return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py index 81ddc3fa..a870b40c 100644 --- a/tools/docgen/utils.py +++ b/tools/docgen/utils.py @@ -4,14 +4,32 @@ from typing import Any from typing import Dict from typing import List from typing import Optional +from typing import Set from typing import Type +from ytdl_sub.config.validators.options import OptionsValidator + LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} -def section(name: str, level: int, as_code: bool = False) -> str: - if as_code: - name = f"``{name}``" +def _should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + "script", + "unresolvable", + ) + + +def section(name: str, level: Optional[int]) -> str: + if level is None: + return f"\n{name}\n\n" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" @@ -40,10 +58,20 @@ def camel_case_to_human(string: str) -> str: return output_str +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return "" + + def get_function_docs( function_name: str, obj: Any, - level: int, + level: Optional[int], display_function_name: Optional[str] = None, pre_docstring: Optional[str] = None, ) -> str: @@ -56,11 +84,20 @@ def get_function_docs( return docs -def line() -> str: - return "\n" + ("-" * 100) + "\n" +def generate_options_validator_docs( + name: str, options: Type[OptionsValidator], offset: int, skip_properties: bool +) -> str: + docs = "" + docs += section(name, level=offset + 0) + docs += inspect.cleandoc(options.__doc__) + docs += "\n" -def line_section(section_idx: int) -> str: - if section_idx > 0: - return line() - return "" + if skip_properties: + return docs + + property_names = [prop for prop in properties(options) if not _should_filter_property(prop)] + for property_name in sorted(property_names): + docs += get_function_docs(function_name=property_name, obj=options, level=None, display_function_name=f"``{property_name}``") + + return docs