plugin modularize for config docs

docs/source/config_reference/plugins.rst

@@ -21,7 +21,6 @@ Extracts audio from a video file.
   The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a,
   opus, vorbis, wav, and best to grab the best possible format at runtime.
 
-
 ``enable``
 
 :expected type: Optional[OverridesFormatter]
@@ -30,7 +29,6 @@ Extracts audio from a video file.
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``quality``
 
 :expected type: Float
@@ -38,7 +36,6 @@ Extracts audio from a video file.
   Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9``
   (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k.
 
-
 ----------------------------------------------------------------------------------------------------
 
 chapters
@@ -77,14 +74,12 @@ chapters and remove specific ones. Can also remove chapters using regex.
   Defaults to False. If chapters do not exist in the video/description itself, attempt to
   scrape comments to find the chapters.
 
-
 ``embed_chapters``
 
 :expected type: Optional[Boolean]
 :description:
   Defaults to True. Embed chapters into the file.
 
-
 ``enable``
 
 :expected type: Optional[OverridesFormatter]
@@ -93,7 +88,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``force_key_frames``
 
 :expected type: Optional[Boolean]
@@ -101,7 +95,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
   Defaults to False. Force keyframes at cuts when removing sections. This is slow due to
   needing a re-encode, but the resulting video may have fewer artifacts around the cuts.
 
-
 ``remove_chapters_regex``
 
 :expected type: Optional[List[RegexString]
@@ -109,7 +102,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
   List of regex patterns to match chapter titles against and remove them from the
   entry.
 
-
 ``remove_sponsorblock_categories``
 
 :expected type: Optional[List[String]]
@@ -118,7 +110,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
   categories that are specified in ``sponsorblock_categories`` or "all", which removes
   everything specified in ``sponsorblock_categories``.
 
-
 ``sponsorblock_categories``
 
 :expected type: Optional[List[String]]
@@ -127,7 +118,6 @@ chapters and remove specific ones. Can also remove chapters using regex.
   "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
   "poi_highlight", or "all" to include all categories.
 
-
 ----------------------------------------------------------------------------------------------------
 
 date_range
@@ -158,14 +148,12 @@ granularity possible.
 :description:
   Only download videos after this datetime.
 
-
 ``before``
 
 :expected type: Optional[OverridesFormatter]
 :description:
   Only download videos before this datetime.
 
-
 ``breaks``
 
 :expected type: Optional[OverridesFormatter]
@@ -173,7 +161,6 @@ granularity possible.
   Toggle to enable breaking subsequent metadata downloads if an entry's upload date
   is out of range. Defaults to True.
 
-
 ``enable``
 
 :expected type: Optional[OverridesFormatter]
@@ -182,7 +169,6 @@ granularity possible.
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ----------------------------------------------------------------------------------------------------
 
 download
@@ -286,7 +272,6 @@ Also supports custom ffmpeg conversions:
     - Video: avi, flv, mkv, mov, mp4, webm
     - Audio: aac, flac, mp3, m4a, opus, vorbis, wav
 
-
 ``convert_with``
 
 :expected type: Optional[String]
@@ -295,7 +280,6 @@ Also supports custom ffmpeg conversions:
   yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified
   with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
 
-
 ``enable``
 
 :expected type: Optional[OverridesFormatter]
@@ -304,7 +288,6 @@ Also supports custom ffmpeg conversions:
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``ffmpeg_post_process_args``
 
 :expected type: Optional[OverridesFormatter]
@@ -317,7 +300,6 @@ Also supports custom ffmpeg conversions:
   The output file will use the extension specified in ``convert_to``. Post-processing args
   can still be set  with ``convert_with`` set to ``yt-dlp``.
 
-
 ----------------------------------------------------------------------------------------------------
 
 filter_exclude
@@ -453,7 +435,6 @@ with a ``.nfo`` extension. You can add any values into the NFO.
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``kodi_safe``
 
 :expected type: Optional[Boolean]
@@ -462,14 +443,12 @@ with a ``.nfo`` extension. You can add any values into the NFO.
   emojis and some foreign language characters. Setting this to True will replace those
   characters with '□'.
 
-
 ``nfo_name``
 
 :expected type: EntryFormatter
 :description:
   The NFO file name.
 
-
 ``nfo_root``
 
 :expected type: EntryFormatter
@@ -482,7 +461,6 @@ with a ``.nfo`` extension. You can add any values into the NFO.
      <episodedetails>
      </episodedetails>
 
-
 ``tags``
 
 :expected type: NfoTags
@@ -519,7 +497,6 @@ with a ``.nfo`` extension. You can add any values into the NFO.
      <genre>Comedy</genre>
      <genre>Drama</genre>
 
-
 ----------------------------------------------------------------------------------------------------
 
 output_directory_nfo_tags
@@ -551,7 +528,6 @@ Usage:
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``kodi_safe``
 
 :expected type: Optional[Boolean]
@@ -560,14 +536,12 @@ Usage:
   emojis and some foreign language characters. Setting this to True will replace those
   characters with '□'.
 
-
 ``nfo_name``
 
 :expected type: EntryFormatter
 :description:
   The NFO file name.
 
-
 ``nfo_root``
 
 :expected type: EntryFormatter
@@ -580,7 +554,6 @@ Usage:
      <tvshow>
      </tvshow>
 
-
 ``tags``
 
 :expected type: NfoTags
@@ -615,7 +588,6 @@ Usage:
      <genre>Comedy</genre>
      <genre>Drama</genre>
 
-
 ----------------------------------------------------------------------------------------------------
 
 output_options
@@ -648,7 +620,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   The file name to store a subscriptions download archive placed relative to
   the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
 
-
 ``file_name``
 
 :expected type: EntryFormatter
@@ -656,7 +627,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   The file name for the media file. This can include directories such as
   ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
 
-
 ``info_json_name``
 
 :expected type: Optional[EntryFormatter]
@@ -665,7 +635,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
   directory. Can be set to empty string or `null` to disable info json writes.
 
-
 ``keep_files_after``
 
 :expected type: Optional[OverridesFormatter]
@@ -677,7 +646,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   files after ``19000101``, which implies all files. Can be used in conjunction with
   ``keep_max_files``.
 
-
 ``keep_files_before``
 
 :expected type: Optional[OverridesFormatter]
@@ -689,7 +657,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   files before ``now``, which implies all files. Can be used in conjunction with
   ``keep_max_files``.
 
-
 ``keep_max_files``
 
 :expected type: Optional[OverridesFormatter]
@@ -699,7 +666,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
   applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
 
-
 ``maintain_download_archive``
 
 :expected type: Optional[Boolean]
@@ -714,7 +680,6 @@ Defines where to output files and thumbnails after all post-processing has compl
 
   Defaults to False.
 
-
 ``migrated_download_archive_name``
 
 :expected type: Optional[OverridesFormatter]
@@ -724,14 +689,12 @@ Defines where to output files and thumbnails after all post-processing has compl
   name first, and fallback to ``download_archive_name``. It will always save to this file
   and remove the original ``download_archive_name``.
 
-
 ``output_directory``
 
 :expected type: OverridesFormatter
 :description:
   The output directory to store all media files downloaded.
 
-
 ``thumbnail_name``
 
 :expected type: Optional[EntryFormatter]
@@ -740,7 +703,6 @@ Defines where to output files and thumbnails after all post-processing has compl
   as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
   directory. Can be set to empty string or `null` to disable thumbnail writes.
 
-
 ----------------------------------------------------------------------------------------------------
 
 overrides
@@ -805,7 +767,6 @@ used with no modifications.
   If a file has no chapters and is set to "pass", then ``chapter_title`` is
   set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1.
 
-
 ----------------------------------------------------------------------------------------------------
 
 subtitles
@@ -833,7 +794,6 @@ It will set the respective language to the correct subtitle file.
 :description:
   Defaults to False. Whether to allow auto generated subtitles.
 
-
 ``embed_subtitles``
 
 :expected type: Optional[Boolean]
@@ -841,7 +801,6 @@ It will set the respective language to the correct subtitle file.
   Defaults to False. Whether to embed the subtitles into the video file. Note that
   webm files can only embed "vtt" subtitle types.
 
-
 ``enable``
 
 :expected type: Optional[OverridesFormatter]
@@ -850,7 +809,6 @@ It will set the respective language to the correct subtitle file.
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``languages``
 
 :expected type: Optional[List[String]]
@@ -858,7 +816,6 @@ It will set the respective language to the correct subtitle file.
   Language code(s) to download for subtitles. Supports a single or list of multiple
   language codes. Defaults to only "en".
 
-
 ``subtitles_name``
 
 :expected type: Optional[EntryFormatter]
@@ -868,14 +825,12 @@ It will set the respective language to the correct subtitle file.
   and will be placed in the output directory. ``lang`` is dynamic since you can download
   multiple subtitles. It will set the respective language to the correct subtitle file.
 
-
 ``subtitles_type``
 
 :expected type: Optional[String]
 :description:
   Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc".
 
-
 ----------------------------------------------------------------------------------------------------
 
 throttle_protection
@@ -910,14 +865,12 @@ scripted.
   this field can be set using an override variable to easily toggle whether this plugin
   is enabled or not via Boolean.
 
-
 ``max_downloads_per_subscription``
 
 :expected type: Optional[Range]
 :description:
   Number of downloads to perform per subscription.
 
-
 ``sleep_per_download_s``
 
 :expected type: Optional[Range]
@@ -925,14 +878,12 @@ scripted.
   Number in seconds to sleep between each download. Does not include time it takes for
   ytdl-sub to perform post-processing.
 
-
 ``sleep_per_subscription_s``
 
 :expected type: Optional[Range]
 :description:
   Number in seconds to sleep between each subscription.
 
-
 ``subscription_download_probability``
 
 :expected type: Optional[Float]
@@ -941,7 +892,6 @@ scripted.
   recommended to set if you run ytdl-sub in a cron-job, that way you are statistically
   guaranteed over time to eventually download the subscription.
 
-
 ----------------------------------------------------------------------------------------------------
 
 video_tags

tools/docgen/configuration.py

@@ -9,11 +9,10 @@ from tools.docgen.utils import camel_case_to_human
 from tools.docgen.utils import get_function_docs
 from tools.docgen.utils import line_section
 from tools.docgen.utils import section
+from ytdl_sub.config.config_validator import ConfigOptions
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
 
-from ytdl_sub.config.config_validator import ConfigOptions
-
 
 def _variable_class_to_name(obj: Type[Any]) -> str:
     assert "VariableDefinitions" in obj.__name__, f"{obj.__name__} doesnt have VariableDefinitions"

tools/docgen/plugins.py

@@ -1,13 +1,11 @@
-import inspect
 from pathlib import Path
-from typing import Any
 from typing import Dict
-from typing import Optional
+from typing import Set
 from typing import Type
 
 from tools.docgen.docgen import DocGen
-from tools.docgen.utils import line_section, get_function_docs
+from tools.docgen.utils import generate_options_validator_docs
-from tools.docgen.utils import properties
+from tools.docgen.utils import line_section
 from tools.docgen.utils import section
 from ytdl_sub.config.overrides import Overrides
 from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
@@ -16,49 +14,16 @@ from ytdl_sub.config.preset_options import YTDLOptions
 from ytdl_sub.config.validators.options import OptionsValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
 
-
+PLUGIN_NAMES_TO_SKIP_PROPERTIES: Set[str] = {
-def should_filter_all_properties(plugin_name: str) -> bool:
+    "format",
-    return plugin_name in (
+    "match_filters",
-        "format",
+    "music_tags",
-        "match_filters",
+    "filter_include",
-        "music_tags",
+    "filter_exclude",
-        "filter_include",
+    "embed_thumbnail",
-        "filter_exclude",
+    "video_tags",
-        "embed_thumbnail",
+    "download",
-        "video_tags",
+}
-        "download",
-    )
-
-
-def should_filter_property(property_name: str) -> bool:
-    return property_name.startswith("_") or property_name in (
-        "value",
-        "source_variable_capture_dict",
-        "dict",
-        "keys",
-        "dict_with_format_strings",
-        "subscription_name",
-        "list",
-        "script",
-        "unresolvable",
-    )
-
-
-def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str:
-    docs = ""
-    docs += section(name, level=offset + 0)
-
-    docs += inspect.cleandoc(options.__doc__)
-    docs += "\n"
-
-    if should_filter_all_properties(name):
-        return docs
-
-    property_names = [prop for prop in properties(options) if not should_filter_property(prop)]
-    for property_name in sorted(property_names):
-        docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1)
-
-    return docs
 
 
 class PluginsDocGen(DocGen):
@@ -81,6 +46,11 @@ class PluginsDocGen(DocGen):
         docs = section("Plugins", level=0)
         for idx, name in enumerate(sorted(options_dict.keys())):
             docs += line_section(section_idx=idx)
-            docs += generate_plugin_docs(name, options_dict[name], offset=1)
+            docs += generate_options_validator_docs(
+                name=name,
+                options=options_dict[name],
+                offset=1,
+                skip_properties=name in PLUGIN_NAMES_TO_SKIP_PROPERTIES,
+            )
 
         return docs

tools/docgen/utils.py

@@ -4,14 +4,32 @@ from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Set
 from typing import Type
 
+from ytdl_sub.config.validators.options import OptionsValidator
+
 LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"}
 
 
-def section(name: str, level: int, as_code: bool = False) -> str:
+def _should_filter_property(property_name: str) -> bool:
-    if as_code:
+    return property_name.startswith("_") or property_name in (
-        name = f"``{name}``"
+        "value",
+        "source_variable_capture_dict",
+        "dict",
+        "keys",
+        "dict_with_format_strings",
+        "subscription_name",
+        "list",
+        "script",
+        "unresolvable",
+    )
+
+
+def section(name: str, level: Optional[int]) -> str:
+    if level is None:
+        return f"\n{name}\n\n"
+
     return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n"
 
 
@@ -40,10 +58,20 @@ def camel_case_to_human(string: str) -> str:
     return output_str
 
 
+def line() -> str:
+    return "\n" + ("-" * 100) + "\n"
+
+
+def line_section(section_idx: int) -> str:
+    if section_idx > 0:
+        return line()
+    return ""
+
+
 def get_function_docs(
     function_name: str,
     obj: Any,
-    level: int,
+    level: Optional[int],
     display_function_name: Optional[str] = None,
     pre_docstring: Optional[str] = None,
 ) -> str:
@@ -56,11 +84,20 @@ def get_function_docs(
     return docs
 
 
-def line() -> str:
+def generate_options_validator_docs(
-    return "\n" + ("-" * 100) + "\n"
+    name: str, options: Type[OptionsValidator], offset: int, skip_properties: bool
+) -> str:
+    docs = ""
+    docs += section(name, level=offset + 0)
 
+    docs += inspect.cleandoc(options.__doc__)
+    docs += "\n"
 
-def line_section(section_idx: int) -> str:
+    if skip_properties:
-    if section_idx > 0:
+        return docs
-        return line()
+
-    return ""
+    property_names = [prop for prop in properties(options) if not _should_filter_property(prop)]
+    for property_name in sorted(property_names):
+        docs += get_function_docs(function_name=property_name, obj=options, level=None, display_function_name=f"``{property_name}``")
+
+    return docs