diff --git a/docs/source/conf.py b/docs/source/conf.py index f754815a..c226b589 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -7,7 +7,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = "ytdl-sub" -copyright = "2024, Jesse Bannon" +copyright = "2026, Jesse Bannon" author = "Jesse Bannon" release = "" @@ -15,10 +15,8 @@ release = "" # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "sphinx.ext.autodoc", "sphinx.ext.autosectionlabel", "sphinx.ext.extlinks", - "sphinx.ext.napoleon", "sphinx_copybutton", "sphinx_design", ] @@ -70,19 +68,3 @@ extlinks = { "lsio-gh": ("https://github.com/linuxserver/%s", "%s image"), "ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"), } - -# -- Options for autodoc ---------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration - -# Automatically extract typehints when specified and place them in -# descriptions of the relevant function/method. -autodoc_default_options = { - "autodoc_typehints_format": "short", - "autodoc_class_signature": "separated", - "add_module_names": False, - # "add_class_names": False, -} - -python_use_unqualified_type_names = True -napoleon_numpy_docstring = True -napoleon_use_rtype = False diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst index 6955c386..ccfa901b 100644 --- a/docs/source/config_reference/config_yaml.rst +++ b/docs/source/config_reference/config_yaml.rst @@ -1,7 +1,13 @@ -================== +.. + WARNING: This RST file is generated from docstrings in: + The respective function docstrings within ytdl_sub/config/config_validator.py + In order to make a change to this file, edit the respective docstring + and run `make docs`. This will automatically sync the Python RST-based + docstrings into this file. If the docstrings and RST file are out of sync, + it will fail TestDocGen tests in GitHub CI. + Configuration File ================== - ytdl-sub is configured using a ``config.yaml`` file. The ``config.yaml`` is made up of two sections: @@ -11,113 +17,118 @@ The ``config.yaml`` is made up of two sections: configuration: presets: -You can jump to any section and subsection of the config using the navigation section to -the left. Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. -If you wish to represent paths like Windows, you will need to -``C:\\double\\bashslash\\paths`` in order to escape the backslash character. - - -configuration -------------- - -The ``configuration`` section contains app-wide configs applied to all presets and -subscriptions. - -.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions() - :members: - :member-order: bysource - :exclude-members: subscription_value, persist_logs, experimental - -persist_logs -~~~~~~~~~~~~ - -Without this key, ``ytdl-sub`` only prints output to it's ``stdout`` and ``stderr``. If -your configuration includes the ``persist_logs:`` key, then ``ytdl-sub`` also writes log -files to disk. - -.. warning:: - - The log files grow rapidly if ``keep_successful_logs:`` is ``true``, the default, and - may fill up disk space. Set ``keep_successful_logs: false`` or prune the log files - regularly. - -For example: +If you prefer to use a Windows backslash, note that it must have +``C:\\double\\bashslash\\paths`` in order to escape the backslash character. This is due +to it being a YAML escape character. .. code-block:: yaml configuration: + dl_aliases: + mv: "--preset music_video" + u: "--download.url" + + experimental: + enable_update_with_info_json: True + + ffmpeg_path: "/usr/bin/ffmpeg" + ffprobe_path: "/usr/bin/ffprobe" + + file_name_max_bytes: 255 + lock_directory: "/tmp" + persist_logs: - logs_directory: "/path/to/log/directory" + keep_successful_logs: True + logs_directory: "/var/log/ytdl-sub-logs" -.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator() - :members: - :member-order: bysource + umask: "022" + working_directory: ".ytdl-sub-working-directory" +dl_aliases +---------- +.. _dl_aliases: -presets -------- - -Each key under ``presets:`` defines a `formula` for how to format downloaded media and -metadata. The key is the name of the preset and the value is a mapping that defines the -preset. - -.. note:: - - The ``presets:`` key at the top of the configuration file contains multiple - user-defined presets, but *each preset* itself may include a ``preset:`` key that - defines *that preset's* base presets. For example: - - .. code-block:: yaml - - presets: - Foo Preset: - preset: - - "Jellyfin TV Show by Date" - - "Only Recent" - -preset -~~~~~~ - -Presets support inheritance by defining one or more parent presets: +Alias definitions to shorten :ref:`dl arguments `. For example, .. code-block:: yaml - presets: - custom_preset: - ... - parent_preset: - ... - child_preset: - preset: - - "parent_preset" + configuration: + dl_aliases: + mv: "--preset music_video" + u: "--download.url" -In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``. -Use parent presets where possible to reduce duplicate yaml definitions. +Simplifies -Presets also support inheritance from multiple presets: +.. code-block:: bash -.. code-block:: yaml + ytdl-sub dl --preset "Jellyfin Music Videos" --download.url "youtube.com/watch?v=a1b2c3" - child_preset: - preset: - - "custom_preset" - - "parent_preset" +to -In this example, ``child_preset`` will inherit all fields from ``custom_preset`` and -``parent_preset`` in that order. The bottom-most preset has the highest priority. More -specifically, presets are merged using `mergedeep`_ via `a TYPESAFE_ADDITIVE merge`_, -which means: +.. code-block:: bash -- if two conflicting keys arent lists or mappings, overwrite the higher priority one -- otherwise, combine then re-evaluate + ytdl-sub dl --mv --u "youtube.com/watch?v=a1b2c3" -If you are only inheriting from one preset, using a single string instead of a list is -valid, for example ``preset: "parent_preset"``, but we recommend always using a list for -consistent readability between presets. +experimental +------------ +Experimental flags reside under the ``experimental`` key. -.. _`mergedeep`: - https://mergedeep.readthedocs.io/en/latest/ -.. _`a TYPESAFE_ADDITIVE merge`: - https://mergedeep.readthedocs.io/en/latest/index.html#merge-strategies +``enable_update_with_info_json`` + +Enables modifying subscription files using info.json files using the argument +``--update-with-info-json``. This feature is still being tested and has the ability to +destroy files. Ensure you have a full backup before usage. You have been warned! + +ffmpeg_path +----------- +Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux, +``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows. + +ffprobe_path +------------ +Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux, +``./ffprobe.exe`` in the same directory as ytdl-sub for Windows. + +file_name_max_bytes +------------------- +Max file name size in bytes. Most OS's typically default to 255 bytes. + +lock_directory +-------------- +The directory to temporarily store file locks, which prevents multiple instances +of ``ytdl-sub`` from running. Note that file locks do not work on +network-mounted directories. Ensure that this directory resides on the host +machine. Defaults to ``/tmp``. + +persist_logs +------------ +By default, no logs are persisted. Specifying this key will enable persisted logs. The following +options are available. + +``keep_successful_logs`` + +Defaults to ``True``. When this key is ``False``, only write log files for failed +subscriptions. + +``logs_directory`` + +Required field. Write log files to this directory with names like +``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. + +umask +----- +Umask in octal format to apply to every created file. Defaults to ``022``. + +working_directory +----------------- +The directory to temporarily store downloaded files before moving them into their final +directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory +that ytdl-sub is invoked from. + +Presets +======= +Custom presets are defined in this section. Refer to the +:ref:`Getting Started Guide` +on how to configure. diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 6b8229e4..75496323 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -28,7 +28,6 @@ Extracts audio from a video file. The codec to output after extracting the audio. Supported codecs are aac, flac, mp3, m4a, opus, vorbis, wav, and best to grab the best possible format at runtime. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -37,7 +36,6 @@ Extracts audio from a video file. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``quality`` :expected type: Float @@ -45,7 +43,6 @@ Extracts audio from a video file. Optional. Specify ffmpeg audio quality. Insert a value between ``0`` (better) and ``9`` (worse) for variable bitrate, or a specific bitrate like ``128`` for 128k. - ---------------------------------------------------------------------------------------------------- chapters @@ -84,14 +81,12 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to False. If chapters do not exist in the video/description itself, attempt to scrape comments to find the chapters. - ``embed_chapters`` :expected type: Optional[Boolean] :description: Defaults to True. Embed chapters into the file. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -100,7 +95,6 @@ chapters and remove specific ones. Can also remove chapters using regex. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``force_key_frames`` :expected type: Optional[Boolean] @@ -108,7 +102,6 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to False. Force keyframes at cuts when removing sections. This is slow due to needing a re-encode, but the resulting video may have fewer artifacts around the cuts. - ``remove_chapters_regex`` :expected type: Optional[List[RegexString] @@ -116,7 +109,6 @@ chapters and remove specific ones. Can also remove chapters using regex. List of regex patterns to match chapter titles against and remove them from the entry. - ``remove_sponsorblock_categories`` :expected type: Optional[List[String]] @@ -125,7 +117,6 @@ chapters and remove specific ones. Can also remove chapters using regex. categories that are specified in ``sponsorblock_categories`` or "all", which removes everything specified in ``sponsorblock_categories``. - ``sponsorblock_categories`` :expected type: Optional[List[String]] @@ -134,7 +125,6 @@ chapters and remove specific ones. Can also remove chapters using regex. "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", "poi_highlight", or "all" to include all categories. - ---------------------------------------------------------------------------------------------------- date_range @@ -169,14 +159,12 @@ intended download files. :description: Only download videos after or on this datetime, inclusive. - ``before`` :expected type: Optional[OverridesFormatter] :description: Only download videos only before this datetime, not inclusive. - ``breaks`` :expected type: Optional[OverridesFormatter] @@ -184,7 +172,6 @@ intended download files. Toggle to enable breaking subsequent metadata downloads if an entry's upload date is out of range. Defaults to True. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -193,7 +180,6 @@ intended download files. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``type`` :expected type: Optional[OverridesFormatter] @@ -201,7 +187,6 @@ intended download files. Which type of date to use. Must be either ``upload_date`` or ``release_date``. Defaults to ``upload_date``. - ---------------------------------------------------------------------------------------------------- download @@ -305,7 +290,6 @@ Also supports custom ffmpeg conversions: - Video: avi, flv, mkv, mov, mp4, webm - Audio: aac, flac, mp3, m4a, opus, vorbis, wav - ``convert_with`` :expected type: Optional[String] @@ -314,7 +298,6 @@ Also supports custom ffmpeg conversions: yt-dlp whereas ``ffmpeg`` specifies it will be converted using a custom command specified with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -323,7 +306,6 @@ Also supports custom ffmpeg conversions: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``ffmpeg_post_process_args`` :expected type: Optional[OverridesFormatter] @@ -336,7 +318,6 @@ Also supports custom ffmpeg conversions: The output file will use the extension specified in ``convert_to``. Post-processing args can still be set with ``convert_with`` set to ``yt-dlp``. - ---------------------------------------------------------------------------------------------------- filter_exclude @@ -472,7 +453,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: OverridesBooleanFormatterValidator @@ -481,14 +461,12 @@ with a ``.nfo`` extension. You can add any values into the NFO. emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -501,7 +479,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. - ``tags`` :expected type: NfoTags @@ -538,7 +515,6 @@ with a ``.nfo`` extension. You can add any values into the NFO. Comedy Drama - ---------------------------------------------------------------------------------------------------- output_directory_nfo_tags @@ -570,7 +546,6 @@ Usage: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: OverridesBooleanFormatterValidator @@ -579,14 +554,12 @@ Usage: emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -599,7 +572,6 @@ Usage: - ``tags`` :expected type: NfoTags @@ -634,7 +606,6 @@ Usage: Comedy Drama - ---------------------------------------------------------------------------------------------------- output_options @@ -669,7 +640,6 @@ Defines where to output files and thumbnails after all post-processing has compl The file name to store a subscriptions download archive placed relative to the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json`` - ``file_name`` :expected type: EntryFormatter @@ -677,7 +647,6 @@ Defines where to output files and thumbnails after all post-processing has compl The file name for the media file. This can include directories such as ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory. - ``info_json_name`` :expected type: Optional[EntryFormatter] @@ -686,7 +655,6 @@ Defines where to output files and thumbnails after all post-processing has compl as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output directory. Can be set to empty string or `null` to disable info json writes. - ``keep_files_after`` :expected type: Optional[OverridesFormatter] @@ -698,7 +666,6 @@ Defines where to output files and thumbnails after all post-processing has compl files after ``19000101``, which implies all files. Can be used in conjunction with ``keep_max_files``. - ``keep_files_before`` :expected type: Optional[OverridesFormatter] @@ -710,7 +677,6 @@ Defines where to output files and thumbnails after all post-processing has compl files before ``now``, which implies all files. Can be used in conjunction with ``keep_max_files``. - ``keep_files_date_eval`` :expected type: str @@ -720,7 +686,6 @@ Defines where to output files and thumbnails after all post-processing has compl perform evaluation for keep_files_before/after and keep_max_files. Defaults to the entry's upload_date_standardized variable. - ``keep_max_files`` :expected type: Optional[OverridesFormatter] @@ -730,7 +695,6 @@ Defines where to output files and thumbnails after all post-processing has compl Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. - ``maintain_download_archive`` :expected type: Optional[Boolean] @@ -745,7 +709,6 @@ Defines where to output files and thumbnails after all post-processing has compl Defaults to False. - ``migrated_download_archive_name`` :expected type: Optional[OverridesFormatter] @@ -755,14 +718,12 @@ Defines where to output files and thumbnails after all post-processing has compl name first, and fallback to ``download_archive_name``. It will always save to this file and remove the original ``download_archive_name``. - ``output_directory`` :expected type: OverridesFormatter :description: The output directory to store all media files downloaded. - ``preserve_mtime`` :expected type: Optional[Boolean] @@ -771,7 +732,6 @@ Defines where to output files and thumbnails after all post-processing has compl When True, sets the file's mtime to match the video's upload_date from yt-dlp metadata. Defaults to False. - ``thumbnail_name`` :expected type: Optional[EntryFormatter] @@ -780,7 +740,6 @@ Defines where to output files and thumbnails after all post-processing has compl as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output directory. Can be set to empty string or `null` to disable thumbnail writes. - ---------------------------------------------------------------------------------------------------- overrides @@ -845,7 +804,6 @@ used with no modifications. If a file has no chapters and is set to "pass", then ``chapter_title`` is set to the entry's title and ``chapter_index``, ``chapter_count`` are both set to 1. - ---------------------------------------------------------------------------------------------------- square_thumbnail @@ -892,7 +850,6 @@ Usage: this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``kodi_safe`` :expected type: OverridesBooleanFormatterValidator @@ -901,14 +858,12 @@ Usage: emojis and some foreign language characters. Setting this to True will replace those characters with '□'. - ``nfo_name`` :expected type: EntryFormatter :description: The NFO file name. - ``nfo_root`` :expected type: EntryFormatter @@ -921,7 +876,6 @@ Usage: - ``tags`` :expected type: NfoTags @@ -935,7 +889,6 @@ Usage: My custom season name! - ---------------------------------------------------------------------------------------------------- subtitles @@ -963,7 +916,6 @@ It will set the respective language to the correct subtitle file. :description: Defaults to False. Whether to allow auto generated subtitles. - ``embed_subtitles`` :expected type: Optional[Boolean] @@ -971,7 +923,6 @@ It will set the respective language to the correct subtitle file. Defaults to False. Whether to embed the subtitles into the video file. Note that webm files can only embed "vtt" subtitle types. - ``enable`` :expected type: Optional[OverridesFormatter] @@ -980,7 +931,6 @@ It will set the respective language to the correct subtitle file. this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``languages`` :expected type: Optional[List[String]] @@ -988,7 +938,6 @@ It will set the respective language to the correct subtitle file. Language code(s) to download for subtitles. Supports a single or list of multiple language codes. Defaults to only "en". - ``languages_required`` :expected type: Optional[List[String]] @@ -996,7 +945,6 @@ It will set the respective language to the correct subtitle file. Language code(s) that are required to be present for downloads to continue. If missing, ytdl-sub will throw an error. NOTE: currently this only checks file-based subtitles. - ``subtitles_name`` :expected type: Optional[EntryFormatter] @@ -1006,14 +954,12 @@ It will set the respective language to the correct subtitle file. and will be placed in the output directory. ``lang`` is dynamic since you can download multiple subtitles. It will set the respective language to the correct subtitle file. - ``subtitles_type`` :expected type: Optional[String] :description: Defaults to "srt". One of the subtitle file types "srt", "vtt", "ass", "lrc". - ---------------------------------------------------------------------------------------------------- throttle_protection @@ -1054,14 +1000,12 @@ Range min and max values support static override variables within their definiti this field can be set using an override variable to easily toggle whether this plugin is enabled or not via Boolean. - ``max_downloads_per_subscription`` :expected type: Optional[Range] :description: Number of downloads to perform per subscription. - ``sleep_per_download_s`` :expected type: Optional[Range] @@ -1069,7 +1013,6 @@ Range min and max values support static override variables within their definiti Number in seconds to sleep between each download. Does not include time it takes for ytdl-sub to perform post-processing. - ``sleep_per_request_s`` :expected type: Optional[Range] @@ -1079,14 +1022,12 @@ Range min and max values support static override variables within their definiti download for the entry. Also, yt-dlp only supports a single value at this time for this, so will always use the max value. - ``sleep_per_subscription_s`` :expected type: Optional[Range] :description: Number in seconds to sleep between each subscription. - ``subscription_download_probability`` :expected type: Optional[Float] @@ -1095,7 +1036,6 @@ Range min and max values support static override variables within their definiti recommended to set if you run ytdl-sub in a cron-job, that way you are statistically guaranteed over time to eventually download the subscription. - ---------------------------------------------------------------------------------------------------- video_tags diff --git a/docs/source/guides/getting_started/first_config.rst b/docs/source/guides/getting_started/first_config.rst index 34dabdb2..0f98b84e 100644 --- a/docs/source/guides/getting_started/first_config.rst +++ b/docs/source/guides/getting_started/first_config.rst @@ -3,13 +3,33 @@ Basic Configuration A configuration file serves two purposes: -1. Set advanced functionality that is not specifiable in a subscription file, such as - working directory location. These are set underneath ``configuration``. -2. Create custom presets, which can drastically simplify your subscription file. These - are defined underneath ``presets``. Presets are intended to be applicable and - reusable across multiple subscriptions. +1. Set application-level functionality that is not specifiable in a subscription file. -Below is a common configuration: + .. note:: + + ytdl-sub does not require a configuration file. However, + certain application settings may be desirable for tweak, such as setting + ``working_directory`` to make ytdl-sub perform the initial download + to an SSD drive. + +2. Create custom presets. + + .. note:: + + In the prior Initial Subscription examples, we leveraged the prebuilt preset + ``Jellyfin TV Show by Date``. This preset is entirely built using the same + YAML configuration system offered to users by using a configuration file. + +The following section attempts to demystify and explain how to... + +- Set an application setting +- Know whether or not custom presets are actually needed +- How to create a custom preset +- How to use a custom preset on subscriptions + +------------- + +how this works, and show-case how .. code-block:: yaml :linenos: diff --git a/docs/source/guides/getting_started/index.rst b/docs/source/guides/getting_started/index.rst index 1288380d..5c04e47f 100644 --- a/docs/source/guides/getting_started/index.rst +++ b/docs/source/guides/getting_started/index.rst @@ -134,7 +134,7 @@ use no preset at all and will just run ``yt-dlp`` without any customization or p processing. The subscriptions file has special support for :ref:`overriding the presets of all subscriptions in the file `. The configuration file supports :ref:`a few special options -` that are not about defining presets. See +` that are not about defining presets. See :doc:`the reference documentation <../../config_reference/index>` for technically complete details, but for almost all of the use cases served by ``ytdl-sub``, the above is accurate and representative. diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 0db65064..cb5b827e 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -87,8 +87,7 @@ Using the command: --overrides.tv_show_name "Rick A" \ --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" -See how to shorten commands using `download aliases -`_. +See how to shorten commands using :ref:`download aliases `. View Options diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py index 073eb23a..0dc2c1e0 100644 --- a/src/ytdl_sub/config/config_validator.py +++ b/src/ytdl_sub/config/config_validator.py @@ -24,6 +24,10 @@ from ytdl_sub.validators.validators import StringValidator class ExperimentalValidator(StrictDictValidator): + """ + Experimental flags reside under the ``experimental`` key. + """ + _optional_keys = {"enable_update_with_info_json"} _allow_extra_keys = True @@ -45,6 +49,11 @@ class ExperimentalValidator(StrictDictValidator): class PersistLogsValidator(StrictDictValidator): + """ + By default, no logs are persisted. Specifying this key will enable persisted logs. The following + options are available. + """ + _required_keys = {"logs_directory"} _optional_keys = {"keep_logs_after", "keep_successful_logs"} @@ -69,8 +78,8 @@ class PersistLogsValidator(StrictDictValidator): @property def logs_directory(self) -> str: """ - Write log files to this directory with names like - ``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. (required) + Required field. Write log files to this directory with names like + ``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``. """ return self._logs_directory.value @@ -95,15 +104,54 @@ class PersistLogsValidator(StrictDictValidator): @property def keep_successful_logs(self) -> bool: """ - If the ``persist_logs:`` key is in the configuration, then ``ytdl-sub`` *always* - writes log files for the subscription both for successful downloads and when it - encounters an error while downloading. When this key is ``False``, only write - log files for errors. (default ``True``) + Defaults to ``True``. When this key is ``False``, only write log files for failed + subscriptions. """ return self._keep_successful_logs.value class ConfigOptions(StrictDictValidator): + """ + ytdl-sub is configured using a ``config.yaml`` file. + + The ``config.yaml`` is made up of two sections: + + .. code-block:: yaml + + configuration: + presets: + + + Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``. + If you prefer to use a Windows backslash, note that it must have + ``C:\\\\double\\\\bashslash\\\\paths`` in order to escape the backslash character. This is due + to it being a YAML escape character. + + .. code-block:: yaml + + configuration: + dl_aliases: + mv: "--preset music_video" + u: "--download.url" + + experimental: + enable_update_with_info_json: True + + ffmpeg_path: "/usr/bin/ffmpeg" + ffprobe_path: "/usr/bin/ffprobe" + + file_name_max_bytes: 255 + lock_directory: "/tmp" + + persist_logs: + keep_successful_logs: True + logs_directory: "/var/log/ytdl-sub-logs" + + umask: "022" + working_directory: ".ytdl-sub-working-directory" + + """ + _optional_keys = { "working_directory", "umask", @@ -159,7 +207,8 @@ class ConfigOptions(StrictDictValidator): def working_directory(self) -> str: """ The directory to temporarily store downloaded files before moving them into their final - directory. (default ``./.ytdl-sub-working-directory``) + directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory + that ytdl-sub is invoked from. """ # Expands tildas to actual paths, use native os sep return os.path.expanduser(self._working_directory.value.replace(posixpath.sep, os.sep)) @@ -167,7 +216,7 @@ class ConfigOptions(StrictDictValidator): @property def umask(self) -> Optional[str]: """ - Umask in octal format to apply to every created file. (default ``022``) + Umask in octal format to apply to every created file. Defaults to ``022``. """ return self._umask.value @@ -176,7 +225,7 @@ class ConfigOptions(StrictDictValidator): """ .. _dl_aliases: - Alias definitions to shorten ``ytdl-sub dl`` arguments. For example, + Alias definitions to shorten :ref:`dl arguments `. For example, .. code-block:: yaml @@ -228,23 +277,23 @@ class ConfigOptions(StrictDictValidator): The directory to temporarily store file locks, which prevents multiple instances of ``ytdl-sub`` from running. Note that file locks do not work on network-mounted directories. Ensure that this directory resides on the host - machine. (default ``/tmp``) + machine. Defaults to ``/tmp``. """ return self._lock_directory.value @property def ffmpeg_path(self) -> str: """ - Path to ffmpeg executable. (default ``/usr/bin/ffmpeg`` for Linux, - ``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows) + Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux, + ``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows. """ return self._ffmpeg_path.value @property def ffprobe_path(self) -> str: """ - Path to ffprobe executable. (default ``/usr/bin/ffprobe`` for Linux, - ``./ffprobe.exe`` in the same directory as ytdl-sub for Windows) + Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux, + ``./ffprobe.exe`` in the same directory as ytdl-sub for Windows. """ return self._ffprobe_path.value diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index b89e1d53..2c8ad348 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -55,6 +55,12 @@ class _PresetShell(StrictDictValidator): class Preset(_PresetShell): + """ + Custom presets are defined in this section. Refer to the + :ref:`Getting Started Guide` + on how to configure. + """ + @classmethod def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None: """ diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py index 5d326839..5cfb8f92 100644 --- a/tests/unit/docgen/test_docgen.py +++ b/tests/unit/docgen/test_docgen.py @@ -1,5 +1,6 @@ from typing import Type +from tools.docgen.configuration import ConfigurationDocGen from tools.docgen.docgen import DocGen from tools.docgen.entry_variables import EntryVariablesDocGen from tools.docgen.plugins import PluginsDocGen @@ -28,3 +29,6 @@ class TestDocGen: def test_plugins_generated(self): _test_doc_gen(PluginsDocGen) + + def test_configuration_generated(self): + _test_doc_gen(ConfigurationDocGen) diff --git a/tools/docgen/configuration.py b/tools/docgen/configuration.py new file mode 100644 index 00000000..958c09a1 --- /dev/null +++ b/tools/docgen/configuration.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import generate_options_validator_docs +from ytdl_sub.config.config_validator import ConfigOptions +from ytdl_sub.config.preset import Preset + + +class ConfigurationDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/config_yaml.rst") + DOCSTRING_LOCATION = ( + "The respective function docstrings within ytdl_sub/config/config_validator.py" + ) + + @classmethod + def generate(cls): + docs = generate_options_validator_docs( + name="Configuration File", + options=ConfigOptions, + offset=0, + skip_properties=False, + recurse_property_options=True, + property_sections=True, + ) + + docs += generate_options_validator_docs( + name="Presets", + options=Preset, + offset=0, + skip_properties=True, + recurse_property_options=False, + property_sections=False, + ) + + return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py index d2be80d3..f0331acc 100644 --- a/tools/docgen/plugins.py +++ b/tools/docgen/plugins.py @@ -1,12 +1,11 @@ -import inspect from pathlib import Path -from typing import Any from typing import Dict +from typing import Set from typing import Type from tools.docgen.docgen import DocGen +from tools.docgen.utils import generate_options_validator_docs from tools.docgen.utils import line_section -from tools.docgen.utils import properties from tools.docgen.utils import section from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_mapping import PluginMapping @@ -15,59 +14,17 @@ from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator - -def should_filter_all_properties(plugin_name: str) -> bool: - return plugin_name in ( - "format", - "match_filters", - "music_tags", - "filter_include", - "filter_exclude", - "embed_thumbnail", - "square_thumbnail", - "video_tags", - "download", - ) - - -def should_filter_property(property_name: str) -> bool: - return property_name.startswith("_") or property_name in ( - "value", - "source_variable_capture_dict", - "dict", - "keys", - "dict_with_format_strings", - "dict_with_parsed_format_strings", - "subscription_name", - "list", - "script", - "unresolvable", - "leaf_name", - ) - - -def get_function_docs(function_name: str, obj: Any, level: int) -> str: - docs = f"\n``{function_name}``\n\n" - docs += inspect.cleandoc(getattr(obj, function_name).__doc__) - docs += "\n\n" - return docs - - -def generate_plugin_docs(name: str, options: Type[OptionsValidator], offset: int) -> str: - docs = "" - docs += section(name, level=offset + 0) - - docs += inspect.cleandoc(options.__doc__) - docs += "\n" - - if should_filter_all_properties(name): - return docs - - property_names = [prop for prop in properties(options) if not should_filter_property(prop)] - for property_name in sorted(property_names): - docs += get_function_docs(function_name=property_name, obj=options, level=offset + 1) - - return docs +PLUGIN_NAMES_TO_SKIP_PROPERTIES: Set[str] = { + "format", + "match_filters", + "music_tags", + "filter_include", + "filter_exclude", + "embed_thumbnail", + "square_thumbnail", + "video_tags", + "download", +} class PluginsDocGen(DocGen): @@ -91,6 +48,11 @@ class PluginsDocGen(DocGen): docs = section("Plugins", level=0) for idx, name in enumerate(sorted(options_dict.keys())): docs += line_section(section_idx=idx) - docs += generate_plugin_docs(name, options_dict[name], offset=1) + docs += generate_options_validator_docs( + name=name, + options=options_dict[name], + offset=1, + skip_properties=name in PLUGIN_NAMES_TO_SKIP_PROPERTIES, + ) return docs diff --git a/tools/docgen/utils.py b/tools/docgen/utils.py index 81ddc3fa..5cd6f357 100644 --- a/tools/docgen/utils.py +++ b/tools/docgen/utils.py @@ -6,12 +6,51 @@ from typing import List from typing import Optional from typing import Type +from ytdl_sub.script.utils.type_checking import get_optional_type +from ytdl_sub.script.utils.type_checking import is_optional +from ytdl_sub.validators.validators import Validator + LEVEL_CHARS: Dict[int, str] = {0: "=", 1: "-", 2: "~", 3: "^"} -def section(name: str, level: int, as_code: bool = False) -> str: - if as_code: - name = f"``{name}``" +def _should_filter_property(property_name: str) -> bool: + return property_name.startswith("_") or property_name in ( + "value", + "source_variable_capture_dict", + "dict", + "keys", + "dict_with_format_strings", + "subscription_name", + "list", + "script", + "unresolvable", + "dict_with_parsed_format_strings", + "leaf_name", + ) + + +def _is_validator_property( + options: Type[Validator], property_name: str +) -> Optional[Type[Validator]]: + property_return_type = inspect.getfullargspec(getattr(options, property_name).fget).annotations[ + "return" + ] + if is_optional(property_return_type): + property_return_type = get_optional_type(property_return_type) + + try: + if issubclass(property_return_type, Validator): + return property_return_type + except TypeError: + return None + + return None + + +def section(name: str, level: Optional[int]) -> str: + if level is None: + return f"\n{name}\n\n" + return f"\n{name}\n{len(name) * LEVEL_CHARS[level]}\n" @@ -40,10 +79,20 @@ def camel_case_to_human(string: str) -> str: return output_str +def line() -> str: + return "\n" + ("-" * 100) + "\n" + + +def line_section(section_idx: int) -> str: + if section_idx > 0: + return line() + return "" + + def get_function_docs( function_name: str, obj: Any, - level: int, + level: Optional[int], display_function_name: Optional[str] = None, pre_docstring: Optional[str] = None, ) -> str: @@ -56,11 +105,42 @@ def get_function_docs( return docs -def line() -> str: - return "\n" + ("-" * 100) + "\n" +def generate_options_validator_docs( + name: str, + options: Type[Validator], + offset: int, + skip_properties: bool, + recurse_property_options: bool = False, + property_sections: bool = False, +) -> str: + docs = "" + docs += section(name, level=offset + 0) + docs += inspect.cleandoc(options.__doc__) + docs += "\n" -def line_section(section_idx: int) -> str: - if section_idx > 0: - return line() - return "" + if skip_properties: + return docs + + property_names = [prop for prop in properties(options) if not _should_filter_property(prop)] + for property_name in sorted(property_names): + maybe_validator_property = ( + _is_validator_property(options, property_name) if recurse_property_options else None + ) + if maybe_validator_property: + docs += generate_options_validator_docs( + name=property_name, + options=maybe_validator_property, + offset=offset + 1, + skip_properties=False, + recurse_property_options=False, + ) + else: + docs += get_function_docs( + function_name=property_name, + obj=options, + level=(offset + 1) if property_sections else None, + display_function_name=f"``{property_name}``" if not property_sections else None, + ) + + return docs