diff --git a/docker/root/defaults/subscriptions.yaml b/docker/root/defaults/subscriptions.yaml index a7e129dc..98535e27 100644 --- a/docker/root/defaults/subscriptions.yaml +++ b/docker/root/defaults/subscriptions.yaml @@ -15,12 +15,12 @@ __preset__: music_video_directory: "/music_videos" # For 'Only Recent' preset, only keep vids within this range and limit - only_recent_date_range: "2months" - only_recent_max_files: 30 + # only_recent_date_range: "2months" + # only_recent_max_files: 30 # Pass any arg directly to yt-dlp's Python API - ytdl_options: - cookiefile: "/config/cookie.txt" + # ytdl_options: + # cookiefile: "/config/cookie.txt" ################################################################### # Subscriptions nested under this will use the @@ -35,52 +35,54 @@ Plex TV Show by Date: # Sets genre tag to "Documentaries" = Documentaries: "NOVA PBS": "https://www.youtube.com/@novapbs" - "National Geographic": "https://www.youtube.com/@NatGeo" - "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U" + # "National Geographic": "https://www.youtube.com/@NatGeo" + # "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U" # Sets genre tag to "Kids", "TV-Y" for content rating - = Kids | = TV-Y: - "Jake Trains": "https://www.youtube.com/@JakeTrains" - "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel" + # = Kids | = TV-Y: + # "Jake Trains": "https://www.youtube.com/@JakeTrains" + # "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel" - = Music: - # TV show subscriptions can support multiple urls and store in the same TV Show - "Rick Beato": - - "https://www.youtube.com/@RickBeato" - - "https://www.youtube.com/@rickbeato240" + # = Music: + # # TV show subscriptions can support multiple urls and store in the same TV Show + # "Rick Beato": + # - "https://www.youtube.com/@RickBeato" + # - "https://www.youtube.com/@rickbeato240" # Set genre tag to "News", use `Only Recent` preset to only store videos uploaded recently - = News | Only Recent: - "BBC News": "https://www.youtube.com/@BBCNews" + # = News | Only Recent: + # "BBC News": "https://www.youtube.com/@BBCNews" ################################################################### # Subscriptions nested under these will use the various prebuilt # music presets -YouTube Releases: - = Jazz: # Sets genre tag to "Jazz" - "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases" -YouTube Full Albums: - = Lofi: - "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i" +# YouTube Releases: +# = Jazz: # Sets genre tag to "Jazz" +# "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases" -SoundCloud Discography: - = Chill Hop: - "UKNOWY": "https://soundcloud.com/uknowymunich" - = Synthwave: - "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords" - "Earmake": "https://soundcloud.com/earmake" +# YouTube Full Albums: +# = Lofi: +# "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i" -Bandcamp: - = Lofi: - "Emily Hopkins": "https://emilyharpist.bandcamp.com/" +# SoundCloud Discography: +# = Chill Hop: +# "UKNOWY": "https://soundcloud.com/uknowymunich" +# = Synthwave: +# "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords" +# "Earmake": "https://soundcloud.com/earmake" + +# Bandcamp: +# = Lofi: +# "Emily Hopkins": "https://emilyharpist.bandcamp.com/" ################################################################### # Can choose between: # - Plex Music Videos: # - Jellyfin Music Videos: # - Kodi Music Videos: -"Plex Music Videos": - = Pop: # Sets genre tag to "Pop" - "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" - "Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" \ No newline at end of file + +# "Plex Music Videos": +# = Pop: # Sets genre tag to "Pop" +# "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc" +# "Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E" diff --git a/docs/source/config_reference/config_yaml.rst b/docs/source/config_reference/config_yaml.rst index 8e4097c3..94233766 100644 --- a/docs/source/config_reference/config_yaml.rst +++ b/docs/source/config_reference/config_yaml.rst @@ -82,9 +82,18 @@ Presets also support inheritance from multiple presets: - "custom_preset" - "parent_preset" -In this example, ``child_preset`` will inherit all fields from ``custom_preset`` -and ``parent_preset`` in that order. The bottom-most preset has the highest -priority. +In this example, ``child_preset`` will inherit all fields from ``custom_preset`` and +``parent_preset`` in that order. The bottom-most preset has the highest priority. More +specifically, presets are merged using `mergedeep`_ via `a TYPESAFE_ADDITIVE merge`_, +which means: + +- if two conflicting keys arent lists or mappings, overwrite the higher priority one +- otherwise, combine then re-evaluate If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is -valid YAML. Inheriting from multiple presets require use of a list. \ No newline at end of file +valid YAML. Inheriting from multiple presets require use of a list. + +.. _`mergedeep`: + https://mergedeep.readthedocs.io/en/latest/ +.. _`a TYPESAFE_ADDITIVE merge`: + https://mergedeep.readthedocs.io/en/latest/index.html#merge-strategies diff --git a/docs/source/config_reference/index.rst b/docs/source/config_reference/index.rst index 0fe2f88c..723efc24 100644 --- a/docs/source/config_reference/index.rst +++ b/docs/source/config_reference/index.rst @@ -4,9 +4,47 @@ Reference This section contains direct references to the code of ``ytdl-sub`` and information on how it functions. + +Terminology +----------- + +Must-know terminology: + +- ``subscription``: URL(s) that you want to download with specific metadata + requirements. +- ``preset``: A media profile comprised of YAML configuration that can specify anything + from metadata layout, media quality, or any feature of ytdl-sub, to apply to + subscriptions. A preset can inherit other presets. +- ``prebuilt preset``: Presets that are included in ytdl-sub. These do most of the work + defining plugins, overrides, etc in order to make downloads ready for player + consumption. +- ``override``: Verb describing the act of overriding something in a preset. For + example, the TV Show presets practically expect you to *override* the URL variable to + tell ytdl-sub where to download from. +- ``override variables``: User-defined variables that are intended to *override* + something. +- ``subscription file``: The file to specify all of your subscriptions and some override + variables. + +Intermediate terminology: + +- ``plugin``: Modular logic to apply to a subscription. To use a plugin, it must be + defined in a preset. +- ``config file``: An optional file where you can define custom presets and other + advanced configuration. +- ``yt-dlp``: The underlying application that handles downloading for ytdl-sub. + +Advanced terminology: + +- ``entry variables``: Variables that derive from a downloaded yt-dlp entry (media). +- ``static variables``: Variables that do not have a dependency to entry variables. +- ``scripting``: Syntax that allows the use of entry variables, static variables, and + functions in override variables. + + .. toctree:: config_yaml subscription_yaml plugins scripting/index - prebuilt_presets/index \ No newline at end of file + prebuilt_presets/index diff --git a/docs/source/guides/getting_started/index.rst b/docs/source/guides/getting_started/index.rst index b5ecf033..11e28528 100644 --- a/docs/source/guides/getting_started/index.rst +++ b/docs/source/guides/getting_started/index.rst @@ -4,8 +4,7 @@ Getting Started Prerequisite Knowledge ---------------------- -As a lower level tool with no GUI, ``ytdl-sub`` requires some technical knowledge. You -must be able to: +Using ``ytdl-sub`` requires some technical knowledge. You must be able to: - do `basic CLI shell navigation`_ - read and write `YAML text files`_ @@ -17,34 +16,105 @@ If you plan on using a :ref:`Docker headless image variant - or bind mount ``/config/`` as a Docker volume and use the editor of your choice from the host +Soon, it's time to start configuring ``ytdl-sub``. We provide a :doc:`./quick_start` +with rigid, rote instructions on how to get a minimal configuration up and running, but +if that serves all your needs, then you're probably better off with :ref:`one of the +more user-friendly yt-dlp wrappers available `. As a lower +level tool with no GUI, most ``ytdl-sub`` users will need to understand at least some of +how ``ytdl-sub`` works, how it "thinks". So before you start configuring ``ytdl-sub``, +`read on `_ to learn how ``ytdl-sub`` works. + .. _`basic CLI shell navigation`: https://developer.mozilla.org/en-US/docs/Learn_web_development/Getting_started/Environment_setup/Command_line .. _`YAML text files`: http://thomasloven.com/blog/2018/08/YAML-For-Nonprogrammers/ -Terminology ------------ +Architecture +------------ -Must-know terminology: +For most users, ``ytdl-sub`` works as follows: -- ``subscription``: URL(s) that you want to download with specific metadata requirements. -- ``preset``: A media profile comprised of YAML configuration that can specify anything from metadata layout, media quality, or any feature of ytdl-sub, to apply to subscriptions. A preset can inherit other presets. -- ``prebuilt preset``: Presets that are included in ytdl-sub. These do most of the work defining plugins, overrides, etc in order to make downloads ready for player consumption. -- ``override``: Verb describing the act of overriding something in a preset. For example, the TV Show presets practically expect you to *override* the URL variable to tell ytdl-sub where to download from. -- ``override variables``: User-defined variables that are intended to *override* something. -- ``subscription file``: The file to specify all of your subscriptions and some override variables. +Subscriptions use presets +^^^^^^^^^^^^^^^^^^^^^^^^^ -Intermediate terminology: +Run ``$ ytdl-sub sub`` to read :doc:`a subscription file <./first_sub>` that defines +what subscriptions to download and place into your media library. Each subscription +selects which :doc:`presets <../../prebuilt_presets/index>` to apply. Those presets +configure how each subscription is downloaded and placed in the media library. -- ``plugin``: Modular logic to apply to a subscription. To use a plugin, it must be defined in a preset. -- ``config file``: An optional file where you can define custom presets and other advanced configuration. -- ``yt-dlp``: The underlying application that handles downloading for ytdl-sub. +Presets configure plugins +^^^^^^^^^^^^^^^^^^^^^^^^^ -Advanced terminology: +:doc:`A preset <../../prebuilt_presets/index>` is effectively a set of plugin +configurations. Specifically, a preset consists of: + +- base presets that it inherits from and extends +- plugin configurations + +When a preset has multiple base presets and more than one of those base presets +configures the same keys for a plugin, the later/lower base preset overrides the plugin +key configurations of earlier/higher base presets. Similarly, when the preset configures +the same keys for a plugin that one of its base plugins configures, the preset +configuration overrides the base presets. + +Plugins do the work +^^^^^^^^^^^^^^^^^^^ + +``ytdl-sub`` applies the plugins that the presets configure when it downloads a +subscription. :doc:`The plugins <../../config_reference/plugins>` control how to run +``yt-dlp``, which media in the subscription to download, how to collect and format +metadata for those media, and how to place the resulting files into your media library, +and more. + +Presets and subscriptions accept overrides +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Presets accept override keys and values and the preset uses those overrides to modify +their plugin configurations. Similarly, individual subscriptions can supply overrides of +their presets for just that subscription. + +Subscriptions are grouped by indentation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Most subscriptions have more in common with each other than not. Thus, defining the presets +and overrides for each subscription would result in mostly repetition and would multiply +the burden of management for the user. The more subscriptions the more work. + +To avoid this redundant work, and so that the subscription configurations describe the +intent of the user, subscriptions are nested/indented under parent/ancestor keys that +define their shared configuration. To support this, ``ytdl-sub`` uses special handling +of the ancestor YAML keys above each subscription. A subscription is the most +nested/indented/descendant key that specifies the URLs for that subscription. The +ancestor keys above that subscription describe the shared presets of that subscription +and all the other descendant subscriptions under them. + +Genres are also more often shared between subscriptions than not. To accommodate that +reality, the ancestor keys of subscriptions may also use the special ``= ...`` prefix to +specify genres that also apply to all descendant subscriptions. + +Finally, ancestor keys may use the ``... | ...`` special character to combine multiple +presets and/or genres for the descendant subscriptions beneath. + +The configuration file extends pre-defined presets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Users define additional presets in :doc:`their configuration file <./first_config>` that +they then use in most of their subscriptions. Most user-defines presets extend the +:doc:`../../prebuilt_presets/index` provided by ``ytdl-sub``. + +Caveats +^^^^^^^ + +Some of these descriptions are not technically complete. For example, a subscription may +use no preset at all and will just run ``yt-dlp`` without any customization or post +processing. The subscriptions file has special support for :ref:`overriding the presets +of all subscriptions in the file `. The +configuration file supports :ref:`a few special options +` that are not about defining presets. See +:doc:`the reference documentation <../../config_reference/index>` for technically +complete details, but for almost all of the use cases served by ``ytdl-sub``, the above +is accurate and representative. -- ``entry variables``: Variables that derive from a downloaded yt-dlp entry (media). -- ``static variables``: Variables that do not have a dependency to entry variables. -- ``scripting``: Syntax that allows the use of entry variables, static variables, and functions in override variables. Ready to Start? --------------- @@ -58,3 +128,8 @@ It is recommended to go through the below sections in order to fully grasp ytdl- first_download automating_downloads first_config + +.. toctree:: + :hidden: + + quick_start diff --git a/docs/source/guides/getting_started/quick_start.rst b/docs/source/guides/getting_started/quick_start.rst new file mode 100644 index 00000000..d0c27169 --- /dev/null +++ b/docs/source/guides/getting_started/quick_start.rst @@ -0,0 +1,72 @@ +Quick Start +=========== + +:ref:`Again `, if the following +serves all your needs, then you're probably better off with :ref:`one of the more +user-friendly yt-dlp wrappers available `. If you still want to +get ``ytdl-sub`` up and running quickly and without understanding, then follow these +instructions to the letter. + +#. Install using :ref:`the official Docker GUI image variant `. + +#. Update the paths for your media library: + + Edit :ref:`the subscriptions file `. Near the + top, under ``__preset__:`` and then ``overrides:``, update the values under the + ``*_directory:`` keys with the correct paths for your media library *as they appear + inside the container*. + +#. Select your media library software: + + Change the ``Plex TV Show by Date:`` *key itself* to the preset for your media + library software. See the comment above for the available options. + +#. Select the genre: + + Under the library software preset key from the previous step, change the ``= + Documentaries`` *key itself* to the genre for this subscription prefixed with ``= + ...``. When adding other subscriptions that have the same genre, place them under the + same key. + +#. Update the subscription name and URL: + + Under the genre key from the previous step, update the ``"NOVA PBS":`` key to the + directory name the downloaded files should be placed beneath. This directory will be + created under the ``tv_show_directory:`` from step #2. Then update the + ``"https://www.youtube.com/@novapbs"`` value to the URL of the channel or playlist + for this subscription. + +#. Preview what ``ytdl-sub`` would do for this subscription: + + Run the :ref:`'sub' sub-command ` but with the ``max_downloads`` + setting from ``yt-dlp`` along with the ``--dry-run`` and ``--match`` options from + ``ytdl-sub`` to minimize requests and prevent actual downloads. Be sure to update the + ``--match="..."`` value with the subscription name:: + + $ ytdl-sub --dry-run sub -o '--ytdl_options.max_downloads 3' --match="NOVA PBS" + + Examine the output carefully. + +#. Review the results of real downloads: + + Run it again without the ``--dry-run`` option to actually download media and place + the files in your library:: + + $ ytdl-sub sub -o '--ytdl_options.max_downloads 3' --match="NOVA PBS" + + Examine the output carefully, then examine how the downloads work in your + library. Repeat with a larger value for ``max_downloads`` and examine the output and + downloads again. + +#. Add the rest of your subscriptions: + + Repeat steps #4-7 for each of your subscriptions. Be sure to repeat the preview and + review steps for each subscription. In general, move slowly and carefully review + everything. It's best to catch issues early to avoid repeating downloads and to + minimize requests to avoid being throttled or banned by servers. + +#. Automate downloads: + + :ref:`Set up ytdl-sub to run periodically + `. diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 8fbfac15..044509bf 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -545,4 +545,4 @@ def test_default_docker_config_and_subscriptions(): default_subs = Subscription.from_file_path( config=default_config, subscription_path=Path("docker/root/defaults/subscriptions.yaml") ) - assert len(default_subs) == 15 + assert len(default_subs) == 1