diff --git a/docs/images/plex_scanner_agent.png b/docs/images/plex_scanner_agent.png new file mode 100644 index 00000000..49ce6b33 Binary files /dev/null and b/docs/images/plex_scanner_agent.png differ diff --git a/docs/images/unraid_badconsole.png b/docs/images/unraid_badconsole.png new file mode 100644 index 00000000..73cc15a9 Binary files /dev/null and b/docs/images/unraid_badconsole.png differ diff --git a/docs/source/config_reference/scripting/index.rst b/docs/source/config_reference/scripting/index.rst index bee9a4e8..4366914b 100644 --- a/docs/source/config_reference/scripting/index.rst +++ b/docs/source/config_reference/scripting/index.rst @@ -9,7 +9,7 @@ contain reference documentation for each built-in variable and scripting functio :maxdepth: 1 entry_variables - override_variables + static_variables scripting_functions scripting_types @@ -30,7 +30,7 @@ considered *static* because it does not depend on anything from an entry. .. code-block:: yaml output_options: - output_directory: "Custom YTDL-SUB TV Show" + output_directory: "/path/to/tv_shows/Custom YTDL-SUB TV Show" Static Variables ~~~~~~~~~~~~~~~~ @@ -41,7 +41,7 @@ We can use this instead of hard-coding it above: .. code-block:: yaml output_options: - output_directory: "{subscription_name}" + output_directory: "/path/to/tv_shows/{subscription_name}" The syntax for variable usage is curly-braces with the variable name within it. Assuming our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub`` @@ -64,7 +64,7 @@ title in its name. We can do that using entry variables: .. code-block:: yaml output_options: - output_directory: "{subscription_name}" + output_directory: "/path/to/tv_shows/{subscription_name}" file_name: "{title}.{ext}" thumbnail_name: "{title}.{thumbnail_ext}" @@ -83,7 +83,7 @@ a ``custom_file_name`` variable to use for the entry file and thumbnail fields: .. code-block:: yaml output_options: - output_directory: "{subscription_name}" + output_directory: "/path/to/tv_shows/{subscription_name}" file_name: "{custom_file_name}.{ext}" thumbnail_name: "{custom_file_name}.{thumbnail_ext}" @@ -104,7 +104,7 @@ are safe by using: .. code-block:: yaml output_options: - output_directory: "{subscription_name_sanitized}" + output_directory: "/path/to/tv_shows/{subscription_name_sanitized}" file_name: "{custom_file_name}.{ext}" thumbnail_name: "{custom_file_name}.{thumbnail_ext}" @@ -115,8 +115,9 @@ Simply add a ``_sanitized`` suffix to any variable name to make it sanitized. .. note:: - Make sure you do not sanitize custom variables that intentionally create directories, otherwise - they will... be sanitized and not resolve to directories! + Make sure you do not sanitize custom variables that intentionally create directories, + (i.e. sanitizing ``/path/to/tv_shows/``) otherwise they will... be sanitized and not resolve to + directories! Using Scripting Functions @@ -130,7 +131,7 @@ Let's suppose you are an avid command-line user, and like all of your file names .. code-block:: yaml output_options: - output_directory: "{subscription_name_sanitized}" + output_directory: "/path/to/tv_shows/{subscription_name_sanitized}" file_name: "{custom_file_name}.{ext}" thumbnail_name: "{custom_file_name}.{thumbnail_ext}" @@ -147,7 +148,7 @@ saying: - Allow a string to be multi-lined, and do not include newlines before or after it. -See for yourself `here `_. +See for yourself `here `_. Any whitespace within curly-braces is okay since it will be parsed out. This is needed to make scripting function usage readable. @@ -161,12 +162,41 @@ Advanced Scripting Accessing ``info.json`` Fields ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -WIP +The entirety of an entry's ``info.json`` file resides in the +`Map `_ +variable +`entry_metadata `_. + +Any field can be accessed by using the +`map_get `_ +function like so: + +.. code-block:: yaml + :caption: Fetches the 'artist' value from the .info.json, returns null if it does not exist. + + artist: >- + { %map_get( entry_metadata, "artist", null ) } Creating Custom Functions ~~~~~~~~~~~~~~~~~~~~~~~~~ -WIP +Custom functions can be created in the overrides section using the following syntax: -Parsing Maps and Arrays -~~~~~~~~~~~~~~~~~~~~~~~ -WIP +.. code-block:: yaml + + overrides: + "%get_entry_metadata_field": >- + { %map_get( entry_metadata, $0, null ) } + +Custom function definitions must have ``%`` as a prefix to the function name, be surrounded by +quotes to make YAML parsing happy, and can support arguments using ``$0``, ``$1``, ... to indicate +their first argument, second argument, etc. + +Using our new custom function, we can simply the ``artist`` variable definition above to: + +.. code-block:: yaml + + overrides: + "%get_entry_metadata_field": >- + { %map_get( entry_metadata, $0, null ) } + artist: >- + { get_entry_metadata_field("artist") } diff --git a/docs/source/config_reference/scripting/scripting_types.rst b/docs/source/config_reference/scripting/scripting_types.rst index d389c2a9..381f8cda 100644 --- a/docs/source/config_reference/scripting/scripting_types.rst +++ b/docs/source/config_reference/scripting/scripting_types.rst @@ -8,23 +8,22 @@ Types String ~~~~~~ -Strings are a series of characters surrounded by quotes and can be defined in a few ways, including: +Strings are a series of characters surrounded by quotes. + +.. code-block:: yaml + + string_variable: "This is a String variable" + +.. note:: + + For non-String types, they must be defined as parameters to scripting functions. This is because + anything in a variable definition that is not within curly-braces gets evaluated as a String. + +We can define Strings within curly-braces by setting them as parameters to a function: .. tab-set:: - .. tab-item:: Literal - - .. code-block:: yaml - - string_variable: "This is a String variable" - - .. tab-item:: In-Line - - .. code-block:: yaml - - string_variable: "{ %string('This is a String variable') }" - - .. tab-item:: Single Quote + .. tab-item:: Multi-Line Single Quote .. code-block:: yaml @@ -33,7 +32,7 @@ Strings are a series of characters surrounded by quotes and can be defined in a %string('This is a String variable') } - .. tab-item:: Double Quote + .. tab-item:: Multi-Line Double Quote .. code-block:: yaml @@ -42,13 +41,42 @@ Strings are a series of characters surrounded by quotes and can be defined in a %string("This is a String variable") } - .. tab-item:: Triple Quote +There are a few ways to make variables that use curly braces more compact, including: + +.. tab-set:: + + .. tab-item:: New-Line Single Quote + + .. code-block:: yaml + + string_variable: >- + { %string('This is a String variable') } + + .. tab-item:: New-Line Double Quote + + .. code-block:: yaml + + string_variable: >- + { %string("This is a String variable") } + + .. tab-item:: Same-Line + + .. code-block:: yaml + + string_variable: "{ %string('This is a String variable') }" + +In the case that you want to define a string variable that contains both single and double quotes, +triple-quotes can be used to avoid *closing* the String. + +.. tab-set:: + + .. tab-item:: Triple-Single Quote .. code-block:: yaml string_variable: >- { - %string('''This is a String variable''') + %string('''This has both " and ' in it.''') } .. tab-item:: Triple-Double Quote @@ -57,14 +85,9 @@ Strings are a series of characters surrounded by quotes and can be defined in a string_variable: >- { - %string("""This is a String variable""") + %string("""This has both " and ' in it.""") } -.. note:: - - For non-String types, they must be defined as parameters to scripting functions. This is because - anything in a variable definition that is not within curly-braces gets evaluated as a String. - Integer ~~~~~~~ @@ -72,7 +95,7 @@ Integers are whole numbers with no decimal. .. tab-set:: - .. tab-item:: Literal + .. tab-item:: Multi-Line .. code-block:: yaml @@ -81,7 +104,15 @@ Integers are whole numbers with no decimal. %int(2022) } - .. tab-item:: In-Line + .. tab-item:: New-Line + + .. code-block:: yaml + + int_variable: >- + { %int(2022) } + + + .. tab-item:: Same-Line .. code-block:: yaml @@ -94,7 +125,7 @@ Floats are floating-point decimals numbers. .. tab-set:: - .. tab-item:: Literal + .. tab-item:: Multi-Line .. code-block:: yaml @@ -103,7 +134,14 @@ Floats are floating-point decimals numbers. %float(3.14) } - .. tab-item:: In-Line + .. tab-item:: New-Line + + .. code-block:: yaml + + float_variable: >- + { %float(3.14) } + + .. tab-item:: Same-Line .. code-block:: yaml @@ -116,7 +154,7 @@ A type is considered boolean if it spells out ``True`` or ``False``, case-insens .. tab-set:: - .. tab-item:: Literal + .. tab-item:: Multi-Line .. code-block:: yaml @@ -125,7 +163,14 @@ A type is considered boolean if it spells out ``True`` or ``False``, case-insens %bool(True) } - .. tab-item:: In-Line + .. tab-item:: New-Line + + .. code-block:: yaml + + bool_variable: >- + { %bool(True) } + + .. tab-item:: Same-Line .. code-block:: yaml @@ -139,7 +184,7 @@ Arrays are defined using brackets (``[ ]``), and are accessed using zero-based i .. tab-set:: - .. tab-item:: Literal + .. tab-item:: Multi-Line .. code-block:: yaml @@ -157,7 +202,16 @@ Arrays are defined using brackets (``[ ]``), and are accessed using zero-based i %array_at(array_variable, 0) } - .. tab-item:: In-Line + .. tab-item:: New-Line + + .. code-block:: yaml + + array_variable: >- + { ["element with index 0", 1, 2.0, ["Nested Array 3"]] } + element_0: >- + { %array_at(array_variable, 0) } + + .. tab-item:: Same-Line .. code-block:: yaml @@ -168,11 +222,11 @@ Map ~~~ A Map is a key-value store, containing mappings between keys and values. -Maps are defined using curley-braces (``{ }``), and are accessed using their keys. +Maps are defined using curly-braces (``{ }``), and are accessed using their keys. .. tab-set:: - .. tab-item:: Literal + .. tab-item:: Multi-Line .. code-block:: yaml @@ -189,7 +243,16 @@ Maps are defined using curley-braces (``{ }``), and are accessed using their key %map_get(map_variable, "string_key") } - .. tab-item:: In-Line + .. tab-item:: New-Line + + .. code-block:: yaml + + map_variable: >- + { {"string_key": "string_value", 1: "int_key", "list_value": ["elem0", 1, 2.0]} } + string_value: >- + { %map_get(map_variable, "string_key") } + + .. tab-item:: Same-Line .. code-block:: yaml @@ -209,7 +272,14 @@ case-insensitive. null_variable: "" - .. tab-item:: In-Line + .. tab-item:: New-Line + + .. code-block:: yaml + + null_variable: >- + { %string(null) } + + .. tab-item:: Same-Line .. code-block:: yaml @@ -276,8 +346,9 @@ it expects the lambda function to have two input arguments. These are denoted us LambdaReduce ~~~~~~~~~~~~ -LambdaReduce is special type of lambda that reduces an Array to a single value by calling the -LabmdaReduce function repeatedly on two elements in the Array until it is reduced to a single value. +LambdaReduce parameters are a reference to a function that will perform a *reduce* - an operation +that reduces an Array to a single value by calling the LambdaReduce function repeatedly on two +elements in the Array until it is reduced to a single value. In this example, @@ -294,11 +365,9 @@ on the input array, using `add `_ as the LambdaReduce function. This will reduce the Array to a single value by internally calling -.. code-block:: - - - %add(1, 2) = 3 - - %add(3, 3) = 6 - - %add(6, 4) = 10 +- *reduce-call 1*: ``%add(1, 2) = 3`` (first two elements) +- *reduce-call 2*: ``%add(3, 3) = 6`` (output from first two and third element) +- *reduce-call 3*: ``%add(6, 4) = 10`` (output from first three elements and fourth element) And evaluate to ``10``. diff --git a/docs/source/config_reference/scripting/override_variables.rst b/docs/source/config_reference/scripting/static_variables.rst similarity index 79% rename from docs/source/config_reference/scripting/override_variables.rst rename to docs/source/config_reference/scripting/static_variables.rst index c7a9991b..e6e41527 100644 --- a/docs/source/config_reference/scripting/override_variables.rst +++ b/docs/source/config_reference/scripting/static_variables.rst @@ -1,9 +1,12 @@ -Override Variables -================== +Static Variables +================ + +Subscription Variables +---------------------- subscription_indent_i ---------------------- +~~~~~~~~~~~~~~~~~~~~~ For subscriptions in the form of .. code-block:: yaml @@ -16,7 +19,7 @@ For subscriptions in the form of ``Indent Value 1`` and ``Indent Value 2``. subscription_map ----------------- +~~~~~~~~~~~~~~~~ For subscriptions in the form of .. code-block:: yaml @@ -42,11 +45,12 @@ Stores all the contents under the subscription name into the override variable } subscription_name ------------------ -Name of the subscription +~~~~~~~~~~~~~~~~~ +Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``), +the prefix and all whitespace afterwards is stripped from the subscription name. subscription_value ------------------- +~~~~~~~~~~~~~~~~~~ For subscriptions in the form of .. code-block:: yaml @@ -56,7 +60,7 @@ For subscriptions in the form of ``subscription_value`` gets set to ``https://...``. subscription_value_i --------------------- +~~~~~~~~~~~~~~~~~~~~ For subscriptions in the form of .. code-block:: yaml diff --git a/docs/source/faq/index.rst b/docs/source/faq/index.rst index c728b7ca..489240de 100644 --- a/docs/source/faq/index.rst +++ b/docs/source/faq/index.rst @@ -1,8 +1,8 @@ +=== FAQ === -Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as -more questions get asked. +Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as more questions get asked. .. contents:: Frequently Asked Questions :depth: 3 @@ -10,12 +10,25 @@ more questions get asked. How do I... ----------- +...get support or reach out to contribute? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you need support, you can: + +* :ytdl-sub-gh:`Open an issue on GitHub ` + +* `Join our Discord `_ + +If you would like to contribute, we're happy to accept any help, even non-coders! To find out how you can help this project, you can: + +* `Join our Discord `_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute. + +* If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull request ` with information on what issue you're resolving and it will be reviewed as soon as possible. + ...download age-restricted YouTube videos? -'''''''''''''''''''''''''''''''''''''''''' -See -`ytdls recommended way `_ -to download your YouTube cookie, then add it to your -`ytdl options `_ section of your config: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See `yt-dl's recommended way `_ to download your YouTube cookie, then add it to your :ref:`ytdl options ` section of your config: .. code-block:: yaml @@ -23,14 +36,16 @@ to download your YouTube cookie, then add it to your cookiefile: "/path/to/cookies/file.txt" ...automate my downloads? -''''''''''''''''''''''''' -`This part of the wiki `_ shows how to set up ``ytdl-sub`` to run in a cron job within Docker. +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:doc:`This page ` shows how to set up ``ytdl-sub`` to run automatically on various platforms. There is a bug where... ----------------------- ...date_range is not downloading older videos after I changed the range -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether. .. code-block:: yaml @@ -38,11 +53,12 @@ Your preset most likely has ``break_on_existing`` set to True, which will stop d ytdl_options: break_on_existing: False -After your download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads. +After you download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads. ...it is downloading non-English title and description metadata -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' -Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using. .. code-block:: yaml @@ -53,7 +69,19 @@ Most likely the video has a non-English language set to its 'native' language. Y - "en" ...Plex is not showing my TV shows correctly -'''''''''''''''''''''''''''''''''''''''''''' -Set the following -`Scanner and Agent `_ -for your library. \ No newline at end of file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set the following for your ytdl-sub library that has been added to Plex. + +.. figure:: ../../images/plex_scanner_agent.png + :alt: The Plex library editor, under the advanced settings, showing the required options for Plex to show the TV shows correctly. + +**Scanner:** Plex Series Scanner + +**Agent:** Personal Media shows + +**Visibility:** Exclude from home screen and global search + +**Episode sorting:** Library default + +**YES** Enable video preview thumbnails \ No newline at end of file diff --git a/docs/source/guides/install/docker.rst b/docs/source/guides/install/docker.rst index 9c89e9be..04b5e456 100644 --- a/docs/source/guides/install/docker.rst +++ b/docs/source/guides/install/docker.rst @@ -17,14 +17,14 @@ The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install yt GUI Image --------- -The GUI image uses LSIO's :lsio-gh:`docker-code-server image` for its base image. More info on other code-server environment variables can be found within its documentation. +The GUI image uses LSIO's :lsio-gh:`docker-code-server image <\ >` for its base image. More info on other code-server environment variables can be found within its documentation. -After starting, code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``. +After starting, the code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``. Headless Image -------------- -The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image` for its base image. Execute the following command to access and interact with ``ytdl-sub``: +The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image <\ >` for its base image. Execute the following command to access and interact with ``ytdl-sub``: .. code-block:: bash diff --git a/docs/source/guides/install/unraid.rst b/docs/source/guides/install/unraid.rst index ac979a99..d27535dd 100644 --- a/docs/source/guides/install/unraid.rst +++ b/docs/source/guides/install/unraid.rst @@ -1,3 +1,16 @@ Unraid -------------- -You can install our :unraid:`unraid community apps ` through the `Unraid Community Apps plugin `_. Uses Docker under the hood. \ No newline at end of file +You can install our :unraid:`unraid community apps ` through the `Unraid Community Apps plugin `_. + + +If you installed the ``ytdl-sub-gui`` app, the code-server will be running at http://localhost:8443 (replace ``localhost`` with the IP of the computer running Unraid if you aren't trying to access ``ytdl-sub`` on that computer). Open this page in a browser to access and interact with ``ytdl-sub``. + +If you installed the ``ytdl-sub`` app (headless), open the normal app-specific console to access and interact with ``ytdl-sub``. Once open, you must first run ``su abc -s /bin/bash`` to change to the non-root user. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``. + + +.. warning:: + + If you use the below option to access the ``ytdl-sub`` console, be sure to run ``su abc -s /bin/bash`` first thing. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``. Do **NOT** run ``ytdl-sub`` as the root user! Running as root will set the owner of all modified files to root, which prevents most media managers and players from accessing the files. + + .. figure:: ../../../images/unraid_badconsole.png + :alt: The Unraid community app plugin GUI, with an arrow pointing at the "Console" option in the dropdown after selecting ytdl-sub-gui \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d78c5633..a26368d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,24 +22,6 @@ disable = [ load-plugins = "pylint.extensions.docparams" -[tool.pydocstyle] -inherit = false -match = "[^test_].*\\.py" -ignore = [ - "D100", # docstring in public module - "D101", # Missing docstring in public class (covered by pylint) - "D104", # docstring in public package - "D107", # docstring in init - "D200", # One-line should fit on one line - "D203", # 1 blank line before class docstring - "D205", # 1 blank line between summary and description - "D212", # Multi-line should start at first line - "D400", # Should end with a period - "D401", # Return vs Returns - "D413", # Missing blank line after last section - "D415", # Should end with a period -] - [tool.coverage.run] include = [ "src/*" diff --git a/setup.cfg b/setup.cfg index db9e581d..5692d3f1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ package_dir = packages=find: install_requires = - yt-dlp==2023.11.16 + yt-dlp==2023.12.30 argparse==1.4.0 colorama==0.4.6 mergedeep==1.3.4 diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 59b787b7..3372e936 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -8,7 +8,7 @@ import mergedeep from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.variables.override_variables import OverrideHelpers -from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved @@ -56,7 +56,7 @@ class Overrides(DictFormatterValidator, Scriptable): def __init__(self, name, value): DictFormatterValidator.__init__(self, name, value) - Scriptable.__init__(self) + Scriptable.__init__(self, initialize_base_script=True) for key in self._keys: self.ensure_variable_name_valid(key) @@ -135,7 +135,7 @@ class Overrides(DictFormatterValidator, Scriptable): """ self.script.add( ScriptUtils.add_sanitized_variables( - {OverrideVariables.subscription_name(): subscription_name} + {SubscriptionVariables.subscription_name(): subscription_name} ) ) self.script.add( diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index 1b2ea1e4..cc554a70 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -14,7 +14,7 @@ from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS -from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import validate_formatters @@ -67,7 +67,9 @@ def _get_added_and_modified_variables( def _override_variables(overrides: Overrides) -> Set[str]: - return set(list(overrides.initial_variables().keys())) | {OverrideVariables.subscription_name()} + return set(list(overrides.initial_variables().keys())) | { + SubscriptionVariables.subscription_name() + } def _entry_variables() -> Set[str]: diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index 509a4f8d..71f3c996 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -152,7 +152,9 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): for file_name in entry_file_names: ext = get_file_extension(file_name) file_path = Path(self.output_directory) / file_name - working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}" + working_directory_file_path = Path(self.working_directory) / entry.base_filename( + ext=ext + ) # NFO files will always get rewritten, so ignore if ext == "nfo": diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 67feabd0..b1be016b 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -349,39 +349,43 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ) def _iterate_child_entries( - self, url_validator: UrlValidator, entries: List[Entry] + self, entries: List[Entry], download_reversed: bool ) -> Iterator[Entry]: - entries_to_iterate = entries - if url_validator.download_reverse: - entries_to_iterate = reversed(entries) + # Iterate a list of entries, and delete the entries after yielding + indices = list(range(len(entries))) + if download_reversed: + indices = reversed(indices) - for entry in entries_to_iterate: + for idx in indices: self._url_state.entries_downloaded += 1 - if self._is_downloaded(entry): + if self._is_downloaded(entries[idx]): download_logger.info( "Already downloaded entry %d/%d: %s", self._url_state.entries_downloaded, self._url_state.entries_total, - entry.title, + entries[idx].title, ) + del entries[idx] continue - yield entry - self._mark_downloaded(entry) + yield entries[idx] + self._mark_downloaded(entries[idx]) + + del entries[idx] def _iterate_parent_entry( - self, url_validator: UrlValidator, parent: EntryParent + self, parent: EntryParent, download_reversed: bool ) -> Iterator[Entry]: for entry_child in self._iterate_child_entries( - url_validator=url_validator, entries=parent.entry_children() + entries=parent.entry_children(), download_reversed=download_reversed ): yield entry_child # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): for entry_child in self._iterate_parent_entry( - url_validator=url_validator, parent=parent_child + parent=parent_child, download_reversed=download_reversed ): yield entry_child @@ -415,9 +419,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): def _iterate_entries( self, - url_validator: UrlValidator, parents: List[EntryParent], orphans: List[Entry], + download_reversed: bool, ) -> Iterator[Entry]: """ Downloads the leaf entries from EntryParent trees @@ -426,11 +430,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): with self._separate_download_archives(clear_info_json_files=True): for parent in parents: for entry_child in self._iterate_parent_entry( - url_validator=url_validator, parent=parent + parent=parent, download_reversed=download_reversed ): yield entry_child - for orphan in self._iterate_child_entries(url_validator=url_validator, entries=orphans): + for orphan in self._iterate_child_entries( + entries=orphans, download_reversed=download_reversed + ): yield orphan def download_metadata(self) -> Iterable[Entry]: @@ -454,7 +460,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): "Beginning downloads for %s", self.overrides.apply_formatter(collection_url.url) ) for entry in self._iterate_entries( - url_validator=collection_url, parents=parents, orphans=orphan_entries + parents=parents, + orphans=orphan_entries, + download_reversed=collection_url.download_reverse, ): entry.initialize_script(self.overrides).add( {v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)} diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index c4243d63..1d8acd3e 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -8,6 +8,8 @@ from typing import Type from typing import TypeVar from typing import final +from yt_dlp.utils import sanitize_filename + from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions @@ -45,6 +47,19 @@ class BaseEntry(ABC): """ return str(self._kwargs[v.uid.metadata_key]) + @property + def uid_sanitized(self) -> str: + """ + Sanitized version, used in filenames + """ + return sanitize_filename(self.uid) + + def base_filename(self, ext: str): + """ + The base filename of all yt-dlp downloaded entry files + """ + return f"{self.uid_sanitized}.{ext}" + @property def download_archive_extractor(self) -> str: """ @@ -101,30 +116,13 @@ class BaseEntry(ABC): """ return self._working_directory - def add_kwargs(self, variables_to_add: Dict[str, Any]) -> "BaseEntry": - """ - Adds variables to kwargs. Use with caution since yt-dlp data can be overwritten. - Plugins should use ``add_variables``. - - Parameters - ---------- - variables_to_add - Variables to add to kwargs - - Returns - ------- - self - """ - self._kwargs = dict(self._kwargs, **variables_to_add) - return self - def get_download_info_json_name(self) -> str: """ Returns ------- The download info json's file name """ - return f"{self.uid}.{self.info_json_ext}" + return self.base_filename(ext=self.info_json_ext) def get_download_info_json_path(self) -> str: """ diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index c7442992..831bda6c 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -44,12 +44,6 @@ class Entry(BaseEntry, Scriptable): BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory) Scriptable.__init__(self) - def _add_entry_kwargs_to_script(self) -> None: - # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized - self.unresolvable.remove(v.entry_metadata.variable_name) - self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) - self.update_script() - def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry": """ Initializes the entry script using the Overrides script, then adding @@ -57,12 +51,20 @@ class Entry(BaseEntry, Scriptable): """ # Overrides contains added variables that are unresolvable, add them here if other: - self.script = copy.deepcopy(other.script) - self.unresolvable = copy.deepcopy(other.unresolvable) + self._script = copy.deepcopy(other.script) + self._unresolvable = copy.deepcopy(other.unresolvable) + else: + self.initialize_base_script() self._add_entry_kwargs_to_script() return self + def _add_entry_kwargs_to_script(self) -> None: + # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized + self.unresolvable.remove(v.entry_metadata.variable_name) + self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) + self.update_script() + def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT: """ Gets a variable of an expected type. Will error if it does not exist or is not resolved. @@ -113,7 +115,8 @@ class Entry(BaseEntry, Scriptable): """ ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key] for possible_ext in [ext, "mkv"]: - file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}") + file_name = self.base_filename(ext=possible_ext) + file_path = str(Path(self.working_directory()) / file_name) if os.path.isfile(file_path): return possible_ext @@ -125,7 +128,7 @@ class Entry(BaseEntry, Scriptable): ------- The entry's file name """ - return f"{self.uid}.{self.ext}" + return self.base_filename(ext=self.ext) def get_download_file_path(self) -> str: """Returns the entry's file path to where it was downloaded""" @@ -137,7 +140,7 @@ class Entry(BaseEntry, Scriptable): ------- The download thumbnail's file name """ - return f"{self.uid}.{self.get(v.thumbnail_ext, str)}" + return self.base_filename(ext=self.get(v.thumbnail_ext, str)) def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" @@ -155,7 +158,10 @@ class Entry(BaseEntry, Scriptable): possible_thumbnail_exts.add(thumbnail["url"].split(".")[-1]) for ext in possible_thumbnail_exts: - possible_thumbnail_path = str(Path(self.working_directory()) / f"{self.uid}.{ext}") + possible_thumbnail_filename = self.base_filename(ext=ext) + possible_thumbnail_path = str( + Path(self.working_directory()) / possible_thumbnail_filename + ) if os.path.isfile(possible_thumbnail_path): return possible_thumbnail_path @@ -202,7 +208,7 @@ class Entry(BaseEntry, Scriptable): # HACK: yt-dlp does not record extracted/converted extensions anywhere. If the file is not # found, try it using all possible extensions if not file_exists: - for ext in AUDIO_CODEC_EXTS.union(VIDEO_CODEC_EXTS): + for ext in AUDIO_CODEC_EXTS | VIDEO_CODEC_EXTS: if os.path.isfile(self.get_download_file_path().removesuffix(self.ext) + ext): file_exists = True break diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 38f7160c..007aaf2b 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -7,11 +7,12 @@ from ytdl_sub.script.utils.name_validation import is_valid_name SUBSCRIPTION_ARRAY = "subscription_array" -class OverrideVariables: +class SubscriptionVariables: @staticmethod def subscription_name() -> str: """ - Name of the subscription + Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``), + the prefix and all whitespace afterwards is stripped from the subscription name. """ return "subscription_name" diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index b3ef63e2..9a7c8bab 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -13,7 +13,7 @@ from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.validators import BoolValidator -logger = Logger.get("embed_thumbnail") +logger = Logger.get("embed-thumbnail") class EmbedThumbnailOptions(BoolValidator, OptionsValidator): diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index 7f381db1..72ad6c88 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -11,7 +11,7 @@ from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive -logger = Logger.get("conditional") +logger = Logger.get("filter-exclude") class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): @@ -55,6 +55,11 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]): for formatter in self.plugin_options.list: out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) if bool(out): + logger.info( + "Filtering '%s' from the filter %s evaluating to True", + entry.title, + formatter.format_string, + ) return None return entry diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 41a98f78..5a54e8ec 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -11,7 +11,7 @@ from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive -logger = Logger.get("conditional") +logger = Logger.get("filter-include") class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): @@ -63,6 +63,11 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]): for formatter in self.plugin_options.list: out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) if not bool(out): + logger.info( + "Filtering '%s' from the filter %s evaluating to False", + entry.title, + formatter.format_string, + ) return None return entry diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index fe10a69b..90211e2c 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -8,7 +8,7 @@ from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator -logger = Logger.get("match_filters") +logger = Logger.get("match-filters") def default_filters() -> Tuple[List[str], List[str]]: diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index f6098517..6ad24e37 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import BoolValidator v: VariableDefinitions = VARIABLES -logger = Logger.get("music_tags") +logger = Logger.get("music-tags") def _is_multi_field(tag_name: str) -> bool: diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 9c53a9bd..0e74d448 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -208,7 +208,6 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}") if self.plugin_options.subtitles_name: for lang in langs: - subtitle_file_name = f"{entry.uid}.{lang}.{self.plugin_options.subtitles_type}" output_subtitle_file_name = self.overrides.apply_formatter( formatter=self.plugin_options.subtitles_name, entry=entry, @@ -216,7 +215,9 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): ) self.save_file( - file_name=subtitle_file_name, + file_name=entry.base_filename( + ext=f"{lang}.{self.plugin_options.subtitles_type}" + ), output_file_name=output_subtitle_file_name, entry=entry, ) @@ -225,9 +226,8 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): # Can happen for both file and embedded subs for lang in langs: for possible_ext in SUBTITLE_EXTENSIONS: - possible_subs_file = ( - Path(self.working_directory) / f"{entry.uid}.{lang}.{possible_ext}" - ) + possible_subs_filename = entry.base_filename(ext=f"{lang}.{possible_ext}") + possible_subs_file = Path(self.working_directory) / possible_subs_filename FileHandler.delete(possible_subs_file) return file_metadata diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index c3b2e6ec..0d4614a8 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -10,7 +10,7 @@ from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator -logger = Logger.get("video_tags") +logger = Logger.get("video-tags") class VideoTagsOptions(OptionsDictValidator): diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml index 239495ae..746578ea 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml @@ -1,5 +1,3 @@ - - presets: ################################################################################## @@ -12,7 +10,7 @@ presets: # # "Subscription Name": "url" # -# where url tries to grab channel avatar + banner +# where the first url tries to grab channel avatar + banner # _multi_url: download: @@ -41,9 +39,91 @@ presets: - url: "{url18}" - url: "{url19}" - url: "{url20}" + - url: "{url21}" + - url: "{url22}" + - url: "{url23}" + - url: "{url24}" + - url: "{url25}" + - url: "{url26}" + - url: "{url27}" + - url: "{url28}" + - url: "{url29}" + - url: "{url30}" + - url: "{url31}" + - url: "{url32}" + - url: "{url33}" + - url: "{url34}" + - url: "{url35}" + - url: "{url36}" + - url: "{url37}" + - url: "{url38}" + - url: "{url39}" + - url: "{url40}" + - url: "{url41}" + - url: "{url42}" + - url: "{url43}" + - url: "{url44}" + - url: "{url45}" + - url: "{url46}" + - url: "{url47}" + - url: "{url48}" + - url: "{url49}" + - url: "{url50}" + - url: "{url51}" + - url: "{url52}" + - url: "{url53}" + - url: "{url54}" + - url: "{url55}" + - url: "{url56}" + - url: "{url57}" + - url: "{url58}" + - url: "{url59}" + - url: "{url60}" + - url: "{url61}" + - url: "{url62}" + - url: "{url63}" + - url: "{url64}" + - url: "{url65}" + - url: "{url66}" + - url: "{url67}" + - url: "{url68}" + - url: "{url69}" + - url: "{url70}" + - url: "{url71}" + - url: "{url72}" + - url: "{url73}" + - url: "{url74}" + - url: "{url75}" + - url: "{url76}" + - url: "{url77}" + - url: "{url78}" + - url: "{url79}" + - url: "{url80}" + - url: "{url81}" + - url: "{url82}" + - url: "{url83}" + - url: "{url84}" + - url: "{url85}" + - url: "{url86}" + - url: "{url87}" + - url: "{url88}" + - url: "{url89}" + - url: "{url90}" + - url: "{url91}" + - url: "{url92}" + - url: "{url93}" + - url: "{url94}" + - url: "{url95}" + - url: "{url96}" + - url: "{url97}" + - url: "{url98}" + - url: "{url99}" + - url: "{url100}" + overrides: avatar_uncropped_thumbnail_file_name: "" banner_uncropped_thumbnail_file_name: "" + subscription_value: "" subscription_value_2: "" subscription_value_3: "" @@ -64,6 +144,87 @@ presets: subscription_value_18: "" subscription_value_19: "" subscription_value_20: "" + subscription_value_21: "" + subscription_value_22: "" + subscription_value_23: "" + subscription_value_24: "" + subscription_value_25: "" + subscription_value_26: "" + subscription_value_27: "" + subscription_value_28: "" + subscription_value_29: "" + subscription_value_30: "" + subscription_value_31: "" + subscription_value_32: "" + subscription_value_33: "" + subscription_value_34: "" + subscription_value_35: "" + subscription_value_36: "" + subscription_value_37: "" + subscription_value_38: "" + subscription_value_39: "" + subscription_value_40: "" + subscription_value_41: "" + subscription_value_42: "" + subscription_value_43: "" + subscription_value_44: "" + subscription_value_45: "" + subscription_value_46: "" + subscription_value_47: "" + subscription_value_48: "" + subscription_value_49: "" + subscription_value_50: "" + subscription_value_51: "" + subscription_value_52: "" + subscription_value_53: "" + subscription_value_54: "" + subscription_value_55: "" + subscription_value_56: "" + subscription_value_57: "" + subscription_value_58: "" + subscription_value_59: "" + subscription_value_60: "" + subscription_value_61: "" + subscription_value_62: "" + subscription_value_63: "" + subscription_value_64: "" + subscription_value_65: "" + subscription_value_66: "" + subscription_value_67: "" + subscription_value_68: "" + subscription_value_69: "" + subscription_value_70: "" + subscription_value_71: "" + subscription_value_72: "" + subscription_value_73: "" + subscription_value_74: "" + subscription_value_75: "" + subscription_value_76: "" + subscription_value_77: "" + subscription_value_78: "" + subscription_value_79: "" + subscription_value_80: "" + subscription_value_81: "" + subscription_value_82: "" + subscription_value_83: "" + subscription_value_84: "" + subscription_value_85: "" + subscription_value_86: "" + subscription_value_87: "" + subscription_value_88: "" + subscription_value_89: "" + subscription_value_90: "" + subscription_value_91: "" + subscription_value_92: "" + subscription_value_93: "" + subscription_value_94: "" + subscription_value_95: "" + subscription_value_96: "" + subscription_value_97: "" + subscription_value_98: "" + subscription_value_99: "" + subscription_value_100: "" + url: "{subscription_value}" url2: "{subscription_value_2}" url3: "{subscription_value_3}" @@ -84,3 +245,83 @@ presets: url18: "{subscription_value_18}" url19: "{subscription_value_19}" url20: "{subscription_value_20}" + url21: "{subscription_value_21}" + url22: "{subscription_value_22}" + url23: "{subscription_value_23}" + url24: "{subscription_value_24}" + url25: "{subscription_value_25}" + url26: "{subscription_value_26}" + url27: "{subscription_value_27}" + url28: "{subscription_value_28}" + url29: "{subscription_value_29}" + url30: "{subscription_value_30}" + url31: "{subscription_value_31}" + url32: "{subscription_value_32}" + url33: "{subscription_value_33}" + url34: "{subscription_value_34}" + url35: "{subscription_value_35}" + url36: "{subscription_value_36}" + url37: "{subscription_value_37}" + url38: "{subscription_value_38}" + url39: "{subscription_value_39}" + url40: "{subscription_value_40}" + url41: "{subscription_value_41}" + url42: "{subscription_value_42}" + url43: "{subscription_value_43}" + url44: "{subscription_value_44}" + url45: "{subscription_value_45}" + url46: "{subscription_value_46}" + url47: "{subscription_value_47}" + url48: "{subscription_value_48}" + url49: "{subscription_value_49}" + url50: "{subscription_value_50}" + url51: "{subscription_value_51}" + url52: "{subscription_value_52}" + url53: "{subscription_value_53}" + url54: "{subscription_value_54}" + url55: "{subscription_value_55}" + url56: "{subscription_value_56}" + url57: "{subscription_value_57}" + url58: "{subscription_value_58}" + url59: "{subscription_value_59}" + url60: "{subscription_value_60}" + url61: "{subscription_value_61}" + url62: "{subscription_value_62}" + url63: "{subscription_value_63}" + url64: "{subscription_value_64}" + url65: "{subscription_value_65}" + url66: "{subscription_value_66}" + url67: "{subscription_value_67}" + url68: "{subscription_value_68}" + url69: "{subscription_value_69}" + url70: "{subscription_value_70}" + url71: "{subscription_value_71}" + url72: "{subscription_value_72}" + url73: "{subscription_value_73}" + url74: "{subscription_value_74}" + url75: "{subscription_value_75}" + url76: "{subscription_value_76}" + url77: "{subscription_value_77}" + url78: "{subscription_value_78}" + url79: "{subscription_value_79}" + url80: "{subscription_value_80}" + url81: "{subscription_value_81}" + url82: "{subscription_value_82}" + url83: "{subscription_value_83}" + url84: "{subscription_value_84}" + url85: "{subscription_value_85}" + url86: "{subscription_value_86}" + url87: "{subscription_value_87}" + url88: "{subscription_value_88}" + url89: "{subscription_value_89}" + url90: "{subscription_value_90}" + url91: "{subscription_value_91}" + url92: "{subscription_value_92}" + url93: "{subscription_value_93}" + url94: "{subscription_value_94}" + url95: "{subscription_value_95}" + url96: "{subscription_value_96}" + url97: "{subscription_value_97}" + url98: "{subscription_value_98}" + url99: "{subscription_value_99}" + url100: "{subscription_value_100}" diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 7bbd09ca..56e93954 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -343,6 +343,7 @@ class SubscriptionDownload(BaseSubscription, ABC): preset=self._preset_options, plugins=plugins, enhanced_download_archive=self._enhanced_download_archive, + overrides=self.overrides, working_directory=self.working_directory, dry_run=dry_run, ) @@ -395,6 +396,7 @@ class SubscriptionDownload(BaseSubscription, ABC): preset=self._preset_options, plugins=plugins, enhanced_download_archive=self._enhanced_download_archive, + overrides=self.overrides, working_directory=self.working_directory, dry_run=dry_run, ) diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 62445552..c247779a 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -9,7 +9,7 @@ from typing import final from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.overrides import Overrides -from ytdl_sub.entries.variables.override_variables import OverrideVariables +from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.validators import DictValidator @@ -32,7 +32,7 @@ class SubscriptionOutput(Validator, ABC): indent overrides to merge with the preset dict's overrides """ return { - OverrideVariables.subscription_indent_i(i): self._indent_overrides[i] + SubscriptionVariables.subscription_indent_i(i): self._indent_overrides[i] for i in range(len(self._indent_overrides)) } @@ -143,7 +143,7 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add[OverrideVariables.subscription_value()] = self.value + self._overrides_to_add[SubscriptionVariables.subscription_value()] = self.value class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator): @@ -168,10 +168,12 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid for idx, list_value in enumerate(self.list): # Write the first list value into subscription_value as well if idx == 0: - self._overrides_to_add[OverrideVariables.subscription_value()] = list_value.value + self._overrides_to_add[ + SubscriptionVariables.subscription_value() + ] = list_value.value self._overrides_to_add[ - OverrideVariables.subscription_value_i(index=idx) + SubscriptionVariables.subscription_value_i(index=idx) ] = list_value.value @@ -215,7 +217,7 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add[OverrideVariables.subscription_map()] = ScriptUtils.to_script( + self._overrides_to_add[SubscriptionVariables.subscription_map()] = ScriptUtils.to_script( self.dict ) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 14916c2e..77ec1c6b 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -7,6 +7,7 @@ from typing import TypeVar from yt_dlp import match_filter_func +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder @@ -33,12 +34,14 @@ class SubscriptionYTDLOptions: preset: Preset, plugins: List[Plugin], enhanced_download_archive: EnhancedDownloadArchive, + overrides: Overrides, working_directory: str, dry_run: bool, ): self._preset = preset self._plugins = plugins self._enhanced_download_archive = enhanced_download_archive + self._overrides = overrides self._working_directory = working_directory self._dry_run = dry_run @@ -56,8 +59,8 @@ class SubscriptionYTDLOptions: ytdl-options to apply to every run no matter what """ ytdl_options = { - # Download all files in the format of {id}.{ext} - "outtmpl": str(Path(self._working_directory) / "%(id)s.%(ext)s"), + # Download all files in the format of {id}.{ext}, where id is sanitized + "outtmpl": str(Path(self._working_directory) / "%(id)S.%(ext)s"), # Always write thumbnails "writethumbnail": True, "ffmpeg_location": FFMPEG.ffmpeg_path(), @@ -78,6 +81,7 @@ class SubscriptionYTDLOptions: "skip_download": True, "writethumbnail": False, "writeinfojson": True, + "extract_flat": "discard", # do not store info.json in mem since its in file } @property @@ -90,6 +94,11 @@ class SubscriptionYTDLOptions: if self._preset.output_options.maintain_download_archive: ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path + if self._preset.output_options.keep_max_files: + # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure + ytdl_options["max_downloads"] = max( + int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2 + ) return ytdl_options diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index aba9d1fc..b1dc20e7 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -2,6 +2,7 @@ import copy from abc import ABC from typing import Any from typing import Dict +from typing import Optional from typing import Set from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS @@ -13,21 +14,47 @@ from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.script import ScriptUtils +_BASE_SCRIPT: Script = Script( + ScriptUtils.add_sanitized_variables( + dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) + ) +) + class Scriptable(ABC): """ Shared class between Entry and Overrides to manage their underlying Script. """ - _BASE_SCRIPT: Script = Script( - ScriptUtils.add_sanitized_variables( - dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) - ) - ) + def __init__(self, initialize_base_script: bool = False): + self._script: Optional[Script] = None + self._unresolvable: Optional[Set[str]] = None - def __init__(self): - self.script = copy.deepcopy(Scriptable._BASE_SCRIPT) - self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES) + if initialize_base_script: + self.initialize_base_script() + + def initialize_base_script(self): + """ + Initializes with base values + """ + self._script = copy.deepcopy(_BASE_SCRIPT) + self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES) + + @property + def script(self) -> Script: + """ + Initialized script + """ + assert self._script is not None, "Not initialized" + return self._script + + @property + def unresolvable(self) -> Set[str]: + """ + Initialized unresolvable variables + """ + assert self._unresolvable is not None, "Not initialized" + return self._unresolvable def update_script(self) -> None: """ @@ -45,7 +72,7 @@ class Scriptable(ABC): for var, definition in values.items() } - self.unresolvable -= set(list(values_as_str.keys())) + self._unresolvable -= set(list(values_as_str.keys())) self.script.add( ScriptUtils.add_sanitized_variables( { diff --git a/tests/unit/docgen/test_docgen.py b/tests/unit/docgen/test_docgen.py index 82768347..5d326839 100644 --- a/tests/unit/docgen/test_docgen.py +++ b/tests/unit/docgen/test_docgen.py @@ -2,9 +2,9 @@ from typing import Type from tools.docgen.docgen import DocGen from tools.docgen.entry_variables import EntryVariablesDocGen -from tools.docgen.override_variables import OverrideVariablesDocGen from tools.docgen.plugins import PluginsDocGen from tools.docgen.scripting_functions import ScriptingFunctionsDocGen +from tools.docgen.static_variables import StaticVariablesDocGen from ytdl_sub.utils.file_handler import get_md5_hash @@ -20,8 +20,8 @@ class TestDocGen: def test_entry_variables_generated(self): _test_doc_gen(EntryVariablesDocGen) - def test_override_variables_generated(self): - _test_doc_gen(OverrideVariablesDocGen) + def test_static_variables_generated(self): + _test_doc_gen(StaticVariablesDocGen) def test_scripting_functions_generated(self): _test_doc_gen(ScriptingFunctionsDocGen) diff --git a/tools/docgen/override_variables.py b/tools/docgen/override_variables.py deleted file mode 100644 index e919877c..00000000 --- a/tools/docgen/override_variables.py +++ /dev/null @@ -1,25 +0,0 @@ -from pathlib import Path - -from tools.docgen.docgen import DocGen -from tools.docgen.utils import get_function_docs -from tools.docgen.utils import section -from tools.docgen.utils import static_methods -from ytdl_sub.entries.variables.override_variables import OverrideVariables - - -class OverrideVariablesDocGen(DocGen): - - LOCATION = Path("docs/source/config_reference/scripting/override_variables.rst") - - @classmethod - def generate(cls) -> str: - docs = section("Override Variables", level=0) - - for name in static_methods(OverrideVariables): - docs += get_function_docs( - function_name=name, - obj=OverrideVariables, - level=1, - ) - - return docs diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py index 10323066..8080c297 100644 --- a/tools/docgen/plugins.py +++ b/tools/docgen/plugins.py @@ -39,6 +39,8 @@ def should_filter_property(property_name: str) -> bool: "dict_with_format_strings", "subscription_name", "list", + "script", + "unresolvable", ) diff --git a/tools/docgen/static_variables.py b/tools/docgen/static_variables.py new file mode 100644 index 00000000..254179cc --- /dev/null +++ b/tools/docgen/static_variables.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from tools.docgen.docgen import DocGen +from tools.docgen.utils import get_function_docs +from tools.docgen.utils import section +from tools.docgen.utils import static_methods +from ytdl_sub.entries.variables.override_variables import SubscriptionVariables + + +class StaticVariablesDocGen(DocGen): + + LOCATION = Path("docs/source/config_reference/scripting/static_variables.rst") + + @classmethod + def generate(cls) -> str: + docs = section("Static Variables", level=0) + + docs += section("Subscription Variables", level=1) + for name in static_methods(SubscriptionVariables): + docs += get_function_docs( + function_name=name, + obj=SubscriptionVariables, + level=2, + ) + + return docs