diff --git a/docs/images/plex_scanner_agent.png b/docs/images/plex_scanner_agent.png new file mode 100644 index 00000000..49ce6b33 Binary files /dev/null and b/docs/images/plex_scanner_agent.png differ diff --git a/docs/images/unraid_badconsole.png b/docs/images/unraid_badconsole.png new file mode 100644 index 00000000..73cc15a9 Binary files /dev/null and b/docs/images/unraid_badconsole.png differ diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index c271669e..1e62d137 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -38,10 +38,11 @@ array_apply_fixed array_at ~~~~~~~~ -:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument`` +:spec: ``array_at(array: Array, idx: Integer, default: Optional[AnyArgument]) -> AnyArgument`` :description: - Return the element in the Array at index ``idx``. + Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length, + either return ``default`` if provided or throw an error. array_contains ~~~~~~~~~~~~~~ @@ -225,6 +226,27 @@ xor Conditional Functions --------------------- +elif +~~~~ +:spec: ``elif(if_elif_else: AnyArgument, ...) -> AnyArgument`` + +:description: + Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via + adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to + supply at least one conditional and an else. +:usage: + + .. code-block:: python + + %elif( + condition1, + return1, + condition2, + return2, + ... + else_return + ) + if ~~ :spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]`` @@ -543,6 +565,13 @@ slice :description: Returns the slice of the Array. +split +~~~~~ +:spec: ``split(string: String, sep: String, max_split: Optional[Integer]) -> Array`` + +:description: + Splits the input string into multiple strings. + string ~~~~~~ :spec: ``string(value: AnyArgument) -> String`` diff --git a/docs/source/faq/index.rst b/docs/source/faq/index.rst index c728b7ca..489240de 100644 --- a/docs/source/faq/index.rst +++ b/docs/source/faq/index.rst @@ -1,8 +1,8 @@ +=== FAQ === -Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as -more questions get asked. +Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as more questions get asked. .. contents:: Frequently Asked Questions :depth: 3 @@ -10,12 +10,25 @@ more questions get asked. How do I... ----------- +...get support or reach out to contribute? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you need support, you can: + +* :ytdl-sub-gh:`Open an issue on GitHub ` + +* `Join our Discord `_ + +If you would like to contribute, we're happy to accept any help, even non-coders! To find out how you can help this project, you can: + +* `Join our Discord `_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute. + +* If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull request ` with information on what issue you're resolving and it will be reviewed as soon as possible. + ...download age-restricted YouTube videos? -'''''''''''''''''''''''''''''''''''''''''' -See -`ytdls recommended way `_ -to download your YouTube cookie, then add it to your -`ytdl options `_ section of your config: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See `yt-dl's recommended way `_ to download your YouTube cookie, then add it to your :ref:`ytdl options ` section of your config: .. code-block:: yaml @@ -23,14 +36,16 @@ to download your YouTube cookie, then add it to your cookiefile: "/path/to/cookies/file.txt" ...automate my downloads? -''''''''''''''''''''''''' -`This part of the wiki `_ shows how to set up ``ytdl-sub`` to run in a cron job within Docker. +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:doc:`This page ` shows how to set up ``ytdl-sub`` to run automatically on various platforms. There is a bug where... ----------------------- ...date_range is not downloading older videos after I changed the range -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether. .. code-block:: yaml @@ -38,11 +53,12 @@ Your preset most likely has ``break_on_existing`` set to True, which will stop d ytdl_options: break_on_existing: False -After your download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads. +After you download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads. ...it is downloading non-English title and description metadata -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' -Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using. .. code-block:: yaml @@ -53,7 +69,19 @@ Most likely the video has a non-English language set to its 'native' language. Y - "en" ...Plex is not showing my TV shows correctly -'''''''''''''''''''''''''''''''''''''''''''' -Set the following -`Scanner and Agent `_ -for your library. \ No newline at end of file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set the following for your ytdl-sub library that has been added to Plex. + +.. figure:: ../../images/plex_scanner_agent.png + :alt: The Plex library editor, under the advanced settings, showing the required options for Plex to show the TV shows correctly. + +**Scanner:** Plex Series Scanner + +**Agent:** Personal Media shows + +**Visibility:** Exclude from home screen and global search + +**Episode sorting:** Library default + +**YES** Enable video preview thumbnails \ No newline at end of file diff --git a/docs/source/guides/install/docker.rst b/docs/source/guides/install/docker.rst index 9c89e9be..04b5e456 100644 --- a/docs/source/guides/install/docker.rst +++ b/docs/source/guides/install/docker.rst @@ -17,14 +17,14 @@ The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install yt GUI Image --------- -The GUI image uses LSIO's :lsio-gh:`docker-code-server image` for its base image. More info on other code-server environment variables can be found within its documentation. +The GUI image uses LSIO's :lsio-gh:`docker-code-server image <\ >` for its base image. More info on other code-server environment variables can be found within its documentation. -After starting, code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``. +After starting, the code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``. Headless Image -------------- -The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image` for its base image. Execute the following command to access and interact with ``ytdl-sub``: +The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image <\ >` for its base image. Execute the following command to access and interact with ``ytdl-sub``: .. code-block:: bash diff --git a/docs/source/guides/install/unraid.rst b/docs/source/guides/install/unraid.rst index ac979a99..d27535dd 100644 --- a/docs/source/guides/install/unraid.rst +++ b/docs/source/guides/install/unraid.rst @@ -1,3 +1,16 @@ Unraid -------------- -You can install our :unraid:`unraid community apps ` through the `Unraid Community Apps plugin `_. Uses Docker under the hood. \ No newline at end of file +You can install our :unraid:`unraid community apps ` through the `Unraid Community Apps plugin `_. + + +If you installed the ``ytdl-sub-gui`` app, the code-server will be running at http://localhost:8443 (replace ``localhost`` with the IP of the computer running Unraid if you aren't trying to access ``ytdl-sub`` on that computer). Open this page in a browser to access and interact with ``ytdl-sub``. + +If you installed the ``ytdl-sub`` app (headless), open the normal app-specific console to access and interact with ``ytdl-sub``. Once open, you must first run ``su abc -s /bin/bash`` to change to the non-root user. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``. + + +.. warning:: + + If you use the below option to access the ``ytdl-sub`` console, be sure to run ``su abc -s /bin/bash`` first thing. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``. Do **NOT** run ``ytdl-sub`` as the root user! Running as root will set the owner of all modified files to root, which prevents most media managers and players from accessing the files. + + .. figure:: ../../../images/unraid_badconsole.png + :alt: The Unraid community app plugin GUI, with an arrow pointing at the "Console" option in the dropdown after selecting ytdl-sub-gui \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 0a9f1d3e..b9865cce 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -25,6 +25,8 @@ General options must be specified before the command (i.e. ``sub``). path to store the transaction log output of all files added, modified, deleted -st, --suppress-transaction-log do not output transaction logs to console or file + -m MATCH [MATCH ...], --match MATCH [MATCH ...] + match subscription names to one or more substrings, and only run those subscriptions Sub Options ----------- @@ -37,6 +39,14 @@ Download all subscriptions specified in each ``SUBPATH``. ``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided. It will use the config specified by ``--config``, or ``config.yaml`` if not provided. +.. code-block:: text + :caption: Additional Options + + -u, --update-with-info-json + update all subscriptions with the current config using info.json files + -o DL_OVERRIDE, --dl-override DL_OVERRIDE + override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3' + Download Options ----------------- Download a single subscription in the form of CLI arguments. @@ -67,7 +77,7 @@ Using the command: --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" See how to shorten commands using -`download aliases `_. +`download aliases `_. View Options ----------------- @@ -76,6 +86,7 @@ View Options ytdl-sub view [-sc] [URL] .. code-block:: text + :caption: Additional Options -sc, --split-chapters View source variables after splitting by chapters diff --git a/pyproject.toml b/pyproject.toml index d78c5633..a26368d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,24 +22,6 @@ disable = [ load-plugins = "pylint.extensions.docparams" -[tool.pydocstyle] -inherit = false -match = "[^test_].*\\.py" -ignore = [ - "D100", # docstring in public module - "D101", # Missing docstring in public class (covered by pylint) - "D104", # docstring in public package - "D107", # docstring in init - "D200", # One-line should fit on one line - "D203", # 1 blank line before class docstring - "D205", # 1 blank line between summary and description - "D212", # Multi-line should start at first line - "D400", # Should end with a period - "D401", # Return vs Returns - "D413", # Missing blank line after last section - "D415", # Should end with a period -] - [tool.coverage.run] include = [ "src/*" diff --git a/setup.cfg b/setup.cfg index db9e581d..5692d3f1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ package_dir = packages=find: install_requires = - yt-dlp==2023.11.16 + yt-dlp==2023.12.30 argparse==1.4.0 colorama==0.4.6 mergedeep==1.3.4 diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index bc7fc04e..1d6a19af 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -3,6 +3,7 @@ import os import sys from datetime import datetime from pathlib import Path +from typing import Dict from typing import List from typing import Optional @@ -67,7 +68,12 @@ def _maybe_write_subscription_log_file( def _download_subscriptions_from_yaml_files( - config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool + config: ConfigFile, + subscription_paths: List[str], + subscription_matches: List[str], + subscription_override_dict: Dict, + update_with_info_json: bool, + dry_run: bool, ) -> List[Subscription]: """ Downloads all subscriptions from one or many subscription yaml files. @@ -78,6 +84,8 @@ def _download_subscriptions_from_yaml_files( Configuration file subscription_paths Path to subscription files to download + subscription_matches + Optional list of substrings to match subscription names to (only run if matched) update_with_info_json Whether to actually download or update using existing info json dry_run @@ -96,7 +104,17 @@ def _download_subscriptions_from_yaml_files( # Load all the subscriptions first to perform all validation before downloading for path in subscription_paths: - subscriptions += Subscription.from_file_path(config=config, subscription_path=path) + subscriptions += Subscription.from_file_path( + config=config, + subscription_path=path, + subscription_override_dict=subscription_override_dict, + ) + + if subscriptions and subscription_matches: + logger.info("Filtering subscriptions by name based on --match arguments") + subscriptions = [ + sub for sub in subscriptions if any(match in sub.name for match in subscription_matches) + ] for subscription in subscriptions: with subscription.exception_handling(): @@ -119,7 +137,7 @@ def _download_subscriptions_from_yaml_files( exception=subscription.exception, ) - Logger.cleanup(cleanup_error_log=False) + Logger.cleanup(has_error=False) gc.collect() # Garbage collect after each subscription download return subscriptions @@ -221,10 +239,18 @@ def main() -> List[Subscription]: "full backup before usage. You have been warned!", ) + subscription_override_dict = {} + if args.dl_override: + subscription_override_dict = DownloadArgsParser.from_dl_override( + override=args.dl_override, config=config + ).to_subscription_dict() + logger.info("Validating subscriptions...") subscriptions = _download_subscriptions_from_yaml_files( config=config, subscription_paths=args.subscription_paths, + subscription_matches=args.match, + subscription_override_dict=subscription_override_dict, update_with_info_json=args.update_with_info_json, dry_run=args.dry_run, ) diff --git a/src/ytdl_sub/cli/parsers/dl.py b/src/ytdl_sub/cli/parsers/dl.py index a3ada3b3..d9e1eb1a 100644 --- a/src/ytdl_sub/cli/parsers/dl.py +++ b/src/ytdl_sub/cli/parsers/dl.py @@ -9,6 +9,7 @@ from typing import Tuple from mergedeep import mergedeep from ytdl_sub.cli.parsers.main import MainArguments +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.utils.exceptions import InvalidDlArguments @@ -247,3 +248,12 @@ class DownloadArgsParser: """ hash_string = str(sorted(self._unknown_arguments)) return hashlib.sha256(hash_string.encode()).hexdigest()[-8:] + + @classmethod + def from_dl_override(cls, override: str, config: ConfigFile) -> "DownloadArgsParser": + """ + Create a DownloadArgsParser from a sub --override argument value + """ + return DownloadArgsParser( + extra_arguments=override.split(), config_options=config.config_options + ) diff --git a/src/ytdl_sub/cli/parsers/main.py b/src/ytdl_sub/cli/parsers/main.py index ccb50734..99f6c231 100644 --- a/src/ytdl_sub/cli/parsers/main.py +++ b/src/ytdl_sub/cli/parsers/main.py @@ -40,6 +40,10 @@ class MainArguments: long="--suppress-transaction-log", is_positional=True, ) + MATCH = CLIArgument( + short="-m", + long="--match", + ) @classmethod def all(cls) -> List[CLIArgument]: @@ -54,6 +58,7 @@ class MainArguments: cls.LOG_LEVEL, cls.TRANSACTION_LOG, cls.SUPPRESS_TRANSACTION_LOG, + cls.MATCH, ] @classmethod @@ -124,6 +129,16 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults help="do not output transaction logs to console or file", default=argparse.SUPPRESS if suppress_defaults else False, ) + arg_parser.add_argument( + MainArguments.MATCH.short, + MainArguments.MATCH.long, + dest="match", + nargs="+", + action="extend", + type=str, + help="match subscription names to one or more substrings, and only run those subscriptions", + default=argparse.SUPPRESS if suppress_defaults else [], + ) ################################################################################################### @@ -142,6 +157,10 @@ class SubArguments: short="-u", long="--update-with-info-json", ) + OVERRIDE = CLIArgument( + short="-o", + long="--dl-override", + ) subscription_parser = subparsers.add_parser("sub") @@ -160,6 +179,13 @@ subscription_parser.add_argument( help="update all subscriptions with the current config using info.json files", default=False, ) +subscription_parser.add_argument( + SubArguments.OVERRIDE.short, + SubArguments.OVERRIDE.long, + type=str, + help="override all subscription config values using `dl` syntax, " + "i.e. --dl-override='--ytdl_options.max_downloads 3'", +) ################################################################################################### # DOWNLOAD PARSER diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index f68ce727..3372e936 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -56,7 +56,7 @@ class Overrides(DictFormatterValidator, Scriptable): def __init__(self, name, value): DictFormatterValidator.__init__(self, name, value) - Scriptable.__init__(self) + Scriptable.__init__(self, initialize_base_script=True) for key in self._keys: self.ensure_variable_name_valid(key) diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index 509a4f8d..71f3c996 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -152,7 +152,9 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): for file_name in entry_file_names: ext = get_file_extension(file_name) file_path = Path(self.output_directory) / file_name - working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}" + working_directory_file_path = Path(self.working_directory) / entry.base_filename( + ext=ext + ) # NFO files will always get rewritten, so ignore if ext == "nfo": diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 67feabd0..a0ceb535 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -112,13 +112,6 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. """ - if playlist_metadata := entry.get(v.playlist_metadata, dict): - self._download_parent_thumbnails( - thumbnail_list_info=collection_url.playlist_thumbnails, - entry=entry, - parent=EntryParent(playlist_metadata, working_directory=self.working_directory), - ) - if source_metadata := entry.get(v.source_metadata, dict): self._download_parent_thumbnails( thumbnail_list_info=collection_url.source_thumbnails, @@ -126,6 +119,13 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): parent=EntryParent(source_metadata, working_directory=self.working_directory), ) + if playlist_metadata := entry.get(v.playlist_metadata, dict): + self._download_parent_thumbnails( + thumbnail_list_info=collection_url.playlist_thumbnails, + entry=entry, + parent=EntryParent(playlist_metadata, working_directory=self.working_directory), + ) + def modify_entry(self, entry: Entry) -> Optional[Entry]: """ Use the entry to download thumbnails (or move if LATEST_ENTRY). @@ -349,39 +349,43 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ) def _iterate_child_entries( - self, url_validator: UrlValidator, entries: List[Entry] + self, entries: List[Entry], download_reversed: bool ) -> Iterator[Entry]: - entries_to_iterate = entries - if url_validator.download_reverse: - entries_to_iterate = reversed(entries) + # Iterate a list of entries, and delete the entries after yielding + indices = list(range(len(entries))) + if download_reversed: + indices = reversed(indices) - for entry in entries_to_iterate: + for idx in indices: self._url_state.entries_downloaded += 1 - if self._is_downloaded(entry): + if self._is_downloaded(entries[idx]): download_logger.info( "Already downloaded entry %d/%d: %s", self._url_state.entries_downloaded, self._url_state.entries_total, - entry.title, + entries[idx].title, ) + del entries[idx] continue - yield entry - self._mark_downloaded(entry) + yield entries[idx] + self._mark_downloaded(entries[idx]) + + del entries[idx] def _iterate_parent_entry( - self, url_validator: UrlValidator, parent: EntryParent + self, parent: EntryParent, download_reversed: bool ) -> Iterator[Entry]: for entry_child in self._iterate_child_entries( - url_validator=url_validator, entries=parent.entry_children() + entries=parent.entry_children(), download_reversed=download_reversed ): yield entry_child # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): for entry_child in self._iterate_parent_entry( - url_validator=url_validator, parent=parent_child + parent=parent_child, download_reversed=download_reversed ): yield entry_child @@ -415,9 +419,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): def _iterate_entries( self, - url_validator: UrlValidator, parents: List[EntryParent], orphans: List[Entry], + download_reversed: bool, ) -> Iterator[Entry]: """ Downloads the leaf entries from EntryParent trees @@ -426,11 +430,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): with self._separate_download_archives(clear_info_json_files=True): for parent in parents: for entry_child in self._iterate_parent_entry( - url_validator=url_validator, parent=parent + parent=parent, download_reversed=download_reversed ): yield entry_child - for orphan in self._iterate_child_entries(url_validator=url_validator, entries=orphans): + for orphan in self._iterate_child_entries( + entries=orphans, download_reversed=download_reversed + ): yield orphan def download_metadata(self) -> Iterable[Entry]: @@ -454,7 +460,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): "Beginning downloads for %s", self.overrides.apply_formatter(collection_url.url) ) for entry in self._iterate_entries( - url_validator=collection_url, parents=parents, orphans=orphan_entries + parents=parents, + orphans=orphan_entries, + download_reversed=collection_url.download_reverse, ): entry.initialize_script(self.overrides).add( {v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)} diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index a7912232..0f43b606 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -216,11 +216,14 @@ class YTDLP: **kwargs arguments passed directory to YoutubeDL extract_info """ + parent_dict: Dict = {} try: with cls._listen_and_log_downloaded_info_json( working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl ): - _ = cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) + parent_dict = cls.extract_info( + ytdl_options_overrides=ytdl_options_overrides, **kwargs + ) except RejectedVideoReached: cls.logger.debug( "RejectedVideoReached, stopping additional downloads " @@ -234,4 +237,19 @@ class YTDLP: except MaxDownloadsReached: cls.logger.info("MaxDownloadsReached, stopping additional downloads.") + # For YouTube playlists in particular, channel metadata is not fetched. Attempt to get + # channel metadata via grabbing uploader_url info json a max of 3 times + current_iter = 0 + url = kwargs.get("url") + uploader_url = parent_dict.get("uploader_url") + while current_iter < 3 and uploader_url and url != uploader_url: + cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + parent_dict = cls.extract_info( + ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"}, + url=uploader_url, + ) + current_iter += 1 + url = uploader_url + uploader_url = parent_dict.get("uploader_url") + return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index c4243d63..1d8acd3e 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -8,6 +8,8 @@ from typing import Type from typing import TypeVar from typing import final +from yt_dlp.utils import sanitize_filename + from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions @@ -45,6 +47,19 @@ class BaseEntry(ABC): """ return str(self._kwargs[v.uid.metadata_key]) + @property + def uid_sanitized(self) -> str: + """ + Sanitized version, used in filenames + """ + return sanitize_filename(self.uid) + + def base_filename(self, ext: str): + """ + The base filename of all yt-dlp downloaded entry files + """ + return f"{self.uid_sanitized}.{ext}" + @property def download_archive_extractor(self) -> str: """ @@ -101,30 +116,13 @@ class BaseEntry(ABC): """ return self._working_directory - def add_kwargs(self, variables_to_add: Dict[str, Any]) -> "BaseEntry": - """ - Adds variables to kwargs. Use with caution since yt-dlp data can be overwritten. - Plugins should use ``add_variables``. - - Parameters - ---------- - variables_to_add - Variables to add to kwargs - - Returns - ------- - self - """ - self._kwargs = dict(self._kwargs, **variables_to_add) - return self - def get_download_info_json_name(self) -> str: """ Returns ------- The download info json's file name """ - return f"{self.uid}.{self.info_json_ext}" + return self.base_filename(ext=self.info_json_ext) def get_download_info_json_path(self) -> str: """ diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index c7442992..831bda6c 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -44,12 +44,6 @@ class Entry(BaseEntry, Scriptable): BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory) Scriptable.__init__(self) - def _add_entry_kwargs_to_script(self) -> None: - # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized - self.unresolvable.remove(v.entry_metadata.variable_name) - self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) - self.update_script() - def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry": """ Initializes the entry script using the Overrides script, then adding @@ -57,12 +51,20 @@ class Entry(BaseEntry, Scriptable): """ # Overrides contains added variables that are unresolvable, add them here if other: - self.script = copy.deepcopy(other.script) - self.unresolvable = copy.deepcopy(other.unresolvable) + self._script = copy.deepcopy(other.script) + self._unresolvable = copy.deepcopy(other.unresolvable) + else: + self.initialize_base_script() self._add_entry_kwargs_to_script() return self + def _add_entry_kwargs_to_script(self) -> None: + # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized + self.unresolvable.remove(v.entry_metadata.variable_name) + self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) + self.update_script() + def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT: """ Gets a variable of an expected type. Will error if it does not exist or is not resolved. @@ -113,7 +115,8 @@ class Entry(BaseEntry, Scriptable): """ ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key] for possible_ext in [ext, "mkv"]: - file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}") + file_name = self.base_filename(ext=possible_ext) + file_path = str(Path(self.working_directory()) / file_name) if os.path.isfile(file_path): return possible_ext @@ -125,7 +128,7 @@ class Entry(BaseEntry, Scriptable): ------- The entry's file name """ - return f"{self.uid}.{self.ext}" + return self.base_filename(ext=self.ext) def get_download_file_path(self) -> str: """Returns the entry's file path to where it was downloaded""" @@ -137,7 +140,7 @@ class Entry(BaseEntry, Scriptable): ------- The download thumbnail's file name """ - return f"{self.uid}.{self.get(v.thumbnail_ext, str)}" + return self.base_filename(ext=self.get(v.thumbnail_ext, str)) def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" @@ -155,7 +158,10 @@ class Entry(BaseEntry, Scriptable): possible_thumbnail_exts.add(thumbnail["url"].split(".")[-1]) for ext in possible_thumbnail_exts: - possible_thumbnail_path = str(Path(self.working_directory()) / f"{self.uid}.{ext}") + possible_thumbnail_filename = self.base_filename(ext=ext) + possible_thumbnail_path = str( + Path(self.working_directory()) / possible_thumbnail_filename + ) if os.path.isfile(possible_thumbnail_path): return possible_thumbnail_path @@ -202,7 +208,7 @@ class Entry(BaseEntry, Scriptable): # HACK: yt-dlp does not record extracted/converted extensions anywhere. If the file is not # found, try it using all possible extensions if not file_exists: - for ext in AUDIO_CODEC_EXTS.union(VIDEO_CODEC_EXTS): + for ext in AUDIO_CODEC_EXTS | VIDEO_CODEC_EXTS: if os.path.isfile(self.get_download_file_path().removesuffix(self.ext) + ext): file_exists = True break diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 6cf7dcb1..232b2ab4 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -157,13 +157,16 @@ class EntryParent(BaseEntry): def _uid_is_uploader_id(parent: "EntryParent"): return parent.uid == parent.uploader_id - top_level_parents = [ - parent for parent in parents if parent.num_children() == 0 and _url_matches(parent) - ] + top_level_parents = [parent for parent in parents if parent.num_children() == 0] # If more than 1 parent exists, assume the uploader_id is the root parent if len(top_level_parents) > 1: - top_level_parents = [parent for parent in parents if _uid_is_uploader_id(parent)] + top_level_parents = [ + parent for parent in top_level_parents if _uid_is_uploader_id(parent) + ] + + if len(top_level_parents) > 1: + top_level_parents = [parent for parent in top_level_parents if _url_matches(parent)] match len(top_level_parents): case 0: diff --git a/src/ytdl_sub/main.py b/src/ytdl_sub/main.py index b4e7e776..44258a5c 100644 --- a/src/ytdl_sub/main.py +++ b/src/ytdl_sub/main.py @@ -27,7 +27,7 @@ def main(): """ try: return_code = _main() - Logger.cleanup(cleanup_error_log=return_code == 0) + Logger.cleanup(has_error=return_code != 0) sys.exit(return_code) except Exception as exc: # pylint: disable=broad-except Logger.log_exception(exception=exc) diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index b3ef63e2..9a7c8bab 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -13,7 +13,7 @@ from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.validators import BoolValidator -logger = Logger.get("embed_thumbnail") +logger = Logger.get("embed-thumbnail") class EmbedThumbnailOptions(BoolValidator, OptionsValidator): diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index 7f381db1..72ad6c88 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -11,7 +11,7 @@ from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive -logger = Logger.get("conditional") +logger = Logger.get("filter-exclude") class FilterExcludeOptions(ListFormatterValidator, OptionsValidator): @@ -55,6 +55,11 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]): for formatter in self.plugin_options.list: out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) if bool(out): + logger.info( + "Filtering '%s' from the filter %s evaluating to True", + entry.title, + formatter.format_string, + ) return None return entry diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 41a98f78..5a54e8ec 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -11,7 +11,7 @@ from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive -logger = Logger.get("conditional") +logger = Logger.get("filter-include") class FilterIncludeOptions(ListFormatterValidator, OptionsValidator): @@ -63,6 +63,11 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]): for formatter in self.plugin_options.list: out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) if not bool(out): + logger.info( + "Filtering '%s' from the filter %s evaluating to False", + entry.title, + formatter.format_string, + ) return None return entry diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index fe10a69b..90211e2c 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -8,7 +8,7 @@ from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator -logger = Logger.get("match_filters") +logger = Logger.get("match-filters") def default_filters() -> Tuple[List[str], List[str]]: diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index f6098517..6ad24e37 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import BoolValidator v: VariableDefinitions = VARIABLES -logger = Logger.get("music_tags") +logger = Logger.get("music-tags") def _is_multi_field(tag_name: str) -> bool: diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 9c53a9bd..0e74d448 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -208,7 +208,6 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}") if self.plugin_options.subtitles_name: for lang in langs: - subtitle_file_name = f"{entry.uid}.{lang}.{self.plugin_options.subtitles_type}" output_subtitle_file_name = self.overrides.apply_formatter( formatter=self.plugin_options.subtitles_name, entry=entry, @@ -216,7 +215,9 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): ) self.save_file( - file_name=subtitle_file_name, + file_name=entry.base_filename( + ext=f"{lang}.{self.plugin_options.subtitles_type}" + ), output_file_name=output_subtitle_file_name, entry=entry, ) @@ -225,9 +226,8 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): # Can happen for both file and embedded subs for lang in langs: for possible_ext in SUBTITLE_EXTENSIONS: - possible_subs_file = ( - Path(self.working_directory) / f"{entry.uid}.{lang}.{possible_ext}" - ) + possible_subs_filename = entry.base_filename(ext=f"{lang}.{possible_ext}") + possible_subs_file = Path(self.working_directory) / possible_subs_filename FileHandler.delete(possible_subs_file) return file_metadata diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index c3b2e6ec..0d4614a8 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -10,7 +10,7 @@ from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator -logger = Logger.get("video_tags") +logger = Logger.get("video-tags") class VideoTagsOptions(OptionsDictValidator): diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml index 3ed19608..d3e646ca 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml @@ -10,7 +10,7 @@ presets: # # "Subscription Name": "url" # -# where the first url tries to grab channel avatar + banner +# where the first the first url tries to grab channel avatar + banner # _multi_url: download: @@ -20,6 +20,11 @@ presets: uid: "avatar_uncropped" - name: "{banner_uncropped_thumbnail_file_name}" uid: "banner_uncropped" + source_thumbnails: + - name: "{avatar_uncropped_thumbnail_file_name}" + uid: "avatar_uncropped" + - name: "{banner_uncropped_thumbnail_file_name}" + uid: "banner_uncropped" - url: "{url2}" - url: "{url3}" - url: "{url4}" @@ -326,3 +331,84 @@ presets: url99: "{subscription_value_99}" url100: "{subscription_value_100}" + url21: "{subscription_value_21}" + url22: "{subscription_value_22}" + url23: "{subscription_value_23}" + url24: "{subscription_value_24}" + url25: "{subscription_value_25}" + url26: "{subscription_value_26}" + url27: "{subscription_value_27}" + url28: "{subscription_value_28}" + url29: "{subscription_value_29}" + url30: "{subscription_value_30}" + url31: "{subscription_value_31}" + url32: "{subscription_value_32}" + url33: "{subscription_value_33}" + url34: "{subscription_value_34}" + url35: "{subscription_value_35}" + url36: "{subscription_value_36}" + url37: "{subscription_value_37}" + url38: "{subscription_value_38}" + url39: "{subscription_value_39}" + url40: "{subscription_value_40}" + url41: "{subscription_value_41}" + url42: "{subscription_value_42}" + url43: "{subscription_value_43}" + url44: "{subscription_value_44}" + url45: "{subscription_value_45}" + url46: "{subscription_value_46}" + url47: "{subscription_value_47}" + url48: "{subscription_value_48}" + url49: "{subscription_value_49}" + url50: "{subscription_value_50}" + url51: "{subscription_value_51}" + url52: "{subscription_value_52}" + url53: "{subscription_value_53}" + url54: "{subscription_value_54}" + url55: "{subscription_value_55}" + url56: "{subscription_value_56}" + url57: "{subscription_value_57}" + url58: "{subscription_value_58}" + url59: "{subscription_value_59}" + url60: "{subscription_value_60}" + url61: "{subscription_value_61}" + url62: "{subscription_value_62}" + url63: "{subscription_value_63}" + url64: "{subscription_value_64}" + url65: "{subscription_value_65}" + url66: "{subscription_value_66}" + url67: "{subscription_value_67}" + url68: "{subscription_value_68}" + url69: "{subscription_value_69}" + url70: "{subscription_value_70}" + url71: "{subscription_value_71}" + url72: "{subscription_value_72}" + url73: "{subscription_value_73}" + url74: "{subscription_value_74}" + url75: "{subscription_value_75}" + url76: "{subscription_value_76}" + url77: "{subscription_value_77}" + url78: "{subscription_value_78}" + url79: "{subscription_value_79}" + url80: "{subscription_value_80}" + url81: "{subscription_value_81}" + url82: "{subscription_value_82}" + url83: "{subscription_value_83}" + url84: "{subscription_value_84}" + url85: "{subscription_value_85}" + url86: "{subscription_value_86}" + url87: "{subscription_value_87}" + url88: "{subscription_value_88}" + url89: "{subscription_value_89}" + url90: "{subscription_value_90}" + url91: "{subscription_value_91}" + url92: "{subscription_value_92}" + url93: "{subscription_value_93}" + url94: "{subscription_value_94}" + url95: "{subscription_value_95}" + url96: "{subscription_value_96}" + url97: "{subscription_value_97}" + url98: "{subscription_value_98}" + url99: "{subscription_value_99}" + url100: "{subscription_value_100}" + diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index 3c8da03a..82efaee6 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -45,6 +45,11 @@ presets: uid: "avatar_uncropped" - name: "{tv_show_fanart_file_name}" uid: "banner_uncropped" + source_thumbnails: + - name: "{tv_show_poster_file_name}" + uid: "avatar_uncropped" + - name: "{tv_show_fanart_file_name}" + uid: "banner_uncropped" output_directory_nfo_tags: tags: diff --git a/src/ytdl_sub/script/functions/array_functions.py b/src/ytdl_sub/script/functions/array_functions.py index 4af6f63c..a07585e7 100644 --- a/src/ytdl_sub/script/functions/array_functions.py +++ b/src/ytdl_sub/script/functions/array_functions.py @@ -73,12 +73,18 @@ class ArrayFunctions: return Array(output) @staticmethod - def array_at(array: Array, idx: Integer) -> AnyArgument: + def array_at(array: Array, idx: Integer, default: Optional[AnyArgument] = None) -> AnyArgument: """ :description: - Return the element in the Array at index ``idx``. + Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length, + either return ``default`` if provided or throw an error. """ - return array.value[idx.value] + try: + return array.value[idx.value] + except IndexError: + if default is not None: + return default + raise @staticmethod def array_first(array: Array, fallback: AnyArgument) -> AnyArgument: diff --git a/src/ytdl_sub/script/functions/conditional_functions.py b/src/ytdl_sub/script/functions/conditional_functions.py index e532031b..8b9da74b 100644 --- a/src/ytdl_sub/script/functions/conditional_functions.py +++ b/src/ytdl_sub/script/functions/conditional_functions.py @@ -1,8 +1,10 @@ from typing import Union +from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import ReturnableArgumentA from ytdl_sub.script.types.resolvable import ReturnableArgumentB +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException class ConditionalFunctions: @@ -19,6 +21,39 @@ class ConditionalFunctions: return true return false + @staticmethod + def elif_(*if_elif_else: AnyArgument) -> AnyArgument: + """ + :description: + Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via + adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to + supply at least one conditional and an else. + :usage: + + .. code-block:: python + + %elif( + condition1, + return1, + condition2, + return2, + ... + else_return + ) + """ + arguments = list(if_elif_else) + if len(arguments) < 3: + raise FunctionRuntimeException("elif requires at least 3 arguments") + + if len(arguments) % 2 == 0: + raise FunctionRuntimeException("elif must have an odd number of arguments") + + for idx in range(0, len(arguments) - 1, 2): + if bool(arguments[idx].value): + return arguments[idx + 1] + + return arguments[-1] + @staticmethod def if_passthrough( maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index 1f0280b5..fd0cac9f 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -1,5 +1,6 @@ from typing import Optional +from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.resolvable import AnyArgument from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Integer @@ -80,6 +81,22 @@ class StringFunctions: return String(string.value.replace(old.value, new.value)) + @staticmethod + def split(string: String, sep: String, max_split: Optional[Integer] = None) -> Array: + """ + :description: + Splits the input string into multiple strings. + """ + if max_split is not None: + return Array( + [ + String(split_val) + for split_val in string.value.split(sep=sep.value, maxsplit=max_split.value) + ] + ) + + return Array([String(split_val) for split_val in string.value.split(sep=sep.value)]) + @staticmethod def concat(*values: String) -> String: """ diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 50b2f319..37d0168f 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -3,6 +3,9 @@ from pathlib import Path from typing import Any from typing import Dict from typing import List +from typing import Optional + +from mergedeep import mergedeep from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset @@ -69,7 +72,10 @@ class Subscription(SubscriptionDownload): @classmethod def from_file_path( - cls, config: ConfigFile, subscription_path: str | Path + cls, + config: ConfigFile, + subscription_path: str | Path, + subscription_override_dict: Optional[Dict] = None, ) -> List["Subscription"]: """ Loads subscriptions from a file. @@ -80,6 +86,8 @@ class Subscription(SubscriptionDownload): Validated instance of the config subscription_path: File path to the subscription yaml file + subscription_override_dict: + Optional dict containing overrides to every subscription Returns ------- @@ -122,6 +130,13 @@ class Subscription(SubscriptionDownload): ) for subscription_key, subscription_object in subscriptions_dicts.items(): + # Hard-override subscriptions here + mergedeep.merge( + subscription_object, + subscription_override_dict or {}, + strategy=mergedeep.Strategy.ADDITIVE, + ) + subscriptions.append( cls.from_dict( config=config, diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 7bbd09ca..56e93954 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -343,6 +343,7 @@ class SubscriptionDownload(BaseSubscription, ABC): preset=self._preset_options, plugins=plugins, enhanced_download_archive=self._enhanced_download_archive, + overrides=self.overrides, working_directory=self.working_directory, dry_run=dry_run, ) @@ -395,6 +396,7 @@ class SubscriptionDownload(BaseSubscription, ABC): preset=self._preset_options, plugins=plugins, enhanced_download_archive=self._enhanced_download_archive, + overrides=self.overrides, working_directory=self.working_directory, dry_run=dry_run, ) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 14916c2e..77ec1c6b 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -7,6 +7,7 @@ from typing import TypeVar from yt_dlp import match_filter_func +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder @@ -33,12 +34,14 @@ class SubscriptionYTDLOptions: preset: Preset, plugins: List[Plugin], enhanced_download_archive: EnhancedDownloadArchive, + overrides: Overrides, working_directory: str, dry_run: bool, ): self._preset = preset self._plugins = plugins self._enhanced_download_archive = enhanced_download_archive + self._overrides = overrides self._working_directory = working_directory self._dry_run = dry_run @@ -56,8 +59,8 @@ class SubscriptionYTDLOptions: ytdl-options to apply to every run no matter what """ ytdl_options = { - # Download all files in the format of {id}.{ext} - "outtmpl": str(Path(self._working_directory) / "%(id)s.%(ext)s"), + # Download all files in the format of {id}.{ext}, where id is sanitized + "outtmpl": str(Path(self._working_directory) / "%(id)S.%(ext)s"), # Always write thumbnails "writethumbnail": True, "ffmpeg_location": FFMPEG.ffmpeg_path(), @@ -78,6 +81,7 @@ class SubscriptionYTDLOptions: "skip_download": True, "writethumbnail": False, "writeinfojson": True, + "extract_flat": "discard", # do not store info.json in mem since its in file } @property @@ -90,6 +94,11 @@ class SubscriptionYTDLOptions: if self._preset.output_options.maintain_download_archive: ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path + if self._preset.output_options.keep_max_files: + # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure + ytdl_options["max_downloads"] = max( + int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2 + ) return ytdl_options diff --git a/src/ytdl_sub/utils/logger.py b/src/ytdl_sub/utils/logger.py index 0b531a32..62e40ba1 100644 --- a/src/ytdl_sub/utils/logger.py +++ b/src/ytdl_sub/utils/logger.py @@ -210,6 +210,14 @@ class Logger: finally: redirect_stream.flush() + @classmethod + def _append_to_error_log(cls): + # Any time an exception occurs, dump all debug logs into the error log + with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open( + cls.error_log_filename(), mode="a", encoding="utf-8" + ) as error_logs: + error_logs.writelines(debug_logs.readlines()) + @classmethod def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None): """ @@ -248,14 +256,10 @@ class Logger: log_filepath if log_filepath else Logger.error_log_filename(), ) - # Any time an exception occurs, dump all debug logs into the error log - with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open( - cls.error_log_filename(), mode="a", encoding="utf-8" - ) as error_logs: - error_logs.writelines(debug_logs.readlines()) + cls._append_to_error_log() @classmethod - def cleanup(cls, cleanup_error_log: bool = False): + def cleanup(cls, has_error: bool = False): """ Cleans up debug log file left behind """ @@ -263,9 +267,11 @@ class Logger: for handler in logger.handlers: handler.close() - cls._DEBUG_LOGGER_FILE.close() - FileHandler.delete(cls.debug_log_filename()) - - if cleanup_error_log: + if has_error: + cls._append_to_error_log() + else: cls._ERROR_LOG_FILE.close() FileHandler.delete(cls.error_log_filename()) + + cls._DEBUG_LOGGER_FILE.close() + FileHandler.delete(cls.debug_log_filename()) diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index aba9d1fc..b1dc20e7 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -2,6 +2,7 @@ import copy from abc import ABC from typing import Any from typing import Dict +from typing import Optional from typing import Set from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS @@ -13,21 +14,47 @@ from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.script import ScriptUtils +_BASE_SCRIPT: Script = Script( + ScriptUtils.add_sanitized_variables( + dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) + ) +) + class Scriptable(ABC): """ Shared class between Entry and Overrides to manage their underlying Script. """ - _BASE_SCRIPT: Script = Script( - ScriptUtils.add_sanitized_variables( - dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) - ) - ) + def __init__(self, initialize_base_script: bool = False): + self._script: Optional[Script] = None + self._unresolvable: Optional[Set[str]] = None - def __init__(self): - self.script = copy.deepcopy(Scriptable._BASE_SCRIPT) - self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES) + if initialize_base_script: + self.initialize_base_script() + + def initialize_base_script(self): + """ + Initializes with base values + """ + self._script = copy.deepcopy(_BASE_SCRIPT) + self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES) + + @property + def script(self) -> Script: + """ + Initialized script + """ + assert self._script is not None, "Not initialized" + return self._script + + @property + def unresolvable(self) -> Set[str]: + """ + Initialized unresolvable variables + """ + assert self._unresolvable is not None, "Not initialized" + return self._unresolvable def update_script(self) -> None: """ @@ -45,7 +72,7 @@ class Scriptable(ABC): for var, definition in values.items() } - self.unresolvable -= set(list(values_as_str.keys())) + self._unresolvable -= set(list(values_as_str.keys())) self.script.add( ScriptUtils.add_sanitized_variables( { diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 7d8eca5e..3c8e142f 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -199,3 +199,26 @@ class TestPlaylist: dry_run=dry_run, expected_download_summary_file_name="youtube/test_playlist.json", ) + + def test_playlist_download_from_cli_sub_with_override_arg( + self, + preset_dict_to_subscription_yaml_generator, + playlist_preset_dict, + output_directory, + ): + # TODO: Fix CLI parsing on windows when dealing with spaces + if IS_WINDOWS: + return + + # No config needed when using only prebuilt presets + with preset_dict_to_subscription_yaml_generator( + subscription_name="music_video_playlist_test", preset_dict=playlist_preset_dict + ) as subscription_path: + args = ( + f"--dry-run sub '{subscription_path}' --dl-override '--date_range.after 20240101'" + ) + + subscriptions = mock_run_from_cli(args=args) + + assert len(subscriptions) == 1 + assert subscriptions[0].transaction_log.is_empty diff --git a/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json b/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json index 1ff13886..49314662 100644 --- a/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json +++ b/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json @@ -1,6 +1,6 @@ { ".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec", "JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6", - "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "f401b98c332b76ee1c87065e195d73ce", + "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "068526b2d8f85fdcf914df3e23d0b1fa", "JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json b/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json index f6777ba6..03a1f924 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json @@ -1,6 +1,6 @@ { ".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4", "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969", - "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "8c12640f0c5c280c7a77423431b4ecb1", + "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "526b6df52a8aaf11dfe56f25ac35a567", "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index c4fb6cca..872a49bd 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -12,6 +12,8 @@ "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8", + "JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee", + "JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json index c5490aa6..81618c34 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist_archive_migrated.json @@ -12,6 +12,8 @@ "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6", "JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8", + "JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee", + "JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73", "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", "JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc" } \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/youtube/test_playlist.txt b/tests/resources/transaction_log_summaries/youtube/test_playlist.txt index 3213a124..ae53c14d 100644 --- a/tests/resources/transaction_log_summaries/youtube/test_playlist.txt +++ b/tests/resources/transaction_log_summaries/youtube/test_playlist.txt @@ -2,6 +2,8 @@ Files created: ---------------------------------------- {output_directory} .ytdl-sub-music_video_playlist_test-download-archive.json + fanart.jpg + poster.jpg season01-poster.jpg tvshow.nfo NFO tags: diff --git a/tests/unit/cli/test_entrypoint.py b/tests/unit/cli/test_entrypoint.py index 7a4c66e7..31c4c443 100644 --- a/tests/unit/cli/test_entrypoint.py +++ b/tests/unit/cli/test_entrypoint.py @@ -2,6 +2,7 @@ import re import sys from pathlib import Path from typing import Callable +from typing import List from unittest.mock import patch import pytest @@ -22,6 +23,7 @@ from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled @pytest.mark.parametrize("dry_run", [True, False]) @pytest.mark.parametrize("mock_success_output", [True, False]) @pytest.mark.parametrize("keep_successful_logs", [True, False]) +@pytest.mark.parametrize("match", [[], ["Rick", "Michael"]]) def test_subscription_logs_write_to_file( persist_logs_directory: str, persist_logs_config_factory: Callable, @@ -30,8 +32,11 @@ def test_subscription_logs_write_to_file( dry_run: bool, mock_success_output: bool, keep_successful_logs: bool, + match: List[str], ): - subscripton_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] + subscription_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] + if match: + subscription_names = ["Rick Astley", "Michael Jackson"] num_runs = 2 config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs) @@ -47,6 +52,8 @@ def test_subscription_logs_write_to_file( _download_subscriptions_from_yaml_files( config=config, subscription_paths=subscription_paths, + subscription_matches=match, + subscription_override_dict={}, update_with_info_json=False, dry_run=dry_run, ) @@ -61,8 +68,8 @@ def test_subscription_logs_write_to_file( return # If not success, expect 2 log files for both sub errors elif not mock_success_output: - assert len(log_directory_files) == (num_runs * len(subscripton_names)) - for log_path, subscription_name in zip(log_directory_files, subscripton_names): + assert len(log_directory_files) == (num_runs * len(subscription_names)) + for log_path, subscription_name in zip(log_directory_files, subscription_names): subscription_log_file_name = subscription_name.lower().replace(" ", "_") assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name)) @@ -74,9 +81,9 @@ def test_subscription_logs_write_to_file( ) # If success and success logging, expect 3 log files else: - assert len(log_directory_files) == (num_runs * len(subscripton_names)) + assert len(log_directory_files) == (num_runs * len(subscription_names)) for log_file_path, subscription_name in zip( - log_directory_files, subscripton_names * num_runs + log_directory_files, subscription_names * num_runs ): subscription_log_file_name = subscription_name.lower().replace(" ", "_") diff --git a/tests/unit/main/test_main.py b/tests/unit/main/test_main.py index a42e21a5..a88faa13 100644 --- a/tests/unit/main/test_main.py +++ b/tests/unit/main/test_main.py @@ -47,8 +47,8 @@ def test_main_exit_code(mock_sys_exit, return_code: int): main() assert mock_logger_cleanup.call_count == 1 - assert mock_logger_cleanup.call_args.kwargs["cleanup_error_log"] == ( - True if return_code == 0 else False + assert mock_logger_cleanup.call_args.kwargs["has_error"] == ( + True if return_code != 0 else False ) @@ -107,6 +107,58 @@ def test_args_after_sub_work(mock_sys_exit, tv_show_config_path): assert mock_sub.call_count == 1 assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == [] + assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE + + +def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path): + with mock_sys_exit(expected_exit_code=0), patch.object( + sys, + "argv", + [ + "ytdl-sub", + "--match", + "testA", + "testB", + "-c", + tv_show_config_path, + "sub", + "--log-level", + "verbose", + ], + ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub: + main() + + assert mock_sub.call_count == 1 + assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] + assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"] + assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE + + +def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path): + with mock_sys_exit(expected_exit_code=0), patch.object( + sys, + "argv", + [ + "ytdl-sub", + "-c", + tv_show_config_path, + "sub", + "--log-level", + "verbose", + "--match", + "testA", + "--match", + "testB", + ], + ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub: + main() + + assert mock_sub.call_count == 1 + assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] + assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"] assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE diff --git a/tests/unit/script/functions/test_array_functions.py b/tests/unit/script/functions/test_array_functions.py index 06a3b150..bfee05da 100644 --- a/tests/unit/script/functions/test_array_functions.py +++ b/tests/unit/script/functions/test_array_functions.py @@ -26,6 +26,14 @@ class TestArrayFunctions: output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}") assert output == "b" + def test_array_at_default(self): + output = single_variable_output("{%array_at(['a', 'b', 'c'], 30, 'd')}") + assert output == "d" + + def test_array_at_error(self): + with pytest.raises(FunctionRuntimeException): + single_variable_output("{%array_at(['a', 'b', 'c'], 30)}") + def test_array_flatten(self): output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}") assert output == ["a", "b", "c"] diff --git a/tests/unit/script/functions/test_conditional_functions.py b/tests/unit/script/functions/test_conditional_functions.py index f7217915..5cf04a18 100644 --- a/tests/unit/script/functions/test_conditional_functions.py +++ b/tests/unit/script/functions/test_conditional_functions.py @@ -1,6 +1,10 @@ +import re + import pytest from unit.script.conftest import single_variable_output +from ytdl_sub.script.utils.exceptions import FunctionRuntimeException + class TestConditionalFunction: @pytest.mark.parametrize( @@ -33,3 +37,67 @@ class TestConditionalFunction: }""" ) assert output == "winner" + + def test_elif_function(self): + output = single_variable_output( + """{ + %elif( + False, + "nope", + False, + "still nope", + True, + "yes", + "default value" + ) + }""" + ) + assert output == "yes" + + def test_elif_function_default_value(self): + output = single_variable_output( + """{ + %elif( + False, + "nope", + False, + "still nope", + False, + "will be default", + "default value" + ) + }""" + ) + assert output == "default value" + + def test_elif_function_errors_lt3(self): + with pytest.raises( + FunctionRuntimeException, + match=re.escape("elif requires at least 3 arguments"), + ): + single_variable_output( + """ + { + %elif( + False, + "only two args" + ) + }""" + ) + + def test_elif_function_errors_odd(self): + with pytest.raises( + FunctionRuntimeException, + match=re.escape("elif must have an odd number of arguments"), + ): + single_variable_output( + """ + { + %elif( + False, + "1", + False, + "even number args bad" + ) + }""" + ) diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py index 9a48079a..cfa5175e 100644 --- a/tests/unit/script/functions/test_string_functions.py +++ b/tests/unit/script/functions/test_string_functions.py @@ -1,3 +1,6 @@ +from typing import List +from typing import Optional + import pytest from unit.script.conftest import single_variable_output @@ -114,3 +117,21 @@ class TestNumericFunctions: def test_contains(self, value, expected_output): output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}") assert output == expected_output + + @pytest.mark.parametrize( + "input_string, split, max_split, expected_output", + [ + ("no splits", " | ", None, ["no splits"]), + ("one | split", " | ", None, ["one", "split"]), + ("max | split | one", " | ", 1, ["max", "split | one"]), + ], + ) + def test_split( + self, input_string: str, split: str, max_split: Optional[int], expected_output: List[str] + ): + if max_split: + output = single_variable_output(f"{{%split('{input_string}', '{split}', {max_split})}}") + else: + output = single_variable_output(f"{{%split('{input_string}', '{split}')}}") + + assert output == expected_output diff --git a/tests/unit/script/types/test_function.py b/tests/unit/script/types/test_function.py index 014e4a85..e13828f4 100644 --- a/tests/unit/script/types/test_function.py +++ b/tests/unit/script/types/test_function.py @@ -49,7 +49,11 @@ class TestFunction: @pytest.mark.parametrize( "function_str, expected_types, received_types", [ - ("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"), + ( + "{%array_at({'a': 'dict?'}, 1)}", + "array: Array, idx: Integer, default: Optional[AnyArgument]", + "Map, Integer", + ), ("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"), ( "{%replace('hi mom', 'mom', 'dad', 1, 0)}", diff --git a/tests/unit/utils/test_logger.py b/tests/unit/utils/test_logger.py index 5a75e5c3..97ce6bbd 100644 --- a/tests/unit/utils/test_logger.py +++ b/tests/unit/utils/test_logger.py @@ -111,8 +111,8 @@ class TestLogger: Logger.cleanup() assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name) - @pytest.mark.parametrize("clean_error_log", [True, False]) - def test_logger_can_be_cleaned_during_execution(self, clean_error_log: bool): + @pytest.mark.parametrize("has_error", [True, False]) + def test_logger_can_be_cleaned_during_execution(self, has_error: bool): Logger._LOGGER_LEVEL = LoggerLevels.INFO logger = Logger.get(name="name_test") @@ -133,11 +133,11 @@ class TestLogger: except ValueError as exc: Logger.log_exception(exception=exc) - Logger.cleanup(cleanup_error_log=clean_error_log) + Logger.cleanup(has_error=has_error) assert not os.path.isfile(Logger.debug_log_filename()) - assert clean_error_log == (not os.path.isfile(Logger.error_log_filename())) - if not clean_error_log: + assert not has_error == (not os.path.isfile(Logger.error_log_filename())) + if has_error: with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file: err_logs = err_file.readlines() expected = [ diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py index 10323066..8080c297 100644 --- a/tools/docgen/plugins.py +++ b/tools/docgen/plugins.py @@ -39,6 +39,8 @@ def should_filter_property(property_name: str) -> bool: "dict_with_format_strings", "subscription_name", "list", + "script", + "unresolvable", )