Merge branch 'master' into j/unstructured

Makefile

@@ -22,7 +22,8 @@ check_lint:
 		&& black . --check  \
 		&& pylint src/
 wheel: clean
-	$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py)
+	$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"" > src/ytdl_sub/__init__.py)
+	$(shell echo "__local_version__ = \"$(LOCAL_VERSION)\"" >> src/ytdl_sub/__init__.py)
 	cat src/ytdl_sub/__init__.py
 	pip3 install build
 	python3 -m build
@@ -39,6 +40,7 @@ executable: clean
 	pyinstaller ytdl-sub.spec
 	mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX}
 docs:
+	REGENERATE_DOCS=1 pytest tests/unit/docgen/test_docgen.py
 	sphinx-build -M html docs/source/ docs/build/
 clean:
 	rm -rf \

docs/source/config_reference/scripting/scripting_functions.rst

@@ -560,6 +560,15 @@ regex_search
   the string as the first element of the Array. If there are capture groups, returns each
   group as a subsequent element in the Array.
 
+regex_sub
+~~~~~~~~~
+:spec: ``regex_sub(regex: String, replacement: String, string: String) -> String``
+
+:description:
+  Returns the string obtained by replacing the leftmost non-overlapping occurrences of the
+  pattern in string by the replacement string. The replacement string can reference the
+  match groups via backslash escapes. Callables as replacement argument are not supported.
+
 ----------------------------------------------------------------------------------------------------
 
 String Functions

docs/source/faq/index.rst

@@ -33,7 +33,7 @@ If you need support, you can:
 
 If you would like to contribute, we're happy to accept any help, even non-coders! To find out how you can help this project, you can:
 
-* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute. 
+* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute.
 
 * If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull request <compare>` with information on what issue you're resolving and it will be reviewed as soon as possible.
 
@@ -83,17 +83,18 @@ Most likely the video has a non-English language set to its 'native' language. Y
 ...Plex is not showing my TV shows correctly
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Set the following for your ytdl-sub library that has been added to Plex.
+1. Set the following for your ytdl-sub library that has been added to Plex.
 
 .. figure:: ../../images/plex_scanner_agent.png
   :alt: The Plex library editor, under the advanced settings, showing the required options for Plex to show the TV shows correctly.
 
-**Scanner:** Plex Series Scanner
+- **Scanner:** Plex Series Scanner
+- **Agent:** Personal Media shows
+- **Visibility:** Exclude from home screen and global search
+- **Episode sorting:** Library default
+- **YES** Enable video preview thumbnails
 
-**Agent:** Personal Media shows
+2. Under **Settings** > **Agents**, confirm Plex Personal Media Shows/Movies scanner has **Local Media Assets** enabled.
 
-**Visibility:** Exclude from home screen and global search
+.. figure:: ../../images/plex_agent_sources.png
-
+  :alt: The Plex Agents settings page has Local Media Assets enabled for Personal Media Shows and Movies tabs.
-**Episode sorting:** Library default
-
-**YES** Enable video preview thumbnails

pyproject.toml

@@ -1,3 +1,69 @@
+[project]
+name ="ytdl-sub"
+dynamic = [ "version" ]
+authors = [ { name = "Jesse Bannon" } ]
+description = "Automate downloading metadata generation with YoutubeDL"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+classifiers = [
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Multimedia :: Video",
+    "License :: Public Domain",
+    "Environment :: Console",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+    "yt-dlp==2024.04.09",
+    "colorama~=0.4",
+    "mergedeep~=1.3",
+    "mediafile~=0.12",
+    "PyYAML~=6.0",
+]
+urls = { Homepage = "https://github.com/jmbannon/ytdl-sub" }
+
+[build-system]
+requires = [ "setuptools >= 67.0" ]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+platforms = [ "Unix" ]
+
+[tool.setuptools.dynamic]
+version = { attr = "ytdl_sub.__pypi_version__" }
+[tool.setuptools.package-dir]
+"" = "src"
+[tool.setuptools.package-data]
+"*" = ["*.yaml"]
+[tool.setuptools.packages.find]
+where = ["src"]
+
+
+[project.optional-dependencies]
+test = [
+    "coverage[toml]>=6.3,<8.0",
+    "pytest>=7.2,<9.0",
+    "pytest-rerunfailures~=14.0",
+]
+lint = [
+    "black==24.4.2",
+    "isort==5.13.2",
+    "pylint==2.13.5",
+]
+docs = [
+    "sphinx~=7.0",
+    "sphinx-rtd-theme~=2.0",
+    "sphinx-book-theme~=1.0",
+]
+build = [
+    "build~=1.2",
+    "twine~=5.0",
+    "pyinstaller~=6.5",
+]
+[project.scripts]
+ytdl-sub = "ytdl_sub.main:main"
+
 [tool.isort]
 profile = "black"
 line_length = 100
@@ -30,4 +96,4 @@ include = [
 [tool.coverage.report]
 exclude_also = [
     "raise UNREACHABLE.*",
-]
+]

setup.cfg

@@ -1,60 +0,0 @@
-[metadata]
-name = ytdl-sub
-version = attr:ytdl_sub.__pypi_version__
-author = Jesse Bannon
-description = Automate downloading and metadata generation with YoutubeDL
-long_description = file: README.md
-long_description_content_type= text/markdown
-author_email = use_github_issues@nope.com
-url = https://github.com/jmbannon/ytdl-sub
-license = GNUv3
-platforms = Unix
-classifiers =
-    Topic :: Multimedia :: Sound/Audio
-    Topic :: Multimedia :: Video
-    License :: Public Domain
-    Environment :: Console
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-
-[options.entry_points]
-console_scripts =
-    ytdl-sub = ytdl_sub.main:main
-
-[options]
-package_dir =
-    = src
-packages=find:
-
-install_requires =
-    yt-dlp==2024.3.10
-    argparse==1.4.0
-    colorama==0.4.6
-    mergedeep==1.3.4
-    mediafile==0.12.0
-    PyYAML==5.3.1
-
-[options.package_data]
-* = *.yaml
-
-[options.packages.find]
-where=src
-
-[options.extras_require]
-test =
-    coverage[toml]==6.3.2
-    pytest==7.1.1
-    pytest-rerunfailures==12.0
-lint =
-    black==22.3.0
-    isort==5.10.1
-    pylint==2.13.5
-docs =
-    sphinx==7.2.6
-    sphinx-rtd-theme==2.0.0
-    sphinx-book-theme==1.1.0
-build =
-    build
-    twine
-    pyinstaller
-

src/ytdl_sub/cli/parsers/main.py

@@ -150,6 +150,8 @@ parser.add_argument("-v", "--version", action="version", version="%(prog)s " + _
 _add_shared_arguments(parser, suppress_defaults=False)
 
 subparsers = parser.add_subparsers(dest="subparser")
+
+
 ###################################################################################################
 # SUBSCRIPTION PARSER
 class SubArguments:

src/ytdl_sub/config/validators/variable_validation.py

@@ -25,7 +25,7 @@ _DUMMY_ENTRY_VARIABLES: Dict[str, str] = {
     name: to_variable_dependency_format_string(
         # pylint: disable=protected-access
         script=BASE_SCRIPT,
-        parsed_format_string=BASE_SCRIPT._variables[name]
+        parsed_format_string=BASE_SCRIPT._variables[name],
         # pylint: enable=protected-access
     )
     for name in BASE_SCRIPT.variable_names

src/ytdl_sub/downloaders/url/downloader.py

@@ -248,9 +248,11 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
             self._download_ytdl_options_builder.clone()
             .add(self.ytdl_option_defaults(), before=True)
             .add(
-                self.plugin_options.urls.list[url_idx].ytdl_options.dict
+                (
-                if url_idx is not None
+                    self.plugin_options.urls.list[url_idx].ytdl_options.dict
-                else None,
+                    if url_idx is not None
+                    else None
+                ),
                 before=True,
             )
             .to_dict()
@@ -352,9 +354,11 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
                 url_idx=entry.get(v.ytdl_sub_input_url_index, int)
             ),
             is_downloaded_fn=None if self.is_dry_run else entry.is_downloaded,
-            is_thumbnail_downloaded_fn=None
+            is_thumbnail_downloaded_fn=(
-            if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
+                None
-            else entry.is_thumbnail_downloaded_via_ytdlp,
+                if (self.is_dry_run or not self.is_entry_thumbnails_enabled)
+                else entry.is_thumbnail_downloaded_via_ytdlp
+            ),
             url=entry.webpage_url,
         )
         return Entry(

src/ytdl_sub/plugins/chapters.py

@@ -263,13 +263,13 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
                 "force_keyframes": self.plugin_options.force_key_frames,
             }
             if self.plugin_options.remove_sponsorblock_categories is not None:
-                remove_chapters_post_processor[
+                remove_chapters_post_processor["remove_sponsor_segments"] = (
-                    "remove_sponsor_segments"
+                    self.plugin_options.remove_sponsorblock_categories
-                ] = self.plugin_options.remove_sponsorblock_categories
+                )
             if self.plugin_options.remove_chapters_regex is not None:
-                remove_chapters_post_processor[
+                remove_chapters_post_processor["remove_chapters_patterns"] = (
-                    "remove_chapters_patterns"
+                    self.plugin_options.remove_chapters_regex
-                ] = self.plugin_options.remove_chapters_regex
+                )
 
             if self.plugin_options.embed_chapters:
                 builder.add(

src/ytdl_sub/plugins/split_by_chapters.py

@@ -159,9 +159,9 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
 
         metadata_value_dict = {}
         if self.is_dry_run:
-            metadata_value_dict[
+            metadata_value_dict["Warning"] = (
-                "Warning"
+                "Dry-run assumes embedded chapters with no modifications"
-            ] = "Dry-run assumes embedded chapters with no modifications"
+            )
 
         metadata_value_dict["Source Title"] = new_entry.title
         metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}"

src/ytdl_sub/script/functions/regex_functions.py

@@ -52,3 +52,13 @@ class RegexFunctions:
           Returns number of capture groups in regex
         """
         return Integer(re.compile(regex.value).groups)
+
+    @staticmethod
+    def regex_sub(regex: String, replacement: String, string: String) -> String:
+        """
+        :description:
+          Returns the string obtained by replacing the leftmost non-overlapping occurrences of the
+          pattern in string by the replacement string. The replacement string can reference the
+          match groups via backslash escapes. Callables as replacement argument are not supported.
+        """
+        return String(re.sub(regex.value, replacement.value, string.value))

src/ytdl_sub/subscriptions/base_subscription.py

@@ -74,13 +74,13 @@ class BaseSubscription(ABC):
             }
         )
 
-        self._enhanced_download_archive: Optional[
+        self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = (
-            EnhancedDownloadArchive
+            _initialize_download_archive(
-        ] = _initialize_download_archive(
+                output_options=self.output_options,
-            output_options=self.output_options,
+                overrides=self.overrides,
-            overrides=self.overrides,
+                working_directory=self.working_directory,
-            working_directory=self.working_directory,
+                output_directory=self.output_directory,
-            output_directory=self.output_directory,
+            )
         )
 
         # Add post-archive variables

src/ytdl_sub/subscriptions/subscription_validators.py

@@ -143,9 +143,9 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator):
             presets=presets,
             indent_overrides=indent_overrides,
         )
-        self._overrides_to_add[
+        self._overrides_to_add[SubscriptionVariables.subscription_value().variable_name] = (
-            SubscriptionVariables.subscription_value().variable_name
+            self.value
-        ] = self.value
+        )
 
 
 class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator):
@@ -170,9 +170,9 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid
         for idx, list_value in enumerate(self.list):
             # Write the first list value into subscription_value as well
             if idx == 0:
-                self._overrides_to_add[
+                self._overrides_to_add[SubscriptionVariables.subscription_value().variable_name] = (
-                    SubscriptionVariables.subscription_value().variable_name
+                    list_value.value
-                ] = list_value.value
+                )
 
             self._overrides_to_add[
                 SubscriptionVariables.subscription_value_i(index=idx).variable_name
@@ -219,9 +219,9 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator):
             presets=presets,
             indent_overrides=indent_overrides,
         )
-        self._overrides_to_add[
+        self._overrides_to_add[SubscriptionVariables.subscription_map().variable_name] = (
-            SubscriptionVariables.subscription_map().variable_name
+            ScriptUtils.to_script(self.dict)
-        ] = ScriptUtils.to_script(self.dict)
+        )
 
 
 class SubscriptionValidator(SubscriptionOutput):

src/ytdl_sub/utils/logger.py

@@ -213,9 +213,10 @@ class Logger:
     @classmethod
     def _append_to_error_log(cls):
         # Any time an exception occurs, dump all debug logs into the error log
-        with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open(
+        with (
-            cls.error_log_filename(), mode="a", encoding="utf-8"
+            open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs,
-        ) as error_logs:
+            open(cls.error_log_filename(), mode="a", encoding="utf-8") as error_logs,
+        ):
             error_logs.writelines(debug_logs.readlines())
 
     @classmethod

src/ytdl_sub/utils/retry.py

@@ -34,12 +34,14 @@ def retry(times: int, exceptions: Tuple[Type[Exception], ...], wait_sec: int = 5
             while attempt < times:
                 try:
                     return func(*args, **kwargs)
-                except exceptions:
+                except exceptions as exc:
                     logger.debug(
-                        "Exception thrown when attempting to run %s, attempt %d of %d",
+                        "Exception thrown when attempting to run %s, attempt %d of %d\n"
+                        "Exception:\n%s",
                         func.__name__,
                         attempt + 1,
                         times,
+                        str(exc),
                     )
                     attempt += 1
                     sleep(wait_sec)

tests/e2e/youtube/test_video.py

@@ -122,11 +122,13 @@ class TestYoutubeVideo:
             try_convert_download_thumbnail(entry=entry)
 
         # Pretend the thumbnail did not download via returning nothing for its downloaded path
-        with patch.object(YTDLP, "_EXTRACT_ENTRY_NUM_RETRIES", 1), patch.object(
+        with (
-            Entry, "try_get_ytdlp_download_thumbnail_path"
+            patch.object(YTDLP, "_EXTRACT_ENTRY_NUM_RETRIES", 1),
-        ) as mock_ytdlp_path, patch(
+            patch.object(Entry, "try_get_ytdlp_download_thumbnail_path") as mock_ytdlp_path,
-            "ytdl_sub.downloaders.url.downloader.try_convert_download_thumbnail",
+            patch(
-            side_effect=delete_entry_thumb,
+                "ytdl_sub.downloaders.url.downloader.try_convert_download_thumbnail",
+                side_effect=delete_entry_thumb,
+            ),
         ):
             mock_ytdlp_path.return_value = None
             transaction_log = single_video_subscription.download(dry_run=False)

tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json

@@ -1,6 +1,6 @@
 {
   ".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec",
   "JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6",
-  "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "068526b2d8f85fdcf914df3e23d0b1fa",
+  "JMC/Move 78 - Automated Improvisation [Full Album].mp4": "63119bd17a035574263aeec188e11d5e",
   "JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac"
 }

tests/resources/expected_downloads_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_pass.txt

@@ -1,5 +1,5 @@
 {
   ".ytdl-sub-split_by_chapters_with_regex_video_no_chapters-download-archive.json": "4008e43668447f1a3a6a55520a6ff475",
-  "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "7813b727a1d3df89effe45c42e7c7e63",
+  "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/01 - Oblivion Mod ＂Falcor＂ p.1.mp3": "f1c2e04fb84768fed69ed981d9a2dbcd",
   "Project Zombie/[2010] Oblivion Mod ＂Falcor＂ p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
 }

tests/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json

@@ -1,6 +1,6 @@
 {
   ".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4",
   "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969",
-  "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "53590935ec5f801fd7e1b5aacf28fa5d",
+  "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "0539ff781c824f17ae3ffbd7dc830a8f",
   "JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4"
 }

tests/resources/expected_downloads_summaries/soundcloud/test_soundcloud_discography.json

@@ -1,19 +1,19 @@
 {
   ".ytdl-sub-j_b-download-archive.json": "1a99156e9ece62539fb2608416a07200",
-  "j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3": "3b686f82ebb2ffe4bb7a491b00ea8137",
+  "j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3": "b35d01351e5dd1195e90a41b2edeb5b7",
   "j_b/[2021] Baby Santana's Dorian Groove/folder.jpg": "967892be44b8c47e1be73f055a7c6f08",
-  "j_b/[2021] Purple Clouds/01 - Purple Clouds.mp3": "64ded79214390867c7ddc08d290183f4",
+  "j_b/[2021] Purple Clouds/01 - Purple Clouds.mp3": "5c107c122048389aa40936c4fba0436e",
   "j_b/[2021] Purple Clouds/folder.jpg": "967892be44b8c47e1be73f055a7c6f08",
-  "j_b/[2022] Acoustic Treats/01 - 20160426 184214.mp3": "ef254985af511b8917fbe32feb2bf1a6",
+  "j_b/[2022] Acoustic Treats/01 - 20160426 184214.mp3": "e737c2ba118920643cec9eac09a283ea",
-  "j_b/[2022] Acoustic Treats/02 - 20160502 123150.mp3": "b668004a76e06871ae8aa5a757f02928",
+  "j_b/[2022] Acoustic Treats/02 - 20160502 123150.mp3": "d5def3e4329b9b95c5354c838dd3bc91",
-  "j_b/[2022] Acoustic Treats/03 - 20160504 143832.mp3": "0d40f6261cb638d65e473d5c3172d1fa",
+  "j_b/[2022] Acoustic Treats/03 - 20160504 143832.mp3": "1694c4a2889aa7dacafc5f2fdb4fe1d7",
-  "j_b/[2022] Acoustic Treats/04 - 20160601 221234.mp3": "670f2f35e83f588023cabdaade2f5537",
+  "j_b/[2022] Acoustic Treats/04 - 20160601 221234.mp3": "9c8ddee6aaf88b38706ce6bbdf907878",
-  "j_b/[2022] Acoustic Treats/05 - 20160601 222440.mp3": "032f59d0f2c7c3ce352a677ce5d30ee4",
+  "j_b/[2022] Acoustic Treats/05 - 20160601 222440.mp3": "4bf9e700137cb1b58e730dd312e00bb4",
-  "j_b/[2022] Acoustic Treats/06 - 20170604 190236.mp3": "faffcfb3d1b87b18be77ab4f86dd298f",
+  "j_b/[2022] Acoustic Treats/06 - 20170604 190236.mp3": "c1920254046eda0d84b36dd34f9bafdb",
-  "j_b/[2022] Acoustic Treats/07 - 20170612 193646.mp3": "a7976b0380ec7b0c32193a58dc15cfa2",
+  "j_b/[2022] Acoustic Treats/07 - 20170612 193646.mp3": "26f2eb8395b23cecc4000e2fd58304bf",
-  "j_b/[2022] Acoustic Treats/08 - 20170628 215206.mp3": "b8de5803604102592564c0ebc46000a3",
+  "j_b/[2022] Acoustic Treats/08 - 20170628 215206.mp3": "e3ceb5ae25c115d037a298b4620bce9f",
-  "j_b/[2022] Acoustic Treats/09 - Finding Home.mp3": "29b40c0c66a3bce2da6fb86d9bad1b42",
+  "j_b/[2022] Acoustic Treats/09 - Finding Home.mp3": "f3dcfe7c8569853a02eb12ad21b0e248",
-  "j_b/[2022] Acoustic Treats/10 - Shallow Water WIP.mp3": "8de2a91d10ce54deaa81980c03457963",
+  "j_b/[2022] Acoustic Treats/10 - Shallow Water WIP.mp3": "4fb9a18b3fdd788ce9d9f4c8a07b6028",
-  "j_b/[2022] Acoustic Treats/11 - Untold History.mp3": "966c57fcf80ab88e2f083625baf0b8bc",
+  "j_b/[2022] Acoustic Treats/11 - Untold History.mp3": "ad186ee1d0029f8de02357e8b8712a4b",
   "j_b/[2022] Acoustic Treats/folder.jpg": "967892be44b8c47e1be73f055a7c6f08"
 }

tests/resources/expected_downloads_summaries/youtube/test_channel_full.json

@@ -2,7 +2,7 @@
   "Project ⧸ Zombie/.ytdl-sub-pz-download-archive.json": "aadb59c92dcf14ee6617c77423a14584",
   "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
   "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json": "INFO_JSON",
-  "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "246fa05b6443337785575987904848df",
+  "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4": "0571a944a25791bc3c6cbbf436cb3778",
   "Project ⧸ Zombie/Season 2010/s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo": "a1970f06fbc4743fca6db0627de779f3",
   "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1",
   "Project ⧸ Zombie/Season 2010/s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json": "INFO_JSON",

tests/resources/expected_downloads_summaries/youtube/test_video.json

@@ -1,5 +1,5 @@
 {
   "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e",
-  "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6",
+  "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "318faf3eb1d7666491553cdfd2a2e9fb",
   "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
 }

tests/resources/expected_downloads_summaries/youtube/test_video_cli.json

@@ -1,5 +1,5 @@
 {
   "JMC/Oblivion Mod ＂Falcor＂ p.1.jpg": "fb95b510681676e81c321171fc23143e",
-  "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "f2be699684854bdb6e09c02d24bdd5b6",
+  "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "318faf3eb1d7666491553cdfd2a2e9fb",
   "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
 }

tests/resources/expected_downloads_summaries/youtube/test_video_missing_thumb.json

@@ -1,4 +1,4 @@
 {
-  "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d9d2d12feee44ee97729b39ba981c542",
+  "JMC/Oblivion Mod ＂Falcor＂ p.1.mp4": "d3bdda5ec6822ea3b4b4bd8908eb327a",
   "JMC/Oblivion Mod ＂Falcor＂ p.1.nfo": "58c2be339869b5d071c1758d55c72ddb"
 }

tests/resources/transaction_log_summaries/plugins/chapters/test_chapters_from_comments.txt

@@ -5,16 +5,6 @@ Files created:
 {output_directory}/JMC
   Move 78 - Automated Improvisation [Full Album].jpg
   Move 78 - Automated Improvisation [Full Album].mp4
-    Chapters from comments:
-      0:00: 01. The Lonely Tears of Lee Seedol
-      4:30: 02. But What If We're Wrong
-      9:16: 03. Follow the Earworm Pt.2
-      12:25: 04. Keyword Salad
-      16:48: 05. Ultra Natural
-      20:47: 06. Flight Instructions
-      25:46: 07. Dawn of the Useless Class
-      29:58: 08. Schnitzel Whisperer
-      32:16: 09. Teilo
     Embedded subtitles with lang(s) en, de
     Video Tags:
       album: Music Videos

tests/unit/cli/test_entrypoint.py

@@ -42,12 +42,15 @@ def test_subscription_logs_write_to_file(
     config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs)
     subscription_paths = [str(music_video_subscription_path)] * num_runs
 
-    with patch.object(
+    with (
-        Subscription,
+        patch.object(
-        "download",
+            Subscription,
-        new=mock_subscription_download_factory(mock_success_output=mock_success_output),
+            "download",
-        # mock datetime to be an index to be able to run instantly
+            new=mock_subscription_download_factory(mock_success_output=mock_success_output),
-    ), patch("ytdl_sub.cli.entrypoint._log_time", side_effect=[str(idx) for idx in range(10)]):
+            # mock datetime to be an index to be able to run instantly
+        ),
+        patch("ytdl_sub.cli.entrypoint._log_time", side_effect=[str(idx) for idx in range(10)]),
+    ):
         try:
             _download_subscriptions_from_yaml_files(
                 config=config,
@@ -101,16 +104,19 @@ def test_update_with_info_json_requires_experimental_flag(
     default_config_path: Path,
     music_video_subscription_path: Path,
 ) -> None:
-    with patch.object(
+    with (
-        sys,
+        patch.object(
-        "argv",
+            sys,
-        [
+            "argv",
-            "ytdl-sub",
+            [
-            "--config",
+                "ytdl-sub",
-            str(default_config_path),
+                "--config",
-            "sub",
+                str(default_config_path),
-            str(music_video_subscription_path),
+                "sub",
-            "--update-with-info-json",
+                str(music_video_subscription_path),
-        ],
+                "--update-with-info-json",
-    ), pytest.raises(ExperimentalFeatureNotEnabled):
+            ],
+        ),
+        pytest.raises(ExperimentalFeatureNotEnabled),
+    ):
         _ = main()

tests/unit/cli/test_output_transaction_log.py

@@ -32,19 +32,22 @@ def test_suppress_transaction_log(
     music_video_subscription_path: Path,
     file_transaction_log: Optional[str],
 ) -> None:
-    with patch.object(
+    with (
-        sys,
+        patch.object(
-        "argv",
+            sys,
-        [
+            "argv",
-            "ytdl-sub",
+            [
-            "--config",
+                "ytdl-sub",
-            str(default_config_path),
+                "--config",
-            "sub",
+                str(default_config_path),
-            str(music_video_subscription_path),
+                "sub",
-            "--suppress-transaction-log",
+                str(music_video_subscription_path),
-        ]
+                "--suppress-transaction-log",
-        + (["--transaction-log", file_transaction_log] if file_transaction_log else []),
+            ]
-    ), patch("ytdl_sub.cli.output_transaction_log.output_transaction_log") as mock_transaction_log:
+            + (["--transaction-log", file_transaction_log] if file_transaction_log else []),
+        ),
+        patch("ytdl_sub.cli.output_transaction_log.output_transaction_log") as mock_transaction_log,
+    ):
         subscriptions = main()
 
         assert subscriptions
@@ -82,20 +85,23 @@ def test_transaction_log_to_logger(
     default_config_path: Path,
     music_video_subscription_path: Path,
 ) -> None:
-    with patch.object(
+    with (
-        sys,
+        patch.object(
-        "argv",
+            sys,
-        [
+            "argv",
-            "ytdl-sub",
+            [
-            "--config",
+                "ytdl-sub",
-            str(default_config_path),
+                "--config",
-            "sub",
+                str(default_config_path),
-            str(music_video_subscription_path),
+                "sub",
-        ],
+                str(music_video_subscription_path),
-    ), assert_logs(
+            ],
-        logger=transaction_logger,
+        ),
-        expected_message="Transaction log for Rick Astley:\n",
+        assert_logs(
-        log_level="info",
+            logger=transaction_logger,
+            expected_message="Transaction log for Rick Astley:\n",
+            log_level="info",
+        ),
     ):
         subscriptions = main()
         assert subscriptions

tests/unit/config/test_subscription.py

@@ -375,38 +375,47 @@ def test_subscription_file_value_applies_from_config_and_nested_and_indent_varia
     config_file: ConfigFile,
     preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors: Dict,
 ):
-    with mock_load_yaml(
+    with (
-        preset_dict=preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors
+        mock_load_yaml(
-    ), pytest.raises(
+            preset_dict=preset_with_subscription_value_nested_presets_and_indent_variables_same_line_old_format_errors
-        ValidationException,
+        ),
-        match=re.escape(
+        pytest.raises(
-            "Validation error in parent_preset_2.=INDENT_1: 'INDENT_3' in '= INDENT_2  | INDENT_3' is not a preset name. "
+            ValidationException,
-            "To use as a subscription indent value, define it as '= INDENT_3'"
+            match=re.escape(
+                "Validation error in parent_preset_2.=INDENT_1: 'INDENT_3' in '= INDENT_2  | INDENT_3' is not a preset name. "
+                "To use as a subscription indent value, define it as '= INDENT_3'"
+            ),
         ),
     ):
         Subscription.from_file_path(config=config_file, subscription_path="mocked")
 
 
 def test_subscription_file_using_conflicting_preset_name(config_file: ConfigFile):
-    with mock_load_yaml(
+    with (
-        preset_dict={
+        mock_load_yaml(
-            "= INDENTS_IN_ERR_MSG ": {"=ANOTHER": {"jellyfin_tv_show_by_date": "single value"}}
+            preset_dict={
-        }
+                "= INDENTS_IN_ERR_MSG ": {"=ANOTHER": {"jellyfin_tv_show_by_date": "single value"}}
-    ), pytest.raises(
+            }
-        ValidationException,
+        ),
-        match=re.escape(
+        pytest.raises(
-            "Validation error in = INDENTS_IN_ERR_MSG .=ANOTHER.jellyfin_tv_show_by_date: "
+            ValidationException,
-            "jellyfin_tv_show_by_date conflicts with an existing preset name and cannot be used "
+            match=re.escape(
-            "as a subscription name"
+                "Validation error in = INDENTS_IN_ERR_MSG .=ANOTHER.jellyfin_tv_show_by_date: "
+                "jellyfin_tv_show_by_date conflicts with an existing preset name and cannot be used "
+                "as a subscription name"
+            ),
         ),
     ):
         _ = Subscription.from_file_path(config=config_file, subscription_path="mocked")
 
 
 def test_subscription_file_invalid_form(config_file: ConfigFile):
-    with mock_load_yaml(preset_dict={"sub_name": 4332}), pytest.raises(
+    with (
-        ValidationException,
+        mock_load_yaml(preset_dict={"sub_name": 4332}),
-        match=re.escape(f"Subscription value should either be a string, list, or object"),
+        pytest.raises(
+            ValidationException,
+            match=re.escape(f"Subscription value should either be a string, list, or object"),
+        ),
     ):
         _ = Subscription.from_file_path(config=config_file, subscription_path="mocked")
 

tests/unit/conftest.py

@@ -228,10 +228,11 @@ def mock_download_collection_entries(
                 ),
             ]
 
-        with patch.object(
+        with (
-            YTDLP, "extract_info_via_info_json", new=_write_entries_to_working_dir
+            patch.object(YTDLP, "extract_info_via_info_json", new=_write_entries_to_working_dir),
-        ), patch.object(
+            patch.object(
-            MultiUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
+                MultiUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
+            ),
         ):
             # Stub out metadata. TODO: update this if we do metadata plugins
             yield

tests/unit/main/test_main.py

@@ -40,9 +40,11 @@ def mock_sys_exit():
 
 @pytest.mark.parametrize("return_code", [0, 1])
 def test_main_exit_code(mock_sys_exit, return_code: int):
-    with mock_sys_exit(expected_exit_code=return_code), patch(
+    with (
-        "src.ytdl_sub.main._main"
+        mock_sys_exit(expected_exit_code=return_code),
-    ) as mock_inner_main, patch.object(Logger, "cleanup") as mock_logger_cleanup:
+        patch("src.ytdl_sub.main._main") as mock_inner_main,
+        patch.object(Logger, "cleanup") as mock_logger_cleanup,
+    ):
         mock_inner_main.return_value = return_code
         main()
 
@@ -54,9 +56,11 @@ def test_main_exit_code(mock_sys_exit, return_code: int):
 
 def test_main_validation_error(capsys, mock_sys_exit):
     validation_exception = ValidationException("test exc")
-    with mock_sys_exit(expected_exit_code=1), patch(
+    with (
-        "src.ytdl_sub.main._main", side_effect=validation_exception
+        mock_sys_exit(expected_exit_code=1),
-    ), patch.object(logging.Logger, "error") as mock_logger:
+        patch("src.ytdl_sub.main._main", side_effect=validation_exception),
+        patch.object(logging.Logger, "error") as mock_logger,
+    ):
         main()
 
     assert mock_logger.call_count == 1
@@ -65,11 +69,12 @@ def test_main_validation_error(capsys, mock_sys_exit):
 
 def test_main_uncaught_error(capsys, mock_sys_exit, expected_uncaught_error_message):
     uncaught_error = ValueError("test")
-    with mock_sys_exit(expected_exit_code=1), patch(
+    with (
-        "src.ytdl_sub.main._main", side_effect=uncaught_error
+        mock_sys_exit(expected_exit_code=1),
-    ), patch.object(logging.Logger, "exception") as mock_exception, patch.object(
+        patch("src.ytdl_sub.main._main", side_effect=uncaught_error),
-        logging.Logger, "error"
+        patch.object(logging.Logger, "exception") as mock_exception,
-    ) as mock_error:
+        patch.object(logging.Logger, "error") as mock_error,
+    ):
         main()
 
     assert mock_exception.call_count == 1
@@ -83,9 +88,11 @@ def test_main_uncaught_error(capsys, mock_sys_exit, expected_uncaught_error_mess
 
 def test_main_permission_error(capsys, mock_sys_exit, expected_uncaught_error_message):
     permission_error = PermissionError("test")
-    with mock_sys_exit(expected_exit_code=1), patch(
+    with (
-        "src.ytdl_sub.main._main", side_effect=permission_error
+        mock_sys_exit(expected_exit_code=1),
-    ), patch.object(logging.Logger, "error") as mock_error:
+        patch("src.ytdl_sub.main._main", side_effect=permission_error),
+        patch.object(logging.Logger, "error") as mock_error,
+    ):
         main()
 
     assert mock_error.call_count == 1
@@ -97,11 +104,15 @@ def test_main_permission_error(capsys, mock_sys_exit, expected_uncaught_error_me
 
 
 def test_args_after_sub_work(mock_sys_exit, tv_show_config_path):
-    with mock_sys_exit(expected_exit_code=0), patch.object(
+    with (
-        sys,
+        mock_sys_exit(expected_exit_code=0),
-        "argv",
+        patch.object(
-        ["ytdl-sub", "-c", tv_show_config_path, "sub", "--log-level", "verbose"],
+            sys,
-    ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
+            "argv",
+            ["ytdl-sub", "-c", tv_show_config_path, "sub", "--log-level", "verbose"],
+        ),
+        patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub,
+    ):
         main()
 
         assert mock_sub.call_count == 1
@@ -112,21 +123,25 @@ def test_args_after_sub_work(mock_sys_exit, tv_show_config_path):
 
 
 def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path):
-    with mock_sys_exit(expected_exit_code=0), patch.object(
+    with (
-        sys,
+        mock_sys_exit(expected_exit_code=0),
-        "argv",
+        patch.object(
-        [
+            sys,
-            "ytdl-sub",
+            "argv",
-            "--match",
+            [
-            "testA",
+                "ytdl-sub",
-            "testB",
+                "--match",
-            "-c",
+                "testA",
-            tv_show_config_path,
+                "testB",
-            "sub",
+                "-c",
-            "--log-level",
+                tv_show_config_path,
-            "verbose",
+                "sub",
-        ],
+                "--log-level",
-    ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
+                "verbose",
+            ],
+        ),
+        patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub,
+    ):
         main()
 
         assert mock_sub.call_count == 1
@@ -137,22 +152,26 @@ def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path):
 
 
 def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path):
-    with mock_sys_exit(expected_exit_code=0), patch.object(
+    with (
-        sys,
+        mock_sys_exit(expected_exit_code=0),
-        "argv",
+        patch.object(
-        [
+            sys,
-            "ytdl-sub",
+            "argv",
-            "-c",
+            [
-            tv_show_config_path,
+                "ytdl-sub",
-            "sub",
+                "-c",
-            "--log-level",
+                tv_show_config_path,
-            "verbose",
+                "sub",
-            "--match",
+                "--log-level",
-            "testA",
+                "verbose",
-            "--match",
+                "--match",
-            "testB",
+                "testA",
-        ],
+                "--match",
-    ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
+                "testB",
+            ],
+        ),
+        patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub,
+    ):
         main()
 
         assert mock_sub.call_count == 1
@@ -163,11 +182,15 @@ def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path):
 
 
 def test_no_config_works(mock_sys_exit):
-    with mock_sys_exit(expected_exit_code=0), patch.object(
+    with (
-        sys,
+        mock_sys_exit(expected_exit_code=0),
-        "argv",
+        patch.object(
-        ["ytdl-sub", "sub", "--log-level", "verbose"],
+            sys,
-    ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
+            "argv",
+            ["ytdl-sub", "sub", "--log-level", "verbose"],
+        ),
+        patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub,
+    ):
         main()
 
         assert mock_sub.call_count == 1
@@ -183,14 +206,19 @@ def test_uses_default_config_if_present(mock_sys_exit):
         open(DEFAULT_CONFIG_FILE_NAME, "a").close()
 
     try:
-        with mock_sys_exit(expected_exit_code=0), patch.object(
+        with (
-            sys,
+            mock_sys_exit(expected_exit_code=0),
-            "argv",
+            patch.object(
-            ["ytdl-sub", "sub", "--log-level", "verbose"],
+                sys,
-        ), patch(
+                "argv",
-            "ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files"
+                ["ytdl-sub", "sub", "--log-level", "verbose"],
-        ) as mock_sub, patch.object(
+            ),
-            ConfigFile, "from_file_path", new=lambda _: ConfigFile(name="test default", value={})
+            patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub,
+            patch.object(
+                ConfigFile,
+                "from_file_path",
+                new=lambda _: ConfigFile(name="test default", value={}),
+            ),
         ):
             main()
 
@@ -204,11 +232,15 @@ def test_uses_default_config_if_present(mock_sys_exit):
 
 
 def test_no_positional_arg_command(mock_sys_exit, tv_show_config_path):
-    with mock_sys_exit(expected_exit_code=1), patch.object(
+    with (
-        sys,
+        mock_sys_exit(expected_exit_code=1),
-        "argv",
+        patch.object(
-        ["ytdl-sub", "-c", tv_show_config_path, "--log-level", "verbose"],
+            sys,
-    ), patch.object(logging.Logger, "error") as mock_error:
+            "argv",
+            ["ytdl-sub", "-c", tv_show_config_path, "--log-level", "verbose"],
+        ),
+        patch.object(logging.Logger, "error") as mock_error,
+    ):
         main()
 
         assert mock_error.call_count == 1
@@ -216,11 +248,15 @@ def test_no_positional_arg_command(mock_sys_exit, tv_show_config_path):
 
 
 def test_bad_config_path(mock_sys_exit):
-    with mock_sys_exit(expected_exit_code=1), patch.object(
+    with (
-        sys,
+        mock_sys_exit(expected_exit_code=1),
-        "argv",
+        patch.object(
-        ["ytdl-sub", "-c", "does_not_exist.yaml", "sub", "--log-level", "verbose"],
+            sys,
-    ), patch.object(logging.Logger, "error") as mock_error:
+            "argv",
+            ["ytdl-sub", "-c", "does_not_exist.yaml", "sub", "--log-level", "verbose"],
+        ),
+        patch.object(logging.Logger, "error") as mock_error,
+    ):
         main()
 
         assert mock_error.call_count == 1

tests/unit/plugins/test_throttle_protection.py

@@ -39,23 +39,29 @@ class TestThrottleProtectionPlugin:
             preset_dict=preset_dict,
         )
 
-        with mock_download_collection_entries(
+        with (
-            is_youtube_channel=False, num_urls=1, is_extracted_audio=False
+            mock_download_collection_entries(
-        ), assert_logs(
+                is_youtube_channel=False, num_urls=1, is_extracted_audio=False
-            logger=throttle_protection_logger,
+            ),
-            expected_message="Sleeping between downloads for %0.2f seconds",
+            assert_logs(
-            log_level="debug",
+                logger=throttle_protection_logger,
-            expected_occurrences=4,
+                expected_message="Sleeping between downloads for %0.2f seconds",
+                log_level="debug",
+                expected_occurrences=4,
+            ),
         ):
             _ = subscription.download(dry_run=False)
 
-        with mock_download_collection_entries(
+        with (
-            is_youtube_channel=False, num_urls=1, is_extracted_audio=False
+            mock_download_collection_entries(
-        ), assert_logs(
+                is_youtube_channel=False, num_urls=1, is_extracted_audio=False
-            logger=throttle_protection_logger,
+            ),
-            expected_message="Sleeping between subscriptions for %0.2f seconds",
+            assert_logs(
-            log_level="debug",
+                logger=throttle_protection_logger,
-            expected_occurrences=1,
+                expected_message="Sleeping between subscriptions for %0.2f seconds",
+                log_level="debug",
+                expected_occurrences=1,
+            ),
         ):
             _ = subscription.download(dry_run=False)
 
@@ -105,12 +111,15 @@ class TestThrottleProtectionPlugin:
             preset_dict=preset_dict,
         )
 
-        with mock_download_collection_entries(
+        with (
-            is_youtube_channel=False, num_urls=1, is_extracted_audio=False
+            mock_download_collection_entries(
-        ), assert_logs(
+                is_youtube_channel=False, num_urls=1, is_extracted_audio=False
-            logger=throttle_protection_logger,
+            ),
-            expected_message="Sleeping between downloads for %0.2f seconds",
+            assert_logs(
-            log_level="debug",
+                logger=throttle_protection_logger,
-            expected_occurrences=0,
+                expected_message="Sleeping between downloads for %0.2f seconds",
+                log_level="debug",
+                expected_occurrences=0,
+            ),
         ):
             _ = subscription.download(dry_run=False)

tests/unit/script/functions/test_regex_functions.py

@@ -43,3 +43,19 @@ class TestNumericFunctions:
     def test_regex_fullmatch(self, values: str, expected_output: str):
         output = single_variable_output(f"{{%regex_fullmatch({values})}}")
         assert output == expected_output
+
+    @pytest.mark.parametrize(
+        "values, expected_output",
+        [
+            ("'[^A-Za-z0-9 ]', '', 'This title is AWESOME!!'", "This title is AWESOME"),
+            ("'\s+', '_', 'Consolidate   spaces'", "Consolidate_spaces"),
+            (
+                "'(words) are (reordered)', '\\2 are \\1', 'Oh words are reordered'",
+                "Oh reordered are words",
+            ),
+            ("'MATCH', '', 'matcha is great'", "matcha is great"),
+        ],
+    )
+    def test_regex_sub(self, values: str, expected_output: str):
+        output = single_variable_output(f"{{%regex_sub({values})}}")
+        assert output == expected_output

tools/docgen/docgen.py

@@ -1,7 +1,8 @@
+import os
 from abc import abstractmethod
 from pathlib import Path
 
-REGENERATE_DOCS: bool = False
+REGENERATE_DOCS: bool = bool(os.environ.get("REGENERATE_DOCS", 0))
 
 
 class DocGen: