[FEATURE] inspect subcommand (#1407)

Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions.
You can now unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*.

```
ytdl-sub inspect --config /path/to/config.yaml --match "BBC News" /path/to/subscriptions.yaml
```

This can be utilized for numerous purposes including:

* Ensuring your custom preset is getting applied correctly.
* Figuring out which variables set things like file names, metadata, etc.
* Understanding how subscription syntax translates to preset representation.

The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will present the subscription's raw layout with no fill.

docs/source/guides/getting_started/first_config.rst

@@ -242,3 +242,22 @@ Be sure to tell ytdl-sub to use your config by using the argument ``--config
 
 If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``,
 it will use it by default.
+
+Visualizing a subscription in Preset form
+-----------------------------------------
+
+Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions.
+You can unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*.
+
+.. code-block:: bash
+
+   ytdl-sub inspect --config /path/to/config.yaml --match "BBC News" /path/to/subscriptions.yaml
+
+This can be utilized for numerous purposes including:
+
+* Ensuring your custom preset is getting applied correctly.
+* Figuring out which variables set things like file names, metadata, etc.
+* Understanding how subscription syntax translates to preset representation.
+
+The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will
+present the subscription's raw layout with no fill.

docs/source/usage.rst

@@ -112,3 +112,35 @@ Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments.
 .. code-block::
 
    ytdl-sub cli-to-sub [YT-DLP ARGS]
+
+Inspect
+-------
+Inspect a single subscription's underlying preset representation.
+This can be utilized for numerous purposes including:
+
+* Ensuring your custom preset is getting applied correctly.
+* Figuring out which variables set things like file names, metadata, etc.
+* Understanding how subscription syntax translates to preset representation.
+
+Usage:
+
+.. code-block:: bash
+
+   ytdl-sub inspect --match "Game Chops" -mock 'title=Lets Play' examples/music_subscriptions.yaml
+
+.. code-block:: text
+  :caption: Additional Options
+
+     -l 0,1,2,3, --level 0,1,2,3
+                           level of inspection to perform:
+                               0 - original   present the subscription as-is
+                               1 - fill       fill in defined values
+                               2 - resolve    resolve all possible variables (default)
+                               3 - internal   resolve all variables to their internal representation
+
+     -m MATCH [MATCH ...], --match MATCH [MATCH ...]
+                           match subscription names to one or more substrings, and only run those subscriptions
+     -o DL_OVERRIDE, --dl-override DL_OVERRIDE
+                           override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
+     -k VAR=VALUE, --mock VAR=VALUE
+                           ability to mock one or more variable values, i.e. --mock 'title=Lets Play'

src/ytdl_sub/cli/entrypoint.py

@@ -17,12 +17,15 @@ from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
 from ytdl_sub.cli.parsers.dl import DownloadArgsParser
 from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME, parser
 from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.config.validators.variable_validation import ResolutionLevel
 from ytdl_sub.subscriptions.subscription import Subscription
 from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled, ValidationException
 from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.file_lock import working_directory_lock
 from ytdl_sub.utils.logger import Logger
 
+# pylint: disable=too-many-branches
+
 logger = Logger.get()
 
 # View is a command to run a simple dry-run on a URL using the `_view` preset.
@@ -202,6 +205,44 @@ def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Su
     return subscription
 
 
+def _parse_inspect_mocks(mocks: Optional[List[str]]) -> Dict[str, str]:
+    out: Dict[str, str] = {}
+    for mock in mocks or []:
+        spl = mock.split("=", 1)
+        if len(spl) == 1:
+            raise ValidationException("inspect mock must be in the form of VAR=VALUE")
+        out[spl[0].strip()] = spl[1]
+
+    return out
+
+
+def _inspect(
+    config: ConfigFile,
+    subscription_paths: List[str],
+    subscription_matches: List[str],
+    subscription_override_dict: Dict,
+    inspection_level: int,
+    mocks: Dict[str, str],
+) -> None:
+
+    subscriptions: List[Subscription] = []
+    for path in subscription_paths:
+        subscriptions += Subscription.from_file_path(
+            config=config,
+            subscription_path=path,
+            subscription_matches=subscription_matches,
+            subscription_override_dict=subscription_override_dict,
+        )
+
+    if len(subscriptions) > 1:
+        print(
+            "inspect can only inspect a single subscription. Use --match to filter for a single one"
+        )
+        return
+
+    print(subscriptions[0].resolved_yaml(resolution_level=inspection_level, mocks=mocks))
+
+
 def main() -> List[Subscription]:
     """
     Entrypoint for ytdl-sub, without the error handling
@@ -228,6 +269,23 @@ def main() -> List[Subscription]:
 
     subscriptions: List[Subscription] = []
 
+    if args.subparser == "inspect":
+        subscription_override_dict = {}
+        if args.dl_override:
+            subscription_override_dict = DownloadArgsParser.from_dl_override(
+                override=args.dl_override, config=config
+            ).to_subscription_dict()
+
+        _inspect(
+            config=config,
+            subscription_paths=args.subscription_paths,
+            subscription_matches=args.match,
+            subscription_override_dict=subscription_override_dict,
+            inspection_level=ResolutionLevel.level_number(args.inspection_level),
+            mocks=_parse_inspect_mocks(args.mock),
+        )
+        return []
+
     # If transaction log file is specified, make sure we can open it
     _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)
 

src/ytdl_sub/cli/parsers/main.py

@@ -1,6 +1,6 @@
 import argparse
 import dataclasses
-from typing import List
+from typing import Dict, List
 
 from ytdl_sub import __local_version__
 from ytdl_sub.utils.logger import LoggerLevels
@@ -236,3 +236,77 @@ view_parser.add_argument("url", help="URL to view source variables for")
 ###################################################################################################
 # CLI-TO-SUB PARSER
 cli_to_sub_parser = subparsers.add_parser("cli-to-sub")
+
+###################################################################################################
+# INSPECT PARSER
+
+
+class InspectArguments:
+    LEVEL = CLIArgument(
+        short="-l",
+        long="--level",
+    )
+    LevelChoices: Dict[str, str] = {
+        "0": "original",
+        "1": "fill",
+        "2": "resolve",
+        "3": "internal",
+    }
+
+    MOCK = CLIArgument(
+        short="-k",
+        long="--mock",
+    )
+
+
+inspect_parser = subparsers.add_parser("inspect", formatter_class=argparse.RawTextHelpFormatter)
+inspect_parser.add_argument(
+    InspectArguments.LEVEL.short,
+    InspectArguments.LEVEL.long,
+    metavar=",".join(str(i) for i in range(4)),
+    type=str,
+    help="""level of inspection to perform:
+    0 - original   present the subscription as-is
+    1 - fill       fill in defined values
+    2 - resolve    resolve all possible variables (default)
+    3 - internal   resolve all variables to their internal representation
+    """,
+    default="resolve",
+    choices=list(InspectArguments.LevelChoices.keys())
+    + list(InspectArguments.LevelChoices.values()),
+    dest="inspection_level",
+)
+inspect_parser.add_argument(
+    MainArguments.MATCH.short,
+    MainArguments.MATCH.long,
+    dest="match",
+    nargs="+",
+    action="extend",
+    type=str,
+    help="match subscription names to one or more substrings, and only run those subscriptions",
+    default=[],
+)
+
+inspect_parser.add_argument(
+    "subscription_paths",
+    metavar="SUBPATH",
+    nargs="*",
+    help="path to subscription files, uses subscriptions.yaml if not provided",
+    default=["subscriptions.yaml"],
+)
+
+inspect_parser.add_argument(
+    SubArguments.OVERRIDE.short,
+    SubArguments.OVERRIDE.long,
+    type=str,
+    help="override all subscription config values using `dl` syntax, "
+    "i.e. --dl-override='--ytdl_options.max_downloads 3'",
+)
+
+inspect_parser.add_argument(
+    InspectArguments.MOCK.short,
+    InspectArguments.MOCK.long,
+    metavar="VAR=VALUE",
+    action="append",
+    help="ability to mock one or more variable values, i.e. --mock 'title=Lets Play'",
+)

src/ytdl_sub/config/validators/variable_validation.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Set
+from typing import Dict, List, Optional, Set
 
 from ytdl_sub.config.overrides import Overrides
 from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
@@ -35,6 +35,21 @@ class ResolutionLevel:
             return "internal"
         raise ValueError("Invalid resolution level")
 
+    @classmethod
+    def level_number(cls, resolution_arg: str) -> int:
+        """
+        Numeric resolution level
+        """
+        if resolution_arg in ("0", "original"):
+            return 0
+        if resolution_arg in ("1", "fill"):
+            return 1
+        if resolution_arg in ("2", "resolve"):
+            return 2
+        if resolution_arg in ("3", "internal"):
+            return 3
+        raise ValueError("Invalid resolution level")
+
     @classmethod
     def all(cls) -> List[int]:
         """
@@ -53,7 +68,7 @@ class VariableValidation:
             if name not in self.unresolved_variables and not name.endswith("_sanitized")
         }
 
-    def _apply_resolution_level(self) -> None:
+    def _apply_resolution_level(self, mocks: Optional[Dict[str, str]]) -> None:
         if self._resolution_level == ResolutionLevel.FILL:
             self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
             # Only partial resolve definitions that are already resolved
@@ -71,6 +86,16 @@ class VariableValidation:
         else:
             raise ValueError("Invalid resolution level for validation")
 
+        if mocks is not None:
+            for mock_name in mocks.keys():
+                if mock_name in self.unresolved_variables:
+                    self.unresolved_variables.remove(mock_name)
+
+            self.script.add(
+                variables=mocks,
+                unresolvable=self.unresolved_variables,
+            )
+
         self.script = self.script.resolve_partial(
             unresolvable=self.unresolved_variables,
             output_filter=self._get_resolve_partial_filter(),
@@ -83,6 +108,7 @@ class VariableValidation:
         output_options: OutputOptions,
         plugins: PresetPlugins,
         resolution_level: int = ResolutionLevel.RESOLVE,
+        mocks: Optional[Dict[str, str]] = None,
     ):
         self.overrides = overrides
         self.downloader_options = downloader_options
@@ -100,8 +126,7 @@ class VariableValidation:
             additional_options=[self.output_options, self.downloader_options]
         )
         self._resolution_level = resolution_level
-
+        self._apply_resolution_level(mocks=mocks)
-        self._apply_resolution_level()
 
     def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
         """
@@ -114,6 +139,20 @@ class VariableValidation:
 
         self.unresolved_runtime_variables -= added_variables | modified_variables
 
+    def _output_override_variables(self) -> Dict:
+        output = {}
+        for name in self.overrides.keys:
+            value = self.script.definition_of(name)
+            if name in self.script.function_names:
+                # Keep custom functions as-is
+                output[name] = self.overrides.dict_with_format_strings[name]
+            elif resolved := value.maybe_resolvable:
+                output[name] = resolved.native
+            else:
+                output[name] = ScriptUtils.to_native_script(value)
+
+        return output
+
     def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict:
         """
         Validate variables resolve as plugins are executed, and return
@@ -174,19 +213,6 @@ class VariableValidation:
             if url_output["url"]:
                 resolved_subscription["download"].append(url_output)
 
-        # TODO: make function
+        resolved_subscription["overrides"] = self._output_override_variables()
-        resolved_subscription["overrides"] = {}
-        for name in self.overrides.keys:
-            value = self.script.definition_of(name)
-            if name in self.script.function_names:
-                # Keep custom functions as-is
-                resolved_subscription["overrides"][name] = self.overrides.dict_with_format_strings[
-                    name
-                ]
-            elif resolved := value.maybe_resolvable:
-                resolved_subscription["overrides"][name] = resolved.native
-            else:
-                resolved_subscription["overrides"][name] = ScriptUtils.to_native_script(value)
 
-        assert not self.unresolved_runtime_variables
         return resolved_subscription

src/ytdl_sub/main.py

@@ -1,7 +1,7 @@
 import sys
 
 from ytdl_sub.cli.parsers.main import parser
-from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.logger import Logger, LoggerLevels
 
 
 def _main() -> int:
@@ -10,6 +10,11 @@ def _main() -> int:
     args, _ = parser.parse_known_args()
     Logger.set_log_level(log_level_name=args.ytdl_sub_log_level)
 
+    # Suppress all logs during inspection since the output of the subcommand itself
+    # is all that is necessary
+    if args.subparser == "inspect":
+        Logger.set_log_level(log_level_name=LoggerLevels.QUIET.name)
+
     # pylint: disable=import-outside-toplevel
     import ytdl_sub.cli.entrypoint
 

src/ytdl_sub/subscriptions/base_subscription.py

@@ -1,6 +1,6 @@
 from abc import ABC
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 from ytdl_sub.config.config_validator import ConfigOptions
 from ytdl_sub.config.overrides import Overrides
@@ -254,7 +254,11 @@ class BaseSubscription(ABC):
         """
         return self._preset_options.yaml(subscription_only=False)
 
-    def resolved_yaml(self, resolution_level: int = ResolutionLevel.RESOLVE) -> str:
+    def resolved_yaml(
+        self,
+        resolution_level: int = ResolutionLevel.RESOLVE,
+        mocks: Optional[Dict[str, str]] = None,
+    ) -> str:
         """
         Returns
         -------
@@ -269,5 +273,6 @@ class BaseSubscription(ABC):
             output_options=self.output_options,
             plugins=self.plugins,
             resolution_level=resolution_level,
+            mocks=mocks,
         ).ensure_proper_usage(partial_resolve_formatters=True)
         return dump_yaml(out)

src/ytdl_sub/utils/script.py

@@ -180,7 +180,6 @@ class ScriptUtils:
                 return "" if top_level else "''"
 
             quote = cls._get_quote_char(arg.native)
-
             return arg.native if top_level else f"{quote}{arg.native}{quote}"
 
         arg = cls._maybe_to_optimized_sanitize(arg)

src/ytdl_sub/validators/string_formatter_validators.py

@@ -293,7 +293,7 @@ def _validate_formatter(
             )
 
         if maybe_resolved := parsed.maybe_resolvable:
-            return formatter_validator.post_process(maybe_resolved)
+            return formatter_validator.post_process(maybe_resolved.native)
 
         return ScriptUtils.to_native_script(parsed)
     except RuntimeException as exc:

tests/conftest.py

@@ -79,7 +79,7 @@ def working_directory() -> str:
 @pytest.fixture()
 def output_directory() -> str:
     with tempfile.TemporaryDirectory() as temp_dir:
-        yield temp_dir
+        yield os.path.normpath(temp_dir)
 
 
 @pytest.fixture()
@@ -165,10 +165,11 @@ def subscription_yaml_file_generator() -> Callable:
         with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file:
             tmp_file.write(json.dumps(yaml_dict).encode("utf-8"))
 
+        file_path = os.path.normpath(tmp_file.name)
         try:
-            yield tmp_file.name
+            yield file_path
         finally:
-            FileHandler.delete(tmp_file.name)
+            FileHandler.delete(file_path)
 
     return _subscription_yaml_file_generator
 
@@ -269,10 +270,11 @@ def default_config_path(default_config) -> str:
     with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file:
         tmp_file.write(json.dumps(default_config._value).encode("utf-8"))
 
+    file_path = os.path.normpath(tmp_file.name)
     try:
-        yield tmp_file.name
+        yield file_path
     finally:
-        FileHandler.delete(tmp_file.name)
+        FileHandler.delete(file_path)
 
 
 @pytest.fixture()

tests/integration/cli/test_inspect.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+import pytest
+import yaml
+from conftest import mock_run_from_cli
+from unit.config.test_subscription_resolution import compare_resolved_yaml
+
+from ytdl_sub.config.validators.variable_validation import ResolutionLevel
+from ytdl_sub.utils.system import IS_WINDOWS
+
+
+class TestInspect:
+    @pytest.mark.parametrize("config_provided", [True, False])
+    @pytest.mark.parametrize(
+        "inspect_level", ["0", "original", "1", "fill", "2", "resolve", "3", "internal"]
+    )
+    def test_inspect_command(
+        self,
+        capsys,
+        default_config_path: str,
+        tv_show_subscriptions_path: Path,
+        output_directory: str,
+        config_provided: bool,
+        inspect_level: str,
+    ):
+        # TODO: fix mock_run_from_cli in windows to handle file paths correctly
+        if IS_WINDOWS:
+            return
+
+        # Shares same test fixture as `test_subscription_resolution.py`
+        args = f"--config {default_config_path} " if config_provided else ""
+        args += f"inspect {tv_show_subscriptions_path} --match 'NOVA PBS' "
+        args += f"--level {inspect_level} "
+
+        subscriptions = mock_run_from_cli(args=args)
+        assert len(subscriptions) == 0
+
+        out = yaml.safe_load(capsys.readouterr().out)
+        compare_resolved_yaml(
+            out=out,
+            output_directory=output_directory,
+            subscription_name="NOVA PBS",
+            preset_type="tv_show",
+            resolution_level=ResolutionLevel.level_number(resolution_arg=inspect_level),
+        )

tests/unit/config/test_subscription_resolution.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Dict
 
 import pytest
 import yaml
@@ -10,12 +11,13 @@ from ytdl_sub.subscriptions.subscription import Subscription
 from ytdl_sub.utils.file_path import FilePathTruncater
 
 
-def _ensure_resolved_yaml(
+def compare_resolved_yaml(
-    sub: Subscription, output_directory: str, preset_type: str, resolution_level: int
+    out: Dict,
+    output_directory: str,
+    subscription_name: str,
+    preset_type: str,
+    resolution_level: int,
 ) -> None:
-    output_yaml = sub.resolved_yaml(resolution_level=resolution_level)
-    out = yaml.safe_load(output_yaml)
-
     expected_out_filename = (
         f"{preset_type}/inspect_sub_{ResolutionLevel.name_of(resolution_level)}.json"
     )
@@ -24,7 +26,7 @@ def _ensure_resolved_yaml(
     if resolution_level > ResolutionLevel.ORIGINAL:
         output_path = Path(output_directory)
         if "tv_show_directory" in expected_out["overrides"]:
-            output_path = output_path / sub.name
+            output_path = output_path / subscription_name
 
         expected_out["output_options"]["output_directory"] = FilePathTruncater.to_native_filepath(
             str(output_path)
@@ -40,6 +42,21 @@ def _ensure_resolved_yaml(
     assert out == expected_out
 
 
+def _ensure_resolved_yaml(
+    sub: Subscription, output_directory: str, preset_type: str, resolution_level: int
+) -> None:
+    output_yaml = sub.resolved_yaml(resolution_level=resolution_level)
+    out = yaml.safe_load(output_yaml)
+
+    compare_resolved_yaml(
+        out=out,
+        output_directory=output_directory,
+        subscription_name=sub.name,
+        preset_type=preset_type,
+        resolution_level=resolution_level,
+    )
+
+
 @pytest.mark.parametrize("resolution_level", ResolutionLevel.all())
 class TestResolution:
     def test_resolution_tv_show(