diff --git a/docs/source/guides/getting_started/first_config.rst b/docs/source/guides/getting_started/first_config.rst index 0f98b84e..fffbaa85 100644 --- a/docs/source/guides/getting_started/first_config.rst +++ b/docs/source/guides/getting_started/first_config.rst @@ -242,3 +242,22 @@ Be sure to tell ytdl-sub to use your config by using the argument ``--config If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``, it will use it by default. + +Visualizing a subscription in Preset form +----------------------------------------- + +Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions. +You can unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*. + +.. code-block:: bash + + ytdl-sub inspect --config /path/to/config.yaml --match "BBC News" /path/to/subscriptions.yaml + +This can be utilized for numerous purposes including: + +* Ensuring your custom preset is getting applied correctly. +* Figuring out which variables set things like file names, metadata, etc. +* Understanding how subscription syntax translates to preset representation. + +The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will +present the subscription's raw layout with no fill. \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst index cb5b827e..dc59cc99 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -112,3 +112,35 @@ Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments. .. code-block:: ytdl-sub cli-to-sub [YT-DLP ARGS] + +Inspect +------- +Inspect a single subscription's underlying preset representation. +This can be utilized for numerous purposes including: + +* Ensuring your custom preset is getting applied correctly. +* Figuring out which variables set things like file names, metadata, etc. +* Understanding how subscription syntax translates to preset representation. + +Usage: + +.. code-block:: bash + + ytdl-sub inspect --match "Game Chops" -mock 'title=Lets Play' examples/music_subscriptions.yaml + +.. code-block:: text + :caption: Additional Options + + -l 0,1,2,3, --level 0,1,2,3 + level of inspection to perform: + 0 - original present the subscription as-is + 1 - fill fill in defined values + 2 - resolve resolve all possible variables (default) + 3 - internal resolve all variables to their internal representation + + -m MATCH [MATCH ...], --match MATCH [MATCH ...] + match subscription names to one or more substrings, and only run those subscriptions + -o DL_OVERRIDE, --dl-override DL_OVERRIDE + override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3' + -k VAR=VALUE, --mock VAR=VALUE + ability to mock one or more variable values, i.e. --mock 'title=Lets Play' diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index ade74a42..5fdf868d 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -17,12 +17,15 @@ from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub from ytdl_sub.cli.parsers.dl import DownloadArgsParser from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME, parser from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.validators.variable_validation import ResolutionLevel from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled, ValidationException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_lock import working_directory_lock from ytdl_sub.utils.logger import Logger +# pylint: disable=too-many-branches + logger = Logger.get() # View is a command to run a simple dry-run on a URL using the `_view` preset. @@ -202,6 +205,44 @@ def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Su return subscription +def _parse_inspect_mocks(mocks: Optional[List[str]]) -> Dict[str, str]: + out: Dict[str, str] = {} + for mock in mocks or []: + spl = mock.split("=", 1) + if len(spl) == 1: + raise ValidationException("inspect mock must be in the form of VAR=VALUE") + out[spl[0].strip()] = spl[1] + + return out + + +def _inspect( + config: ConfigFile, + subscription_paths: List[str], + subscription_matches: List[str], + subscription_override_dict: Dict, + inspection_level: int, + mocks: Dict[str, str], +) -> None: + + subscriptions: List[Subscription] = [] + for path in subscription_paths: + subscriptions += Subscription.from_file_path( + config=config, + subscription_path=path, + subscription_matches=subscription_matches, + subscription_override_dict=subscription_override_dict, + ) + + if len(subscriptions) > 1: + print( + "inspect can only inspect a single subscription. Use --match to filter for a single one" + ) + return + + print(subscriptions[0].resolved_yaml(resolution_level=inspection_level, mocks=mocks)) + + def main() -> List[Subscription]: """ Entrypoint for ytdl-sub, without the error handling @@ -228,6 +269,23 @@ def main() -> List[Subscription]: subscriptions: List[Subscription] = [] + if args.subparser == "inspect": + subscription_override_dict = {} + if args.dl_override: + subscription_override_dict = DownloadArgsParser.from_dl_override( + override=args.dl_override, config=config + ).to_subscription_dict() + + _inspect( + config=config, + subscription_paths=args.subscription_paths, + subscription_matches=args.match, + subscription_override_dict=subscription_override_dict, + inspection_level=ResolutionLevel.level_number(args.inspection_level), + mocks=_parse_inspect_mocks(args.mock), + ) + return [] + # If transaction log file is specified, make sure we can open it _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log) diff --git a/src/ytdl_sub/cli/parsers/main.py b/src/ytdl_sub/cli/parsers/main.py index eb9b49ca..5fecb463 100644 --- a/src/ytdl_sub/cli/parsers/main.py +++ b/src/ytdl_sub/cli/parsers/main.py @@ -1,6 +1,6 @@ import argparse import dataclasses -from typing import List +from typing import Dict, List from ytdl_sub import __local_version__ from ytdl_sub.utils.logger import LoggerLevels @@ -236,3 +236,77 @@ view_parser.add_argument("url", help="URL to view source variables for") ################################################################################################### # CLI-TO-SUB PARSER cli_to_sub_parser = subparsers.add_parser("cli-to-sub") + +################################################################################################### +# INSPECT PARSER + + +class InspectArguments: + LEVEL = CLIArgument( + short="-l", + long="--level", + ) + LevelChoices: Dict[str, str] = { + "0": "original", + "1": "fill", + "2": "resolve", + "3": "internal", + } + + MOCK = CLIArgument( + short="-k", + long="--mock", + ) + + +inspect_parser = subparsers.add_parser("inspect", formatter_class=argparse.RawTextHelpFormatter) +inspect_parser.add_argument( + InspectArguments.LEVEL.short, + InspectArguments.LEVEL.long, + metavar=",".join(str(i) for i in range(4)), + type=str, + help="""level of inspection to perform: + 0 - original present the subscription as-is + 1 - fill fill in defined values + 2 - resolve resolve all possible variables (default) + 3 - internal resolve all variables to their internal representation + """, + default="resolve", + choices=list(InspectArguments.LevelChoices.keys()) + + list(InspectArguments.LevelChoices.values()), + dest="inspection_level", +) +inspect_parser.add_argument( + MainArguments.MATCH.short, + MainArguments.MATCH.long, + dest="match", + nargs="+", + action="extend", + type=str, + help="match subscription names to one or more substrings, and only run those subscriptions", + default=[], +) + +inspect_parser.add_argument( + "subscription_paths", + metavar="SUBPATH", + nargs="*", + help="path to subscription files, uses subscriptions.yaml if not provided", + default=["subscriptions.yaml"], +) + +inspect_parser.add_argument( + SubArguments.OVERRIDE.short, + SubArguments.OVERRIDE.long, + type=str, + help="override all subscription config values using `dl` syntax, " + "i.e. --dl-override='--ytdl_options.max_downloads 3'", +) + +inspect_parser.add_argument( + InspectArguments.MOCK.short, + InspectArguments.MOCK.long, + metavar="VAR=VALUE", + action="append", + help="ability to mock one or more variable values, i.e. --mock 'title=Lets Play'", +) diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index 8fbac314..e682dee2 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Set +from typing import Dict, List, Optional, Set from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_mapping import PluginMapping @@ -35,6 +35,21 @@ class ResolutionLevel: return "internal" raise ValueError("Invalid resolution level") + @classmethod + def level_number(cls, resolution_arg: str) -> int: + """ + Numeric resolution level + """ + if resolution_arg in ("0", "original"): + return 0 + if resolution_arg in ("1", "fill"): + return 1 + if resolution_arg in ("2", "resolve"): + return 2 + if resolution_arg in ("3", "internal"): + return 3 + raise ValueError("Invalid resolution level") + @classmethod def all(cls) -> List[int]: """ @@ -53,7 +68,7 @@ class VariableValidation: if name not in self.unresolved_variables and not name.endswith("_sanitized") } - def _apply_resolution_level(self) -> None: + def _apply_resolution_level(self, mocks: Optional[Dict[str, str]]) -> None: if self._resolution_level == ResolutionLevel.FILL: self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True) # Only partial resolve definitions that are already resolved @@ -71,6 +86,16 @@ class VariableValidation: else: raise ValueError("Invalid resolution level for validation") + if mocks is not None: + for mock_name in mocks.keys(): + if mock_name in self.unresolved_variables: + self.unresolved_variables.remove(mock_name) + + self.script.add( + variables=mocks, + unresolvable=self.unresolved_variables, + ) + self.script = self.script.resolve_partial( unresolvable=self.unresolved_variables, output_filter=self._get_resolve_partial_filter(), @@ -83,6 +108,7 @@ class VariableValidation: output_options: OutputOptions, plugins: PresetPlugins, resolution_level: int = ResolutionLevel.RESOLVE, + mocks: Optional[Dict[str, str]] = None, ): self.overrides = overrides self.downloader_options = downloader_options @@ -100,8 +126,7 @@ class VariableValidation: additional_options=[self.output_options, self.downloader_options] ) self._resolution_level = resolution_level - - self._apply_resolution_level() + self._apply_resolution_level(mocks=mocks) def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None: """ @@ -114,6 +139,20 @@ class VariableValidation: self.unresolved_runtime_variables -= added_variables | modified_variables + def _output_override_variables(self) -> Dict: + output = {} + for name in self.overrides.keys: + value = self.script.definition_of(name) + if name in self.script.function_names: + # Keep custom functions as-is + output[name] = self.overrides.dict_with_format_strings[name] + elif resolved := value.maybe_resolvable: + output[name] = resolved.native + else: + output[name] = ScriptUtils.to_native_script(value) + + return output + def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict: """ Validate variables resolve as plugins are executed, and return @@ -174,19 +213,6 @@ class VariableValidation: if url_output["url"]: resolved_subscription["download"].append(url_output) - # TODO: make function - resolved_subscription["overrides"] = {} - for name in self.overrides.keys: - value = self.script.definition_of(name) - if name in self.script.function_names: - # Keep custom functions as-is - resolved_subscription["overrides"][name] = self.overrides.dict_with_format_strings[ - name - ] - elif resolved := value.maybe_resolvable: - resolved_subscription["overrides"][name] = resolved.native - else: - resolved_subscription["overrides"][name] = ScriptUtils.to_native_script(value) + resolved_subscription["overrides"] = self._output_override_variables() - assert not self.unresolved_runtime_variables return resolved_subscription diff --git a/src/ytdl_sub/main.py b/src/ytdl_sub/main.py index 44258a5c..ad1c54f3 100644 --- a/src/ytdl_sub/main.py +++ b/src/ytdl_sub/main.py @@ -1,7 +1,7 @@ import sys from ytdl_sub.cli.parsers.main import parser -from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.logger import Logger, LoggerLevels def _main() -> int: @@ -10,6 +10,11 @@ def _main() -> int: args, _ = parser.parse_known_args() Logger.set_log_level(log_level_name=args.ytdl_sub_log_level) + # Suppress all logs during inspection since the output of the subcommand itself + # is all that is necessary + if args.subparser == "inspect": + Logger.set_log_level(log_level_name=LoggerLevels.QUIET.name) + # pylint: disable=import-outside-toplevel import ytdl_sub.cli.entrypoint diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 55b96240..d991b287 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -1,6 +1,6 @@ from abc import ABC from pathlib import Path -from typing import Optional +from typing import Dict, Optional from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.config.overrides import Overrides @@ -254,7 +254,11 @@ class BaseSubscription(ABC): """ return self._preset_options.yaml(subscription_only=False) - def resolved_yaml(self, resolution_level: int = ResolutionLevel.RESOLVE) -> str: + def resolved_yaml( + self, + resolution_level: int = ResolutionLevel.RESOLVE, + mocks: Optional[Dict[str, str]] = None, + ) -> str: """ Returns ------- @@ -269,5 +273,6 @@ class BaseSubscription(ABC): output_options=self.output_options, plugins=self.plugins, resolution_level=resolution_level, + mocks=mocks, ).ensure_proper_usage(partial_resolve_formatters=True) return dump_yaml(out) diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py index ee5c0d2a..0e228327 100644 --- a/src/ytdl_sub/utils/script.py +++ b/src/ytdl_sub/utils/script.py @@ -180,7 +180,6 @@ class ScriptUtils: return "" if top_level else "''" quote = cls._get_quote_char(arg.native) - return arg.native if top_level else f"{quote}{arg.native}{quote}" arg = cls._maybe_to_optimized_sanitize(arg) diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 780976ba..400e3d82 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -293,7 +293,7 @@ def _validate_formatter( ) if maybe_resolved := parsed.maybe_resolvable: - return formatter_validator.post_process(maybe_resolved) + return formatter_validator.post_process(maybe_resolved.native) return ScriptUtils.to_native_script(parsed) except RuntimeException as exc: diff --git a/tests/conftest.py b/tests/conftest.py index 0a2a14e4..dbf7378e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,7 +79,7 @@ def working_directory() -> str: @pytest.fixture() def output_directory() -> str: with tempfile.TemporaryDirectory() as temp_dir: - yield temp_dir + yield os.path.normpath(temp_dir) @pytest.fixture() @@ -165,10 +165,11 @@ def subscription_yaml_file_generator() -> Callable: with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file: tmp_file.write(json.dumps(yaml_dict).encode("utf-8")) + file_path = os.path.normpath(tmp_file.name) try: - yield tmp_file.name + yield file_path finally: - FileHandler.delete(tmp_file.name) + FileHandler.delete(file_path) return _subscription_yaml_file_generator @@ -269,10 +270,11 @@ def default_config_path(default_config) -> str: with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file: tmp_file.write(json.dumps(default_config._value).encode("utf-8")) + file_path = os.path.normpath(tmp_file.name) try: - yield tmp_file.name + yield file_path finally: - FileHandler.delete(tmp_file.name) + FileHandler.delete(file_path) @pytest.fixture() diff --git a/tests/integration/cli/test_inspect.py b/tests/integration/cli/test_inspect.py new file mode 100644 index 00000000..d51b3a3f --- /dev/null +++ b/tests/integration/cli/test_inspect.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import pytest +import yaml +from conftest import mock_run_from_cli +from unit.config.test_subscription_resolution import compare_resolved_yaml + +from ytdl_sub.config.validators.variable_validation import ResolutionLevel +from ytdl_sub.utils.system import IS_WINDOWS + + +class TestInspect: + @pytest.mark.parametrize("config_provided", [True, False]) + @pytest.mark.parametrize( + "inspect_level", ["0", "original", "1", "fill", "2", "resolve", "3", "internal"] + ) + def test_inspect_command( + self, + capsys, + default_config_path: str, + tv_show_subscriptions_path: Path, + output_directory: str, + config_provided: bool, + inspect_level: str, + ): + # TODO: fix mock_run_from_cli in windows to handle file paths correctly + if IS_WINDOWS: + return + + # Shares same test fixture as `test_subscription_resolution.py` + args = f"--config {default_config_path} " if config_provided else "" + args += f"inspect {tv_show_subscriptions_path} --match 'NOVA PBS' " + args += f"--level {inspect_level} " + + subscriptions = mock_run_from_cli(args=args) + assert len(subscriptions) == 0 + + out = yaml.safe_load(capsys.readouterr().out) + compare_resolved_yaml( + out=out, + output_directory=output_directory, + subscription_name="NOVA PBS", + preset_type="tv_show", + resolution_level=ResolutionLevel.level_number(resolution_arg=inspect_level), + ) diff --git a/tests/unit/config/test_subscription_resolution.py b/tests/unit/config/test_subscription_resolution.py index 12781bb8..dd6b77e8 100644 --- a/tests/unit/config/test_subscription_resolution.py +++ b/tests/unit/config/test_subscription_resolution.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Dict import pytest import yaml @@ -10,12 +11,13 @@ from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.file_path import FilePathTruncater -def _ensure_resolved_yaml( - sub: Subscription, output_directory: str, preset_type: str, resolution_level: int +def compare_resolved_yaml( + out: Dict, + output_directory: str, + subscription_name: str, + preset_type: str, + resolution_level: int, ) -> None: - output_yaml = sub.resolved_yaml(resolution_level=resolution_level) - out = yaml.safe_load(output_yaml) - expected_out_filename = ( f"{preset_type}/inspect_sub_{ResolutionLevel.name_of(resolution_level)}.json" ) @@ -24,7 +26,7 @@ def _ensure_resolved_yaml( if resolution_level > ResolutionLevel.ORIGINAL: output_path = Path(output_directory) if "tv_show_directory" in expected_out["overrides"]: - output_path = output_path / sub.name + output_path = output_path / subscription_name expected_out["output_options"]["output_directory"] = FilePathTruncater.to_native_filepath( str(output_path) @@ -40,6 +42,21 @@ def _ensure_resolved_yaml( assert out == expected_out +def _ensure_resolved_yaml( + sub: Subscription, output_directory: str, preset_type: str, resolution_level: int +) -> None: + output_yaml = sub.resolved_yaml(resolution_level=resolution_level) + out = yaml.safe_load(output_yaml) + + compare_resolved_yaml( + out=out, + output_directory=output_directory, + subscription_name=sub.name, + preset_type=preset_type, + resolution_level=resolution_level, + ) + + @pytest.mark.parametrize("resolution_level", ResolutionLevel.all()) class TestResolution: def test_resolution_tv_show(