danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] inspect subcommand (#1407)

Browse source 
Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions.
You can now unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*.

```
ytdl-sub inspect --config /path/to/config.yaml --match "BBC News" /path/to/subscriptions.yaml
```

This can be utilized for numerous purposes including:

* Ensuring your custom preset is getting applied correctly.
* Figuring out which variables set things like file names, metadata, etc.
* Understanding how subscription syntax translates to preset representation.

The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will present the subscription's raw layout with no fill.
This commit is contained in:
Jesse Bannon 2026-04-13 10:10:58 -07:00 committed by GitHub
parent e34b9b6295
commit 970c74ba45
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 317 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
docs/source/guides/getting_started/first_config.rst
View file

@ -242,3 +242,22 @@ Be sure to tell ytdl-sub to use your config by using the argument ``--config
If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``,
it will use it by default.
Visualizing a subscription in Preset form
-----------------------------------------
Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions.
You can unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*.
.. code-block:: bash
ytdl-sub inspect --config /path/to/config.yaml --match "BBC News" /path/to/subscriptions.yaml
This can be utilized for numerous purposes including:
* Ensuring your custom preset is getting applied correctly.
* Figuring out which variables set things like file names, metadata, etc.
* Understanding how subscription syntax translates to preset representation.
The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will
present the subscription's raw layout with no fill.

32
docs/source/usage.rst
View file

@ -112,3 +112,35 @@ Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments.
.. code-block::
ytdl-sub cli-to-sub [YT-DLP ARGS]
Inspect
-------
Inspect a single subscription's underlying preset representation.
This can be utilized for numerous purposes including:
* Ensuring your custom preset is getting applied correctly.
* Figuring out which variables set things like file names, metadata, etc.
* Understanding how subscription syntax translates to preset representation.
Usage:
.. code-block:: bash
ytdl-sub inspect --match "Game Chops" -mock 'title=Lets Play' examples/music_subscriptions.yaml
.. code-block:: text
:caption: Additional Options
-l 0,1,2,3, --level 0,1,2,3
level of inspection to perform:
0 - original present the subscription as-is
1 - fill fill in defined values
2 - resolve resolve all possible variables (default)
3 - internal resolve all variables to their internal representation
-m MATCH [MATCH ...], --match MATCH [MATCH ...]
match subscription names to one or more substrings, and only run those subscriptions
-o DL_OVERRIDE, --dl-override DL_OVERRIDE
override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
-k VAR=VALUE, --mock VAR=VALUE
ability to mock one or more variable values, i.e. --mock 'title=Lets Play'

58
src/ytdl_sub/cli/entrypoint.py
View file

@ -17,12 +17,15 @@ from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
from ytdl_sub.cli.parsers.dl import DownloadArgsParser
from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME, parser
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.validators.variable_validation import ResolutionLevel
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled, ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_lock import working_directory_lock
from ytdl_sub.utils.logger import Logger
# pylint: disable=too-many-branches
logger = Logger.get()
# View is a command to run a simple dry-run on a URL using the `_view` preset.
@ -202,6 +205,44 @@ def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Su
return subscription
def _parse_inspect_mocks(mocks: Optional[List[str]]) -> Dict[str, str]:
out: Dict[str, str] = {}
for mock in mocks or []:
spl = mock.split("=", 1)
if len(spl) == 1:
raise ValidationException("inspect mock must be in the form of VAR=VALUE")
out[spl[0].strip()] = spl[1]
return out
def _inspect(
config: ConfigFile,
subscription_paths: List[str],
subscription_matches: List[str],
subscription_override_dict: Dict,
inspection_level: int,
mocks: Dict[str, str],
) -> None:
subscriptions: List[Subscription] = []
for path in subscription_paths:
subscriptions += Subscription.from_file_path(
config=config,
subscription_path=path,
subscription_matches=subscription_matches,
subscription_override_dict=subscription_override_dict,
)
if len(subscriptions) > 1:
print(
"inspect can only inspect a single subscription. Use --match to filter for a single one"
)
return
print(subscriptions[0].resolved_yaml(resolution_level=inspection_level, mocks=mocks))
def main() -> List[Subscription]:
"""
Entrypoint for ytdl-sub, without the error handling
@ -228,6 +269,23 @@ def main() -> List[Subscription]:
subscriptions: List[Subscription] = []
if args.subparser == "inspect":
subscription_override_dict = {}
if args.dl_override:
subscription_override_dict = DownloadArgsParser.from_dl_override(
override=args.dl_override, config=config
).to_subscription_dict()
_inspect(
config=config,
subscription_paths=args.subscription_paths,
subscription_matches=args.match,
subscription_override_dict=subscription_override_dict,
inspection_level=ResolutionLevel.level_number(args.inspection_level),
mocks=_parse_inspect_mocks(args.mock),
)
return []
# If transaction log file is specified, make sure we can open it
_maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)

76
src/ytdl_sub/cli/parsers/main.py
View file

@ -1,6 +1,6 @@
import argparse
import dataclasses
from typing import List
from typing import Dict, List
from ytdl_sub import __local_version__
from ytdl_sub.utils.logger import LoggerLevels
@ -236,3 +236,77 @@ view_parser.add_argument("url", help="URL to view source variables for")
###################################################################################################
# CLI-TO-SUB PARSER
cli_to_sub_parser = subparsers.add_parser("cli-to-sub")
###################################################################################################
# INSPECT PARSER
class InspectArguments:
LEVEL = CLIArgument(
short="-l",
long="--level",
)
LevelChoices: Dict[str, str] = {
"0": "original",
"1": "fill",
"2": "resolve",
"3": "internal",
}
MOCK = CLIArgument(
short="-k",
long="--mock",
)
inspect_parser = subparsers.add_parser("inspect", formatter_class=argparse.RawTextHelpFormatter)
inspect_parser.add_argument(
InspectArguments.LEVEL.short,
InspectArguments.LEVEL.long,
metavar=",".join(str(i) for i in range(4)),
type=str,
help="""level of inspection to perform:
0 - original present the subscription as-is
1 - fill fill in defined values
2 - resolve resolve all possible variables (default)
3 - internal resolve all variables to their internal representation
""",
default="resolve",
choices=list(InspectArguments.LevelChoices.keys())
+ list(InspectArguments.LevelChoices.values()),
dest="inspection_level",
)
inspect_parser.add_argument(
MainArguments.MATCH.short,
MainArguments.MATCH.long,
dest="match",
nargs="+",
action="extend",
type=str,
help="match subscription names to one or more substrings, and only run those subscriptions",
default=[],
)
inspect_parser.add_argument(
"subscription_paths",
metavar="SUBPATH",
nargs="*",
help="path to subscription files, uses subscriptions.yaml if not provided",
default=["subscriptions.yaml"],
)
inspect_parser.add_argument(
SubArguments.OVERRIDE.short,
SubArguments.OVERRIDE.long,
type=str,
help="override all subscription config values using `dl` syntax, "
"i.e. --dl-override='--ytdl_options.max_downloads 3'",
)
inspect_parser.add_argument(
InspectArguments.MOCK.short,
InspectArguments.MOCK.long,
metavar="VAR=VALUE",
action="append",
help="ability to mock one or more variable values, i.e. --mock 'title=Lets Play'",
)

62
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -1,4 +1,4 @@
from typing import Dict, List, Set
from typing import Dict, List, Optional, Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_mapping import PluginMapping
@ -35,6 +35,21 @@ class ResolutionLevel:
return "internal"
raise ValueError("Invalid resolution level")
@classmethod
def level_number(cls, resolution_arg: str) -> int:
"""
Numeric resolution level
"""
if resolution_arg in ("0", "original"):
return 0
if resolution_arg in ("1", "fill"):
return 1
if resolution_arg in ("2", "resolve"):
return 2
if resolution_arg in ("3", "internal"):
return 3
raise ValueError("Invalid resolution level")
@classmethod
def all(cls) -> List[int]:
"""
@ -53,7 +68,7 @@ class VariableValidation:
if name not in self.unresolved_variables and not name.endswith("_sanitized")
}
def _apply_resolution_level(self) -> None:
def _apply_resolution_level(self, mocks: Optional[Dict[str, str]]) -> None:
if self._resolution_level == ResolutionLevel.FILL:
self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True)
# Only partial resolve definitions that are already resolved
@ -71,6 +86,16 @@ class VariableValidation:
else:
raise ValueError("Invalid resolution level for validation")
if mocks is not None:
for mock_name in mocks.keys():
if mock_name in self.unresolved_variables:
self.unresolved_variables.remove(mock_name)
self.script.add(
variables=mocks,
unresolvable=self.unresolved_variables,
)
self.script = self.script.resolve_partial(
unresolvable=self.unresolved_variables,
output_filter=self._get_resolve_partial_filter(),
@ -83,6 +108,7 @@ class VariableValidation:
output_options: OutputOptions,
plugins: PresetPlugins,
resolution_level: int = ResolutionLevel.RESOLVE,
mocks: Optional[Dict[str, str]] = None,
):
self.overrides = overrides
self.downloader_options = downloader_options
@ -100,8 +126,7 @@ class VariableValidation:
additional_options=[self.output_options, self.downloader_options]
)
self._resolution_level = resolution_level
self._apply_resolution_level()
self._apply_resolution_level(mocks=mocks)
def _add_runtime_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
"""
@ -114,6 +139,20 @@ class VariableValidation:
self.unresolved_runtime_variables -= added_variables | modified_variables
def _output_override_variables(self) -> Dict:
output = {}
for name in self.overrides.keys:
value = self.script.definition_of(name)
if name in self.script.function_names:
# Keep custom functions as-is
output[name] = self.overrides.dict_with_format_strings[name]
elif resolved := value.maybe_resolvable:
output[name] = resolved.native
else:
output[name] = ScriptUtils.to_native_script(value)
return output
def ensure_proper_usage(self, partial_resolve_formatters: bool = False) -> Dict:
"""
Validate variables resolve as plugins are executed, and return
@ -174,19 +213,6 @@ class VariableValidation:
if url_output["url"]:
resolved_subscription["download"].append(url_output)
# TODO: make function
resolved_subscription["overrides"] = {}
for name in self.overrides.keys:
value = self.script.definition_of(name)
if name in self.script.function_names:
# Keep custom functions as-is
resolved_subscription["overrides"][name] = self.overrides.dict_with_format_strings[
name
]
elif resolved := value.maybe_resolvable:
resolved_subscription["overrides"][name] = resolved.native
else:
resolved_subscription["overrides"][name] = ScriptUtils.to_native_script(value)
resolved_subscription["overrides"] = self._output_override_variables()
assert not self.unresolved_runtime_variables
return resolved_subscription

7
src/ytdl_sub/main.py
View file

@ -1,7 +1,7 @@
import sys
from ytdl_sub.cli.parsers.main import parser
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.logger import Logger, LoggerLevels
def _main() -> int:
@ -10,6 +10,11 @@ def _main() -> int:
args, _ = parser.parse_known_args()
Logger.set_log_level(log_level_name=args.ytdl_sub_log_level)
# Suppress all logs during inspection since the output of the subcommand itself
# is all that is necessary
if args.subparser == "inspect":
Logger.set_log_level(log_level_name=LoggerLevels.QUIET.name)
# pylint: disable=import-outside-toplevel
import ytdl_sub.cli.entrypoint

9
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -1,6 +1,6 @@
from abc import ABC
from pathlib import Path
from typing import Optional
from typing import Dict, Optional
from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.config.overrides import Overrides
@ -254,7 +254,11 @@ class BaseSubscription(ABC):
"""
return self._preset_options.yaml(subscription_only=False)
def resolved_yaml(self, resolution_level: int = ResolutionLevel.RESOLVE) -> str:
def resolved_yaml(
self,
resolution_level: int = ResolutionLevel.RESOLVE,
mocks: Optional[Dict[str, str]] = None,
) -> str:
"""
Returns
-------
@ -269,5 +273,6 @@ class BaseSubscription(ABC):
output_options=self.output_options,
plugins=self.plugins,
resolution_level=resolution_level,
mocks=mocks,
).ensure_proper_usage(partial_resolve_formatters=True)
return dump_yaml(out)

1
src/ytdl_sub/utils/script.py
View file

@ -180,7 +180,6 @@ class ScriptUtils:
return "" if top_level else "''"
quote = cls._get_quote_char(arg.native)
return arg.native if top_level else f"{quote}{arg.native}{quote}"
arg = cls._maybe_to_optimized_sanitize(arg)

2
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -293,7 +293,7 @@ def _validate_formatter(
)
if maybe_resolved := parsed.maybe_resolvable:
return formatter_validator.post_process(maybe_resolved)
return formatter_validator.post_process(maybe_resolved.native)
return ScriptUtils.to_native_script(parsed)
except RuntimeException as exc:

12
tests/conftest.py
View file

@ -79,7 +79,7 @@ def working_directory() -> str:
@pytest.fixture()
def output_directory() -> str:
with tempfile.TemporaryDirectory() as temp_dir:
yield temp_dir
yield os.path.normpath(temp_dir)
@pytest.fixture()
@ -165,10 +165,11 @@ def subscription_yaml_file_generator() -> Callable:
with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file:
tmp_file.write(json.dumps(yaml_dict).encode("utf-8"))
file_path = os.path.normpath(tmp_file.name)
try:
yield tmp_file.name
yield file_path
finally:
FileHandler.delete(tmp_file.name)
FileHandler.delete(file_path)
return _subscription_yaml_file_generator
@ -269,10 +270,11 @@ def default_config_path(default_config) -> str:
with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp_file:
tmp_file.write(json.dumps(default_config._value).encode("utf-8"))
file_path = os.path.normpath(tmp_file.name)
try:
yield tmp_file.name
yield file_path
finally:
FileHandler.delete(tmp_file.name)
FileHandler.delete(file_path)
@pytest.fixture()

45
tests/integration/cli/test_inspect.py Normal file
View file

@ -0,0 +1,45 @@
from pathlib import Path
import pytest
import yaml
from conftest import mock_run_from_cli
from unit.config.test_subscription_resolution import compare_resolved_yaml
from ytdl_sub.config.validators.variable_validation import ResolutionLevel
from ytdl_sub.utils.system import IS_WINDOWS
class TestInspect:
@pytest.mark.parametrize("config_provided", [True, False])
@pytest.mark.parametrize(
"inspect_level", ["0", "original", "1", "fill", "2", "resolve", "3", "internal"]
)
def test_inspect_command(
self,
capsys,
default_config_path: str,
tv_show_subscriptions_path: Path,
output_directory: str,
config_provided: bool,
inspect_level: str,
):
# TODO: fix mock_run_from_cli in windows to handle file paths correctly
if IS_WINDOWS:
return
# Shares same test fixture as `test_subscription_resolution.py`
args = f"--config {default_config_path} " if config_provided else ""
args += f"inspect {tv_show_subscriptions_path} --match 'NOVA PBS' "
args += f"--level {inspect_level} "
subscriptions = mock_run_from_cli(args=args)
assert len(subscriptions) == 0
out = yaml.safe_load(capsys.readouterr().out)
compare_resolved_yaml(
out=out,
output_directory=output_directory,
subscription_name="NOVA PBS",
preset_type="tv_show",
resolution_level=ResolutionLevel.level_number(resolution_arg=inspect_level),
)

29
tests/unit/config/test_subscription_resolution.py
View file

@ -1,4 +1,5 @@
from pathlib import Path
from typing import Dict
import pytest
import yaml
@ -10,12 +11,13 @@ from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.file_path import FilePathTruncater
def _ensure_resolved_yaml(
sub: Subscription, output_directory: str, preset_type: str, resolution_level: int
def compare_resolved_yaml(
out: Dict,
output_directory: str,
subscription_name: str,
preset_type: str,
resolution_level: int,
) -> None:
output_yaml = sub.resolved_yaml(resolution_level=resolution_level)
out = yaml.safe_load(output_yaml)
expected_out_filename = (
f"{preset_type}/inspect_sub_{ResolutionLevel.name_of(resolution_level)}.json"
)
@ -24,7 +26,7 @@ def _ensure_resolved_yaml(
if resolution_level > ResolutionLevel.ORIGINAL:
output_path = Path(output_directory)
if "tv_show_directory" in expected_out["overrides"]:
output_path = output_path / sub.name
output_path = output_path / subscription_name
expected_out["output_options"]["output_directory"] = FilePathTruncater.to_native_filepath(
str(output_path)
@ -40,6 +42,21 @@ def _ensure_resolved_yaml(
assert out == expected_out
def _ensure_resolved_yaml(
sub: Subscription, output_directory: str, preset_type: str, resolution_level: int
) -> None:
output_yaml = sub.resolved_yaml(resolution_level=resolution_level)
out = yaml.safe_load(output_yaml)
compare_resolved_yaml(
out=out,
output_directory=output_directory,
subscription_name=sub.name,
preset_type=preset_type,
resolution_level=resolution_level,
)
@pytest.mark.parametrize("resolution_level", ResolutionLevel.all())
class TestResolution:
def test_resolution_tv_show(