Merge branch 'master' into prebuilt_presets
This commit is contained in:
commit
ef62e03e07
51 changed files with 729 additions and 160 deletions
BIN
docs/images/plex_scanner_agent.png
Normal file
BIN
docs/images/plex_scanner_agent.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 94 KiB |
BIN
docs/images/unraid_badconsole.png
Normal file
BIN
docs/images/unraid_badconsole.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
|
|
@ -38,10 +38,11 @@ array_apply_fixed
|
||||||
|
|
||||||
array_at
|
array_at
|
||||||
~~~~~~~~
|
~~~~~~~~
|
||||||
:spec: ``array_at(array: Array, idx: Integer) -> AnyArgument``
|
:spec: ``array_at(array: Array, idx: Integer, default: Optional[AnyArgument]) -> AnyArgument``
|
||||||
|
|
||||||
:description:
|
:description:
|
||||||
Return the element in the Array at index ``idx``.
|
Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length,
|
||||||
|
either return ``default`` if provided or throw an error.
|
||||||
|
|
||||||
array_contains
|
array_contains
|
||||||
~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~
|
||||||
|
|
@ -225,6 +226,27 @@ xor
|
||||||
Conditional Functions
|
Conditional Functions
|
||||||
---------------------
|
---------------------
|
||||||
|
|
||||||
|
elif
|
||||||
|
~~~~
|
||||||
|
:spec: ``elif(if_elif_else: AnyArgument, ...) -> AnyArgument``
|
||||||
|
|
||||||
|
:description:
|
||||||
|
Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via
|
||||||
|
adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to
|
||||||
|
supply at least one conditional and an else.
|
||||||
|
:usage:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
%elif(
|
||||||
|
condition1,
|
||||||
|
return1,
|
||||||
|
condition2,
|
||||||
|
return2,
|
||||||
|
...
|
||||||
|
else_return
|
||||||
|
)
|
||||||
|
|
||||||
if
|
if
|
||||||
~~
|
~~
|
||||||
:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
|
:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
|
||||||
|
|
@ -543,6 +565,13 @@ slice
|
||||||
:description:
|
:description:
|
||||||
Returns the slice of the Array.
|
Returns the slice of the Array.
|
||||||
|
|
||||||
|
split
|
||||||
|
~~~~~
|
||||||
|
:spec: ``split(string: String, sep: String, max_split: Optional[Integer]) -> Array``
|
||||||
|
|
||||||
|
:description:
|
||||||
|
Splits the input string into multiple strings.
|
||||||
|
|
||||||
string
|
string
|
||||||
~~~~~~
|
~~~~~~
|
||||||
:spec: ``string(value: AnyArgument) -> String``
|
:spec: ``string(value: AnyArgument) -> String``
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
|
===
|
||||||
FAQ
|
FAQ
|
||||||
===
|
===
|
||||||
|
|
||||||
Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as
|
Since ytdl-sub is relatively new to the public, there has not been many question asked yet. We will update this as more questions get asked.
|
||||||
more questions get asked.
|
|
||||||
|
|
||||||
.. contents:: Frequently Asked Questions
|
.. contents:: Frequently Asked Questions
|
||||||
:depth: 3
|
:depth: 3
|
||||||
|
|
@ -10,12 +10,25 @@ more questions get asked.
|
||||||
How do I...
|
How do I...
|
||||||
-----------
|
-----------
|
||||||
|
|
||||||
|
...get support or reach out to contribute?
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
If you need support, you can:
|
||||||
|
|
||||||
|
* :ytdl-sub-gh:`Open an issue on GitHub <issues/new>`
|
||||||
|
|
||||||
|
* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_
|
||||||
|
|
||||||
|
If you would like to contribute, we're happy to accept any help, even non-coders! To find out how you can help this project, you can:
|
||||||
|
|
||||||
|
* `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in #development with where you think you can assist or what skills you would like to contribute.
|
||||||
|
|
||||||
|
* If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull request <compare>` with information on what issue you're resolving and it will be reviewed as soon as possible.
|
||||||
|
|
||||||
...download age-restricted YouTube videos?
|
...download age-restricted YouTube videos?
|
||||||
''''''''''''''''''''''''''''''''''''''''''
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
See
|
|
||||||
`ytdls recommended way <https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_
|
See `yt-dl's recommended way <https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_ to download your YouTube cookie, then add it to your :ref:`ytdl options <config_reference/plugins:ytdl_options>` section of your config:
|
||||||
to download your YouTube cookie, then add it to your
|
|
||||||
`ytdl options <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl-options>`_ section of your config:
|
|
||||||
|
|
||||||
.. code-block:: yaml
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
|
@ -23,14 +36,16 @@ to download your YouTube cookie, then add it to your
|
||||||
cookiefile: "/path/to/cookies/file.txt"
|
cookiefile: "/path/to/cookies/file.txt"
|
||||||
|
|
||||||
...automate my downloads?
|
...automate my downloads?
|
||||||
'''''''''''''''''''''''''
|
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
`This part of the wiki <https://github.com/jmbannon/ytdl-sub/wiki/7.-Automate-Downloading-New-Content-Using-Your-Configs>`_ shows how to set up ``ytdl-sub`` to run in a cron job within Docker.
|
|
||||||
|
:doc:`This page </guides/getting_started/automating_downloads>` shows how to set up ``ytdl-sub`` to run automatically on various platforms.
|
||||||
|
|
||||||
There is a bug where...
|
There is a bug where...
|
||||||
-----------------------
|
-----------------------
|
||||||
|
|
||||||
...date_range is not downloading older videos after I changed the range
|
...date_range is not downloading older videos after I changed the range
|
||||||
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether.
|
Your preset most likely has ``break_on_existing`` set to True, which will stop downloading additional metadata/videos if the video exists in your download archive. Set the following in your config to skip downloading videos that exist instead of stopping altogether.
|
||||||
|
|
||||||
.. code-block:: yaml
|
.. code-block:: yaml
|
||||||
|
|
@ -38,11 +53,12 @@ Your preset most likely has ``break_on_existing`` set to True, which will stop d
|
||||||
ytdl_options:
|
ytdl_options:
|
||||||
break_on_existing: False
|
break_on_existing: False
|
||||||
|
|
||||||
After your download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads.
|
After you download your new date_range duration, re-enable ``break_on_existing`` to speed up successive downloads.
|
||||||
|
|
||||||
...it is downloading non-English title and description metadata
|
...it is downloading non-English title and description metadata
|
||||||
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using
|
|
||||||
|
Most likely the video has a non-English language set to its 'native' language. You can tell yt-dlp to explicitly download English metadata using.
|
||||||
|
|
||||||
.. code-block:: yaml
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
|
@ -53,7 +69,19 @@ Most likely the video has a non-English language set to its 'native' language. Y
|
||||||
- "en"
|
- "en"
|
||||||
|
|
||||||
...Plex is not showing my TV shows correctly
|
...Plex is not showing my TV shows correctly
|
||||||
''''''''''''''''''''''''''''''''''''''''''''
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
Set the following
|
|
||||||
`Scanner and Agent <https://i.imgur.com/zdZhCLZ.png>`_
|
Set the following for your ytdl-sub library that has been added to Plex.
|
||||||
for your library.
|
|
||||||
|
.. figure:: ../../images/plex_scanner_agent.png
|
||||||
|
:alt: The Plex library editor, under the advanced settings, showing the required options for Plex to show the TV shows correctly.
|
||||||
|
|
||||||
|
**Scanner:** Plex Series Scanner
|
||||||
|
|
||||||
|
**Agent:** Personal Media shows
|
||||||
|
|
||||||
|
**Visibility:** Exclude from home screen and global search
|
||||||
|
|
||||||
|
**Episode sorting:** Library default
|
||||||
|
|
||||||
|
**YES** Enable video preview thumbnails
|
||||||
|
|
@ -17,14 +17,14 @@ The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install yt
|
||||||
GUI Image
|
GUI Image
|
||||||
---------
|
---------
|
||||||
|
|
||||||
The GUI image uses LSIO's :lsio-gh:`docker-code-server image` for its base image. More info on other code-server environment variables can be found within its documentation.
|
The GUI image uses LSIO's :lsio-gh:`docker-code-server image <\ >` for its base image. More info on other code-server environment variables can be found within its documentation.
|
||||||
|
|
||||||
After starting, code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``.
|
After starting, the code-server will be running at http://localhost:8443. Open this page in a browser to access and interact with ``ytdl-sub``.
|
||||||
|
|
||||||
Headless Image
|
Headless Image
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image` for its base image. Execute the following command to access and interact with ``ytdl-sub``:
|
The headless image uses LSIO's :lsio-gh:`docker-baseimage-alpine image <\ >` for its base image. Execute the following command to access and interact with ``ytdl-sub``:
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,16 @@
|
||||||
Unraid
|
Unraid
|
||||||
--------------
|
--------------
|
||||||
You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>` through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_. Uses Docker under the hood.
|
You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>` through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_.
|
||||||
|
|
||||||
|
|
||||||
|
If you installed the ``ytdl-sub-gui`` app, the code-server will be running at http://localhost:8443 (replace ``localhost`` with the IP of the computer running Unraid if you aren't trying to access ``ytdl-sub`` on that computer). Open this page in a browser to access and interact with ``ytdl-sub``.
|
||||||
|
|
||||||
|
If you installed the ``ytdl-sub`` app (headless), open the normal app-specific console to access and interact with ``ytdl-sub``. Once open, you must first run ``su abc -s /bin/bash`` to change to the non-root user. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``.
|
||||||
|
|
||||||
|
|
||||||
|
.. warning::
|
||||||
|
|
||||||
|
If you use the below option to access the ``ytdl-sub`` console, be sure to run ``su abc -s /bin/bash`` first thing. You can confirm that this command worked by running ``whoami`` and verifying that the result is ``abc``. Do **NOT** run ``ytdl-sub`` as the root user! Running as root will set the owner of all modified files to root, which prevents most media managers and players from accessing the files.
|
||||||
|
|
||||||
|
.. figure:: ../../../images/unraid_badconsole.png
|
||||||
|
:alt: The Unraid community app plugin GUI, with an arrow pointing at the "Console" option in the dropdown after selecting ytdl-sub-gui
|
||||||
|
|
@ -25,6 +25,8 @@ General options must be specified before the command (i.e. ``sub``).
|
||||||
path to store the transaction log output of all files added, modified, deleted
|
path to store the transaction log output of all files added, modified, deleted
|
||||||
-st, --suppress-transaction-log
|
-st, --suppress-transaction-log
|
||||||
do not output transaction logs to console or file
|
do not output transaction logs to console or file
|
||||||
|
-m MATCH [MATCH ...], --match MATCH [MATCH ...]
|
||||||
|
match subscription names to one or more substrings, and only run those subscriptions
|
||||||
|
|
||||||
Sub Options
|
Sub Options
|
||||||
-----------
|
-----------
|
||||||
|
|
@ -37,6 +39,14 @@ Download all subscriptions specified in each ``SUBPATH``.
|
||||||
``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided.
|
``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided.
|
||||||
It will use the config specified by ``--config``, or ``config.yaml`` if not provided.
|
It will use the config specified by ``--config``, or ``config.yaml`` if not provided.
|
||||||
|
|
||||||
|
.. code-block:: text
|
||||||
|
:caption: Additional Options
|
||||||
|
|
||||||
|
-u, --update-with-info-json
|
||||||
|
update all subscriptions with the current config using info.json files
|
||||||
|
-o DL_OVERRIDE, --dl-override DL_OVERRIDE
|
||||||
|
override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
|
||||||
|
|
||||||
Download Options
|
Download Options
|
||||||
-----------------
|
-----------------
|
||||||
Download a single subscription in the form of CLI arguments.
|
Download a single subscription in the form of CLI arguments.
|
||||||
|
|
@ -67,7 +77,7 @@ Using the command:
|
||||||
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
|
--overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
|
||||||
|
|
||||||
See how to shorten commands using
|
See how to shorten commands using
|
||||||
`download aliases <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
|
`download aliases <https://ytdl-sub.readthedocs.io/en/latest/config_reference/config_yaml.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
|
||||||
|
|
||||||
View Options
|
View Options
|
||||||
-----------------
|
-----------------
|
||||||
|
|
@ -76,6 +86,7 @@ View Options
|
||||||
ytdl-sub view [-sc] [URL]
|
ytdl-sub view [-sc] [URL]
|
||||||
|
|
||||||
.. code-block:: text
|
.. code-block:: text
|
||||||
|
:caption: Additional Options
|
||||||
|
|
||||||
-sc, --split-chapters
|
-sc, --split-chapters
|
||||||
View source variables after splitting by chapters
|
View source variables after splitting by chapters
|
||||||
|
|
|
||||||
|
|
@ -22,24 +22,6 @@ disable = [
|
||||||
|
|
||||||
load-plugins = "pylint.extensions.docparams"
|
load-plugins = "pylint.extensions.docparams"
|
||||||
|
|
||||||
[tool.pydocstyle]
|
|
||||||
inherit = false
|
|
||||||
match = "[^test_].*\\.py"
|
|
||||||
ignore = [
|
|
||||||
"D100", # docstring in public module
|
|
||||||
"D101", # Missing docstring in public class (covered by pylint)
|
|
||||||
"D104", # docstring in public package
|
|
||||||
"D107", # docstring in init
|
|
||||||
"D200", # One-line should fit on one line
|
|
||||||
"D203", # 1 blank line before class docstring
|
|
||||||
"D205", # 1 blank line between summary and description
|
|
||||||
"D212", # Multi-line should start at first line
|
|
||||||
"D400", # Should end with a period
|
|
||||||
"D401", # Return vs Returns
|
|
||||||
"D413", # Missing blank line after last section
|
|
||||||
"D415", # Should end with a period
|
|
||||||
]
|
|
||||||
|
|
||||||
[tool.coverage.run]
|
[tool.coverage.run]
|
||||||
include = [
|
include = [
|
||||||
"src/*"
|
"src/*"
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ package_dir =
|
||||||
packages=find:
|
packages=find:
|
||||||
|
|
||||||
install_requires =
|
install_requires =
|
||||||
yt-dlp==2023.11.16
|
yt-dlp==2023.12.30
|
||||||
argparse==1.4.0
|
argparse==1.4.0
|
||||||
colorama==0.4.6
|
colorama==0.4.6
|
||||||
mergedeep==1.3.4
|
mergedeep==1.3.4
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import os
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Dict
|
||||||
from typing import List
|
from typing import List
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
|
|
@ -67,7 +68,12 @@ def _maybe_write_subscription_log_file(
|
||||||
|
|
||||||
|
|
||||||
def _download_subscriptions_from_yaml_files(
|
def _download_subscriptions_from_yaml_files(
|
||||||
config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool
|
config: ConfigFile,
|
||||||
|
subscription_paths: List[str],
|
||||||
|
subscription_matches: List[str],
|
||||||
|
subscription_override_dict: Dict,
|
||||||
|
update_with_info_json: bool,
|
||||||
|
dry_run: bool,
|
||||||
) -> List[Subscription]:
|
) -> List[Subscription]:
|
||||||
"""
|
"""
|
||||||
Downloads all subscriptions from one or many subscription yaml files.
|
Downloads all subscriptions from one or many subscription yaml files.
|
||||||
|
|
@ -78,6 +84,8 @@ def _download_subscriptions_from_yaml_files(
|
||||||
Configuration file
|
Configuration file
|
||||||
subscription_paths
|
subscription_paths
|
||||||
Path to subscription files to download
|
Path to subscription files to download
|
||||||
|
subscription_matches
|
||||||
|
Optional list of substrings to match subscription names to (only run if matched)
|
||||||
update_with_info_json
|
update_with_info_json
|
||||||
Whether to actually download or update using existing info json
|
Whether to actually download or update using existing info json
|
||||||
dry_run
|
dry_run
|
||||||
|
|
@ -96,7 +104,17 @@ def _download_subscriptions_from_yaml_files(
|
||||||
|
|
||||||
# Load all the subscriptions first to perform all validation before downloading
|
# Load all the subscriptions first to perform all validation before downloading
|
||||||
for path in subscription_paths:
|
for path in subscription_paths:
|
||||||
subscriptions += Subscription.from_file_path(config=config, subscription_path=path)
|
subscriptions += Subscription.from_file_path(
|
||||||
|
config=config,
|
||||||
|
subscription_path=path,
|
||||||
|
subscription_override_dict=subscription_override_dict,
|
||||||
|
)
|
||||||
|
|
||||||
|
if subscriptions and subscription_matches:
|
||||||
|
logger.info("Filtering subscriptions by name based on --match arguments")
|
||||||
|
subscriptions = [
|
||||||
|
sub for sub in subscriptions if any(match in sub.name for match in subscription_matches)
|
||||||
|
]
|
||||||
|
|
||||||
for subscription in subscriptions:
|
for subscription in subscriptions:
|
||||||
with subscription.exception_handling():
|
with subscription.exception_handling():
|
||||||
|
|
@ -119,7 +137,7 @@ def _download_subscriptions_from_yaml_files(
|
||||||
exception=subscription.exception,
|
exception=subscription.exception,
|
||||||
)
|
)
|
||||||
|
|
||||||
Logger.cleanup(cleanup_error_log=False)
|
Logger.cleanup(has_error=False)
|
||||||
gc.collect() # Garbage collect after each subscription download
|
gc.collect() # Garbage collect after each subscription download
|
||||||
|
|
||||||
return subscriptions
|
return subscriptions
|
||||||
|
|
@ -221,10 +239,18 @@ def main() -> List[Subscription]:
|
||||||
"full backup before usage. You have been warned!",
|
"full backup before usage. You have been warned!",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
subscription_override_dict = {}
|
||||||
|
if args.dl_override:
|
||||||
|
subscription_override_dict = DownloadArgsParser.from_dl_override(
|
||||||
|
override=args.dl_override, config=config
|
||||||
|
).to_subscription_dict()
|
||||||
|
|
||||||
logger.info("Validating subscriptions...")
|
logger.info("Validating subscriptions...")
|
||||||
subscriptions = _download_subscriptions_from_yaml_files(
|
subscriptions = _download_subscriptions_from_yaml_files(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_paths=args.subscription_paths,
|
subscription_paths=args.subscription_paths,
|
||||||
|
subscription_matches=args.match,
|
||||||
|
subscription_override_dict=subscription_override_dict,
|
||||||
update_with_info_json=args.update_with_info_json,
|
update_with_info_json=args.update_with_info_json,
|
||||||
dry_run=args.dry_run,
|
dry_run=args.dry_run,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ from typing import Tuple
|
||||||
from mergedeep import mergedeep
|
from mergedeep import mergedeep
|
||||||
|
|
||||||
from ytdl_sub.cli.parsers.main import MainArguments
|
from ytdl_sub.cli.parsers.main import MainArguments
|
||||||
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.config.config_validator import ConfigOptions
|
from ytdl_sub.config.config_validator import ConfigOptions
|
||||||
from ytdl_sub.utils.exceptions import InvalidDlArguments
|
from ytdl_sub.utils.exceptions import InvalidDlArguments
|
||||||
|
|
||||||
|
|
@ -247,3 +248,12 @@ class DownloadArgsParser:
|
||||||
"""
|
"""
|
||||||
hash_string = str(sorted(self._unknown_arguments))
|
hash_string = str(sorted(self._unknown_arguments))
|
||||||
return hashlib.sha256(hash_string.encode()).hexdigest()[-8:]
|
return hashlib.sha256(hash_string.encode()).hexdigest()[-8:]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dl_override(cls, override: str, config: ConfigFile) -> "DownloadArgsParser":
|
||||||
|
"""
|
||||||
|
Create a DownloadArgsParser from a sub --override argument value
|
||||||
|
"""
|
||||||
|
return DownloadArgsParser(
|
||||||
|
extra_arguments=override.split(), config_options=config.config_options
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,10 @@ class MainArguments:
|
||||||
long="--suppress-transaction-log",
|
long="--suppress-transaction-log",
|
||||||
is_positional=True,
|
is_positional=True,
|
||||||
)
|
)
|
||||||
|
MATCH = CLIArgument(
|
||||||
|
short="-m",
|
||||||
|
long="--match",
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def all(cls) -> List[CLIArgument]:
|
def all(cls) -> List[CLIArgument]:
|
||||||
|
|
@ -54,6 +58,7 @@ class MainArguments:
|
||||||
cls.LOG_LEVEL,
|
cls.LOG_LEVEL,
|
||||||
cls.TRANSACTION_LOG,
|
cls.TRANSACTION_LOG,
|
||||||
cls.SUPPRESS_TRANSACTION_LOG,
|
cls.SUPPRESS_TRANSACTION_LOG,
|
||||||
|
cls.MATCH,
|
||||||
]
|
]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
@ -124,6 +129,16 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults
|
||||||
help="do not output transaction logs to console or file",
|
help="do not output transaction logs to console or file",
|
||||||
default=argparse.SUPPRESS if suppress_defaults else False,
|
default=argparse.SUPPRESS if suppress_defaults else False,
|
||||||
)
|
)
|
||||||
|
arg_parser.add_argument(
|
||||||
|
MainArguments.MATCH.short,
|
||||||
|
MainArguments.MATCH.long,
|
||||||
|
dest="match",
|
||||||
|
nargs="+",
|
||||||
|
action="extend",
|
||||||
|
type=str,
|
||||||
|
help="match subscription names to one or more substrings, and only run those subscriptions",
|
||||||
|
default=argparse.SUPPRESS if suppress_defaults else [],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
###################################################################################################
|
###################################################################################################
|
||||||
|
|
@ -142,6 +157,10 @@ class SubArguments:
|
||||||
short="-u",
|
short="-u",
|
||||||
long="--update-with-info-json",
|
long="--update-with-info-json",
|
||||||
)
|
)
|
||||||
|
OVERRIDE = CLIArgument(
|
||||||
|
short="-o",
|
||||||
|
long="--dl-override",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
subscription_parser = subparsers.add_parser("sub")
|
subscription_parser = subparsers.add_parser("sub")
|
||||||
|
|
@ -160,6 +179,13 @@ subscription_parser.add_argument(
|
||||||
help="update all subscriptions with the current config using info.json files",
|
help="update all subscriptions with the current config using info.json files",
|
||||||
default=False,
|
default=False,
|
||||||
)
|
)
|
||||||
|
subscription_parser.add_argument(
|
||||||
|
SubArguments.OVERRIDE.short,
|
||||||
|
SubArguments.OVERRIDE.long,
|
||||||
|
type=str,
|
||||||
|
help="override all subscription config values using `dl` syntax, "
|
||||||
|
"i.e. --dl-override='--ytdl_options.max_downloads 3'",
|
||||||
|
)
|
||||||
|
|
||||||
###################################################################################################
|
###################################################################################################
|
||||||
# DOWNLOAD PARSER
|
# DOWNLOAD PARSER
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ class Overrides(DictFormatterValidator, Scriptable):
|
||||||
|
|
||||||
def __init__(self, name, value):
|
def __init__(self, name, value):
|
||||||
DictFormatterValidator.__init__(self, name, value)
|
DictFormatterValidator.__init__(self, name, value)
|
||||||
Scriptable.__init__(self)
|
Scriptable.__init__(self, initialize_base_script=True)
|
||||||
|
|
||||||
for key in self._keys:
|
for key in self._keys:
|
||||||
self.ensure_variable_name_valid(key)
|
self.ensure_variable_name_valid(key)
|
||||||
|
|
|
||||||
|
|
@ -152,7 +152,9 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
|
||||||
for file_name in entry_file_names:
|
for file_name in entry_file_names:
|
||||||
ext = get_file_extension(file_name)
|
ext = get_file_extension(file_name)
|
||||||
file_path = Path(self.output_directory) / file_name
|
file_path = Path(self.output_directory) / file_name
|
||||||
working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}"
|
working_directory_file_path = Path(self.working_directory) / entry.base_filename(
|
||||||
|
ext=ext
|
||||||
|
)
|
||||||
|
|
||||||
# NFO files will always get rewritten, so ignore
|
# NFO files will always get rewritten, so ignore
|
||||||
if ext == "nfo":
|
if ext == "nfo":
|
||||||
|
|
|
||||||
|
|
@ -112,13 +112,6 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
|
||||||
directory, run this function. This lets the downloader add any extra files directly to the
|
directory, run this function. This lets the downloader add any extra files directly to the
|
||||||
output directory, for things like YT channel image, banner.
|
output directory, for things like YT channel image, banner.
|
||||||
"""
|
"""
|
||||||
if playlist_metadata := entry.get(v.playlist_metadata, dict):
|
|
||||||
self._download_parent_thumbnails(
|
|
||||||
thumbnail_list_info=collection_url.playlist_thumbnails,
|
|
||||||
entry=entry,
|
|
||||||
parent=EntryParent(playlist_metadata, working_directory=self.working_directory),
|
|
||||||
)
|
|
||||||
|
|
||||||
if source_metadata := entry.get(v.source_metadata, dict):
|
if source_metadata := entry.get(v.source_metadata, dict):
|
||||||
self._download_parent_thumbnails(
|
self._download_parent_thumbnails(
|
||||||
thumbnail_list_info=collection_url.source_thumbnails,
|
thumbnail_list_info=collection_url.source_thumbnails,
|
||||||
|
|
@ -126,6 +119,13 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
|
||||||
parent=EntryParent(source_metadata, working_directory=self.working_directory),
|
parent=EntryParent(source_metadata, working_directory=self.working_directory),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if playlist_metadata := entry.get(v.playlist_metadata, dict):
|
||||||
|
self._download_parent_thumbnails(
|
||||||
|
thumbnail_list_info=collection_url.playlist_thumbnails,
|
||||||
|
entry=entry,
|
||||||
|
parent=EntryParent(playlist_metadata, working_directory=self.working_directory),
|
||||||
|
)
|
||||||
|
|
||||||
def modify_entry(self, entry: Entry) -> Optional[Entry]:
|
def modify_entry(self, entry: Entry) -> Optional[Entry]:
|
||||||
"""
|
"""
|
||||||
Use the entry to download thumbnails (or move if LATEST_ENTRY).
|
Use the entry to download thumbnails (or move if LATEST_ENTRY).
|
||||||
|
|
@ -349,39 +349,43 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
||||||
)
|
)
|
||||||
|
|
||||||
def _iterate_child_entries(
|
def _iterate_child_entries(
|
||||||
self, url_validator: UrlValidator, entries: List[Entry]
|
self, entries: List[Entry], download_reversed: bool
|
||||||
) -> Iterator[Entry]:
|
) -> Iterator[Entry]:
|
||||||
entries_to_iterate = entries
|
# Iterate a list of entries, and delete the entries after yielding
|
||||||
if url_validator.download_reverse:
|
indices = list(range(len(entries)))
|
||||||
entries_to_iterate = reversed(entries)
|
if download_reversed:
|
||||||
|
indices = reversed(indices)
|
||||||
|
|
||||||
for entry in entries_to_iterate:
|
for idx in indices:
|
||||||
self._url_state.entries_downloaded += 1
|
self._url_state.entries_downloaded += 1
|
||||||
|
|
||||||
if self._is_downloaded(entry):
|
if self._is_downloaded(entries[idx]):
|
||||||
download_logger.info(
|
download_logger.info(
|
||||||
"Already downloaded entry %d/%d: %s",
|
"Already downloaded entry %d/%d: %s",
|
||||||
self._url_state.entries_downloaded,
|
self._url_state.entries_downloaded,
|
||||||
self._url_state.entries_total,
|
self._url_state.entries_total,
|
||||||
entry.title,
|
entries[idx].title,
|
||||||
)
|
)
|
||||||
|
del entries[idx]
|
||||||
continue
|
continue
|
||||||
|
|
||||||
yield entry
|
yield entries[idx]
|
||||||
self._mark_downloaded(entry)
|
self._mark_downloaded(entries[idx])
|
||||||
|
|
||||||
|
del entries[idx]
|
||||||
|
|
||||||
def _iterate_parent_entry(
|
def _iterate_parent_entry(
|
||||||
self, url_validator: UrlValidator, parent: EntryParent
|
self, parent: EntryParent, download_reversed: bool
|
||||||
) -> Iterator[Entry]:
|
) -> Iterator[Entry]:
|
||||||
for entry_child in self._iterate_child_entries(
|
for entry_child in self._iterate_child_entries(
|
||||||
url_validator=url_validator, entries=parent.entry_children()
|
entries=parent.entry_children(), download_reversed=download_reversed
|
||||||
):
|
):
|
||||||
yield entry_child
|
yield entry_child
|
||||||
|
|
||||||
# Recursion the parent's parent entries
|
# Recursion the parent's parent entries
|
||||||
for parent_child in reversed(parent.parent_children()):
|
for parent_child in reversed(parent.parent_children()):
|
||||||
for entry_child in self._iterate_parent_entry(
|
for entry_child in self._iterate_parent_entry(
|
||||||
url_validator=url_validator, parent=parent_child
|
parent=parent_child, download_reversed=download_reversed
|
||||||
):
|
):
|
||||||
yield entry_child
|
yield entry_child
|
||||||
|
|
||||||
|
|
@ -415,9 +419,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
||||||
|
|
||||||
def _iterate_entries(
|
def _iterate_entries(
|
||||||
self,
|
self,
|
||||||
url_validator: UrlValidator,
|
|
||||||
parents: List[EntryParent],
|
parents: List[EntryParent],
|
||||||
orphans: List[Entry],
|
orphans: List[Entry],
|
||||||
|
download_reversed: bool,
|
||||||
) -> Iterator[Entry]:
|
) -> Iterator[Entry]:
|
||||||
"""
|
"""
|
||||||
Downloads the leaf entries from EntryParent trees
|
Downloads the leaf entries from EntryParent trees
|
||||||
|
|
@ -426,11 +430,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
||||||
with self._separate_download_archives(clear_info_json_files=True):
|
with self._separate_download_archives(clear_info_json_files=True):
|
||||||
for parent in parents:
|
for parent in parents:
|
||||||
for entry_child in self._iterate_parent_entry(
|
for entry_child in self._iterate_parent_entry(
|
||||||
url_validator=url_validator, parent=parent
|
parent=parent, download_reversed=download_reversed
|
||||||
):
|
):
|
||||||
yield entry_child
|
yield entry_child
|
||||||
|
|
||||||
for orphan in self._iterate_child_entries(url_validator=url_validator, entries=orphans):
|
for orphan in self._iterate_child_entries(
|
||||||
|
entries=orphans, download_reversed=download_reversed
|
||||||
|
):
|
||||||
yield orphan
|
yield orphan
|
||||||
|
|
||||||
def download_metadata(self) -> Iterable[Entry]:
|
def download_metadata(self) -> Iterable[Entry]:
|
||||||
|
|
@ -454,7 +460,9 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
|
||||||
"Beginning downloads for %s", self.overrides.apply_formatter(collection_url.url)
|
"Beginning downloads for %s", self.overrides.apply_formatter(collection_url.url)
|
||||||
)
|
)
|
||||||
for entry in self._iterate_entries(
|
for entry in self._iterate_entries(
|
||||||
url_validator=collection_url, parents=parents, orphans=orphan_entries
|
parents=parents,
|
||||||
|
orphans=orphan_entries,
|
||||||
|
download_reversed=collection_url.download_reverse,
|
||||||
):
|
):
|
||||||
entry.initialize_script(self.overrides).add(
|
entry.initialize_script(self.overrides).add(
|
||||||
{v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)}
|
{v.ytdl_sub_input_url: self.overrides.apply_formatter(collection_url.url)}
|
||||||
|
|
|
||||||
|
|
@ -216,11 +216,14 @@ class YTDLP:
|
||||||
**kwargs
|
**kwargs
|
||||||
arguments passed directory to YoutubeDL extract_info
|
arguments passed directory to YoutubeDL extract_info
|
||||||
"""
|
"""
|
||||||
|
parent_dict: Dict = {}
|
||||||
try:
|
try:
|
||||||
with cls._listen_and_log_downloaded_info_json(
|
with cls._listen_and_log_downloaded_info_json(
|
||||||
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
|
working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl
|
||||||
):
|
):
|
||||||
_ = cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
|
parent_dict = cls.extract_info(
|
||||||
|
ytdl_options_overrides=ytdl_options_overrides, **kwargs
|
||||||
|
)
|
||||||
except RejectedVideoReached:
|
except RejectedVideoReached:
|
||||||
cls.logger.debug(
|
cls.logger.debug(
|
||||||
"RejectedVideoReached, stopping additional downloads "
|
"RejectedVideoReached, stopping additional downloads "
|
||||||
|
|
@ -234,4 +237,19 @@ class YTDLP:
|
||||||
except MaxDownloadsReached:
|
except MaxDownloadsReached:
|
||||||
cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
|
cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
|
||||||
|
|
||||||
|
# For YouTube playlists in particular, channel metadata is not fetched. Attempt to get
|
||||||
|
# channel metadata via grabbing uploader_url info json a max of 3 times
|
||||||
|
current_iter = 0
|
||||||
|
url = kwargs.get("url")
|
||||||
|
uploader_url = parent_dict.get("uploader_url")
|
||||||
|
while current_iter < 3 and uploader_url and url != uploader_url:
|
||||||
|
cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url)
|
||||||
|
parent_dict = cls.extract_info(
|
||||||
|
ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"},
|
||||||
|
url=uploader_url,
|
||||||
|
)
|
||||||
|
current_iter += 1
|
||||||
|
url = uploader_url
|
||||||
|
uploader_url = parent_dict.get("uploader_url")
|
||||||
|
|
||||||
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
|
return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory)
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,8 @@ from typing import Type
|
||||||
from typing import TypeVar
|
from typing import TypeVar
|
||||||
from typing import final
|
from typing import final
|
||||||
|
|
||||||
|
from yt_dlp.utils import sanitize_filename
|
||||||
|
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||||
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
|
||||||
|
|
||||||
|
|
@ -45,6 +47,19 @@ class BaseEntry(ABC):
|
||||||
"""
|
"""
|
||||||
return str(self._kwargs[v.uid.metadata_key])
|
return str(self._kwargs[v.uid.metadata_key])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def uid_sanitized(self) -> str:
|
||||||
|
"""
|
||||||
|
Sanitized version, used in filenames
|
||||||
|
"""
|
||||||
|
return sanitize_filename(self.uid)
|
||||||
|
|
||||||
|
def base_filename(self, ext: str):
|
||||||
|
"""
|
||||||
|
The base filename of all yt-dlp downloaded entry files
|
||||||
|
"""
|
||||||
|
return f"{self.uid_sanitized}.{ext}"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def download_archive_extractor(self) -> str:
|
def download_archive_extractor(self) -> str:
|
||||||
"""
|
"""
|
||||||
|
|
@ -101,30 +116,13 @@ class BaseEntry(ABC):
|
||||||
"""
|
"""
|
||||||
return self._working_directory
|
return self._working_directory
|
||||||
|
|
||||||
def add_kwargs(self, variables_to_add: Dict[str, Any]) -> "BaseEntry":
|
|
||||||
"""
|
|
||||||
Adds variables to kwargs. Use with caution since yt-dlp data can be overwritten.
|
|
||||||
Plugins should use ``add_variables``.
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
variables_to_add
|
|
||||||
Variables to add to kwargs
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
self
|
|
||||||
"""
|
|
||||||
self._kwargs = dict(self._kwargs, **variables_to_add)
|
|
||||||
return self
|
|
||||||
|
|
||||||
def get_download_info_json_name(self) -> str:
|
def get_download_info_json_name(self) -> str:
|
||||||
"""
|
"""
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
The download info json's file name
|
The download info json's file name
|
||||||
"""
|
"""
|
||||||
return f"{self.uid}.{self.info_json_ext}"
|
return self.base_filename(ext=self.info_json_ext)
|
||||||
|
|
||||||
def get_download_info_json_path(self) -> str:
|
def get_download_info_json_path(self) -> str:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -44,12 +44,6 @@ class Entry(BaseEntry, Scriptable):
|
||||||
BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory)
|
BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory)
|
||||||
Scriptable.__init__(self)
|
Scriptable.__init__(self)
|
||||||
|
|
||||||
def _add_entry_kwargs_to_script(self) -> None:
|
|
||||||
# Add entry metadata, but avoid the `.add()` helper since it also adds sanitized
|
|
||||||
self.unresolvable.remove(v.entry_metadata.variable_name)
|
|
||||||
self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)})
|
|
||||||
self.update_script()
|
|
||||||
|
|
||||||
def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry":
|
def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry":
|
||||||
"""
|
"""
|
||||||
Initializes the entry script using the Overrides script, then adding
|
Initializes the entry script using the Overrides script, then adding
|
||||||
|
|
@ -57,12 +51,20 @@ class Entry(BaseEntry, Scriptable):
|
||||||
"""
|
"""
|
||||||
# Overrides contains added variables that are unresolvable, add them here
|
# Overrides contains added variables that are unresolvable, add them here
|
||||||
if other:
|
if other:
|
||||||
self.script = copy.deepcopy(other.script)
|
self._script = copy.deepcopy(other.script)
|
||||||
self.unresolvable = copy.deepcopy(other.unresolvable)
|
self._unresolvable = copy.deepcopy(other.unresolvable)
|
||||||
|
else:
|
||||||
|
self.initialize_base_script()
|
||||||
|
|
||||||
self._add_entry_kwargs_to_script()
|
self._add_entry_kwargs_to_script()
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
def _add_entry_kwargs_to_script(self) -> None:
|
||||||
|
# Add entry metadata, but avoid the `.add()` helper since it also adds sanitized
|
||||||
|
self.unresolvable.remove(v.entry_metadata.variable_name)
|
||||||
|
self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)})
|
||||||
|
self.update_script()
|
||||||
|
|
||||||
def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT:
|
def get(self, variable: Variable, expected_type: Type[TypeT]) -> TypeT:
|
||||||
"""
|
"""
|
||||||
Gets a variable of an expected type. Will error if it does not exist or is not resolved.
|
Gets a variable of an expected type. Will error if it does not exist or is not resolved.
|
||||||
|
|
@ -113,7 +115,8 @@ class Entry(BaseEntry, Scriptable):
|
||||||
"""
|
"""
|
||||||
ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key]
|
ext = self.try_get(v.ext, str) or self._kwargs[v.ext.metadata_key]
|
||||||
for possible_ext in [ext, "mkv"]:
|
for possible_ext in [ext, "mkv"]:
|
||||||
file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}")
|
file_name = self.base_filename(ext=possible_ext)
|
||||||
|
file_path = str(Path(self.working_directory()) / file_name)
|
||||||
if os.path.isfile(file_path):
|
if os.path.isfile(file_path):
|
||||||
return possible_ext
|
return possible_ext
|
||||||
|
|
||||||
|
|
@ -125,7 +128,7 @@ class Entry(BaseEntry, Scriptable):
|
||||||
-------
|
-------
|
||||||
The entry's file name
|
The entry's file name
|
||||||
"""
|
"""
|
||||||
return f"{self.uid}.{self.ext}"
|
return self.base_filename(ext=self.ext)
|
||||||
|
|
||||||
def get_download_file_path(self) -> str:
|
def get_download_file_path(self) -> str:
|
||||||
"""Returns the entry's file path to where it was downloaded"""
|
"""Returns the entry's file path to where it was downloaded"""
|
||||||
|
|
@ -137,7 +140,7 @@ class Entry(BaseEntry, Scriptable):
|
||||||
-------
|
-------
|
||||||
The download thumbnail's file name
|
The download thumbnail's file name
|
||||||
"""
|
"""
|
||||||
return f"{self.uid}.{self.get(v.thumbnail_ext, str)}"
|
return self.base_filename(ext=self.get(v.thumbnail_ext, str))
|
||||||
|
|
||||||
def get_download_thumbnail_path(self) -> str:
|
def get_download_thumbnail_path(self) -> str:
|
||||||
"""Returns the entry's thumbnail's file path to where it was downloaded"""
|
"""Returns the entry's thumbnail's file path to where it was downloaded"""
|
||||||
|
|
@ -155,7 +158,10 @@ class Entry(BaseEntry, Scriptable):
|
||||||
possible_thumbnail_exts.add(thumbnail["url"].split(".")[-1])
|
possible_thumbnail_exts.add(thumbnail["url"].split(".")[-1])
|
||||||
|
|
||||||
for ext in possible_thumbnail_exts:
|
for ext in possible_thumbnail_exts:
|
||||||
possible_thumbnail_path = str(Path(self.working_directory()) / f"{self.uid}.{ext}")
|
possible_thumbnail_filename = self.base_filename(ext=ext)
|
||||||
|
possible_thumbnail_path = str(
|
||||||
|
Path(self.working_directory()) / possible_thumbnail_filename
|
||||||
|
)
|
||||||
if os.path.isfile(possible_thumbnail_path):
|
if os.path.isfile(possible_thumbnail_path):
|
||||||
return possible_thumbnail_path
|
return possible_thumbnail_path
|
||||||
|
|
||||||
|
|
@ -202,7 +208,7 @@ class Entry(BaseEntry, Scriptable):
|
||||||
# HACK: yt-dlp does not record extracted/converted extensions anywhere. If the file is not
|
# HACK: yt-dlp does not record extracted/converted extensions anywhere. If the file is not
|
||||||
# found, try it using all possible extensions
|
# found, try it using all possible extensions
|
||||||
if not file_exists:
|
if not file_exists:
|
||||||
for ext in AUDIO_CODEC_EXTS.union(VIDEO_CODEC_EXTS):
|
for ext in AUDIO_CODEC_EXTS | VIDEO_CODEC_EXTS:
|
||||||
if os.path.isfile(self.get_download_file_path().removesuffix(self.ext) + ext):
|
if os.path.isfile(self.get_download_file_path().removesuffix(self.ext) + ext):
|
||||||
file_exists = True
|
file_exists = True
|
||||||
break
|
break
|
||||||
|
|
|
||||||
|
|
@ -157,13 +157,16 @@ class EntryParent(BaseEntry):
|
||||||
def _uid_is_uploader_id(parent: "EntryParent"):
|
def _uid_is_uploader_id(parent: "EntryParent"):
|
||||||
return parent.uid == parent.uploader_id
|
return parent.uid == parent.uploader_id
|
||||||
|
|
||||||
top_level_parents = [
|
top_level_parents = [parent for parent in parents if parent.num_children() == 0]
|
||||||
parent for parent in parents if parent.num_children() == 0 and _url_matches(parent)
|
|
||||||
]
|
|
||||||
|
|
||||||
# If more than 1 parent exists, assume the uploader_id is the root parent
|
# If more than 1 parent exists, assume the uploader_id is the root parent
|
||||||
if len(top_level_parents) > 1:
|
if len(top_level_parents) > 1:
|
||||||
top_level_parents = [parent for parent in parents if _uid_is_uploader_id(parent)]
|
top_level_parents = [
|
||||||
|
parent for parent in top_level_parents if _uid_is_uploader_id(parent)
|
||||||
|
]
|
||||||
|
|
||||||
|
if len(top_level_parents) > 1:
|
||||||
|
top_level_parents = [parent for parent in top_level_parents if _url_matches(parent)]
|
||||||
|
|
||||||
match len(top_level_parents):
|
match len(top_level_parents):
|
||||||
case 0:
|
case 0:
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ def main():
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return_code = _main()
|
return_code = _main()
|
||||||
Logger.cleanup(cleanup_error_log=return_code == 0)
|
Logger.cleanup(has_error=return_code != 0)
|
||||||
sys.exit(return_code)
|
sys.exit(return_code)
|
||||||
except Exception as exc: # pylint: disable=broad-except
|
except Exception as exc: # pylint: disable=broad-except
|
||||||
Logger.log_exception(exception=exc)
|
Logger.log_exception(exception=exc)
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
|
from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
|
||||||
from ytdl_sub.validators.validators import BoolValidator
|
from ytdl_sub.validators.validators import BoolValidator
|
||||||
|
|
||||||
logger = Logger.get("embed_thumbnail")
|
logger = Logger.get("embed-thumbnail")
|
||||||
|
|
||||||
|
|
||||||
class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
|
class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
logger = Logger.get("conditional")
|
logger = Logger.get("filter-exclude")
|
||||||
|
|
||||||
|
|
||||||
class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
|
class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
|
||||||
|
|
@ -55,6 +55,11 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
|
||||||
for formatter in self.plugin_options.list:
|
for formatter in self.plugin_options.list:
|
||||||
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
|
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
|
||||||
if bool(out):
|
if bool(out):
|
||||||
|
logger.info(
|
||||||
|
"Filtering '%s' from the filter %s evaluating to True",
|
||||||
|
entry.title,
|
||||||
|
formatter.format_string,
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return entry
|
return entry
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
|
||||||
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
|
||||||
|
|
||||||
logger = Logger.get("conditional")
|
logger = Logger.get("filter-include")
|
||||||
|
|
||||||
|
|
||||||
class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
|
class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
|
||||||
|
|
@ -63,6 +63,11 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
|
||||||
for formatter in self.plugin_options.list:
|
for formatter in self.plugin_options.list:
|
||||||
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
|
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
|
||||||
if not bool(out):
|
if not bool(out):
|
||||||
|
logger.info(
|
||||||
|
"Filtering '%s' from the filter %s evaluating to False",
|
||||||
|
entry.title,
|
||||||
|
formatter.format_string,
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return entry
|
return entry
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ from ytdl_sub.config.validators.options import OptionsDictValidator
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.validators import StringListValidator
|
from ytdl_sub.validators.validators import StringListValidator
|
||||||
|
|
||||||
logger = Logger.get("match_filters")
|
logger = Logger.get("match-filters")
|
||||||
|
|
||||||
|
|
||||||
def default_filters() -> Tuple[List[str], List[str]]:
|
def default_filters() -> Tuple[List[str], List[str]]:
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import BoolValidator
|
||||||
|
|
||||||
v: VariableDefinitions = VARIABLES
|
v: VariableDefinitions = VARIABLES
|
||||||
|
|
||||||
logger = Logger.get("music_tags")
|
logger = Logger.get("music-tags")
|
||||||
|
|
||||||
|
|
||||||
def _is_multi_field(tag_name: str) -> bool:
|
def _is_multi_field(tag_name: str) -> bool:
|
||||||
|
|
|
||||||
|
|
@ -208,7 +208,6 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
|
||||||
file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}")
|
file_metadata = FileMetadata(f"Embedded subtitles with lang(s) {', '.join(langs)}")
|
||||||
if self.plugin_options.subtitles_name:
|
if self.plugin_options.subtitles_name:
|
||||||
for lang in langs:
|
for lang in langs:
|
||||||
subtitle_file_name = f"{entry.uid}.{lang}.{self.plugin_options.subtitles_type}"
|
|
||||||
output_subtitle_file_name = self.overrides.apply_formatter(
|
output_subtitle_file_name = self.overrides.apply_formatter(
|
||||||
formatter=self.plugin_options.subtitles_name,
|
formatter=self.plugin_options.subtitles_name,
|
||||||
entry=entry,
|
entry=entry,
|
||||||
|
|
@ -216,7 +215,9 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
|
||||||
)
|
)
|
||||||
|
|
||||||
self.save_file(
|
self.save_file(
|
||||||
file_name=subtitle_file_name,
|
file_name=entry.base_filename(
|
||||||
|
ext=f"{lang}.{self.plugin_options.subtitles_type}"
|
||||||
|
),
|
||||||
output_file_name=output_subtitle_file_name,
|
output_file_name=output_subtitle_file_name,
|
||||||
entry=entry,
|
entry=entry,
|
||||||
)
|
)
|
||||||
|
|
@ -225,9 +226,8 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
|
||||||
# Can happen for both file and embedded subs
|
# Can happen for both file and embedded subs
|
||||||
for lang in langs:
|
for lang in langs:
|
||||||
for possible_ext in SUBTITLE_EXTENSIONS:
|
for possible_ext in SUBTITLE_EXTENSIONS:
|
||||||
possible_subs_file = (
|
possible_subs_filename = entry.base_filename(ext=f"{lang}.{possible_ext}")
|
||||||
Path(self.working_directory) / f"{entry.uid}.{lang}.{possible_ext}"
|
possible_subs_file = Path(self.working_directory) / possible_subs_filename
|
||||||
)
|
|
||||||
FileHandler.delete(possible_subs_file)
|
FileHandler.delete(possible_subs_file)
|
||||||
|
|
||||||
return file_metadata
|
return file_metadata
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||||
|
|
||||||
logger = Logger.get("video_tags")
|
logger = Logger.get("video-tags")
|
||||||
|
|
||||||
|
|
||||||
class VideoTagsOptions(OptionsDictValidator):
|
class VideoTagsOptions(OptionsDictValidator):
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ presets:
|
||||||
#
|
#
|
||||||
# "Subscription Name": "url"
|
# "Subscription Name": "url"
|
||||||
#
|
#
|
||||||
# where the first url tries to grab channel avatar + banner
|
# where the first the first url tries to grab channel avatar + banner
|
||||||
#
|
#
|
||||||
_multi_url:
|
_multi_url:
|
||||||
download:
|
download:
|
||||||
|
|
@ -20,6 +20,11 @@ presets:
|
||||||
uid: "avatar_uncropped"
|
uid: "avatar_uncropped"
|
||||||
- name: "{banner_uncropped_thumbnail_file_name}"
|
- name: "{banner_uncropped_thumbnail_file_name}"
|
||||||
uid: "banner_uncropped"
|
uid: "banner_uncropped"
|
||||||
|
source_thumbnails:
|
||||||
|
- name: "{avatar_uncropped_thumbnail_file_name}"
|
||||||
|
uid: "avatar_uncropped"
|
||||||
|
- name: "{banner_uncropped_thumbnail_file_name}"
|
||||||
|
uid: "banner_uncropped"
|
||||||
- url: "{url2}"
|
- url: "{url2}"
|
||||||
- url: "{url3}"
|
- url: "{url3}"
|
||||||
- url: "{url4}"
|
- url: "{url4}"
|
||||||
|
|
@ -326,3 +331,84 @@ presets:
|
||||||
url99: "{subscription_value_99}"
|
url99: "{subscription_value_99}"
|
||||||
url100: "{subscription_value_100}"
|
url100: "{subscription_value_100}"
|
||||||
|
|
||||||
|
url21: "{subscription_value_21}"
|
||||||
|
url22: "{subscription_value_22}"
|
||||||
|
url23: "{subscription_value_23}"
|
||||||
|
url24: "{subscription_value_24}"
|
||||||
|
url25: "{subscription_value_25}"
|
||||||
|
url26: "{subscription_value_26}"
|
||||||
|
url27: "{subscription_value_27}"
|
||||||
|
url28: "{subscription_value_28}"
|
||||||
|
url29: "{subscription_value_29}"
|
||||||
|
url30: "{subscription_value_30}"
|
||||||
|
url31: "{subscription_value_31}"
|
||||||
|
url32: "{subscription_value_32}"
|
||||||
|
url33: "{subscription_value_33}"
|
||||||
|
url34: "{subscription_value_34}"
|
||||||
|
url35: "{subscription_value_35}"
|
||||||
|
url36: "{subscription_value_36}"
|
||||||
|
url37: "{subscription_value_37}"
|
||||||
|
url38: "{subscription_value_38}"
|
||||||
|
url39: "{subscription_value_39}"
|
||||||
|
url40: "{subscription_value_40}"
|
||||||
|
url41: "{subscription_value_41}"
|
||||||
|
url42: "{subscription_value_42}"
|
||||||
|
url43: "{subscription_value_43}"
|
||||||
|
url44: "{subscription_value_44}"
|
||||||
|
url45: "{subscription_value_45}"
|
||||||
|
url46: "{subscription_value_46}"
|
||||||
|
url47: "{subscription_value_47}"
|
||||||
|
url48: "{subscription_value_48}"
|
||||||
|
url49: "{subscription_value_49}"
|
||||||
|
url50: "{subscription_value_50}"
|
||||||
|
url51: "{subscription_value_51}"
|
||||||
|
url52: "{subscription_value_52}"
|
||||||
|
url53: "{subscription_value_53}"
|
||||||
|
url54: "{subscription_value_54}"
|
||||||
|
url55: "{subscription_value_55}"
|
||||||
|
url56: "{subscription_value_56}"
|
||||||
|
url57: "{subscription_value_57}"
|
||||||
|
url58: "{subscription_value_58}"
|
||||||
|
url59: "{subscription_value_59}"
|
||||||
|
url60: "{subscription_value_60}"
|
||||||
|
url61: "{subscription_value_61}"
|
||||||
|
url62: "{subscription_value_62}"
|
||||||
|
url63: "{subscription_value_63}"
|
||||||
|
url64: "{subscription_value_64}"
|
||||||
|
url65: "{subscription_value_65}"
|
||||||
|
url66: "{subscription_value_66}"
|
||||||
|
url67: "{subscription_value_67}"
|
||||||
|
url68: "{subscription_value_68}"
|
||||||
|
url69: "{subscription_value_69}"
|
||||||
|
url70: "{subscription_value_70}"
|
||||||
|
url71: "{subscription_value_71}"
|
||||||
|
url72: "{subscription_value_72}"
|
||||||
|
url73: "{subscription_value_73}"
|
||||||
|
url74: "{subscription_value_74}"
|
||||||
|
url75: "{subscription_value_75}"
|
||||||
|
url76: "{subscription_value_76}"
|
||||||
|
url77: "{subscription_value_77}"
|
||||||
|
url78: "{subscription_value_78}"
|
||||||
|
url79: "{subscription_value_79}"
|
||||||
|
url80: "{subscription_value_80}"
|
||||||
|
url81: "{subscription_value_81}"
|
||||||
|
url82: "{subscription_value_82}"
|
||||||
|
url83: "{subscription_value_83}"
|
||||||
|
url84: "{subscription_value_84}"
|
||||||
|
url85: "{subscription_value_85}"
|
||||||
|
url86: "{subscription_value_86}"
|
||||||
|
url87: "{subscription_value_87}"
|
||||||
|
url88: "{subscription_value_88}"
|
||||||
|
url89: "{subscription_value_89}"
|
||||||
|
url90: "{subscription_value_90}"
|
||||||
|
url91: "{subscription_value_91}"
|
||||||
|
url92: "{subscription_value_92}"
|
||||||
|
url93: "{subscription_value_93}"
|
||||||
|
url94: "{subscription_value_94}"
|
||||||
|
url95: "{subscription_value_95}"
|
||||||
|
url96: "{subscription_value_96}"
|
||||||
|
url97: "{subscription_value_97}"
|
||||||
|
url98: "{subscription_value_98}"
|
||||||
|
url99: "{subscription_value_99}"
|
||||||
|
url100: "{subscription_value_100}"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,11 @@ presets:
|
||||||
uid: "avatar_uncropped"
|
uid: "avatar_uncropped"
|
||||||
- name: "{tv_show_fanart_file_name}"
|
- name: "{tv_show_fanart_file_name}"
|
||||||
uid: "banner_uncropped"
|
uid: "banner_uncropped"
|
||||||
|
source_thumbnails:
|
||||||
|
- name: "{tv_show_poster_file_name}"
|
||||||
|
uid: "avatar_uncropped"
|
||||||
|
- name: "{tv_show_fanart_file_name}"
|
||||||
|
uid: "banner_uncropped"
|
||||||
|
|
||||||
output_directory_nfo_tags:
|
output_directory_nfo_tags:
|
||||||
tags:
|
tags:
|
||||||
|
|
|
||||||
|
|
@ -73,12 +73,18 @@ class ArrayFunctions:
|
||||||
return Array(output)
|
return Array(output)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def array_at(array: Array, idx: Integer) -> AnyArgument:
|
def array_at(array: Array, idx: Integer, default: Optional[AnyArgument] = None) -> AnyArgument:
|
||||||
"""
|
"""
|
||||||
:description:
|
:description:
|
||||||
Return the element in the Array at index ``idx``.
|
Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length,
|
||||||
|
either return ``default`` if provided or throw an error.
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
return array.value[idx.value]
|
return array.value[idx.value]
|
||||||
|
except IndexError:
|
||||||
|
if default is not None:
|
||||||
|
return default
|
||||||
|
raise
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def array_first(array: Array, fallback: AnyArgument) -> AnyArgument:
|
def array_first(array: Array, fallback: AnyArgument) -> AnyArgument:
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
from typing import Union
|
from typing import Union
|
||||||
|
|
||||||
|
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||||
from ytdl_sub.script.types.resolvable import Boolean
|
from ytdl_sub.script.types.resolvable import Boolean
|
||||||
from ytdl_sub.script.types.resolvable import ReturnableArgumentA
|
from ytdl_sub.script.types.resolvable import ReturnableArgumentA
|
||||||
from ytdl_sub.script.types.resolvable import ReturnableArgumentB
|
from ytdl_sub.script.types.resolvable import ReturnableArgumentB
|
||||||
|
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||||
|
|
||||||
|
|
||||||
class ConditionalFunctions:
|
class ConditionalFunctions:
|
||||||
|
|
@ -19,6 +21,39 @@ class ConditionalFunctions:
|
||||||
return true
|
return true
|
||||||
return false
|
return false
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def elif_(*if_elif_else: AnyArgument) -> AnyArgument:
|
||||||
|
"""
|
||||||
|
:description:
|
||||||
|
Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via
|
||||||
|
adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to
|
||||||
|
supply at least one conditional and an else.
|
||||||
|
:usage:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
%elif(
|
||||||
|
condition1,
|
||||||
|
return1,
|
||||||
|
condition2,
|
||||||
|
return2,
|
||||||
|
...
|
||||||
|
else_return
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
arguments = list(if_elif_else)
|
||||||
|
if len(arguments) < 3:
|
||||||
|
raise FunctionRuntimeException("elif requires at least 3 arguments")
|
||||||
|
|
||||||
|
if len(arguments) % 2 == 0:
|
||||||
|
raise FunctionRuntimeException("elif must have an odd number of arguments")
|
||||||
|
|
||||||
|
for idx in range(0, len(arguments) - 1, 2):
|
||||||
|
if bool(arguments[idx].value):
|
||||||
|
return arguments[idx + 1]
|
||||||
|
|
||||||
|
return arguments[-1]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def if_passthrough(
|
def if_passthrough(
|
||||||
maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB
|
maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
|
from ytdl_sub.script.types.array import Array
|
||||||
from ytdl_sub.script.types.resolvable import AnyArgument
|
from ytdl_sub.script.types.resolvable import AnyArgument
|
||||||
from ytdl_sub.script.types.resolvable import Boolean
|
from ytdl_sub.script.types.resolvable import Boolean
|
||||||
from ytdl_sub.script.types.resolvable import Integer
|
from ytdl_sub.script.types.resolvable import Integer
|
||||||
|
|
@ -80,6 +81,22 @@ class StringFunctions:
|
||||||
|
|
||||||
return String(string.value.replace(old.value, new.value))
|
return String(string.value.replace(old.value, new.value))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def split(string: String, sep: String, max_split: Optional[Integer] = None) -> Array:
|
||||||
|
"""
|
||||||
|
:description:
|
||||||
|
Splits the input string into multiple strings.
|
||||||
|
"""
|
||||||
|
if max_split is not None:
|
||||||
|
return Array(
|
||||||
|
[
|
||||||
|
String(split_val)
|
||||||
|
for split_val in string.value.split(sep=sep.value, maxsplit=max_split.value)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
return Array([String(split_val) for split_val in string.value.split(sep=sep.value)])
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def concat(*values: String) -> String:
|
def concat(*values: String) -> String:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,9 @@ from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import List
|
from typing import List
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from mergedeep import mergedeep
|
||||||
|
|
||||||
from ytdl_sub.config.config_file import ConfigFile
|
from ytdl_sub.config.config_file import ConfigFile
|
||||||
from ytdl_sub.config.preset import Preset
|
from ytdl_sub.config.preset import Preset
|
||||||
|
|
@ -69,7 +72,10 @@ class Subscription(SubscriptionDownload):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_file_path(
|
def from_file_path(
|
||||||
cls, config: ConfigFile, subscription_path: str | Path
|
cls,
|
||||||
|
config: ConfigFile,
|
||||||
|
subscription_path: str | Path,
|
||||||
|
subscription_override_dict: Optional[Dict] = None,
|
||||||
) -> List["Subscription"]:
|
) -> List["Subscription"]:
|
||||||
"""
|
"""
|
||||||
Loads subscriptions from a file.
|
Loads subscriptions from a file.
|
||||||
|
|
@ -80,6 +86,8 @@ class Subscription(SubscriptionDownload):
|
||||||
Validated instance of the config
|
Validated instance of the config
|
||||||
subscription_path:
|
subscription_path:
|
||||||
File path to the subscription yaml file
|
File path to the subscription yaml file
|
||||||
|
subscription_override_dict:
|
||||||
|
Optional dict containing overrides to every subscription
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
|
|
@ -122,6 +130,13 @@ class Subscription(SubscriptionDownload):
|
||||||
)
|
)
|
||||||
|
|
||||||
for subscription_key, subscription_object in subscriptions_dicts.items():
|
for subscription_key, subscription_object in subscriptions_dicts.items():
|
||||||
|
# Hard-override subscriptions here
|
||||||
|
mergedeep.merge(
|
||||||
|
subscription_object,
|
||||||
|
subscription_override_dict or {},
|
||||||
|
strategy=mergedeep.Strategy.ADDITIVE,
|
||||||
|
)
|
||||||
|
|
||||||
subscriptions.append(
|
subscriptions.append(
|
||||||
cls.from_dict(
|
cls.from_dict(
|
||||||
config=config,
|
config=config,
|
||||||
|
|
|
||||||
|
|
@ -343,6 +343,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
||||||
preset=self._preset_options,
|
preset=self._preset_options,
|
||||||
plugins=plugins,
|
plugins=plugins,
|
||||||
enhanced_download_archive=self._enhanced_download_archive,
|
enhanced_download_archive=self._enhanced_download_archive,
|
||||||
|
overrides=self.overrides,
|
||||||
working_directory=self.working_directory,
|
working_directory=self.working_directory,
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
)
|
)
|
||||||
|
|
@ -395,6 +396,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
||||||
preset=self._preset_options,
|
preset=self._preset_options,
|
||||||
plugins=plugins,
|
plugins=plugins,
|
||||||
enhanced_download_archive=self._enhanced_download_archive,
|
enhanced_download_archive=self._enhanced_download_archive,
|
||||||
|
overrides=self.overrides,
|
||||||
working_directory=self.working_directory,
|
working_directory=self.working_directory,
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ from typing import TypeVar
|
||||||
|
|
||||||
from yt_dlp import match_filter_func
|
from yt_dlp import match_filter_func
|
||||||
|
|
||||||
|
from ytdl_sub.config.overrides import Overrides
|
||||||
from ytdl_sub.config.plugin.plugin import Plugin
|
from ytdl_sub.config.plugin.plugin import Plugin
|
||||||
from ytdl_sub.config.preset import Preset
|
from ytdl_sub.config.preset import Preset
|
||||||
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
|
||||||
|
|
@ -33,12 +34,14 @@ class SubscriptionYTDLOptions:
|
||||||
preset: Preset,
|
preset: Preset,
|
||||||
plugins: List[Plugin],
|
plugins: List[Plugin],
|
||||||
enhanced_download_archive: EnhancedDownloadArchive,
|
enhanced_download_archive: EnhancedDownloadArchive,
|
||||||
|
overrides: Overrides,
|
||||||
working_directory: str,
|
working_directory: str,
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
):
|
):
|
||||||
self._preset = preset
|
self._preset = preset
|
||||||
self._plugins = plugins
|
self._plugins = plugins
|
||||||
self._enhanced_download_archive = enhanced_download_archive
|
self._enhanced_download_archive = enhanced_download_archive
|
||||||
|
self._overrides = overrides
|
||||||
self._working_directory = working_directory
|
self._working_directory = working_directory
|
||||||
self._dry_run = dry_run
|
self._dry_run = dry_run
|
||||||
|
|
||||||
|
|
@ -56,8 +59,8 @@ class SubscriptionYTDLOptions:
|
||||||
ytdl-options to apply to every run no matter what
|
ytdl-options to apply to every run no matter what
|
||||||
"""
|
"""
|
||||||
ytdl_options = {
|
ytdl_options = {
|
||||||
# Download all files in the format of {id}.{ext}
|
# Download all files in the format of {id}.{ext}, where id is sanitized
|
||||||
"outtmpl": str(Path(self._working_directory) / "%(id)s.%(ext)s"),
|
"outtmpl": str(Path(self._working_directory) / "%(id)S.%(ext)s"),
|
||||||
# Always write thumbnails
|
# Always write thumbnails
|
||||||
"writethumbnail": True,
|
"writethumbnail": True,
|
||||||
"ffmpeg_location": FFMPEG.ffmpeg_path(),
|
"ffmpeg_location": FFMPEG.ffmpeg_path(),
|
||||||
|
|
@ -78,6 +81,7 @@ class SubscriptionYTDLOptions:
|
||||||
"skip_download": True,
|
"skip_download": True,
|
||||||
"writethumbnail": False,
|
"writethumbnail": False,
|
||||||
"writeinfojson": True,
|
"writeinfojson": True,
|
||||||
|
"extract_flat": "discard", # do not store info.json in mem since its in file
|
||||||
}
|
}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|
@ -90,6 +94,11 @@ class SubscriptionYTDLOptions:
|
||||||
|
|
||||||
if self._preset.output_options.maintain_download_archive:
|
if self._preset.output_options.maintain_download_archive:
|
||||||
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
|
ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path
|
||||||
|
if self._preset.output_options.keep_max_files:
|
||||||
|
# yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure
|
||||||
|
ytdl_options["max_downloads"] = max(
|
||||||
|
int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2
|
||||||
|
)
|
||||||
|
|
||||||
return ytdl_options
|
return ytdl_options
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -210,6 +210,14 @@ class Logger:
|
||||||
finally:
|
finally:
|
||||||
redirect_stream.flush()
|
redirect_stream.flush()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _append_to_error_log(cls):
|
||||||
|
# Any time an exception occurs, dump all debug logs into the error log
|
||||||
|
with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open(
|
||||||
|
cls.error_log_filename(), mode="a", encoding="utf-8"
|
||||||
|
) as error_logs:
|
||||||
|
error_logs.writelines(debug_logs.readlines())
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None):
|
def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None):
|
||||||
"""
|
"""
|
||||||
|
|
@ -248,14 +256,10 @@ class Logger:
|
||||||
log_filepath if log_filepath else Logger.error_log_filename(),
|
log_filepath if log_filepath else Logger.error_log_filename(),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Any time an exception occurs, dump all debug logs into the error log
|
cls._append_to_error_log()
|
||||||
with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open(
|
|
||||||
cls.error_log_filename(), mode="a", encoding="utf-8"
|
|
||||||
) as error_logs:
|
|
||||||
error_logs.writelines(debug_logs.readlines())
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def cleanup(cls, cleanup_error_log: bool = False):
|
def cleanup(cls, has_error: bool = False):
|
||||||
"""
|
"""
|
||||||
Cleans up debug log file left behind
|
Cleans up debug log file left behind
|
||||||
"""
|
"""
|
||||||
|
|
@ -263,9 +267,11 @@ class Logger:
|
||||||
for handler in logger.handlers:
|
for handler in logger.handlers:
|
||||||
handler.close()
|
handler.close()
|
||||||
|
|
||||||
cls._DEBUG_LOGGER_FILE.close()
|
if has_error:
|
||||||
FileHandler.delete(cls.debug_log_filename())
|
cls._append_to_error_log()
|
||||||
|
else:
|
||||||
if cleanup_error_log:
|
|
||||||
cls._ERROR_LOG_FILE.close()
|
cls._ERROR_LOG_FILE.close()
|
||||||
FileHandler.delete(cls.error_log_filename())
|
FileHandler.delete(cls.error_log_filename())
|
||||||
|
|
||||||
|
cls._DEBUG_LOGGER_FILE.close()
|
||||||
|
FileHandler.delete(cls.debug_log_filename())
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import copy
|
||||||
from abc import ABC
|
from abc import ABC
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
|
from typing import Optional
|
||||||
from typing import Set
|
from typing import Set
|
||||||
|
|
||||||
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
|
from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS
|
||||||
|
|
@ -13,21 +14,47 @@ from ytdl_sub.script.utils.exceptions import RuntimeException
|
||||||
from ytdl_sub.utils.exceptions import StringFormattingException
|
from ytdl_sub.utils.exceptions import StringFormattingException
|
||||||
from ytdl_sub.utils.script import ScriptUtils
|
from ytdl_sub.utils.script import ScriptUtils
|
||||||
|
|
||||||
|
_BASE_SCRIPT: Script = Script(
|
||||||
|
ScriptUtils.add_sanitized_variables(
|
||||||
|
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Scriptable(ABC):
|
class Scriptable(ABC):
|
||||||
"""
|
"""
|
||||||
Shared class between Entry and Overrides to manage their underlying Script.
|
Shared class between Entry and Overrides to manage their underlying Script.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_BASE_SCRIPT: Script = Script(
|
def __init__(self, initialize_base_script: bool = False):
|
||||||
ScriptUtils.add_sanitized_variables(
|
self._script: Optional[Script] = None
|
||||||
dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS))
|
self._unresolvable: Optional[Set[str]] = None
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def __init__(self):
|
if initialize_base_script:
|
||||||
self.script = copy.deepcopy(Scriptable._BASE_SCRIPT)
|
self.initialize_base_script()
|
||||||
self.unresolvable: Set[str] = copy.deepcopy(UNRESOLVED_VARIABLES)
|
|
||||||
|
def initialize_base_script(self):
|
||||||
|
"""
|
||||||
|
Initializes with base values
|
||||||
|
"""
|
||||||
|
self._script = copy.deepcopy(_BASE_SCRIPT)
|
||||||
|
self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def script(self) -> Script:
|
||||||
|
"""
|
||||||
|
Initialized script
|
||||||
|
"""
|
||||||
|
assert self._script is not None, "Not initialized"
|
||||||
|
return self._script
|
||||||
|
|
||||||
|
@property
|
||||||
|
def unresolvable(self) -> Set[str]:
|
||||||
|
"""
|
||||||
|
Initialized unresolvable variables
|
||||||
|
"""
|
||||||
|
assert self._unresolvable is not None, "Not initialized"
|
||||||
|
return self._unresolvable
|
||||||
|
|
||||||
def update_script(self) -> None:
|
def update_script(self) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -45,7 +72,7 @@ class Scriptable(ABC):
|
||||||
for var, definition in values.items()
|
for var, definition in values.items()
|
||||||
}
|
}
|
||||||
|
|
||||||
self.unresolvable -= set(list(values_as_str.keys()))
|
self._unresolvable -= set(list(values_as_str.keys()))
|
||||||
self.script.add(
|
self.script.add(
|
||||||
ScriptUtils.add_sanitized_variables(
|
ScriptUtils.add_sanitized_variables(
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -199,3 +199,26 @@ class TestPlaylist:
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
expected_download_summary_file_name="youtube/test_playlist.json",
|
expected_download_summary_file_name="youtube/test_playlist.json",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_playlist_download_from_cli_sub_with_override_arg(
|
||||||
|
self,
|
||||||
|
preset_dict_to_subscription_yaml_generator,
|
||||||
|
playlist_preset_dict,
|
||||||
|
output_directory,
|
||||||
|
):
|
||||||
|
# TODO: Fix CLI parsing on windows when dealing with spaces
|
||||||
|
if IS_WINDOWS:
|
||||||
|
return
|
||||||
|
|
||||||
|
# No config needed when using only prebuilt presets
|
||||||
|
with preset_dict_to_subscription_yaml_generator(
|
||||||
|
subscription_name="music_video_playlist_test", preset_dict=playlist_preset_dict
|
||||||
|
) as subscription_path:
|
||||||
|
args = (
|
||||||
|
f"--dry-run sub '{subscription_path}' --dl-override '--date_range.after 20240101'"
|
||||||
|
)
|
||||||
|
|
||||||
|
subscriptions = mock_run_from_cli(args=args)
|
||||||
|
|
||||||
|
assert len(subscriptions) == 1
|
||||||
|
assert subscriptions[0].transaction_log.is_empty
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec",
|
".ytdl-sub-chapters_from_comments-download-archive.json": "2510b2ff3c54aa4813a4f23ea079e1ec",
|
||||||
"JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6",
|
"JMC/Move 78 - Automated Improvisation [Full Album].jpg": "c12e6a6f242680d1096a1a99d74a62c6",
|
||||||
"JMC/Move 78 - Automated Improvisation [Full Album].mp4": "f401b98c332b76ee1c87065e195d73ce",
|
"JMC/Move 78 - Automated Improvisation [Full Album].mp4": "068526b2d8f85fdcf914df3e23d0b1fa",
|
||||||
"JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac"
|
"JMC/Move 78 - Automated Improvisation [Full Album].nfo": "039268e97673a6f2b391772ec3b52fac"
|
||||||
}
|
}
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4",
|
".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "2cb4b9586fd5bb7f1fed76ed9195e6e4",
|
||||||
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969",
|
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.jpg": "b5353a824a4800cc26f884e3025ed969",
|
||||||
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "8c12640f0c5c280c7a77423431b4ecb1",
|
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "526b6df52a8aaf11dfe56f25ac35a567",
|
||||||
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4"
|
"JMC/This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "ae73ec18a9f0e5a54c90061ccd32e7f4"
|
||||||
}
|
}
|
||||||
|
|
@ -12,6 +12,8 @@
|
||||||
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
|
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
|
||||||
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6",
|
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6",
|
||||||
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
|
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
|
||||||
|
"JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee",
|
||||||
|
"JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
|
||||||
"JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
|
"JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
|
||||||
"JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc"
|
"JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc"
|
||||||
}
|
}
|
||||||
|
|
@ -12,6 +12,8 @@
|
||||||
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
|
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "INFO_JSON",
|
||||||
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6",
|
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "7f1a1d5c1d94938a9b6f565e8159c3d6",
|
||||||
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
|
"JMC/Season 01/s01.e11032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "c56083e2f3545fa2cafc4d67cbfdacf8",
|
||||||
|
"JMC/fanart.jpg": "129c6639b47299bc48062f0365e670ee",
|
||||||
|
"JMC/poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
|
||||||
"JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
|
"JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
|
||||||
"JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc"
|
"JMC/tvshow.nfo": "e92e4a2c01522dd9a9c3423f0f9304dc"
|
||||||
}
|
}
|
||||||
|
|
@ -2,6 +2,8 @@ Files created:
|
||||||
----------------------------------------
|
----------------------------------------
|
||||||
{output_directory}
|
{output_directory}
|
||||||
.ytdl-sub-music_video_playlist_test-download-archive.json
|
.ytdl-sub-music_video_playlist_test-download-archive.json
|
||||||
|
fanart.jpg
|
||||||
|
poster.jpg
|
||||||
season01-poster.jpg
|
season01-poster.jpg
|
||||||
tvshow.nfo
|
tvshow.nfo
|
||||||
NFO tags:
|
NFO tags:
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import re
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable
|
from typing import Callable
|
||||||
|
from typing import List
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -22,6 +23,7 @@ from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
|
||||||
@pytest.mark.parametrize("dry_run", [True, False])
|
@pytest.mark.parametrize("dry_run", [True, False])
|
||||||
@pytest.mark.parametrize("mock_success_output", [True, False])
|
@pytest.mark.parametrize("mock_success_output", [True, False])
|
||||||
@pytest.mark.parametrize("keep_successful_logs", [True, False])
|
@pytest.mark.parametrize("keep_successful_logs", [True, False])
|
||||||
|
@pytest.mark.parametrize("match", [[], ["Rick", "Michael"]])
|
||||||
def test_subscription_logs_write_to_file(
|
def test_subscription_logs_write_to_file(
|
||||||
persist_logs_directory: str,
|
persist_logs_directory: str,
|
||||||
persist_logs_config_factory: Callable,
|
persist_logs_config_factory: Callable,
|
||||||
|
|
@ -30,8 +32,11 @@ def test_subscription_logs_write_to_file(
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
mock_success_output: bool,
|
mock_success_output: bool,
|
||||||
keep_successful_logs: bool,
|
keep_successful_logs: bool,
|
||||||
|
match: List[str],
|
||||||
):
|
):
|
||||||
subscripton_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"]
|
subscription_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"]
|
||||||
|
if match:
|
||||||
|
subscription_names = ["Rick Astley", "Michael Jackson"]
|
||||||
num_runs = 2
|
num_runs = 2
|
||||||
|
|
||||||
config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs)
|
config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs)
|
||||||
|
|
@ -47,6 +52,8 @@ def test_subscription_logs_write_to_file(
|
||||||
_download_subscriptions_from_yaml_files(
|
_download_subscriptions_from_yaml_files(
|
||||||
config=config,
|
config=config,
|
||||||
subscription_paths=subscription_paths,
|
subscription_paths=subscription_paths,
|
||||||
|
subscription_matches=match,
|
||||||
|
subscription_override_dict={},
|
||||||
update_with_info_json=False,
|
update_with_info_json=False,
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
)
|
)
|
||||||
|
|
@ -61,8 +68,8 @@ def test_subscription_logs_write_to_file(
|
||||||
return
|
return
|
||||||
# If not success, expect 2 log files for both sub errors
|
# If not success, expect 2 log files for both sub errors
|
||||||
elif not mock_success_output:
|
elif not mock_success_output:
|
||||||
assert len(log_directory_files) == (num_runs * len(subscripton_names))
|
assert len(log_directory_files) == (num_runs * len(subscription_names))
|
||||||
for log_path, subscription_name in zip(log_directory_files, subscripton_names):
|
for log_path, subscription_name in zip(log_directory_files, subscription_names):
|
||||||
subscription_log_file_name = subscription_name.lower().replace(" ", "_")
|
subscription_log_file_name = subscription_name.lower().replace(" ", "_")
|
||||||
|
|
||||||
assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name))
|
assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name))
|
||||||
|
|
@ -74,9 +81,9 @@ def test_subscription_logs_write_to_file(
|
||||||
)
|
)
|
||||||
# If success and success logging, expect 3 log files
|
# If success and success logging, expect 3 log files
|
||||||
else:
|
else:
|
||||||
assert len(log_directory_files) == (num_runs * len(subscripton_names))
|
assert len(log_directory_files) == (num_runs * len(subscription_names))
|
||||||
for log_file_path, subscription_name in zip(
|
for log_file_path, subscription_name in zip(
|
||||||
log_directory_files, subscripton_names * num_runs
|
log_directory_files, subscription_names * num_runs
|
||||||
):
|
):
|
||||||
subscription_log_file_name = subscription_name.lower().replace(" ", "_")
|
subscription_log_file_name = subscription_name.lower().replace(" ", "_")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -47,8 +47,8 @@ def test_main_exit_code(mock_sys_exit, return_code: int):
|
||||||
main()
|
main()
|
||||||
|
|
||||||
assert mock_logger_cleanup.call_count == 1
|
assert mock_logger_cleanup.call_count == 1
|
||||||
assert mock_logger_cleanup.call_args.kwargs["cleanup_error_log"] == (
|
assert mock_logger_cleanup.call_args.kwargs["has_error"] == (
|
||||||
True if return_code == 0 else False
|
True if return_code != 0 else False
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -107,6 +107,58 @@ def test_args_after_sub_work(mock_sys_exit, tv_show_config_path):
|
||||||
assert mock_sub.call_count == 1
|
assert mock_sub.call_count == 1
|
||||||
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
|
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
|
||||||
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
|
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
|
||||||
|
assert mock_sub.call_args.kwargs["subscription_matches"] == []
|
||||||
|
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE
|
||||||
|
|
||||||
|
|
||||||
|
def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path):
|
||||||
|
with mock_sys_exit(expected_exit_code=0), patch.object(
|
||||||
|
sys,
|
||||||
|
"argv",
|
||||||
|
[
|
||||||
|
"ytdl-sub",
|
||||||
|
"--match",
|
||||||
|
"testA",
|
||||||
|
"testB",
|
||||||
|
"-c",
|
||||||
|
tv_show_config_path,
|
||||||
|
"sub",
|
||||||
|
"--log-level",
|
||||||
|
"verbose",
|
||||||
|
],
|
||||||
|
), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
|
||||||
|
main()
|
||||||
|
|
||||||
|
assert mock_sub.call_count == 1
|
||||||
|
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
|
||||||
|
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
|
||||||
|
assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"]
|
||||||
|
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE
|
||||||
|
|
||||||
|
|
||||||
|
def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path):
|
||||||
|
with mock_sys_exit(expected_exit_code=0), patch.object(
|
||||||
|
sys,
|
||||||
|
"argv",
|
||||||
|
[
|
||||||
|
"ytdl-sub",
|
||||||
|
"-c",
|
||||||
|
tv_show_config_path,
|
||||||
|
"sub",
|
||||||
|
"--log-level",
|
||||||
|
"verbose",
|
||||||
|
"--match",
|
||||||
|
"testA",
|
||||||
|
"--match",
|
||||||
|
"testB",
|
||||||
|
],
|
||||||
|
), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub:
|
||||||
|
main()
|
||||||
|
|
||||||
|
assert mock_sub.call_count == 1
|
||||||
|
assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"]
|
||||||
|
assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path
|
||||||
|
assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"]
|
||||||
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE
|
assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,14 @@ class TestArrayFunctions:
|
||||||
output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}")
|
output = single_variable_output("{%array_at(['a', 'b', 'c'], 1)}")
|
||||||
assert output == "b"
|
assert output == "b"
|
||||||
|
|
||||||
|
def test_array_at_default(self):
|
||||||
|
output = single_variable_output("{%array_at(['a', 'b', 'c'], 30, 'd')}")
|
||||||
|
assert output == "d"
|
||||||
|
|
||||||
|
def test_array_at_error(self):
|
||||||
|
with pytest.raises(FunctionRuntimeException):
|
||||||
|
single_variable_output("{%array_at(['a', 'b', 'c'], 30)}")
|
||||||
|
|
||||||
def test_array_flatten(self):
|
def test_array_flatten(self):
|
||||||
output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}")
|
output = single_variable_output("{%array_flatten(['a', ['b'], [['c']]])}")
|
||||||
assert output == ["a", "b", "c"]
|
assert output == ["a", "b", "c"]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,10 @@
|
||||||
|
import re
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unit.script.conftest import single_variable_output
|
from unit.script.conftest import single_variable_output
|
||||||
|
|
||||||
|
from ytdl_sub.script.utils.exceptions import FunctionRuntimeException
|
||||||
|
|
||||||
|
|
||||||
class TestConditionalFunction:
|
class TestConditionalFunction:
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|
@ -33,3 +37,67 @@ class TestConditionalFunction:
|
||||||
}"""
|
}"""
|
||||||
)
|
)
|
||||||
assert output == "winner"
|
assert output == "winner"
|
||||||
|
|
||||||
|
def test_elif_function(self):
|
||||||
|
output = single_variable_output(
|
||||||
|
"""{
|
||||||
|
%elif(
|
||||||
|
False,
|
||||||
|
"nope",
|
||||||
|
False,
|
||||||
|
"still nope",
|
||||||
|
True,
|
||||||
|
"yes",
|
||||||
|
"default value"
|
||||||
|
)
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
assert output == "yes"
|
||||||
|
|
||||||
|
def test_elif_function_default_value(self):
|
||||||
|
output = single_variable_output(
|
||||||
|
"""{
|
||||||
|
%elif(
|
||||||
|
False,
|
||||||
|
"nope",
|
||||||
|
False,
|
||||||
|
"still nope",
|
||||||
|
False,
|
||||||
|
"will be default",
|
||||||
|
"default value"
|
||||||
|
)
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
assert output == "default value"
|
||||||
|
|
||||||
|
def test_elif_function_errors_lt3(self):
|
||||||
|
with pytest.raises(
|
||||||
|
FunctionRuntimeException,
|
||||||
|
match=re.escape("elif requires at least 3 arguments"),
|
||||||
|
):
|
||||||
|
single_variable_output(
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
%elif(
|
||||||
|
False,
|
||||||
|
"only two args"
|
||||||
|
)
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_elif_function_errors_odd(self):
|
||||||
|
with pytest.raises(
|
||||||
|
FunctionRuntimeException,
|
||||||
|
match=re.escape("elif must have an odd number of arguments"),
|
||||||
|
):
|
||||||
|
single_variable_output(
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
%elif(
|
||||||
|
False,
|
||||||
|
"1",
|
||||||
|
False,
|
||||||
|
"even number args bad"
|
||||||
|
)
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,6 @@
|
||||||
|
from typing import List
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unit.script.conftest import single_variable_output
|
from unit.script.conftest import single_variable_output
|
||||||
|
|
||||||
|
|
@ -114,3 +117,21 @@ class TestNumericFunctions:
|
||||||
def test_contains(self, value, expected_output):
|
def test_contains(self, value, expected_output):
|
||||||
output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}")
|
output = single_variable_output(f"{{%contains('a brown dog', '{value}')}}")
|
||||||
assert output == expected_output
|
assert output == expected_output
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"input_string, split, max_split, expected_output",
|
||||||
|
[
|
||||||
|
("no splits", " | ", None, ["no splits"]),
|
||||||
|
("one | split", " | ", None, ["one", "split"]),
|
||||||
|
("max | split | one", " | ", 1, ["max", "split | one"]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_split(
|
||||||
|
self, input_string: str, split: str, max_split: Optional[int], expected_output: List[str]
|
||||||
|
):
|
||||||
|
if max_split:
|
||||||
|
output = single_variable_output(f"{{%split('{input_string}', '{split}', {max_split})}}")
|
||||||
|
else:
|
||||||
|
output = single_variable_output(f"{{%split('{input_string}', '{split}')}}")
|
||||||
|
|
||||||
|
assert output == expected_output
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,11 @@ class TestFunction:
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"function_str, expected_types, received_types",
|
"function_str, expected_types, received_types",
|
||||||
[
|
[
|
||||||
("{%array_at({'a': 'dict?'}, 1)}", "array: Array, idx: Integer", "Map, Integer"),
|
(
|
||||||
|
"{%array_at({'a': 'dict?'}, 1)}",
|
||||||
|
"array: Array, idx: Integer, default: Optional[AnyArgument]",
|
||||||
|
"Map, Integer",
|
||||||
|
),
|
||||||
("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"),
|
("{%array_extend('not', 'array')}", "arrays: Array, ...", "String, String"),
|
||||||
(
|
(
|
||||||
"{%replace('hi mom', 'mom', 'dad', 1, 0)}",
|
"{%replace('hi mom', 'mom', 'dad', 1, 0)}",
|
||||||
|
|
|
||||||
|
|
@ -111,8 +111,8 @@ class TestLogger:
|
||||||
Logger.cleanup()
|
Logger.cleanup()
|
||||||
assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name)
|
assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name)
|
||||||
|
|
||||||
@pytest.mark.parametrize("clean_error_log", [True, False])
|
@pytest.mark.parametrize("has_error", [True, False])
|
||||||
def test_logger_can_be_cleaned_during_execution(self, clean_error_log: bool):
|
def test_logger_can_be_cleaned_during_execution(self, has_error: bool):
|
||||||
Logger._LOGGER_LEVEL = LoggerLevels.INFO
|
Logger._LOGGER_LEVEL = LoggerLevels.INFO
|
||||||
logger = Logger.get(name="name_test")
|
logger = Logger.get(name="name_test")
|
||||||
|
|
||||||
|
|
@ -133,11 +133,11 @@ class TestLogger:
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
Logger.log_exception(exception=exc)
|
Logger.log_exception(exception=exc)
|
||||||
|
|
||||||
Logger.cleanup(cleanup_error_log=clean_error_log)
|
Logger.cleanup(has_error=has_error)
|
||||||
assert not os.path.isfile(Logger.debug_log_filename())
|
assert not os.path.isfile(Logger.debug_log_filename())
|
||||||
|
|
||||||
assert clean_error_log == (not os.path.isfile(Logger.error_log_filename()))
|
assert not has_error == (not os.path.isfile(Logger.error_log_filename()))
|
||||||
if not clean_error_log:
|
if has_error:
|
||||||
with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file:
|
with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file:
|
||||||
err_logs = err_file.readlines()
|
err_logs = err_file.readlines()
|
||||||
expected = [
|
expected = [
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,8 @@ def should_filter_property(property_name: str) -> bool:
|
||||||
"dict_with_format_strings",
|
"dict_with_format_strings",
|
||||||
"subscription_name",
|
"subscription_name",
|
||||||
"list",
|
"list",
|
||||||
|
"script",
|
||||||
|
"unresolvable",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue