ytdl-sub/src/ytdl_sub/config/preset_options.py

from typing import Any, Dict, Optional, Set

from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.utils.exceptions import SubscriptionPermissionError, ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.validators.file_path_validators import (
    OverridesStringFormatterFilePathValidator,
    StringFormatterFileNameValidator,
)
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import (
    OverridesIntegerFormatterValidator,
    OverridesStringFormatterValidator,
    StandardizedDateValidator,
    StringFormatterValidator,
    UnstructuredOverridesDictFormatterValidator,
)
from ytdl_sub.validators.validators import BoolValidator


class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
    """
    Allows you to add any ytdl argument to ytdl-sub's downloader.
    The argument names can differ slightly from the command-line argument names. See
    `this docstring <https://github.com/yt-dlp/yt-dlp/blob/2022.04.08/yt_dlp/YoutubeDL.py#L197>`_
    for more details.

    :Usage:

    .. code-block:: yaml

           presets:
             my_example_preset:
               ytdl_options:
                 # Ignore any download related errors and continue
                 ignoreerrors: True
                 # Stop downloading additional metadata/videos if it
                 # exists in your download archive
                 break_on_existing: True
                 # Path to your YouTube cookies file to download 18+ restricted content
                 cookiefile: "/path/to/cookies/file.txt"
                 # Only download this number of videos/audio
                 max_downloads: 10
                 # Download and use English title/description/etc YouTube metadata
                 extractor_args:
                   youtube:
                     lang:
                       - "en"


    where each key is a ytdl argument. Include in the example are some popular ytdl_options.
    """

    def to_native_dict(self, overrides: Overrides) -> Dict:
        """
        Materializes the entire ytdl-options dict from OverrideStringFormatters into
        native python.
        """
        out = {
            key: overrides.apply_formatter(val, expected_type=object)
            for key, val in self.dict.items()
        }
        if "cookiefile" in out:
            if not FileHandler.is_file_existent(out["cookiefile"]):
                raise ValidationException(
                    f"Specified cookiefile {out['cookiefile']} but it does not exist as a file."
                )

            if not FileHandler.is_file_readable(out["cookiefile"]):
                raise SubscriptionPermissionError(
                    f"Cannot read cookiefile {out['cookiefile']} due to permissions issue."
                )

        return out


# Disable for proper docstring formatting
# pylint: disable=line-too-long


class OutputOptions(OptionsDictValidator):
    """
    Defines where to output files and thumbnails after all post-processing has completed.

    :Usage:

    .. code-block:: yaml

       presets:
         my_example_preset:
           output_options:
             # required
             output_directory: "/path/to/videos_or_music"
             file_name: "{title_sanitized}.{ext}"
             # optional
             thumbnail_name: "{title_sanitized}.{thumbnail_ext}"
             info_json_name: "{title_sanitized}.{info_json_ext}"
             download_archive_name: ".ytdl-sub-{subscription_name}-download-archive.json"
             migrated_download_archive_name: ".ytdl-sub-{subscription_name_sanitized}-download-archive.json"
             maintain_download_archive: True
             keep_files_before: now
             keep_files_after: 19000101
             keep_max_files: 1000
             keep_files_date_eval: "{upload_date_standardized}"
    """

    _required_keys = {"output_directory", "file_name"}
    _optional_keys = {
        "thumbnail_name",
        "info_json_name",
        "download_archive_name",
        "migrated_download_archive_name",
        "maintain_download_archive",
        "keep_files_before",
        "keep_files_after",
        "keep_max_files",
        "download_archive_standardized_date",
        "keep_files_date_eval",
        "preserve_mtime",
    }

    @classmethod
    def partial_validate(cls, name: str, value: Any) -> None:
        """
        Partially validate output options
        """
        if isinstance(value, dict):
            value["output_directory"] = value.get("output_directory", "placeholder")
            value["file_name"] = value.get("file_name", "placeholder")
            # Set this to True by default in partial validate to avoid failing from keep_files
            value["maintain_download_archive"] = value.get("maintain_download_archive", True)
        _ = cls(name, value)

    def __init__(self, name, value):
        super().__init__(name, value)

        # Output directory should resolve without any entry variables.
        # This is to check the directory for any download-archives before any downloads begin
        self._output_directory = self._validate_key(
            key="output_directory", validator=OverridesStringFormatterFilePathValidator
        )

        # file name and thumbnails however can use entry variables
        self._file_name = self._validate_key(
            key="file_name", validator=StringFormatterFileNameValidator
        )
        self._thumbnail_name = self._validate_key_if_present(
            key="thumbnail_name", validator=StringFormatterFileNameValidator
        )
        self._info_json_name = self._validate_key_if_present(
            key="info_json_name", validator=StringFormatterFileNameValidator
        )

        self._download_archive_name = self._validate_key_if_present(
            key="download_archive_name",
            validator=OverridesStringFormatterValidator,
            default=DEFAULT_DOWNLOAD_ARCHIVE_NAME,
        )
        self._migrated_download_archive_name = self._validate_key_if_present(
            key="migrated_download_archive_name",
            validator=OverridesStringFormatterValidator,
        )

        self._maintain_download_archive = self._validate_key_if_present(
            key="maintain_download_archive", validator=BoolValidator, default=False
        )

        self._keep_files_before = self._validate_key_if_present(
            "keep_files_before", StringDatetimeValidator
        )
        self._keep_files_after = self._validate_key_if_present(
            "keep_files_after", StringDatetimeValidator
        )
        self._keep_max_files = self._validate_key_if_present(
            "keep_max_files", OverridesIntegerFormatterValidator
        )
        self._keep_files_date_eval = self._validate_key(
            "keep_files_date_eval",
            StandardizedDateValidator,
            default=f"{{{v.upload_date_standardized.variable_name}}}",
        )

        self._preserve_mtime = self._validate_key_if_present(
            key="preserve_mtime", validator=BoolValidator, default=False
        )

        if (
            self._keep_files_before or self._keep_files_after or self._keep_max_files
        ) and not self.maintain_download_archive:
            raise self._validation_exception(
                "keep_files/keep_max requires maintain_download_archive set to True"
            )

    @property
    def output_directory(self) -> OverridesStringFormatterValidator:
        """
        :expected type: OverridesFormatter
        :description:
          The output directory to store all media files downloaded.
        """
        return self._output_directory

    @property
    def file_name(self) -> StringFormatterValidator:
        """
        :expected type: EntryFormatter
        :description:
          The file name for the media file. This can include directories such as
          ``"Season {upload_year}/{title}.{ext}"``, and will be placed in the output directory.
        """
        return self._file_name

    @property
    def thumbnail_name(self) -> Optional[StringFormatterValidator]:
        """
        :expected type: Optional[EntryFormatter]
        :description:
          The file name for the media's thumbnail image. This can include directories such
          as ``"Season {upload_year}/{title}.{thumbnail_ext}"``, and will be placed in the output
          directory. Can be set to empty string or `null` to disable thumbnail writes.
        """
        return self._thumbnail_name

    @property
    def info_json_name(self) -> Optional[StringFormatterValidator]:
        """
        :expected type: Optional[EntryFormatter]
        :description:
          The file name for the media's info json file. This can include directories such
          as ``"Season {upload_year}/{title}.{info_json_ext}"``, and will be placed in the output
          directory. Can be set to empty string or `null` to disable info json writes.
        """
        return self._info_json_name

    @property
    def download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
        """
        :expected type: Optional[OverridesFormatter]
        :description:
          The file name to store a subscriptions download archive placed relative to
          the output directory. Defaults to ``.ytdl-sub-{subscription_name}-download-archive.json``
        """
        return self._download_archive_name

    @property
    def migrated_download_archive_name(self) -> Optional[OverridesStringFormatterValidator]:
        """
        :expected type: Optional[OverridesFormatter]
        :description:
          Intended to be used if you are migrating a subscription with either a new
          subscription name or output directory. It will try to load the archive file using this
          name first, and fallback to ``download_archive_name``. It will always save to this file
          and remove the original ``download_archive_name``.
        """
        return self._migrated_download_archive_name

    @property
    def maintain_download_archive(self) -> bool:
        """
        :expected type: Optional[Boolean]
        :description:
          Maintains a download archive file in the output directory for a subscription.
          It is named ``.ytdl-sub-{subscription_name}-download-archive.json``, stored in the
          output directory.

          The download archive contains a mapping of ytdl IDs to downloaded files. This is used to
          create a ytdl download-archive file when invoking a download on a subscription. This will
          prevent ytdl from redownloading media already downloaded.

          Defaults to False.
        """
        return self._maintain_download_archive.value

    @property
    def keep_files_before(self) -> Optional[StringDatetimeValidator]:
        """
        :expected type: Optional[OverridesFormatter]
        :description:
          Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
          ``date_range`` plugin.

          Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
          files before ``now``, which implies all files. Can be used in conjunction with
          ``keep_max_files``.
        """
        return self._keep_files_before

    @property
    def keep_files_after(self) -> Optional[StringDatetimeValidator]:
        """
        :expected type: Optional[OverridesFormatter]
        :description:
          Requires ``maintain_download_archive`` set to True. Uses the same syntax as the
          ``date_range`` plugin.

          Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
          files after ``19000101``, which implies all files. Can be used in conjunction with
          ``keep_max_files``.
        """
        return self._keep_files_after

    @property
    def keep_files_date_eval(self) -> StandardizedDateValidator:
        """
        :expected type: str
        :description:
            Uses this standardized date in the form of YYYY-MM-DD to record in the
            download archive for a given entry. Subsequently, uses this value to
            perform evaluation for keep_files_before/after and keep_max_files. Defaults
            to the entry's upload_date_standardized variable.
        """
        return self._keep_files_date_eval

    @property
    def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
        """
        :expected type: Optional[OverridesFormatter]
        :description:
          Requires ``maintain_download_archive`` set to True.

          Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
          applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
        """
        return self._keep_max_files

    @property
    def preserve_mtime(self) -> bool:
        """
        :expected type: Optional[Boolean]
        :description:
          Preserve the video's original upload time as the file modification time.
          When True, sets the file's mtime to match the video's upload_date from
          yt-dlp metadata. Defaults to False.
        """
        return self._preserve_mtime.value

    def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]:
        return {
            # PluginOperation.MODIFY_ENTRY_METADATA: {
            #     VARIABLES.ytdl_sub_entry_date_eval.variable_name
            # }
        }