676 lines
19 KiB
Python
676 lines
19 KiB
Python
import re
|
|
import time
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from email.utils import formatdate
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from app.features.ytdlp.utils import archive_add, archive_delete, archive_read, get_archive_id
|
|
from app.features.ytdlp.ytdlp_opts import YTDLPOpts
|
|
from app.library.encoder import Encoder
|
|
from app.library.log import get_logger
|
|
from app.library.Utils import clean_item, get_file, get_file_sidecar
|
|
|
|
if TYPE_CHECKING:
|
|
from app.features.presets.schemas import Preset
|
|
|
|
LOG = get_logger()
|
|
|
|
|
|
@dataclass(kw_only=True)
|
|
class Item:
|
|
"""
|
|
This class represents a single download request for data transfer purposes.
|
|
"""
|
|
|
|
url: str
|
|
"""The URL of the item to be downloaded."""
|
|
|
|
preset: str = field(default_factory=lambda: Item._default_preset())
|
|
"""The preset to be used for this download."""
|
|
|
|
folder: str = ""
|
|
"""The folder to save the download to."""
|
|
|
|
cookies: str = ""
|
|
"""The cookies to be used for this download."""
|
|
|
|
template: str = ""
|
|
"""The template to be used for this download."""
|
|
|
|
cli: str = ""
|
|
"""The command options for yt-dlp to be used for this download."""
|
|
|
|
extras: dict = field(default_factory=dict)
|
|
"""Extra data to be added to the download."""
|
|
|
|
requeued: bool = False
|
|
"""If the item has been retried already via conditions."""
|
|
|
|
auto_start: bool = True
|
|
"""If the item should be started automatically."""
|
|
|
|
def serialize(self) -> dict:
|
|
"""
|
|
Serialize the item to a dictionary.
|
|
|
|
Returns:
|
|
dict: The serialized item.
|
|
|
|
"""
|
|
return self.__dict__.copy()
|
|
|
|
def json(self) -> str:
|
|
"""
|
|
Convert the item to a JSON string.
|
|
|
|
Returns:
|
|
str: The JSON string representation of the item.
|
|
|
|
"""
|
|
return Encoder(sort_keys=True, indent=4).encode(self.serialize())
|
|
|
|
def get(self, key: str, default: Any = None) -> Any:
|
|
"""
|
|
Get a value from the item by key.
|
|
|
|
Args:
|
|
key (str): The key to get the value for.
|
|
default (Any): The default value to return if the key is not found.
|
|
|
|
Returns:
|
|
Any: The value for the key, or the default value if the key is not found
|
|
|
|
"""
|
|
return self.__dict__.get(key, default)
|
|
|
|
def has_extras(self) -> bool:
|
|
"""
|
|
Check if the item has any extras data associated with it.
|
|
|
|
Returns:
|
|
bool: True if the item has extras, False otherwise.
|
|
|
|
"""
|
|
return bool(self.extras and len(self.extras) > 0)
|
|
|
|
def add_extras(self, key: str, value: Any) -> None:
|
|
"""
|
|
Add an extra data to the item.
|
|
|
|
Args:
|
|
key (str): The key of the extra data.
|
|
value (Any): The value of the extra data.
|
|
|
|
"""
|
|
if not self.extras:
|
|
self.extras = {}
|
|
|
|
self.extras[key] = value
|
|
|
|
def has_cli(self) -> bool:
|
|
"""
|
|
Check if the item has any command options for yt-dlp associated with it.
|
|
|
|
Returns:
|
|
bool: True if the item has command options for yt, False otherwise.
|
|
|
|
"""
|
|
return bool(self.cli and len(self.cli) > 2)
|
|
|
|
@staticmethod
|
|
def _default_preset() -> str:
|
|
"""
|
|
Get the default preset from the configuration.
|
|
|
|
Returns:
|
|
str: The default preset name.
|
|
|
|
"""
|
|
from .config import Config
|
|
|
|
return Config.get_instance().default_preset
|
|
|
|
def new_with(self, **kwargs) -> "Item":
|
|
"""
|
|
Create a new instance of Item with the given parameters.
|
|
|
|
Args:
|
|
**kwargs: The parameters to be used for creating the new instance.
|
|
|
|
Returns:
|
|
Item: A new instance of Item with the given parameters.
|
|
|
|
"""
|
|
return Item.format({**self.serialize(), **kwargs})
|
|
|
|
@staticmethod
|
|
def format(item: dict) -> "Item":
|
|
"""
|
|
Format the item to be added to the download queue.
|
|
|
|
Args:
|
|
item (dict): The item to be formatted.
|
|
|
|
Raises:
|
|
ValueError: If the url is not provided.
|
|
ValueError: If the command options for yt-cli are invalid.
|
|
|
|
Returns:
|
|
Item: The formatted item.
|
|
|
|
"""
|
|
url = item.get("url")
|
|
|
|
if not url or not isinstance(url, str):
|
|
msg = "url param is required."
|
|
raise ValueError(msg)
|
|
|
|
url = url.strip()
|
|
|
|
# If it's only a YouTube video ID, convert to a full URL.
|
|
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
|
|
url = f"https://www.youtube.com/watch?v={url}"
|
|
|
|
data: dict[str, Any] = {"url": url}
|
|
|
|
preset: str | None = item.get("preset")
|
|
if preset and isinstance(preset, str) and preset != Item._default_preset():
|
|
from app.features.presets.service import Presets
|
|
|
|
if not Presets.get_instance().has(preset):
|
|
msg: str = f"Preset '{preset}' does not exist."
|
|
raise ValueError(msg)
|
|
|
|
data["preset"] = preset
|
|
|
|
folder = item.get("folder")
|
|
if isinstance(folder, str) and folder:
|
|
data["folder"] = folder
|
|
|
|
cookies = item.get("cookies")
|
|
if isinstance(cookies, str) and cookies:
|
|
data["cookies"] = cookies
|
|
|
|
template = item.get("template")
|
|
if isinstance(template, str) and template:
|
|
data["template"] = template
|
|
|
|
if isinstance(item.get("auto_start"), bool):
|
|
data["auto_start"] = item["auto_start"]
|
|
|
|
extras: dict | None = item.get("extras")
|
|
if isinstance(extras, dict) and len(extras) > 0:
|
|
data["extras"] = extras
|
|
|
|
requeued = item.get("requeued")
|
|
if isinstance(requeued, bool):
|
|
data["requeued"] = requeued
|
|
|
|
cli: str | None = item.get("cli")
|
|
if cli and len(cli) > 2:
|
|
try:
|
|
from app.features.ytdlp.utils import arg_converter
|
|
|
|
arg_converter(args=cli, level=True)
|
|
data["cli"] = cli
|
|
except Exception as e:
|
|
msg = f"Failed to parse command options for yt-dlp. {e!s}"
|
|
raise ValueError(msg) from e
|
|
|
|
return Item(**data)
|
|
|
|
def get_preset(self) -> "Preset | None":
|
|
"""
|
|
Get the preset for the item.
|
|
|
|
Returns:
|
|
Preset | None: The preset for the item. If not found, None.
|
|
|
|
"""
|
|
from app.features.presets.service import Presets
|
|
|
|
return Presets.get_instance().get(self.preset or self._default_preset())
|
|
|
|
def get_archive_id(self) -> str | None:
|
|
"""
|
|
Get the archive ID for the download URL.
|
|
|
|
Returns:
|
|
str | None: The archive ID if available, None otherwise.
|
|
|
|
"""
|
|
return get_archive_id(self.url).get("archive_id") if self.url else None
|
|
|
|
def get_extractor(self) -> str | None:
|
|
"""
|
|
Get the extractor key for the download URL.
|
|
|
|
Returns:
|
|
str | None: The extractor key if available, None otherwise.
|
|
|
|
"""
|
|
return get_archive_id(self.url).get("ie_key") if self.url else None
|
|
|
|
def get_ytdlp_opts(self) -> YTDLPOpts:
|
|
"""
|
|
Get the yt-dlp options for the item.
|
|
|
|
Returns:
|
|
YTDLPOpts: The yt-dlp options for the item.
|
|
|
|
"""
|
|
params: YTDLPOpts = YTDLPOpts.get_instance()
|
|
|
|
if self.preset:
|
|
params = params.preset(name=self.preset)
|
|
|
|
if self.cli:
|
|
params = params.add_cli(self.cli, from_user=True)
|
|
|
|
return params
|
|
|
|
def get_archive_file(self) -> str | None:
|
|
"""
|
|
Get the archive file path from the yt-dlp options.
|
|
|
|
Returns:
|
|
str | None: The archive file path if available, None otherwise.
|
|
|
|
"""
|
|
return self.get_ytdlp_opts().get_all().get("download_archive")
|
|
|
|
def is_archived(self) -> bool:
|
|
"""
|
|
Check if the item has been archived.
|
|
|
|
Returns:
|
|
bool: True if the item has been archived, False otherwise.
|
|
|
|
"""
|
|
archive_id: str | None = self.get_archive_id()
|
|
archive_file: str | None = self.get_archive_file()
|
|
return len(archive_read(archive_file, [archive_id])) > 0 if archive_file and archive_id else False
|
|
|
|
def __repr__(self) -> str:
|
|
"""
|
|
Get a short string representation of the item.
|
|
|
|
Returns:
|
|
str: A short string representation of the item.
|
|
|
|
"""
|
|
from .config import Config
|
|
from .Utils import calc_download_path, strip_newline
|
|
|
|
data: dict = {}
|
|
for k, v in self.serialize().items():
|
|
if not v and k not in ("auto_start"):
|
|
continue
|
|
|
|
if k == "cli":
|
|
data[k] = strip_newline(v)
|
|
elif k == "extras":
|
|
data[k] = f"{len(v)} items"
|
|
elif k == "cookies":
|
|
data[k] = f"{len(v)}/chars"
|
|
elif k == "folder":
|
|
data[k] = calc_download_path(base_path=Config.get_instance().download_path, folder=v)
|
|
else:
|
|
data[k] = v
|
|
|
|
items: str = "".join(f'{k}="{v}", ' for k, v in data.items() if v is not None)
|
|
return f"Item({items.strip(', ')})"
|
|
|
|
|
|
@dataclass(kw_only=True)
|
|
class ItemDTO:
|
|
"""
|
|
ItemDTO is a data transfer object that represents a single item in the queue.
|
|
It contains all the required information for both the frontend and the backend to process the item.
|
|
"""
|
|
|
|
_id: str = field(default_factory=lambda: str(uuid.uuid4()), init=False)
|
|
""" Unique identifier for the item. """
|
|
error: str | None = None
|
|
""" Error message if the item failed. """
|
|
id: str
|
|
""" The ID of the item yt-dlp """
|
|
title: str
|
|
""" The title of the item. """
|
|
description: str = ""
|
|
""" The description of the item. """
|
|
url: str
|
|
""" The URL of the item. """
|
|
preset: str = "default"
|
|
""" The preset to be used for the item. """
|
|
folder: str
|
|
""" The folder to save the item to. """
|
|
download_dir: str | None = None
|
|
""" The full path to the download directory. """
|
|
temp_dir: str | None = None
|
|
""" The full path to the temporary directory. """
|
|
status: str | None = None
|
|
""" The status of the item. """
|
|
cookies: str | None = None
|
|
""" The cookies to be used for the item. """
|
|
template: str | None = None
|
|
""" The output template to be used for the item. """
|
|
template_chapter: str | None = None
|
|
""" The output template for chapters to be used for the item. """
|
|
timestamp: float = field(default_factory=lambda: time.time_ns())
|
|
""" The timestamp of the item. """
|
|
is_live: bool | None = None
|
|
""" If the item is a live stream. """
|
|
datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
|
|
""" The datetime of the item. """
|
|
live_in: str | None = None
|
|
""" The time until the live stream starts. """
|
|
file_size: int | None = None
|
|
""" The file size of the item. """
|
|
options: dict = field(default_factory=dict)
|
|
""" The options used for the item. """
|
|
extras: dict = field(default_factory=dict)
|
|
""" Extra data associated with the item. """
|
|
cli: str = ""
|
|
""" The command options for yt-dlp to be used for this download. """
|
|
auto_start: bool = True
|
|
""" If the item should be started automatically. """
|
|
is_archivable: bool | None = None
|
|
""" If the item can be archived. """
|
|
is_archived: bool | None = None
|
|
""" If the item has been archived. """
|
|
archive_id: str | None = None
|
|
""" The archive ID of the item. """
|
|
sidecar: dict = field(default_factory=dict)
|
|
""" Sidecar data associated with the item. """
|
|
download_skipped: bool = False
|
|
""" True when yt-dlp intentionally skips the primary media download. """
|
|
|
|
# yt-dlp injected fields.
|
|
tmpfilename: str | None = None
|
|
filename: str | None = None
|
|
total_bytes: int | None = None
|
|
total_bytes_estimate: int | None = None
|
|
downloaded_bytes: int | None = None
|
|
msg: str | None = None
|
|
percent: int | None = None
|
|
speed: str | None = None
|
|
eta: str | None = None
|
|
postprocessor: str | None = None
|
|
|
|
_recomputed: bool = False
|
|
_archive_file: str | None = None
|
|
|
|
def serialize(self) -> dict:
|
|
"""
|
|
Serialize the item to a dictionary.
|
|
|
|
Returns:
|
|
dict: The serialized item.
|
|
|
|
"""
|
|
if "finished" == self.status and not self._recomputed:
|
|
self.archive_status()
|
|
|
|
item, _ = clean_item(self.__dict__.copy(), ItemDTO.removed_fields())
|
|
return item
|
|
|
|
def json(self) -> str:
|
|
"""
|
|
Convert the item to a JSON string.
|
|
|
|
Returns:
|
|
str: The JSON string representation of the item.
|
|
|
|
"""
|
|
return Encoder(sort_keys=True, indent=4).encode(self.serialize())
|
|
|
|
def get(self, key: str, default: Any = None) -> Any:
|
|
"""
|
|
Get a value from the item by key.
|
|
|
|
Args:
|
|
key (str): The key to get the value for.
|
|
default (Any): The default value to return if the key is not found.
|
|
|
|
Returns:
|
|
Any: The value for the key, or the default value if the key is not found
|
|
|
|
"""
|
|
return self.__dict__.get(key, default)
|
|
|
|
def get_id(self) -> str:
|
|
"""
|
|
Get the unique identifier for the item.
|
|
|
|
Returns:
|
|
str: The unique identifier for the item.
|
|
|
|
"""
|
|
return self._id
|
|
|
|
def name(self) -> str:
|
|
"""
|
|
Get a short string representation of the item.
|
|
|
|
Returns:
|
|
str: A short string representation of the item.
|
|
|
|
"""
|
|
return f'id="{self.id}", title="{self.title}"'
|
|
|
|
def get_file(self, download_path: Path | None = None) -> Path | None:
|
|
"""
|
|
Get the file path of the downloaded item.
|
|
|
|
Args:
|
|
download_path (Path | None): The base download path. If None, it will be taken from the config.
|
|
|
|
Returns:
|
|
Path | None: The file path of the downloaded item, or None if not available.
|
|
|
|
"""
|
|
if not self.filename:
|
|
return None
|
|
|
|
if not download_path:
|
|
from .config import Config
|
|
|
|
base_path = Path(Config.get_instance().download_path)
|
|
else:
|
|
base_path = download_path if isinstance(download_path, Path) else Path(download_path)
|
|
|
|
filename = base_path
|
|
|
|
if self.folder:
|
|
filename: Path = filename / self.folder
|
|
|
|
filename = filename / self.filename
|
|
|
|
try:
|
|
realFile, status = get_file(download_path=base_path, file=str(filename.relative_to(base_path)))
|
|
except Exception:
|
|
return None
|
|
|
|
return Path(realFile) if status in (200, 302) else None
|
|
|
|
def get_archive_id(self) -> str | None:
|
|
"""
|
|
Get the archive ID for the download URL.
|
|
|
|
Returns:
|
|
str | None: The archive ID if available, None otherwise.
|
|
|
|
"""
|
|
if self.archive_id:
|
|
return self.archive_id
|
|
|
|
if not self.url:
|
|
return None
|
|
|
|
idDict: dict = get_archive_id(self.url)
|
|
self.archive_id = idDict.get("archive_id")
|
|
|
|
return self.archive_id
|
|
|
|
def get_extractor(self) -> str | None:
|
|
"""
|
|
Get the extractor key for the download URL.
|
|
|
|
Returns:
|
|
str | None: The extractor key if available, None otherwise.
|
|
|
|
"""
|
|
if self.archive_id:
|
|
return self.archive_id.split(" ")[0]
|
|
|
|
idDict: dict[str, str | None] = get_archive_id(self.url)
|
|
self.archive_id = idDict.get("archive_id")
|
|
|
|
return idDict.get("ie_key") if self.url else None
|
|
|
|
def get_ytdlp_opts(self) -> YTDLPOpts:
|
|
"""
|
|
Get the yt-dlp options for the item.
|
|
|
|
Returns:
|
|
YTDLPOpts: The yt-dlp options for the item.
|
|
|
|
"""
|
|
params: YTDLPOpts = YTDLPOpts.get_instance()
|
|
|
|
if self.preset:
|
|
params = params.preset(name=self.preset)
|
|
|
|
if self.cli:
|
|
params = params.add_cli(self.cli, from_user=True)
|
|
|
|
return params
|
|
|
|
def get_preset(self) -> "Preset | None":
|
|
"""
|
|
Get the preset for the item.
|
|
|
|
Returns:
|
|
Preset | None: The preset for the item. If not found, None.
|
|
|
|
"""
|
|
from app.features.presets.service import Presets
|
|
|
|
return Presets.get_instance().get(self.preset or "default")
|
|
|
|
def archive_status(self, force: bool = False) -> None:
|
|
"""
|
|
Recompute the archive status of the item.
|
|
|
|
Args:
|
|
force (bool): If True, force recomputation even if already computed.
|
|
|
|
"""
|
|
if not force and (self._recomputed or not self.archive_id):
|
|
return
|
|
|
|
if "finished" == self.status:
|
|
self._recomputed = True
|
|
|
|
self.is_archivable = bool(self._archive_file)
|
|
if not self.is_archivable:
|
|
self.is_archived = False
|
|
elif self.archive_id and self._archive_file:
|
|
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
|
|
else:
|
|
self.is_archived = False
|
|
|
|
def get_archive_file(self) -> str | None:
|
|
"""
|
|
Get the archive file path from the yt-dlp options.
|
|
|
|
Returns:
|
|
str | None: The archive file path if available, None otherwise.
|
|
|
|
"""
|
|
if self._archive_file or self._recomputed or not self.archive_id:
|
|
return self._archive_file
|
|
|
|
self._archive_file = self.get_ytdlp_opts().get_all().get("download_archive")
|
|
if self._archive_file:
|
|
self._archive_file = self._archive_file.strip()
|
|
|
|
return self._archive_file
|
|
|
|
def archive_add(self) -> bool:
|
|
"""
|
|
Archive the item by adding its archive ID to the download archive file.
|
|
|
|
Returns:
|
|
bool: True if the item was archived, False otherwise.
|
|
|
|
"""
|
|
if self.is_archived or not self.is_archivable or not self.archive_id or not self._archive_file:
|
|
return False
|
|
|
|
self.is_archived = archive_add(self._archive_file, [self.archive_id])
|
|
|
|
return self.is_archived
|
|
|
|
def archive_delete(self) -> bool:
|
|
"""
|
|
Remove the item's archive ID from the download archive file.
|
|
|
|
Returns:
|
|
bool: True if the item was removed from the archive, False otherwise.
|
|
|
|
"""
|
|
if not self.is_archivable or not self.is_archived or not self.archive_id or not self._archive_file:
|
|
return False
|
|
|
|
archive_delete(self._archive_file, [self.archive_id])
|
|
self.is_archived = False
|
|
|
|
return True
|
|
|
|
def get_file_sidecar(self) -> dict:
|
|
"""
|
|
Get sidecar files associated with the item.
|
|
|
|
Returns:
|
|
dict: A dictionary with sidecar files categorized by type.
|
|
|
|
"""
|
|
if filename := self.get_file():
|
|
self.sidecar = get_file_sidecar(filename)
|
|
|
|
return self.sidecar
|
|
|
|
@staticmethod
|
|
def removed_fields() -> tuple:
|
|
"""
|
|
Fields that once existed but are no longer used.
|
|
|
|
Returns:
|
|
tuple: A tuple of field names that are no longer used.
|
|
|
|
"""
|
|
return (
|
|
"thumbnail",
|
|
"quality",
|
|
"format",
|
|
"ytdlp_cookies",
|
|
"ytdlp_config",
|
|
"output_template",
|
|
"output_template_chapter",
|
|
"config",
|
|
"temp_path",
|
|
"_recomputed",
|
|
"_archive_file",
|
|
)
|
|
|
|
def __post_init__(self):
|
|
"""
|
|
Post-initialization to compute archive status if applicable.
|
|
"""
|
|
self.get_archive_id()
|
|
self.get_archive_file()
|
|
self.archive_status()
|