danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #542 from arabcoders/dev

Browse source
This commit is contained in:
Abdulmohsen 2026-01-14 23:16:23 +03:00 committed by GitHub
commit be6b68efdd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
53 changed files with 4340 additions and 2923 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
FAQ.md
View file

@ -57,6 +57,7 @@ or the `environment:` section in `compose.yaml` file.
| YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` |
| YTP_IGNORE_ARCHIVED_ITEMS | Don't report archived items in the download history. | `false` |
| YTP_CHECK_FOR_UPDATES | Whether to check for application updates. | `true` |
| YTP_EXTRACT_INFO_CONCURRENCY | The number of concurrent extract info operations. | `4` |
> [!NOTE]
> To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`.

2
app/library/DataStore.py
View file

@ -4,7 +4,7 @@ from collections import OrderedDict
from collections.abc import Iterable
from enum import Enum
from .Download import Download
from .downloads import Download
from .ItemDTO import ItemDTO
from .operations import matches_condition
from .sqlite_store import SqliteStore

742
app/library/Download.py
View file

@ -1,742 +0,0 @@
import asyncio
import hashlib
import logging
import multiprocessing
import os
import re
import signal
import time
from datetime import UTC, datetime
from email.utils import formatdate
from pathlib import Path
from typing import Any
import yt_dlp.utils
from .config import Config
from .Events import EventBus, Events
from .ffprobe import ffprobe
from .ItemDTO import ItemDTO
from .Utils import create_cookies_file, delete_dir, extract_info, extract_ytdlp_logs
from .ytdlp import YTDLP
GENERIC_EXTRACTORS = ("HTML5MediaEmbed", "generic")
"Generic extractors identifiers."
class Terminator:
pass
class NestedLogger:
debug_messages: list[str] = ["[debug] ", "[download] "]
def __init__(self, logger: logging.Logger) -> None:
self.logger: logging.Logger = logger
def debug(self, msg: str) -> None:
levelno: int = logging.DEBUG if any(msg.startswith(x) for x in self.debug_messages) else logging.INFO
self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE))
def error(self, msg) -> None:
self.logger.error(msg)
def warning(self, msg) -> None:
self.logger.warning(msg)
class Download:
"""
Download task.
"""
temp_path: str = None
update_task = None
cancel_in_progress: bool = False
final_update = False
bad_live_options: list = [
"concurrent_fragment_downloads",
"fragment_retries",
"skip_unavailable_fragments",
]
"Bad yt-dlp options which are known to cause issues with live stream."
_ytdlp_fields: tuple = (
"tmpfilename",
"filename",
"status",
"msg",
"total_bytes",
"total_bytes_estimate",
"downloaded_bytes",
"speed",
"eta",
)
"Fields to be extracted from yt-dlp progress hook."
def __init__(self, info: ItemDTO, info_dict: dict = None, logs: list | None = None):
"""
Initialize download task.
Args:
info (ItemDTO): ItemDTO object.
info_dict (dict): yt-dlp metadata dict.
logs (list): Logs from yt-dlp.
"""
config: Config = Config.get_instance()
self.download_dir: str = info.download_dir
"Download directory."
self.temp_dir: str | None = info.temp_dir
"Temporary directory."
self.template: str | None = info.template
"Filename template."
self.template_chapter: str | None = info.template_chapter
"Chapter filename template."
self.download_info_expires = int(config.download_info_expires)
"Time in seconds before the download info is considered expired."
self.info: ItemDTO = info
"ItemDTO object."
self.id: str = info._id
"Download ID."
self.debug = bool(config.debug)
"Debug mode."
self.debug_ytdl = bool(config.ytdlp_debug)
"Debug mode for yt-dlp."
self.cancelled = False
"Download cancelled."
self.tmpfilename = None
"Temporary filename."
self.status_queue = None
"Status queue."
self.proc = None
"yt-dlp process."
self._notify: EventBus = EventBus.get_instance()
"Event bus instance."
self.max_workers = int(config.max_workers)
"Maximum number of concurrent downloads."
self.temp_keep = bool(config.temp_keep)
"Keep temp folder after download."
self.temp_disabled = bool(config.temp_disabled)
"Disable the temporary files feature."
self.is_live: bool = bool(info.is_live) or info.live_in is not None
"Is the download a live stream."
self.info_dict: dict = info_dict
"yt-dlp metadata dict."
self.logger: logging.Logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
"Logger for the download task."
self.started_time = 0
"Time when the download started."
self.queue_time: datetime = datetime.now(tz=UTC)
"Time when the download was queued."
self.logs: list[str] = logs if logs else []
"Logs from yt-dlp."
def _progress_hook(self, data: dict) -> None:
if self.debug:
try:
d_safe: dict = {
"status": data.get("status"),
"filename": data.get("filename"),
"info_dict": {
k: v
for k, v in data.get("info_dict", {}).items()
if k not in ["formats", "thumbnails", "description", "tags", "_format_sort_fields"]
and not isinstance(v, (type(None).__bases__[0], type(lambda: None))) # Skip non-serializable
},
}
self.logger.debug(f"PG Hook: {d_safe}")
except Exception as e:
self.logger.debug(f"PG Hook: Error creating debug info: {e}")
self.status_queue.put(
{
"id": self.id,
"action": "progress",
**{k: v for k, v in data.items() if k in self._ytdlp_fields},
}
)
def _postprocessor_hook(self, data: dict) -> None:
if self.debug:
try:
d_safe: dict = {
"postprocessor": data.get("postprocessor"),
"status": data.get("status"),
"info_dict": {
k: v
for k, v in data.get("info_dict", {}).items()
if k not in ["formats", "thumbnails", "description", "tags", "_format_sort_fields"]
and not isinstance(v, (type(None).__bases__[0], type(lambda: None))) # Skip non-serializable
},
}
self.logger.debug(f"PP Hook: {d_safe}")
except Exception as e:
self.logger.debug(f"PP Hook: Error creating debug info: {e}")
if "MoveFiles" == data.get("postprocessor") and "finished" == data.get("status"):
if "__finaldir" in data.get("info_dict", {}) and "filepath" in data.get("info_dict", {}):
filename = str(Path(data["info_dict"]["__finaldir"]) / Path(data["info_dict"]["filepath"]).name)
else:
filename: str | None = data.get("info_dict", {}).get("filepath", data.get("filename"))
self.logger.debug(f"Final filename: '{filename}'.")
self.status_queue.put({"id": self.id, "action": "moved", "status": "finished", "final_name": filename})
return
dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
self.status_queue.put({"id": self.id, "action": "postprocessing", **dataDict, "status": "postprocessing"})
def _post_hooks(self, filename: str | None = None) -> None:
if not filename:
return
self.status_queue.put({"id": self.id, "filename": filename})
def _download(self) -> None:
if not self._notify:
self._notify = EventBus.get_instance()
cookie_file = None
try:
params: dict = (
self.info.get_ytdlp_opts()
.add(
config={
"color": "no_color",
"paths": {
"home": str(self.download_dir),
"temp": str(self.temp_path) if self.temp_path else self.download_dir,
},
"outtmpl": {
"default": self.template,
"chapter": self.template_chapter,
},
"noprogress": True,
"ignoreerrors": False,
},
from_user=False,
)
.get_all()
)
params.update(
{
"progress_hooks": [self._progress_hook],
"postprocessor_hooks": [self._postprocessor_hook],
"post_hooks": [self._post_hooks],
}
)
if self.debug_ytdl:
params["verbose"] = True
params["noprogress"] = False
if self.info.cookies:
try:
cookie_file: Path = (
Path(self.temp_path if self.temp_path else self.temp_dir) / f"cookie_{self.info._id}.txt"
)
self.logger.debug(
f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
)
params["cookiefile"] = str(create_cookies_file(self.info.cookies, cookie_file))
except Exception as e:
err_msg: str = f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'."
self.logger.error(err_msg)
raise ValueError(err_msg) from e
if self.info_dict and isinstance(self.info_dict, dict):
# If info extractor_key is generic, we need to remove download_archive param from default preset.
if self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default:
self.logger.debug(f"Removing 'download_archive' for generic extractor. {self.info.url=}")
params.pop("download_archive", None)
# Safe-guard in-case downloading take too long and the info expires.
if self.download_info_expires > 0:
_ts: int | None = self.info_dict.get("epoch", self.info_dict.get("timestamp", None))
_ts = datetime.fromtimestamp(_ts, tz=UTC) if _ts else None
if not _ts or (datetime.now(tz=UTC) - _ts).total_seconds() > self.download_info_expires:
self.info_dict = None
self.logger.warning(f"Info for '{self.info.url}' has expired, re-extracting info.")
if not self.info_dict or not isinstance(self.info_dict, dict):
self.logger.info(f"Extracting info for '{self.info.url}'.")
self.logs = []
ie_params: dict = params.copy()
ie_params["callback"] = {
"func": lambda _, msg: self.logs.append(msg),
"level": logging.WARNING,
"name": "callback-logger",
}
info: dict = extract_info(
config=params,
url=self.info.url,
debug=self.debug,
no_archive=not params.get("download_archive", False),
follow_redirect=True,
)
if info:
self.info_dict = info
if self.is_live:
deletedOpts: list = []
for opt in self.bad_live_options:
if opt in params:
params.pop(opt, None)
deletedOpts.append(opt)
if len(deletedOpts) > 0:
self.logger.warning(
f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted."
)
if isinstance(self.info_dict, dict) and not (
len(self.info_dict.get("formats", [])) > 0 or self.info_dict.get("url")
):
msg: str = f"Failed to extract any formats for '{self.info.url}'."
if filtered_logs := extract_ytdlp_logs(self.logs):
msg += " " + ", ".join(filtered_logs)
raise ValueError(msg) # noqa: TRY301
self.logger.info(
f'Download {self.info.name()}, preset="{self.info.preset}", cookies="{bool(params.get("cookiefile"))}" started.'
)
if self.debug:
self.logger.debug(f"Params before passing to yt-dlp. {params}")
params["logger"] = NestedLogger(self.logger)
if "continuedl" in params and params["continuedl"] is False:
self.delete_temp(by_pass=True)
cls = YTDLP(params=params)
self.started_time = int(time.time())
if "posix" == os.name:
def mark_cancelled(*_) -> None:
cls._interrupted = True
cls.to_screen("[info] Interrupt received, exiting cleanly...")
raise SystemExit(130) # noqa: TRY301
signal.signal(signal.SIGUSR1, mark_cancelled)
self.status_queue.put({"id": self.id, "status": "downloading"})
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
_dct: dict = self.info_dict.copy()
if isinstance(self.info.extras, dict) and len(self.info.extras) > 0:
_dct.update(
{
k: v
for k, v in self.info.extras.items()
if k not in _dct and v is not None and k not in ("is_live",)
}
)
cls.process_ie_result(ie_result=_dct, download=True)
ret: int = cls._download_retcode
else:
self.logger.debug(f"Downloading using url: {self.info.url}")
ret: int = cls.download(url_list=[self.info.url])
self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"})
except yt_dlp.utils.ExistingVideoReached as exc:
self.logger.error(exc)
self.status_queue.put({"id": self.id, "status": "skip", "msg": "Item has already been downloaded."})
except Exception as exc:
self.logger.exception(exc)
self.logger.error(exc)
self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)})
finally:
self.status_queue.put(Terminator())
if cookie_file and cookie_file.exists():
try:
cookie_file.unlink()
self.logger.debug(f"Deleted cookie file: {cookie_file}")
except Exception as e:
self.logger.error(f"Failed to delete cookie file: {cookie_file}. {e}")
self.logger.info(
f'Task {self.info.name()} preset="{self.info.preset}" cookies="{bool(params.get("cookiefile"))}" completed.'
)
async def start(self) -> None:
self.status_queue: multiprocessing.Queue[Any] = Config.get_manager().Queue()
# Create temp dir for each download.
if not self.temp_disabled:
self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5)
if not self.temp_path.exists():
self.temp_path.mkdir(parents=True, exist_ok=True)
self.info.temp_path = str(self.temp_path)
self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download)
self.proc.start()
self.info.status = "preparing"
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
asyncio.create_task(self.progress_update(), name=f"update-{self.id}")
ret = await asyncio.get_running_loop().run_in_executor(None, self.proc.join)
if self.final_update or self.cancelled:
if self.cancelled:
self.info.status = "cancelled"
return ret
self.status_queue.put(Terminator())
self.logger.debug("Draining status queue.")
try:
drain_count: int = 50 + (self.status_queue.qsize() if hasattr(self.status_queue, "qsize") else 5)
except Exception:
drain_count = 55
for i in range(drain_count):
try:
if self.debug:
self.logger.debug(f"(50/{i}) Draining the status queue...")
if self.final_update:
if self.debug:
self.logger.debug("(50/{i}) Draining stopped. Final update received.")
break
next_status = self.status_queue.get(timeout=0.1)
if next_status is None or isinstance(next_status, Terminator):
continue
await self._process_status_update(next_status)
except Exception: # noqa: S112
continue
return ret
def started(self) -> bool:
return self.proc is not None
def cancel(self) -> bool:
"""
Cancel the download task.
For live streams and unresponsive processes, this will force-kill the process.
"""
if not self.started():
return False
self.cancelled = True
# Actively kill the process instead of just setting a flag
return self.kill()
async def close(self) -> bool:
"""
Close download process.
This method MUST be called to clear resources.
"""
if not self.started() or self.cancel_in_progress:
return False
self.cancel_in_progress = True
procId: int | None = self.proc.ident
if not procId:
if self.proc:
self.proc.close()
self.proc = None
self.logger.warning("Attempted to close download process, but it is not running.")
return False
self.logger.info(f"Closing PID='{procId}' download process.")
try:
try:
if "update_task" in self.__dict__ and self.update_task.done() is False:
self.update_task.cancel()
except Exception as e:
self.logger.error(f"Failed to close status queue: '{procId}'. {e}")
self.kill()
loop = asyncio.get_running_loop()
if self.proc.is_alive():
self.logger.debug(f"Waiting for PID='{procId}' to close.")
await loop.run_in_executor(None, self.proc.join)
self.logger.debug(f"PID='{procId}' closed.")
if self.status_queue:
self.logger.debug(f"Closing status queue for PID='{procId}'.")
self.status_queue.put(Terminator())
self.logger.debug(f"Closed status queue for PID='{procId}'.")
if self.proc:
self.proc.close()
self.proc = None
self.delete_temp()
self.logger.debug(f"Closed PID='{procId}' download process.")
return True
except Exception as e:
self.logger.error(f"Failed to close process: '{procId}'. {e}")
self.logger.exception(e)
return False
def running(self) -> bool:
try:
return self.proc is not None and self.proc.is_alive()
except ValueError:
return False
def is_cancelled(self) -> bool:
return self.cancelled
def kill(self) -> bool:
"""
Kill the download process.
Attempts graceful termination via signal first, then force-kills if necessary.
For live streams that may not respond to signals, uses force termination.
"""
if not self.running():
return False
procId: int | None = self.proc.ident
try:
self.logger.info(f"Killing download process: PID={self.proc.pid}, ident={procId}.")
# First, try graceful termination with SIGUSR1 signal (if on POSIX)
if self.proc.pid and "posix" == os.name:
try:
self.logger.debug(f"Sending SIGUSR1 signal to PID={self.proc.pid}.")
os.kill(self.proc.pid, signal.SIGUSR1)
# Wait briefly for graceful shutdown (1 second timeout for live streams)
start_time = time.time()
timeout = 2 if self.is_live else 5 # Live streams get shorter timeout
while self.proc.is_alive() and (time.time() - start_time) < timeout:
time.sleep(0.1)
if self.proc.is_alive():
self.logger.warning(
f"Process PID={self.proc.pid} did not respond to SIGUSR1 "
f"({'live stream' if self.is_live else 'regular download'}), "
f"forcing termination."
)
else:
self.logger.debug(f"Process PID={self.proc.pid} terminated gracefully.")
return True
except (OSError, AttributeError) as e:
self.logger.debug(f"Failed to send SIGUSR1 signal: {e}")
# Force terminate if still running (for live streams or unresponsive processes)
if self.proc.is_alive():
self.logger.info(f"Force-terminating process PID={self.proc.pid}.")
self.proc.terminate()
# Give it a moment to terminate
start_time = time.time()
timeout = 1 if self.is_live else 2
while self.proc.is_alive() and (time.time() - start_time) < timeout:
time.sleep(0.05)
# If still alive, use hard kill
if self.proc.is_alive():
self.logger.warning(
f"Process PID={self.proc.pid} did not respond to terminate(), killing forcefully."
)
self.proc.kill()
# Give it final moment
start_time = time.time()
while self.proc.is_alive() and (time.time() - start_time) < 1:
time.sleep(0.05)
self.logger.info(f"Process PID={self.proc.pid} killed.")
return True
except Exception as e:
self.logger.error(f"Failed to kill process PID={self.proc.pid}, ident={procId}. {e}")
self.logger.exception(e)
return False
def delete_temp(self, by_pass: bool = False) -> None:
if self.temp_disabled or self.temp_keep is True or not self.temp_path:
return
if (
not by_pass
and "finished" != self.info.status
and self.info.downloaded_bytes
and self.info.downloaded_bytes > 0
):
self.logger.warning(f"Keeping temp folder '{self.temp_path}'. {self.info.status=}.")
return
tmp_dir = Path(self.temp_path)
if not tmp_dir.exists():
return
if str(tmp_dir) == str(self.temp_dir):
self.logger.warning(f"Refusing to delete video temp folder '{self.temp_path}' as it's temp root.")
return
status: bool = delete_dir(tmp_dir)
if by_pass:
tmp_dir.mkdir(parents=True, exist_ok=True)
self.logger.info(f"Temp folder '{self.temp_path}' emptied.")
else:
self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.")
async def _process_status_update(self, status) -> None:
if status.get("id") != self.id or len(status) < 2:
self.logger.warning(f"Received invalid status update. {status}")
return
if self.debug:
self.logger.debug(f"Status Update: {self.info._id=} {status=}")
if isinstance(status, str):
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
return
self.tmpfilename: str | None = status.get("tmpfilename")
fl = None
if "final_name" in status:
fl = Path(status.get("final_name"))
try:
self.info.filename = str(fl.relative_to(Path(self.download_dir)))
except ValueError as ve:
self.logger.debug(f"Failed to get relative path for '{fl}' from '{self.download_dir}'. {ve}")
if self.temp_path:
self.info.filename = str(fl.relative_to(Path(self.temp_path)))
else:
self.info.filename = str(fl)
if fl.is_file() and fl.exists():
self.final_update = True
self.logger.debug(f"Final file name: '{fl}'.")
try:
self.info.file_size = fl.stat().st_size
except FileNotFoundError:
self.info.file_size = 0
self.info.status = status.get("status", self.info.status)
self.info.msg = status.get("msg")
if "error" == self.info.status and "error" in status:
self.info.error = status.get("error")
self._notify.emit(
Events.LOG_ERROR,
data=self.info,
title="Download Error",
message=f"'{self.info.title}' failed to download. {self.info.error}",
)
if "downloaded_bytes" in status and status.get("downloaded_bytes", 0) > 0:
self.info.downloaded_bytes = status.get("downloaded_bytes")
total: float | None = status.get("total_bytes") or status.get("total_bytes_estimate")
if total:
try:
self.info.percent = status["downloaded_bytes"] / total * 100
except ZeroDivisionError:
self.info.percent = 0
self.info.total_bytes = total
self.info.speed = status.get("speed")
self.info.eta = status.get("eta")
if "finished" == self.info.status and fl and fl.is_file() and fl.exists():
self.info.file_size = fl.stat().st_size
self.info.datetime = str(formatdate(time.time()))
try:
ff = await ffprobe(str(fl))
self.info.extras["is_video"] = ff.has_video()
self.info.extras["is_audio"] = ff.has_audio()
if (ff.has_video() or ff.has_audio()) and not self.info.extras.get("duration"):
self.info.extras["duration"] = int(float(ff.metadata.get("duration", 0.0)))
except Exception as e:
self.info.extras["is_video"] = True
self.info.extras["is_audio"] = True
self.logger.exception(e)
self.logger.error(f"Failed to run ffprobe. {status.get}. {e}")
if not self.final_update or fl:
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
async def progress_update(self) -> None:
"""
Update status of download task and notify the client.
"""
while True:
try:
self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
status = await self.update_task
if status is None or isinstance(status, Terminator):
return
await self._process_status_update(status)
except (asyncio.CancelledError, OSError, FileNotFoundError):
return
def is_stale(self) -> bool:
"""
Check if the download task is stale.
A download task is considered stale if it has not been updated for a certain period of time.
Returns:
bool: True if the download task is stale, False otherwise.
"""
if not self.info.auto_start:
return False
if self.started_time < 1:
self.logger.debug(f"Download task '{self.info.name()}' not started yet.")
return False
if int(time.time()) - self.started_time < 300:
self.logger.debug(
f"Download task '{self.info.title}: {self.info.id}' started for '{int(time.time()) - self.started_time}' seconds."
)
return False
if not self.running():
self.logger.warning(
f"Download task '{self.info.title}: {self.info.id}' not started running for '{int(time.time()) - self.started_time}' seconds."
)
return True
if self.info.status not in ["finished", "error", "cancelled", "downloading", "postprocessing"]:
status: str = self.info.status if self.info.status else "unknown"
self.logger.warning(
f"Download task '{self.info.title}: {self.info.id}' has been stuck in '{status}' state for '{int(time.time()) - self.started_time}' seconds."
)
return True
return False
def __getstate__(self) -> dict[str, Any]:
state: dict[str, Any] = self.__dict__.copy()
# Exclude (unpickleable) keys during pickling, this issue arise mostly on Windows.
excluded_keys: tuple[str] = ("_notify",)
for key in excluded_keys:
if key in state:
state[key] = None
return state

1425
app/library/DownloadQueue.py
View file

File diff suppressed because it is too large Load diff

2
app/library/Events.py
View file

@ -9,7 +9,7 @@ from typing import Any
from .BackgroundWorker import BackgroundWorker
from .Singleton import Singleton
LOG: logging.Logger = logging.getLogger("Events")
LOG: logging.Logger = logging.getLogger("library.events")
class Events:

1
app/library/ItemDTO.py
View file

@ -399,6 +399,7 @@ class ItemDTO:
percent: int | None = None
speed: str | None = None
eta: str | None = None
postprocessor: str | None = None
_recomputed: bool = False
_archive_file: str | None = None

2
app/library/Tasks.py
View file

@ -14,7 +14,7 @@ from typing import Any
from aiohttp import web
from .config import Config
from .DownloadQueue import DownloadQueue
from .downloads import DownloadQueue
from .encoder import Encoder
from .Events import EventBus, Events
from .ItemDTO import Item, ItemDTO

3
app/library/config.py
View file

@ -117,6 +117,9 @@ class Config(metaclass=Singleton):
extract_info_timeout: int = 70
"""The timeout to use for extracting video information."""
extract_info_concurrency: int = 4
"""The number of concurrent extract_info calls allowed."""
db_file: str = "{config_path}{os_sep}ytptube.db"
"""The path to the database file."""

8
app/library/downloads/__init__.py Normal file
View file

@ -0,0 +1,8 @@
"""Downloads module - refactored download management system."""
from .core import Download
from .hooks import NestedLogger
from .queue_manager import DownloadQueue
from .types import Terminator
__all__ = ["Download", "DownloadQueue", "NestedLogger", "Terminator"]

406
app/library/downloads/core.py Normal file
View file

@ -0,0 +1,406 @@
"""Core Download class - orchestrates the download process."""
from __future__ import annotations
import asyncio
import logging
import os
import signal
import time
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING, Any
import yt_dlp.utils
from app.library.config import Config
from app.library.Events import EventBus, Events
from app.library.Utils import create_cookies_file, extract_info, extract_ytdlp_logs
from app.library.ytdlp import YTDLP
from .hooks import HookHandlers, NestedLogger
from .process_manager import ProcessManager
from .status_tracker import StatusTracker
from .temp_manager import TempManager
from .types import Terminator
from .utils import BAD_LIVE_STREAM_OPTIONS, GENERIC_EXTRACTORS, is_download_stale
if TYPE_CHECKING:
import multiprocessing
from app.library.ItemDTO import ItemDTO
class Download:
update_task: asyncio.Task | None = None
def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None):
"""
Initialize download task.
Args:
info: ItemDTO object containing download parameters
info_dict: Pre-extracted yt-dlp metadata dict (optional)
logs: Pre-existing logs from yt-dlp (optional)
"""
config: Config = Config.get_instance()
self.download_dir: str = info.download_dir
self.temp_dir: str | None = info.temp_dir
self.template: str | None = info.template
self.template_chapter: str | None = info.template_chapter
self.download_info_expires = int(config.download_info_expires)
self.info: ItemDTO = info
self.id: str = info._id
self.debug = bool(config.debug)
self.debug_ytdl = bool(config.ytdlp_debug)
self.tmpfilename: str | None = None
self.status_queue: multiprocessing.Queue[Any] | None = None
self.max_workers = int(config.max_workers)
self.is_live: bool = bool(info.is_live) or info.live_in is not None
self.info_dict: dict | None = info_dict
self.logger: logging.Logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
self.started_time = 0
self.queue_time: datetime = datetime.now(tz=UTC)
self.logs: list[str] = logs if logs else []
self._process_manager = ProcessManager(
download_id=self.id,
is_live=self.is_live,
logger=self.logger,
)
self._temp_manager = TempManager(
info=self.info,
temp_dir=self.temp_dir,
temp_disabled=bool(config.temp_disabled),
temp_keep=bool(config.temp_keep),
logger=self.logger,
)
self._status_tracker: StatusTracker | None = None
self._hook_handlers: HookHandlers | None = None
def _download(self) -> None:
"""
Execute the download in a subprocess.
This method runs in a separate process and performs the actual
download using yt-dlp.
"""
cookie_file: Path | None = None
try:
params: dict = (
self.info.get_ytdlp_opts()
.add(
config={
"color": "no_color",
"paths": {
"home": str(self.download_dir),
"temp": str(self._temp_manager.temp_path)
if self._temp_manager.temp_path
else self.download_dir,
},
"outtmpl": {
"default": self.template,
"chapter": self.template_chapter,
},
"noprogress": True,
"ignoreerrors": False,
},
from_user=False,
)
.get_all()
)
params.update(
{
"progress_hooks": [self._hook_handlers.progress_hook],
"postprocessor_hooks": [self._hook_handlers.postprocessor_hook],
"post_hooks": [self._hook_handlers.post_hook],
}
)
if self.debug_ytdl:
params["verbose"] = True
params["noprogress"] = False
if self.info.cookies:
try:
cookie_file = (
Path(self._temp_manager.temp_path if self._temp_manager.temp_path else self.temp_dir)
/ f"cookie_{self.info._id}.txt"
)
self.logger.debug(
f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
)
params["cookiefile"] = str(create_cookies_file(self.info.cookies, cookie_file))
except Exception as e:
err_msg: str = f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'."
self.logger.error(err_msg)
raise ValueError(err_msg) from e
if self.info_dict and isinstance(self.info_dict, dict):
if self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default:
self.logger.debug(f"Removing 'download_archive' for generic extractor. url={self.info.url}")
params.pop("download_archive", None)
if self.download_info_expires > 0:
_ts: int | None = self.info_dict.get("epoch", self.info_dict.get("timestamp", None))
_ts_dt = datetime.fromtimestamp(_ts, tz=UTC) if _ts else None
if not _ts_dt or (datetime.now(tz=UTC) - _ts_dt).total_seconds() > self.download_info_expires:
self.info_dict = None
self.logger.warning(f"Info for '{self.info.url}' has expired, re-extracting info.")
if not self.info_dict or not isinstance(self.info_dict, dict):
self.logger.info(f"Extracting info for '{self.info.url}'.")
self.logs = []
ie_params: dict = params.copy()
ie_params["callback"] = {
"func": lambda _, msg: self.logs.append(msg),
"level": logging.WARNING,
"name": "callback-logger",
}
info: dict = extract_info(
config=ie_params,
url=self.info.url,
debug=self.debug,
no_archive=not params.get("download_archive", False),
follow_redirect=True,
)
if info:
self.info_dict = info
if self.is_live:
deletedOpts: list[str] = []
for opt in BAD_LIVE_STREAM_OPTIONS:
if opt in params:
params.pop(opt, None)
deletedOpts.append(opt)
if len(deletedOpts) > 0:
self.logger.warning(f"Live stream detected for '{self.info.title}', deleted opts: {deletedOpts}")
if isinstance(self.info_dict, dict) and not (
len(self.info_dict.get("formats", [])) > 0 or self.info_dict.get("url")
):
msg: str = f"Failed to extract any formats for '{self.info.url}'."
if filtered_logs := extract_ytdlp_logs(self.logs):
msg += " " + ", ".join(filtered_logs)
raise ValueError(msg) # noqa: TRY301
self.logger.info(
f'Download {self.info.name()}, preset="{self.info.preset}", cookies="{bool(params.get("cookiefile"))}" started.'
)
if self.debug:
self.logger.debug(f"Params before passing to yt-dlp. {params}")
params["logger"] = NestedLogger(self.logger)
if "continuedl" in params and params["continuedl"] is False:
self._temp_manager.delete_temp(by_pass=True)
cls = YTDLP(params=params)
if "posix" == os.name:
def mark_cancelled(*_) -> None:
cls._interrupted = True
cls.to_screen("[info] Interrupt received, exiting cleanly...")
raise SystemExit(130) # noqa: TRY301
signal.signal(signal.SIGUSR1, mark_cancelled)
self.status_queue.put({"id": self.id, "status": "downloading"})
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
_dct: dict = self.info_dict.copy()
if isinstance(self.info.extras, dict) and len(self.info.extras) > 0:
_dct.update(
{
k: v
for k, v in self.info.extras.items()
if k not in _dct and v is not None and k not in ("is_live",)
}
)
cls.process_ie_result(ie_result=_dct, download=True)
ret: int = cls._download_retcode
else:
self.logger.debug(f"Downloading using url: {self.info.url}")
ret = cls.download(url_list=[self.info.url])
self.status_queue.put({"id": self.id, "status": "finished" if 0 == ret else "error"})
except yt_dlp.utils.ExistingVideoReached as exc:
self.logger.error(exc)
self.status_queue.put({"id": self.id, "status": "skip", "msg": "Item has already been downloaded."})
except Exception as exc:
self.logger.exception(exc)
self.logger.error(exc)
self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)})
finally:
self.status_queue.put(Terminator())
if cookie_file and cookie_file.exists():
try:
cookie_file.unlink()
self.logger.debug(f"Deleted cookie file: {cookie_file}")
except Exception as e:
self.logger.error(f"Failed to delete cookie file: {cookie_file}. {e}")
self.logger.info(
f'Task {self.info.name()} preset="{self.info.preset}" cookies="{bool(params.get("cookiefile"))}" completed.'
)
async def start(self) -> int | None:
"""
Start the download process.
Creates the subprocess, initializes status tracking, and waits
for completion.
Returns:
Process exit code or None
"""
self.status_queue = Config.get_manager().Queue()
if temp_path := self._temp_manager.create_temp_path():
self.info.temp_path = str(temp_path)
self._status_tracker = StatusTracker(
info=self.info,
download_id=self.id,
download_dir=self.download_dir,
temp_path=temp_path,
status_queue=self.status_queue,
logger=self.logger,
debug=self.debug,
)
self._hook_handlers = HookHandlers(
download_id=self.id,
status_queue=self.status_queue,
logger=self.logger,
debug=self.debug,
)
self._process_manager.create_process(target=self._download)
self._process_manager.start()
self.started_time = int(time.time())
self.info.status = "preparing"
EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info)
asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}")
ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join)
if self._status_tracker.final_update or self._process_manager.is_cancelled():
if self._process_manager.is_cancelled():
self.info.status = "cancelled"
return ret
self._status_tracker.put_terminator()
await self._status_tracker.drain_queue()
return ret
def started(self) -> bool:
"""Check if download process has been started."""
return self._process_manager.started()
def cancel(self) -> bool:
"""Cancel the download task."""
return self._process_manager.cancel()
async def close(self) -> bool:
"""Close download process and clean up resources."""
if not self.started() or self._process_manager.cancel_in_progress:
return False
try:
if self._status_tracker:
self._status_tracker.cancel_update_task()
await self._process_manager.close()
if self._status_tracker:
self._status_tracker.put_terminator()
self._temp_manager.delete_temp()
return True
except Exception as e:
self.logger.error(f"Failed to close process. {e}")
self.logger.exception(e)
return False
def running(self) -> bool:
return self._process_manager.running()
def is_cancelled(self) -> bool:
return self._process_manager.is_cancelled()
def kill(self) -> bool:
return self._process_manager.kill()
def delete_temp(self, by_pass: bool = False) -> None:
self._temp_manager.delete_temp(by_pass=by_pass)
def is_stale(self) -> bool:
"""
Check if the download task is stale.
A download task is considered stale if it hasn't been updated
for a certain period or is stuck in an unexpected state.
Returns:
True if the download task is stale, False otherwise
"""
if self.started_time < 1:
self.logger.debug(f"Download task '{self.info.name()}' not started yet.")
return False
elapsed = int(time.time()) - self.started_time
if elapsed < 300:
self.logger.debug(f"Download task '{self.info.title}: {self.info.id}' started for '{elapsed}' seconds.")
return False
status: str = self.info.status if self.info.status else "unknown"
is_stale = is_download_stale(self.started_time, status, self.running(), self.info.auto_start, min_elapsed=300)
if is_stale:
if not self.running():
self.logger.warning(
f"Download task '{self.info.title}: {self.info.id}' not started running for '{elapsed}' seconds."
)
else:
self.logger.warning(
f"Download task '{self.info.title}: {self.info.id}' has been stuck in '{status}' state for '{elapsed}' seconds."
)
return is_stale
def __getstate__(self) -> dict[str, Any]:
"""
Prepare state for pickling.
Excludes unpickleable objects like EventBus instances,
primarily for Windows compatibility.
"""
state: dict[str, Any] = self.__dict__.copy()
excluded_keys: tuple[str, ...] = ("_notify",)
for key in excluded_keys:
if key in state:
state[key] = None
return state

91
app/library/downloads/hooks.py Normal file
View file

@ -0,0 +1,91 @@
"""Hook handlers for download progress and postprocessing events."""
import logging
import re
from typing import TYPE_CHECKING, Any
from .utils import DEBUG_MESSAGE_PREFIXES, YTDLP_PROGRESS_FIELDS, create_debug_safe_dict
if TYPE_CHECKING:
from multiprocessing import Queue
class HookHandlers:
"""Manages yt-dlp hook callbacks for progress tracking and postprocessing."""
def __init__(self, download_id: str, status_queue: "Queue[Any]", logger: logging.Logger, debug: bool = False):
"""
Initialize hook handlers.
Args:
download_id: Unique identifier for the download
status_queue: Multiprocessing queue for status updates
logger: Logger instance for this download
debug: Whether to enable debug logging
"""
self.id = download_id
self.status_queue = status_queue
self.logger = logger
self.debug = debug
def progress_hook(self, data: dict[str, Any]) -> None:
if self.debug:
try:
d_safe = create_debug_safe_dict(data)
self.logger.debug(f"PG Hook: {d_safe}")
except Exception as e:
self.logger.debug(f"PG Hook: Error creating debug info: {e}")
self.status_queue.put(
{
"id": self.id,
"action": "progress",
**{k: v for k, v in data.items() if k in YTDLP_PROGRESS_FIELDS},
}
)
def postprocessor_hook(self, data: dict[str, Any]) -> None:
if self.debug:
try:
d_safe = create_debug_safe_dict(data)
d_safe["postprocessor"] = data.get("postprocessor")
self.logger.debug(f"PP Hook: {d_safe}")
except Exception as e:
self.logger.debug(f"PP Hook: Error creating debug info: {e}")
self.status_queue.put(
{
"id": self.id,
"action": "postprocessing",
**{k: v for k, v in data.items() if k in YTDLP_PROGRESS_FIELDS},
"status": "postprocessing",
}
)
def post_hook(self, filename: str) -> None:
self.status_queue.put({"id": self.id, "status": "finished", "final_name": filename})
class NestedLogger:
"""
Logger adapter for yt-dlp that adjusts log levels based on message prefixes.
yt-dlp logs everything through a custom logger. This adapter maps certain
message types to appropriate log levels and strips redundant prefixes.
"""
def __init__(self, logger: logging.Logger) -> None:
self.logger: logging.Logger = logger
def debug(self, msg: str) -> None:
levelno: int = logging.DEBUG if any(msg.startswith(x) for x in DEBUG_MESSAGE_PREFIXES) else logging.INFO
self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE))
def error(self, msg: str) -> None:
"""Log an error message."""
self.logger.error(msg)
def warning(self, msg: str) -> None:
"""Log a warning message."""
self.logger.warning(msg)

378
app/library/downloads/item_adder.py Normal file
View file

@ -0,0 +1,378 @@
"""
Item addition and entry routing.
This module handles the complete flow of adding items to the download queue:
- Preset and CLI validation
- Cookie file management
- Archive checking (early and post-extraction)
- Info extraction with yt-dlp
- Conditions matching and re-queuing
- Entry type routing (playlist, video, URL)
"""
import asyncio
import functools
import logging
import time
import uuid
from pathlib import Path
from typing import TYPE_CHECKING
import yt_dlp.utils
from app.library.conditions import Conditions
from app.library.Events import Events
from app.library.ItemDTO import ItemDTO
from app.library.Presets import Presets
from app.library.Utils import (
archive_add,
archive_read,
arg_converter,
create_cookies_file,
extract_info,
get_extras,
merge_dict,
ytdlp_reject,
)
from .core import Download
from .playlist_processor import process_playlist
from .video_processor import add_video
if TYPE_CHECKING:
from app.library.ItemDTO import Item
from app.library.Presets import Preset
from .queue_manager import DownloadQueue
LOG: logging.Logger = logging.getLogger("downloads.add")
async def add_item(
queue: "DownloadQueue",
entry: dict,
item: "Item",
already=None,
logs: list | None = None,
yt_params: dict | None = None,
) -> dict[str, str]:
"""
Route an entry to the appropriate processor based on type.
Args:
queue: DownloadQueue instance
entry: Entry dict from yt-dlp
item: Item configuration
already: Set of already processed URLs
logs: yt-dlp logs
yt_params: yt-dlp parameters
Returns:
dict: Status dict with "status" and optional "msg" keys
"""
if not entry:
return {"status": "error", "msg": "Invalid/empty data was given."}
event_type = entry.get("_type", "video")
if event_type.startswith("playlist"):
return await process_playlist(queue=queue, entry=entry, item=item, already=already, yt_params=yt_params)
if event_type.startswith("url"):
return await add(queue=queue, item=item.new_with(url=entry.get("url")), already=already)
if event_type.startswith("video"):
return await add_video(queue=queue, entry=entry, item=item, logs=logs)
return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
async def add(
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
) -> dict[str, str]:
"""
Add an item to the download queue.
Args:
queue: DownloadQueue instance
item: Item to be added to the queue
already: Set of already downloaded items
entry: Entry associated with the item (if already extracted)
Returns:
dict[str, str]: Status dict with "status" and optional "msg" keys
"""
_preset: Preset | None = Presets.get_instance().get(item.preset)
if item.has_cli():
try:
arg_converter(args=item.cli, level=True)
except Exception as e:
LOG.error(f"Invalid command options for yt-dlp '{item.cli}'. {e!s}")
return {"status": "error", "msg": f"Invalid command options for yt-dlp '{item.cli}'. {e!s}"}
if _preset:
if _preset.folder and not item.folder:
item.folder = _preset.folder
if _preset.template and not item.template:
item.template = _preset.template
yt_conf = {}
cookie_file: Path = Path(queue.config.temp_path) / f"c_{uuid.uuid4().hex}.txt"
LOG.info(f"Adding '{item!r}'.")
already = set() if already is None else already
if item.url in already:
LOG.warning(f"Recursion detected with url '{item.url}' skipping.")
return {"status": "ok"}
already.add(item.url)
try:
logs: list = []
yt_conf: dict = {
"callback": {
"func": lambda _, msg: logs.append(msg),
"level": logging.WARNING,
"name": "callback-logger",
},
**item.get_ytdlp_opts().get_all(),
}
if yt_conf.get("external_downloader"):
LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.")
item.extras.update({"external_downloader": True})
archive_id: str | None = item.get_archive_id()
# Early archive check to avoid unnecessary extraction calls
# This sometimes can be different from the final extracted ID, so we need to verify again after extraction.
if archive_id and item.is_archived():
store_type, dlInfo = await queue.get_item(archive_id=archive_id)
if not store_type and not queue.config.ignore_archived_items:
dlInfo = Download(
info=ItemDTO(
id=archive_id.split()[1],
title=archive_id,
url=item.url,
preset=item.preset,
folder=item.folder,
status="skip",
cookies=item.cookies,
template=item.template,
msg="URL is already downloaded.",
extras=item.extras,
)
)
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
dlInfo.info.msg += f" Found in archive '{archive_file}'."
await queue.done.put(dlInfo)
queue._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
title="Download History Update",
message=f"Download history updated with '{item.url}'.",
)
return {"status": "ok"}
message: str = f"The URL '{item.url}':'{archive_id}' is already downloaded and recorded in archive."
LOG.warning(message)
queue._notify.emit(
Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message
)
return {"status": "error", "msg": message, "hidden": True}
started: float = time.perf_counter()
if item.cookies:
try:
yt_conf["cookiefile"] = str(create_cookies_file(item.cookies, cookie_file))
except Exception as e:
msg = f"Failed to create cookie file for '{item.url}'. '{e!s}'."
LOG.error(msg)
return {"status": "error", "msg": msg}
if entry:
LOG.info(f"[P] Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.")
if not entry:
async with queue.extractors:
LOG.info(f"Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.")
entry: dict | None = await asyncio.wait_for(
fut=asyncio.get_running_loop().run_in_executor(
None,
functools.partial(
extract_info,
config=yt_conf,
url=item.url,
debug=bool(queue.config.ytdlp_debug),
no_archive=False,
follow_redirect=True,
),
),
timeout=queue.config.extract_info_timeout,
)
if not entry:
LOG.error(f"Unable to extract info for '{item.url}'. Logs: {logs}")
return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)}
# Sometimes playlists or extractor returns different ID than what we get from the make_archive_id()
# So, we need to re-check the archive after extraction to be sure the item was not downloaded.
# This also apply to old archive IDs that might have been used before.
if _archive_file := item.get_archive_file():
extra_ids: list[str] = []
if entry.get("_old_archive_ids") and isinstance(entry.get("_old_archive_ids"), list):
extra_ids.extend(entry.get("_old_archive_ids", []))
new_archive_id: str | None = None
if entry.get("extractor_key") and entry.get("id"):
new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}"
if new_archive_id != archive_id:
extra_ids.append(new_archive_id)
if len(extra_ids) > 0:
archive_ids: list[str] = archive_read(_archive_file, extra_ids)
if len(archive_ids) > 0:
store_type = None
for n in archive_ids:
store_type, dlInfo = await queue.get_item(archive_id=n)
if store_type:
break
if not store_type and not queue.config.ignore_archived_items:
new_archive_id = new_archive_id or extra_ids.pop(0)
dlInfo = Download(
info=ItemDTO(
id=new_archive_id.split()[1],
title=new_archive_id,
url=entry.get("url") or entry.get("webpage_url") or item.url,
preset=item.preset,
folder=item.folder,
status="skip",
cookies=item.cookies,
template=item.template,
msg="URL is already downloaded.",
extras=merge_dict(item.extras, get_extras(entry)),
)
)
dlInfo.info.msg += f" Found in archive '{_archive_file}'."
await queue.done.put(dlInfo)
queue._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
title="Download History Update",
message=f"Download history updated with '{item.url}'.",
)
return {"status": "ok"}
message: str = (
f"The URL '{item.url}':'{archive_ids.pop(0)}' is already downloaded and recorded in archive."
)
LOG.warning(message)
queue._notify.emit(
Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message
)
return {"status": "error", "msg": message, "hidden": True}
if not item.requeued and (condition := Conditions.get_instance().match(info=entry)):
already.pop()
message = f"Condition '{condition.name}' matched for '{item!r}'."
if condition.cli:
message += f" Re-queuing with '{condition.cli}'."
LOG.info(message)
if condition.extras.get("ignore_download", False):
extra_msg: str = ""
if _archive_file and not condition.extras.get("no_archive", False):
archive_add(_archive_file, [archive_id])
extra_msg = f" and added to archive file '{_archive_file}'"
_name = entry.get("title", entry.get("id"))
log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}."
store_type, dlInfo = await queue.get_item(archive_id=archive_id)
if not store_type:
dlInfo = Download(
info=ItemDTO(
id=entry.get("id"),
title=_name,
url=item.url,
preset=item.preset,
folder=item.folder,
status="skip",
cookies=item.cookies,
template=item.template,
msg=log_message,
extras=merge_dict(item.extras, get_extras(entry)),
)
)
await queue.done.put(dlInfo)
LOG.info(log_message)
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
queue._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
title="Download History Update",
message=f"Download history updated with '{item.url}'.",
)
return {"status": "ok"}
if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")):
if Presets.get_instance().has(target_preset):
log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'."
LOG.info(log_message)
queue._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message)
item = item.new_with(preset=target_preset)
else:
LOG.warning(
f"Preset '{target_preset}' specified in condition '{condition.name}' does not exist. Ignoring set_preset."
)
return await add(queue=queue, item=item.new_with(requeued=True, cli=condition.cli), already=already)
_status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf)
if not _status:
LOG.debug(_msg)
return {"status": "error", "msg": _msg}
end_time = time.perf_counter() - started
LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.")
except yt_dlp.utils.ExistingVideoReached as exc:
LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.")
return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
except yt_dlp.utils.YoutubeDLError as exc:
LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.")
return {"status": "error", "msg": str(exc)}
except asyncio.exceptions.TimeoutError as exc:
LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.")
return {
"status": "error",
"msg": f"TimeoutError: {queue.config.extract_info_timeout}s reached Unable to extract info.",
}
finally:
if cookie_file and cookie_file.exists():
try:
cookie_file.unlink(missing_ok=True)
yt_conf.pop("cookiefile", None)
except Exception as e:
LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
return await add_item(queue=queue, entry=entry, item=item, already=already, logs=logs, yt_params=yt_conf)

173
app/library/downloads/monitors.py Normal file
View file

@ -0,0 +1,173 @@
"""Queue monitoring functions."""
import logging
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING
from app.library.ag_utils import ag
from app.library.Events import Events
from app.library.ItemDTO import Item, ItemDTO
from app.library.Utils import dt_delta, str_to_dt
if TYPE_CHECKING:
from .queue_manager import DownloadQueue
LOG: logging.Logger = logging.getLogger("downloads.monitors")
async def check_for_stale(queue: "DownloadQueue") -> None:
"""
Monitor pool for stale downloads and cancel them if needed.
Iterates through active queue items and cancels any that have become stale
(not making progress for an extended period).
Args:
queue: DownloadQueue instance
"""
if queue.is_paused() or queue.queue.empty():
return
for _id, item in list(queue.queue.items()):
item_ref = f"{_id=} {item.info.id=} {item.info.title=}"
if not item.is_stale():
continue
try:
LOG.warning(f"Cancelling staled item '{item_ref}' from download queue.")
await queue.cancel([_id])
except Exception as e:
LOG.error(f"Failed to cancel staled item '{item_ref}'. {e!s}")
LOG.exception(e)
async def check_live(queue: "DownloadQueue") -> None:
"""
Monitor the queue for items marked as live events and queue them when time is reached.
Checks history for live streams/premieres that are scheduled to start and
re-adds them to the download queue when their start time arrives.
Args:
queue: DownloadQueue instance
"""
if queue.is_paused() or queue.done.empty():
return
if queue.config.debug:
LOG.debug("Checking history queue for queued live stream links.")
time_now = datetime.now(tz=UTC)
status: list[str] = ["not_live", "is_upcoming", "is_live"]
for id, item in list(queue.done.items()):
if item.info.status not in status:
continue
item_ref: str = f"{id=} {item.info.id=} {item.info.title=}"
if not item.is_live:
LOG.debug(f"Item '{item_ref}' is not a live stream.")
continue
duration: int | None = item.info.extras.get("duration", None)
is_premiere: bool = item.info.extras.get("is_premiere", False)
live_in: str | None = item.info.live_in or ag(item.info.extras, ["live_in", "release_in"], None)
if not live_in:
LOG.debug(
f"Item '{item_ref}' marked as {'premiere video' if is_premiere else 'live stream'}, but no date is set."
)
continue
starts_in = str_to_dt(live_in)
starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC)
if time_now < (starts_in + timedelta(minutes=1)):
LOG.debug(f"Item '{item_ref}' is not yet live. will start at '{dt_delta(starts_in - time_now)}'.")
continue
if queue.config.prevent_live_premiere and is_premiere and duration:
buffer_time = queue.config.live_premiere_buffer if queue.config.live_premiere_buffer >= 0 else 5
premiere_ends: datetime = starts_in + timedelta(minutes=buffer_time, seconds=duration)
if time_now < premiere_ends:
LOG.debug(
f"Item '{item_ref}' is premiering, download will start in '{(starts_in + timedelta(minutes=buffer_time, seconds=duration)).astimezone().isoformat()}'"
)
continue
LOG.info(f"Retrying item '{item_ref} {item.info.extras=}' for download.")
try:
await queue.clear([item.info._id], remove_file=False)
except Exception as e:
LOG.error(f"Failed to clear item '{item_ref}'. {e!s}")
continue
try:
await queue.add(
item=Item(
url=item.info.url,
preset=item.info.preset,
folder=item.info.folder,
cookies=item.info.cookies,
template=item.info.template,
cli=item.info.cli,
extras=item.info.extras,
)
)
except Exception as e:
await queue.done.put(item)
LOG.exception(e)
LOG.error(f"Failed to retry item '{item_ref}'. {e!s}")
async def delete_old_history(queue: "DownloadQueue") -> None:
"""
Automatically delete old download history based on user specified days.
Removes finished downloads from history that are older than the configured
auto_clear_history_days setting. 0 or negative value disables this feature.
Args:
queue: DownloadQueue instance
"""
if queue.config.auto_clear_history_days < 0 or queue.is_paused() or queue.done.empty():
return
cutoff_date: datetime = datetime.now(UTC) - timedelta(days=queue.config.auto_clear_history_days)
items_to_delete: list[tuple[str, ItemDTO]] = []
for key, download in queue.done.items():
info: ItemDTO = download.info
if not info or not info.timestamp:
continue
if "finished" != info.status or not info.filename:
continue
try:
timestamp_seconds: float = info.timestamp / 1e9
item_datetime: datetime = datetime.fromtimestamp(timestamp_seconds, tz=UTC)
if item_datetime < cutoff_date:
items_to_delete.append((key, info))
except (OSError, ValueError, OverflowError) as e:
LOG.error(f"Failed to parse timestamp '{info.timestamp}' for item '{info.title}': {e}")
titles: list[str] = []
for key, info in items_to_delete:
item_name: str = info.title or info.id or info._id
queue._notify.emit(
Events.ITEM_DELETED,
data=info,
title="Download Cleared",
message=f"'{item_name}' record removed from history.",
)
titles.append(item_name)
await queue.done.delete(key)
if titles:
LOG.info(f"Automatically cleared '{', '.join(titles)}' from download history due to age.")

157
app/library/downloads/playlist_processor.py Normal file
View file

@ -0,0 +1,157 @@
"""Playlist processing."""
import asyncio
import logging
from typing import TYPE_CHECKING, Any
from app.library.Utils import merge_dict, ytdlp_reject
from .utils import handle_task_exception
if TYPE_CHECKING:
from app.library.ItemDTO import Item
from .queue_manager import DownloadQueue
LOG: logging.Logger = logging.getLogger("downloads.playlist")
async def process_playlist(
queue: "DownloadQueue", entry: dict, item: "Item", already=None, yt_params: dict | None = None
) -> dict[str, str]:
"""
Process a playlist entry.
Args:
queue: DownloadQueue instance for accessing queue/config
entry: Playlist entry dict from yt-dlp
item: Item configuration
already: Set of already processed IDs
yt_params: yt-dlp parameters
Returns:
dict: Status dict with "status" and optional "msg" keys
"""
if not yt_params:
yt_params = {}
entries = entry.get("entries", [])
playlist_name: str = f"{entry.get('id')}: {entry.get('title')}"
LOG.info(f"Processing '{playlist_name} ({len(entries)})' Playlist.")
playlistCount = entry.get("playlist_count")
playlistCount: int = int(playlistCount) if playlistCount else len(entries)
playlist_keys: dict[str, Any] = {
"playlist_count": playlistCount,
"playlist": entry.get("title") or entry.get("id"),
"playlist_id": entry.get("id"),
"playlist_title": entry.get("title"),
"playlist_uploader": entry.get("uploader"),
"playlist_uploader_id": entry.get("uploader_id"),
"playlist_channel": entry.get("channel"),
"playlist_channel_id": entry.get("channel_id"),
"playlist_webpage_url": entry.get("webpage_url"),
"__last_playlist_index": playlistCount - 1,
"n_entries": len(entries),
}
async def playlist_processor(i: int, etr: dict):
acquired = False
try:
item_name: str = (
f"'{entry.get('title')}: {i}/{playlist_keys['n_entries']}' - '{etr.get('id')}: {etr.get('title')}'"
)
LOG.debug(f"Waiting to acquire lock for {item_name}")
await queue.processors.acquire()
acquired = True
LOG.debug(f"Acquired lock for {item_name}")
LOG.info(f"Processing '{item_name}'.")
_status, _msg = ytdlp_reject(entry=etr, yt_params=yt_params)
if not _status:
return {"status": "error", "msg": _msg}
extras: dict[str, Any] = {
**playlist_keys,
"playlist_index": i,
"playlist_index_number": i,
"playlist_autonumber": i,
}
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
extras[f"playlist_{property}"] = entry.get(property)
extractor_key = entry.get("ie_key") or entry.get("extractor_key") or entry.get("extractor") or ""
if "thumbnail" not in etr and "youtube" in str(extractor_key).lower():
extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**etr)
newItem: Item = item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras)
if ("video" == etr.get("_type") and etr.get("url")) or (
"formats" in etr and isinstance(etr["formats"], list) and len(etr["formats"]) > 0
):
dct = merge_dict(merge_dict({"_type": "video"}, etr), entry)
dct.pop("entries", None)
return await queue.add(item=newItem, entry=dct, already=already)
return await queue.add(item=newItem, already=already)
finally:
if acquired:
queue.processors.release()
max_downloads: int = -1
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int):
max_downloads: int = ytdlp_opts.get("max_downloads")
batch_size: int = max(50, int(queue.config.playlist_items_concurrency) * 10)
async def run_batch(batch: list[tuple[int, dict]]) -> list[dict]:
tasks: list[asyncio.Task] = []
for i, etr in batch:
task = asyncio.create_task(
playlist_processor(i, etr),
name=f"playlist_processor_{etr.get('id')}_{i}",
)
task.add_done_callback(lambda t: handle_task_exception(t, LOG))
tasks.append(task)
batch_results: list[dict] = await asyncio.gather(*tasks)
await asyncio.sleep(0)
return batch_results
results: list[dict] = []
batch: list[tuple[int, dict]] = []
for i, etr in enumerate(entries, start=1):
if max_downloads > 0 and i > max_downloads:
break
batch.append((i, etr))
if len(batch) >= batch_size:
results.extend(await run_batch(batch))
batch.clear()
if batch:
results.extend(await run_batch(batch))
log_msg: str = f"Playlist '{playlist_name}' processing completed with '{len(results)}' entries."
if max_downloads > 0 and len(entries) > max_downloads:
skipped: int = len(entries) - max_downloads
log_msg += f" Limited to '{max_downloads}' items, skipped '{skipped}' remaining items."
LOG.info(log_msg)
if any("error" == res["status"] for res in results):
return {
"status": "error",
"msg": ", ".join(res["msg"] for res in results if "error" == res["status"] and "msg" in res),
}
return {"status": "ok"}

210
app/library/downloads/pool_manager.py Normal file
View file

@ -0,0 +1,210 @@
"""Download pool management - worker coordination and execution."""
import asyncio
import logging
from typing import TYPE_CHECKING
from app.library.Events import EventBus, Events
from app.library.Utils import calc_download_path
from .core import Download
from .utils import get_extractor_limit, handle_task_exception
if TYPE_CHECKING:
from app.library.config import Config
from .queue_manager import DownloadQueue
LOG: logging.Logger = logging.getLogger("downloads.pool")
class PoolManager:
"""
Manages download worker pool and execution.
Coordinates concurrent downloads with semaphore-based limits:
- Global worker limit (max_workers)
- Per-extractor limits (max_workers_per_extractor)
- Live streams bypass all limits
Integrates with DownloadQueue for queue access and configuration.
"""
def __init__(self, queue: "DownloadQueue", config: "Config"):
self.queue = queue
self.config = config
self._notify: EventBus = EventBus.get_instance()
self.workers = asyncio.Semaphore(config.max_workers)
"Semaphore to limit the number of concurrent downloads."
self.paused = asyncio.Event()
"Event to pause the download queue."
self.event = asyncio.Event()
"Event to signal the download queue to start downloading."
self._active: dict[str, Download] = {}
"Dictionary of active downloads."
self.paused.set()
def is_paused(self) -> bool:
return not self.paused.is_set()
def pause(self) -> None:
if self.paused.is_set():
self.paused.clear()
LOG.info("Download pool paused.")
def resume(self) -> None:
if not self.paused.is_set():
self.paused.set()
LOG.info("Download pool resumed.")
def trigger_download(self) -> None:
"""Trigger the download pool to check for new items."""
if self.event:
self.event.set()
def get_active_downloads(self) -> dict[str, Download]:
return self._active.copy()
def get_active_download(self, download_id: str) -> Download | None:
return self._active.get(download_id)
async def start_pool(self) -> None:
asyncio.create_task(self._download_pool(), name="download_pool")
async def shutdown(self) -> None:
if self._active:
LOG.info(f"Cancelling '{len(self._active)}' active downloads.")
await self.queue.cancel(list(self._active.keys()))
async def _download_pool(self) -> None:
adaptive_sleep = 0.2
max_sleep = 5.0
while True:
while not self.queue.queue.has_downloads():
LOG.info("Waiting for item to download.")
await self.event.wait()
self.event.clear()
adaptive_sleep = 0.2
if self.is_paused():
LOG.warning("Download pool is paused.")
await self.paused.wait()
LOG.info("Download pool resumed downloading.")
adaptive_sleep = 0.2
items_processed = 0
for _id, entry in list(self.queue.queue.items()):
if entry.started() or entry.is_cancelled() or entry.info.auto_start is False:
continue
extractor: str = entry.info.get_extractor() or "unknown"
# Live downloads bypass all limits.
if entry.is_live:
task: asyncio.Task[None] = asyncio.create_task(
self._download_file(_id, entry), name=f"download_live_{extractor}_{_id}"
)
task.add_done_callback(lambda t: handle_task_exception(t, LOG))
items_processed += 1
else:
_limit: asyncio.Semaphore = get_extractor_limit(
extractor, self.config.max_workers, self.config.max_workers_per_extractor, LOG
)
# Skip this item in this iteration if no slots are available.
if self.workers.locked() or _limit.locked():
continue
await self.workers.acquire()
await _limit.acquire()
task: asyncio.Task[None] = asyncio.create_task(
self._download_file(_id, entry), name=f"download_file_{extractor}_{_id}"
)
def _release(t: asyncio.Task, sem=_limit) -> None:
sem.release()
self.workers.release()
handle_task_exception(t, LOG)
task.add_done_callback(_release)
items_processed += 1
# No items could be processed, back off a bit to avoid busy-waiting.
if 0 == items_processed:
adaptive_sleep: float = min(adaptive_sleep * 1.5, max_sleep)
LOG.debug(f"No download slots available. Backing off for {adaptive_sleep:.2f}s before next attempt.")
else:
adaptive_sleep = 0.2
await asyncio.sleep(adaptive_sleep)
async def _download_file(self, id: str, entry: Download) -> None:
"""
Download the file.
Args:
id: The id of the download
entry: The download entry
"""
filePath: str = calc_download_path(base_path=self.config.download_path, folder=entry.info.folder)
LOG.info(f"Downloading 'id: {entry.id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' To '{filePath}'.")
try:
self._active[entry.info._id] = entry
await entry.start()
if entry.info.status not in ("finished", "skip", "cancelled"):
if not entry.info.error:
entry.info.error = f"Download ended with unexpected status '{entry.info.status}'."
entry.info.status = "error"
except Exception as e:
entry.info.status = "error"
entry.info.error = str(e)
finally:
if entry.info._id in self._active:
self._active.pop(entry.info._id, None)
await entry.close()
if await self.queue.queue.exists(key=id):
LOG.debug(f"Download Task '{id}' is completed. Removing from queue.")
await self.queue.queue.delete(key=id)
nTitle: str | None = None
nMessage: str | None = None
if entry.is_cancelled() is True:
nTitle = "Download Cancelled"
nMessage = f"Cancelled '{entry.info.title}' download."
self._notify.emit(Events.ITEM_CANCELLED, data=entry.info, title=nTitle, message=nMessage)
entry.info.status = "cancelled"
if entry.info.status == "finished" and entry.info.filename:
nTitle = "Download Completed"
nMessage = f"Completed '{entry.info.title}' download."
if entry.info.is_archivable and not entry.info.is_archived:
entry.info.is_archived = True
self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage)
await self.queue.done.put(entry)
self._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": entry.info.preset, "item": entry.info},
title=nTitle,
message=nMessage,
)
else:
LOG.warning(f"Download '{id}' not found in queue.")
if self.event:
self.event.set()

184
app/library/downloads/process_manager.py Normal file
View file

@ -0,0 +1,184 @@
"""Process lifecycle management for downloads."""
import logging
import multiprocessing
import os
import signal
from collections.abc import Callable
from .utils import wait_for_process_with_timeout
class ProcessManager:
"""
Manages the lifecycle of a download subprocess.
Handles process creation, monitoring, graceful termination, and
force-kill operations. Includes special handling for live streams
which may not respond to standard termination signals.
"""
def __init__(self, download_id: str, is_live: bool, logger: logging.Logger):
"""
Initialize process manager.
Args:
download_id: Unique identifier for this download
is_live: Whether this is a live stream download
logger: Logger instance
"""
self.download_id = download_id
self.is_live = is_live
self.logger = logger
self.proc: multiprocessing.Process | None = None
self.cancelled = False
self.cancel_in_progress = False
def create_process(self, target: Callable[[], None]) -> multiprocessing.Process:
"""
Create a new download process.
Args:
target: Function to execute in the subprocess
Returns:
Created but not started process
"""
self.proc = multiprocessing.Process(name=f"download-{self.download_id}", target=target)
return self.proc
def start(self) -> None:
if self.proc:
self.proc.start()
def started(self) -> bool:
return self.proc is not None
def running(self) -> bool:
try:
return self.proc is not None and self.proc.is_alive()
except ValueError:
return False
def is_cancelled(self) -> bool:
return self.cancelled
def cancel(self) -> bool:
"""
Mark download as cancelled and kill the process.
Returns:
True if cancellation was initiated, False otherwise
"""
if not self.started():
return False
self.cancelled = True
return self.kill()
def kill(self) -> bool:
"""
Kill the download process.
Attempts graceful termination via SIGUSR1 signal first (POSIX only),
then escalates to terminate() and finally kill() if needed.
Live streams get shorter timeouts as they're more likely to be unresponsive.
Returns:
True if process was killed successfully, False otherwise
"""
if not self.running():
return False
procId: int | None = self.proc.ident
try:
self.logger.info(f"Killing download process: PID={self.proc.pid}, ident={procId}.")
if self.proc.pid and "posix" == os.name:
try:
self.logger.debug(f"Sending SIGUSR1 signal to PID={self.proc.pid}.")
os.kill(self.proc.pid, signal.SIGUSR1)
if wait_for_process_with_timeout(self.proc, 2 if self.is_live else 5):
self.logger.debug(f"Process PID={self.proc.pid} terminated gracefully.")
return True
self.logger.warning(
f"Process PID={self.proc.pid} did not respond to SIGUSR1 "
f"({'live stream' if self.is_live else 'regular download'}), "
f"forcing termination."
)
except (OSError, AttributeError) as e:
self.logger.debug(f"Failed to send SIGUSR1 signal: {e}")
if self.proc.is_alive():
self.logger.info(f"Force-terminating process PID={self.proc.pid}.")
self.proc.terminate()
if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2):
self.logger.warning(f"Process PID={self.proc.pid} did not respond, killing forcefully.")
self.proc.kill()
wait_for_process_with_timeout(self.proc, 1)
self.logger.info(f"Process PID={self.proc.pid} killed.")
return True
except Exception as e:
self.logger.error(f"Failed to kill process PID={self.proc.pid}, ident={procId}. {e}")
self.logger.exception(e)
return False
async def close(self) -> bool:
"""
Close the download process and clean up resources.
This method must be called to properly release resources.
Returns:
True if close was successful, False otherwise
"""
if not self.started() or self.cancel_in_progress:
return False
self.cancel_in_progress = True
procId: int | None = self.proc.ident
if not procId:
if self.proc:
self.proc.close()
self.proc = None
self.logger.warning("Attempted to close download process, but it is not running.")
return False
self.logger.info(f"Closing PID='{procId}' download process.")
try:
self.kill()
import asyncio
loop = asyncio.get_running_loop()
if self.proc.is_alive():
self.logger.debug(f"Waiting for PID='{procId}' to close.")
await loop.run_in_executor(None, self.proc.join)
self.logger.debug(f"PID='{procId}' closed.")
if self.proc:
self.proc.close()
self.proc = None
self.logger.debug(f"Closed PID='{procId}' download process.")
return True
except Exception as e:
self.logger.error(f"Failed to close process: '{procId}'. {e}")
self.logger.exception(e)
return False

429
app/library/downloads/queue_manager.py Normal file
View file

@ -0,0 +1,429 @@
import asyncio
import functools
import glob
import logging
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING
from aiohttp import web
from app.library.config import Config
from app.library.Events import EventBus, Events
from app.library.ItemDTO import Item, ItemDTO
from app.library.Scheduler import Scheduler
from app.library.Services import Services
from app.library.Singleton import Singleton
from app.library.sqlite_store import SqliteStore
from app.library.Utils import calc_download_path
from .core import Download
from .item_adder import add as add_impl
from .monitors import check_for_stale, check_live, delete_old_history
from .pool_manager import PoolManager
if TYPE_CHECKING:
from app.library.DataStore import StoreType
LOG: logging.Logger = logging.getLogger("downloads.queue")
class DownloadQueue(metaclass=Singleton):
def __init__(self, config: Config | None = None):
# Import here to avoid circular import with DataStore
from app.library.DataStore import DataStore, StoreType
self.config: Config = config or Config.get_instance()
"Configuration instance."
self._notify: EventBus = EventBus.get_instance()
"Event bus instance."
self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance())
"DataStore for the completed downloads."
self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance())
"DataStore for the download queue."
self.processors = asyncio.Semaphore(self.config.playlist_items_concurrency)
"Semaphore to limit the number of concurrent processors."
self.extractors = asyncio.Semaphore(self.config.extract_info_concurrency)
"Semaphore to limit the number of concurrent extract_info calls."
self.pool = PoolManager(queue=self, config=self.config)
"Pool manager for coordinating download execution."
@staticmethod
def get_instance(config: Config | None = None) -> "DownloadQueue":
return DownloadQueue(config=config)
def attach(self, _: web.Application) -> None:
Services.get_instance().add("queue", self)
async def event_handler(_, __):
await self.initialize()
self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}")
Scheduler.get_instance().add(
timer="* * * * *",
func=functools.partial(check_for_stale, self),
id=check_for_stale.__name__,
)
Scheduler.get_instance().add(
timer="* * * * *",
func=functools.partial(check_live, self),
id=check_live.__name__,
)
if self.config.auto_clear_history_days > 0:
Scheduler.get_instance().add(
timer="8 */1 * * *",
func=functools.partial(delete_old_history, self),
id=delete_old_history.__name__,
)
# app.on_shutdown.append(self.on_shutdown)
async def test(self) -> bool:
await self.done.test()
return True
async def initialize(self) -> None:
await self.queue.load()
LOG.info(
f"Using '{self.config.max_workers}' workers for downloading and '{self.config.max_workers_per_extractor}' per extractor."
)
await self.pool.start_pool()
async def start_items(self, ids: list[str]) -> dict[str, str]:
"""
Start one or more queued downloads that were added with auto_started=False.
Args:
ids (list[str]): List of item IDs to start.
Returns:
dict[str, str]: Dictionary of per-ID results and overall status.
"""
status: dict[str, str] = {"status": "ok"}
started = False
for item_id in ids:
try:
item: Download = await self.queue.get(key=item_id)
except KeyError as e:
status[item_id] = f"not found: {e!s}"
status["status"] = "error"
LOG.warning(f"Start requested for non-existent item {item_id=}.")
continue
if item.info.auto_start:
status[item_id] = "already started"
continue
item.info.auto_start = True
updated: Download = await self.queue.put(item)
self._notify.emit(Events.ITEM_UPDATED, data=updated.info)
self._notify.emit(
Events.ITEM_RESUMED,
data=item.info,
title="Download Resumed",
message=f"Download '{item.info.title}' has been resumed.",
)
status[item_id] = "started"
started = True
if started:
self.pool.trigger_download()
return status
async def pause_items(self, ids: list[str]) -> dict[str, str]:
"""
Pause one or more queued downloads that were added with auto_started=True.
Args:
ids (list[str]): List of item IDs to pause.
Returns:
dict[str, str]: Dictionary of per-ID results and overall status.
"""
status: dict[str, str] = {"status": "ok"}
for item_id in ids:
try:
item: Download = await self.queue.get(key=item_id)
except KeyError as e:
status[item_id] = f"not found: {e!s}"
status["status"] = "error"
LOG.warning(f"Start requested for non-existent item {item_id=}.")
continue
if item.started() or item.is_cancelled():
status[item_id] = "already started"
continue
if item.info.auto_start is False:
status[item_id] = "not started"
continue
item.info.auto_start = False
updated: Download = await self.queue.put(item)
self._notify.emit(Events.ITEM_UPDATED, data=updated.info)
self._notify.emit(
Events.ITEM_PAUSED,
data=item.info,
title="Download Paused",
message=f"Download '{item.info.title}' has been paused.",
)
status[item_id] = "paused"
return status
def pause(self, shutdown: bool = False) -> bool:
"""
Pause the download queue.
Returns:
bool: True if the download is paused, False otherwise
"""
if not self.pool.is_paused():
self.pool.pause()
if not shutdown:
LOG.warning(f"Download paused at. {datetime.now(tz=UTC).isoformat()}")
return True
return False
def resume(self) -> bool:
"""
Resume the download queue.
Returns:
bool: True if the download is resumed, False otherwise
"""
if self.pool.is_paused():
self.pool.resume()
LOG.warning(f"Downloading resumed at. {datetime.now(tz=UTC).isoformat()}")
return True
return False
def is_paused(self) -> bool:
"""
Check if the download queue is paused.
Returns:
bool: True if the download queue is paused, False otherwise
"""
return self.pool.is_paused()
async def on_shutdown(self, _: web.Application):
await self.pool.shutdown()
async def add(self, item: Item, already: set | None = None, entry: dict | None = None) -> dict[str, str]:
"""
Add an item to the download queue.
Args:
item: Item to be added to the queue
already: Set of already downloaded items
entry: Entry associated with the item (if already extracted)
Returns:
dict[str, str]: Status dict with "status" and optional "msg" keys
"""
return await add_impl(queue=self, item=item, already=already, entry=entry)
async def cancel(self, ids: list[str]) -> dict[str, str]:
"""
Cancel the download.
Args:
ids (list): The list of ids to cancel.
Returns:
dict: The status of the operation.
"""
status: dict[str, str] = {}
for id in ids:
try:
item = await self.queue.get(key=id)
except KeyError as e:
status[id] = str(e)
status["status"] = "error"
LOG.warning(f"Requested cancel for non-existent download {id=}. {e!s}")
continue
item_ref = f"{id=} {item.info.id=} {item.info.title=}"
if item.running():
LOG.debug(f"Canceling {item_ref}")
item.cancel()
LOG.info(f"Cancelled {item_ref}")
await item.close()
else:
await item.close()
LOG.debug(f"Deleting from queue {item_ref}")
await self.queue.delete(id)
self._notify.emit(
Events.ITEM_CANCELLED,
data=item.info,
title="Download Cancelled",
message=f"Download '{item.info.title}' has been cancelled.",
)
item.info.status = "cancelled"
await self.done.put(item)
self._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": item.info.preset, "item": item.info},
title="Download Cancelled",
message=f"Download '{item.info.title}' has been cancelled.",
)
LOG.info(f"Deleted from queue {item_ref}")
status[id] = "ok"
return status
async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]:
"""
Clear the download history.
Args:
ids (list): The list of ids to clear.
remove_file (bool): Only considered if config.remove_files is True.
Returns:
dict: The status of the operation.
"""
status: dict[str, str] = {}
for id in ids:
try:
item: Download = await self.done.get(key=id)
except KeyError as e:
status[id] = str(e)
status["status"] = "error"
LOG.warning(f"Requested delete for non-existent download {id=}. {e!s}")
continue
itemRef: str = f"{id=} {item.info.id=} {item.info.title=}"
removed_files = 0
filename: str = ""
if self.config.remove_files is not True:
remove_file = False
LOG.debug(f"{remove_file=} {itemRef} - Removing local files: {item.info.status=}")
if remove_file and "finished" == item.info.status and item.info.filename:
filename = str(item.info.filename)
if item.info.folder:
filename = f"{item.info.folder}/{item.info.filename}"
try:
rf = Path(
calc_download_path(
base_path=Path(self.config.download_path),
folder=filename,
create_path=False,
)
)
if rf.is_file() and rf.exists():
if rf.stem and rf.suffix:
for f in rf.parent.glob(f"{glob.escape(rf.stem)}.*"):
if f.is_file() and f.exists() and not f.name.startswith("."):
removed_files += 1
LOG.debug(f"Removing '{itemRef}' local file '{f.name}'.")
f.unlink(missing_ok=True)
else:
LOG.debug(f"Removing '{itemRef}' local file '{rf.name}'.")
rf.unlink(missing_ok=True)
removed_files += 1
else:
LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.")
except Exception as e:
LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {e!s}")
await self.done.delete(id)
_status: str = "Removed" if removed_files > 0 else "Cleared"
self._notify.emit(
Events.ITEM_DELETED,
data=item.info,
title=f"Download {_status}",
message=f"{_status} '{item.info.title}' from history.",
)
msg = f"Deleted completed download '{itemRef}'."
if removed_files > 0:
msg += f" and removed '{removed_files}' local files."
LOG.info(msg=msg)
status[id] = "ok"
return status
async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
"""
Get the download queue and the download history.
Args:
mode (str): The mode to get the items. Supported modes are 'all', 'queue', 'done'.
Returns:
dict: The download queue and the download history.
"""
items = {"queue": {}, "done": {}}
active = self.pool.get_active_downloads()
if mode in ("all", "queue"):
for k, v in await self.queue.saved_items():
items["queue"][k] = active[k].info if k in active else v
if mode in ("all", "done"):
for k, v in await self.done.saved_items():
items["done"][k] = v
if mode in ("all", "queue"):
for k, v in self.queue.items():
if k not in items["queue"]:
items["queue"][k] = active[k].info if k in active else v
if mode in ("all", "done"):
for k, v in self.done.items():
if k in items["done"]:
continue
items["done"][k] = v.info
return items
async def get_item(self, **kwargs) -> tuple["StoreType", Download] | tuple[None, None]:
"""
Get a specific item from the download queue or history.
Args:
**kwargs: The key-value pair to search for. Supported keys are 'id', 'url'.
Returns:
(StoreType, Download) | None: The requested item if found, otherwise None.
"""
from app.library.DataStore import StoreType
if item := await self.queue.get_item(**kwargs):
return (StoreType.QUEUE, item)
if item := await self.done.get_item(**kwargs):
return (StoreType.HISTORY, item)
return (None, None)

202
app/library/downloads/status_tracker.py Normal file
View file

@ -0,0 +1,202 @@
"""Status tracking and update processing for downloads."""
import asyncio
import logging
import time
from email.utils import formatdate
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.library.Events import EventBus, Events
from app.library.ffprobe import ffprobe
from .types import StatusDict, Terminator
from .utils import safe_relative_path
if TYPE_CHECKING:
from multiprocessing import Queue
from app.library.ItemDTO import ItemDTO
class StatusTracker:
def __init__(
self,
info: "ItemDTO",
download_id: str,
download_dir: str,
temp_path: Path | None,
status_queue: "Queue[Any]",
logger: logging.Logger,
debug: bool = False,
):
"""
Initialize status tracker.
Args:
info: Download information object
download_id: Unique download identifier
download_dir: Base download directory
temp_path: Temporary directory for this download
status_queue: Multiprocessing queue for status updates
logger: Logger instance
debug: Enable debug logging
"""
self.info = info
self.id = download_id
self.download_dir = download_dir
self.temp_path = temp_path
self.status_queue = status_queue
self.logger = logger
self.debug = debug
self._notify: EventBus = EventBus.get_instance()
self.tmpfilename: str | None = None
self.final_update = False
self.update_task: asyncio.Task | None = None
async def process_status_update(self, status: StatusDict) -> None:
"""
Process a single status update from the download subprocess.
Updates the ItemDTO with progress information, file paths, errors,
and completion status.
Args:
status: Status dictionary from yt-dlp hooks
"""
if self.final_update:
return
if status.get("id") != self.id or len(status) < 2:
self.logger.warning(f"Received invalid status update. {status}")
return
if self.debug:
self.logger.debug(f"Status Update: _id={self.info._id} status={status}")
if isinstance(status, str):
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
return
self.tmpfilename = status.get("tmpfilename")
self.info.status = status.get("status", self.info.status)
self.info.msg = status.get("msg")
self.info.postprocessor = status.get("postprocessor", None)
if "error" == self.info.status and "error" in status:
self.info.error = status.get("error")
self._notify.emit(
Events.LOG_ERROR,
data=self.info,
title="Download Error",
message=f"'{self.info.title}' failed to download. {self.info.error}",
)
if "downloaded_bytes" in status and status.get("downloaded_bytes", 0) > 0:
self.info.downloaded_bytes = status.get("downloaded_bytes")
total: float | None = status.get("total_bytes") or status.get("total_bytes_estimate")
if total:
try:
self.info.percent = status["downloaded_bytes"] / total * 100
except ZeroDivisionError:
self.info.percent = 0
self.info.total_bytes = total
self.info.speed = status.get("speed")
self.info.eta = status.get("eta")
if final_name := status.get("final_name", None):
final_name = Path(final_name)
self.info.datetime = str(formatdate(time.time()))
self.info.status = "finished"
self.final_update = True
self.info.filename = safe_relative_path(final_name, Path(self.download_dir), self.temp_path)
self.logger.debug(f"Final file name: '{final_name}'.")
if final_name.is_file() and final_name.exists():
try:
self.info.file_size = final_name.stat().st_size
except FileNotFoundError:
self.info.file_size = 0
try:
ff = await ffprobe(final_name)
self.info.extras["is_video"] = ff.has_video()
self.info.extras["is_audio"] = ff.has_audio()
if ff.has_video() or ff.has_audio():
self.info.extras["duration"] = int(
float(ff.metadata.get("duration", self.info.extras.get("duration", 0.0)))
)
except Exception as e:
self.info.extras["is_video"] = True
self.info.extras["is_audio"] = True
self.logger.exception(e)
self.logger.error(f"Failed to run ffprobe. {e}")
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
async def progress_update(self) -> None:
"""
Continuous loop that processes status updates from the queue.
Runs until a Terminator sentinel is received or the task is cancelled.
"""
while True:
try:
self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
status = await self.update_task
if status is None or isinstance(status, Terminator):
return
await self.process_status_update(status)
except (asyncio.CancelledError, OSError, FileNotFoundError):
return
async def drain_queue(self, max_iterations: int = 50) -> None:
"""
Drain remaining status updates from the queue.
Called after download completion to ensure all status updates are processed.
Args:
max_iterations: Maximum number of items to drain
"""
self.logger.debug("Draining status queue.")
try:
drain_count: int = max_iterations + (
self.status_queue.qsize() if hasattr(self.status_queue, "qsize") else 5
)
except Exception:
drain_count = max_iterations + 5
for i in range(drain_count):
try:
if self.debug:
self.logger.debug(f"({max_iterations}/{i}) Draining the status queue...")
if self.final_update:
if self.debug:
self.logger.debug(f"({max_iterations}/{i}) Draining stopped. Final update received.")
break
next_status = self.status_queue.get(timeout=0.1)
if next_status is None or isinstance(next_status, Terminator):
continue
await self.process_status_update(next_status)
except Exception as e:
self.logger.warning(f"Error processing status update during drain: {e}")
continue
def cancel_update_task(self) -> None:
"""Cancel the progress update task if it's running."""
try:
if self.update_task and not self.update_task.done():
self.update_task.cancel()
except Exception as e:
self.logger.error(f"Failed to cancel update task. {e}")
def put_terminator(self) -> None:
"""Put a terminator sentinel in the status queue."""
if self.status_queue:
self.status_queue.put(Terminator())

100
app/library/downloads/temp_manager.py Normal file
View file

@ -0,0 +1,100 @@
"""Temporary file and directory management for downloads."""
import hashlib
import logging
from pathlib import Path
from typing import TYPE_CHECKING
from app.library.Utils import delete_dir
from .utils import is_safe_to_delete_dir
if TYPE_CHECKING:
from app.library.ItemDTO import ItemDTO
class TempManager:
def __init__(
self,
info: "ItemDTO",
temp_dir: str | None,
temp_disabled: bool,
temp_keep: bool,
logger: logging.Logger,
):
"""
Initialize temp manager.
Args:
info: Download information object
temp_dir: Base temporary directory path
temp_disabled: Whether temporary directory feature is disabled
temp_keep: Whether to keep temp files after download
logger: Logger instance
"""
self.info = info
self.temp_dir = temp_dir
self.temp_disabled = temp_disabled
self.temp_keep = temp_keep
self.logger = logger
self.temp_path: Path | None = None
def create_temp_path(self) -> Path | None:
"""
Create unique temporary directory for this download.
Returns:
Path to created temporary directory, or None if disabled
"""
if self.temp_disabled or not self.temp_dir:
return None
self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5)
if not self.temp_path.exists():
self.temp_path.mkdir(parents=True, exist_ok=True)
return self.temp_path
def delete_temp(self, by_pass: bool = False) -> None:
"""
Delete temporary directory if conditions are met.
Temp directory is kept if:
- temp_keep is enabled
- temp_disabled is enabled
- Download hasn't finished but has partial data (unless by_pass=True)
Args:
by_pass: Force deletion regardless of download state
"""
if self.temp_disabled or self.temp_keep is True or not self.temp_path:
return
if (
not by_pass
and "finished" != self.info.status
and self.info.downloaded_bytes
and self.info.downloaded_bytes > 0
):
self.logger.warning(f"Keeping temp folder '{self.temp_path}'. status={self.info.status}")
return
tmp_dir = Path(self.temp_path)
if not tmp_dir.exists():
return
if not self.temp_dir or not is_safe_to_delete_dir(tmp_dir, self.temp_dir):
self.logger.warning(f"Refusing to delete video temp folder '{self.temp_path}' as it's temp root.")
return
status: bool = delete_dir(tmp_dir)
if by_pass:
tmp_dir.mkdir(parents=True, exist_ok=True)
self.logger.info(f"Temp folder '{self.temp_path}' emptied.")
else:
self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.")

15
app/library/downloads/types.py Normal file
View file

@ -0,0 +1,15 @@
"""Shared types and protocols for the downloads module."""
from typing import Any
class Terminator:
"""
Sentinel class to signal termination of a status queue.
Used to indicate that no more status updates will be sent through.
"""
StatusDict = dict[str, Any]
"""Type alias for status update dictionaries."""

253
app/library/downloads/utils.py Normal file
View file

@ -0,0 +1,253 @@
"""Utility functions and constants for the downloads module."""
import asyncio
import logging
import os
import time
from pathlib import Path
# Global limits container for per-extractor semaphores
LIMITS: dict[str, asyncio.Semaphore] = {}
# Extractor Constants
GENERIC_EXTRACTORS = ("HTML5MediaEmbed", "generic")
# yt-dlp Field Filtering
YTDLP_PROGRESS_FIELDS = (
"tmpfilename",
"filename",
"status",
"msg",
"total_bytes",
"total_bytes_estimate",
"downloaded_bytes",
"speed",
"eta",
"postprocessor",
)
# Live Stream Options (known to cause issues)
BAD_LIVE_STREAM_OPTIONS = [
"concurrent_fragment_downloads",
"fragment_retries",
"skip_unavailable_fragments",
]
# Debug Logging
DEBUG_MESSAGE_PREFIXES = ["[debug] ", "[download] "]
def safe_relative_path(file_path: Path, base_path: Path, fallback_path: Path | None = None) -> str:
"""
Get relative path with fallback handling.
Attempts to compute the relative path from base_path. If that fails,
tries fallback_path. If both fail, returns absolute path as string.
Args:
file_path: The file path to make relative
base_path: The base path to compute relative to
fallback_path: Optional fallback base path
Returns:
str: Relative path string, or absolute path if relative computation fails
"""
try:
return str(file_path.relative_to(base_path))
except ValueError:
if fallback_path:
try:
return str(file_path.relative_to(fallback_path))
except ValueError:
pass
return str(file_path)
def is_safe_to_delete_dir(path: Path, root_path: str | Path) -> bool:
"""
Safety check before deleting directories.
Prevents accidental deletion of the root temporary directory.
Args:
path: The path to check
root_path: The root path that should not be deleted
Returns:
bool: True if safe to delete, False if it's the root
"""
return str(path) != str(root_path)
def wait_for_process_with_timeout(proc, timeout: float, check_interval: float = 0.1) -> bool:
"""
Wait for process to terminate with timeout.
Args:
proc: The multiprocessing.Process to wait for
timeout: Maximum time to wait in seconds
check_interval: How often to check process state
Returns:
bool: True if process terminated, False if timeout reached
"""
start_time = time.time()
while proc.is_alive() and (time.time() - start_time) < timeout:
time.sleep(check_interval)
return not proc.is_alive()
def parse_extractor_limit(
extractor: str,
default_limit: int,
max_workers: int,
logger: logging.Logger | None = None,
) -> int:
"""
Parse and validate environment variable limits for extractors.
Checks for YTP_MAX_WORKERS_FOR_{EXTRACTOR} environment variable,
validates the value, and ensures it doesn't exceed max_workers.
Args:
extractor: The extractor name
default_limit: Default limit if no valid env var found
max_workers: Maximum workers to never exceed
logger: Optional logger for warnings
Returns:
int: The validated limit for this extractor
"""
env_limit: str | None = os.environ.get(f"YTP_MAX_WORKERS_FOR_{extractor.upper()}")
if env_limit and env_limit.isdigit() and 1 <= int(env_limit):
limit: int = min(int(env_limit), max_workers)
else:
if env_limit and logger:
logger.warning(f"Invalid extractor limit '{env_limit}' for '{extractor}', using default limit.")
limit = default_limit
return min(limit, max_workers)
def get_extractor_limit(
extractor: str,
max_workers: int,
max_workers_per_extractor: int,
logger: logging.Logger,
) -> asyncio.Semaphore:
"""
Get or create a semaphore for the given extractor.
Args:
extractor: The extractor name
max_workers: Maximum workers allowed
max_workers_per_extractor: Default per-extractor limit
logger: Logger for info messages
Returns:
asyncio.Semaphore: The semaphore for this extractor
"""
if extractor not in LIMITS:
limit = parse_extractor_limit(extractor, max_workers_per_extractor, max_workers, logger)
LIMITS[extractor] = asyncio.Semaphore(limit)
logger.info(f"Created limits container for extractor '{extractor}': {limit}")
return LIMITS[extractor]
def create_debug_safe_dict(data: dict, exclude_keys: list[str] | None = None) -> dict:
"""
Create sanitized dict for debug logging.
Args:
data: The data dict to sanitize
exclude_keys: Additional keys to exclude from info_dict
Returns:
dict: Sanitized dict safe for logging
"""
if exclude_keys is None:
exclude_keys = ["formats", "thumbnails", "description", "tags", "_format_sort_fields"]
import types
d_safe: dict = {
"status": data.get("status"),
"filename": data.get("filename"),
"info_dict": {
k: v
for k, v in data.get("info_dict", {}).items()
if k not in exclude_keys and v is not None and not isinstance(v, (types.FunctionType, types.LambdaType))
},
}
return d_safe
def is_download_stale(
started_time: int, current_status: str, is_running: bool, auto_start: bool, min_elapsed: int = 300
) -> bool:
"""
Determine if download task has become stale.
A download is stale if it's been running too long without progress,
or if the process died unexpectedly.
Terminal statuses (finished, error, cancelled, downloading, postprocessing)
are never stale - status is source of truth.
Args:
started_time: Unix timestamp when download started
current_status: Current download status
is_running: Whether the process is currently running
auto_start: Whether download was set to auto-start
min_elapsed: Minimum seconds before considering stale
Returns:
bool: True if download is stale and should be cancelled
"""
terminal_statuses = ["finished", "error", "cancelled", "downloading", "postprocessing"]
if current_status in terminal_statuses:
return False
if not auto_start:
return False
if is_running:
return False
elapsed = int(time.time()) - started_time
return elapsed >= min_elapsed
def handle_task_exception(task: asyncio.Task, logger: logging.Logger) -> None:
"""
Handle exceptions from background tasks.
Logs unhandled exceptions from asyncio tasks. Ignores cancelled tasks.
Args:
task: The completed task to check
logger: Logger to use for error output
"""
if task.cancelled():
return
if not (exc := task.exception()):
return
task_name: str = task.get_name() if task.get_name() else "unknown_task"
import traceback
tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
logger.error(f"Unhandled exception in background task '{task_name}': {exc!s}. {tb}")

233
app/library/downloads/video_processor.py Normal file
View file

@ -0,0 +1,233 @@
"""Video entry processing."""
import logging
import time
from datetime import UTC, datetime, timedelta
from email.utils import formatdate
from typing import TYPE_CHECKING
from app.library.Events import Events
from app.library.ItemDTO import ItemDTO
from app.library.Utils import calc_download_path, extract_ytdlp_logs, get_extras, merge_dict, str_to_dt
from .core import Download
if TYPE_CHECKING:
from app.library.ItemDTO import Item
from .queue_manager import DownloadQueue
LOG: logging.Logger = logging.getLogger("downloads.video")
async def add_video(queue: "DownloadQueue", entry: dict, item: "Item", logs: list[str] | None = None) -> dict[str, str]:
"""
Process and add a video entry to the queue.
Args:
queue: DownloadQueue instance
entry: Video entry dict from yt-dlp
item: Item configuration
logs: Optional yt-dlp logs
Returns:
dict: Status dict with "status" and optional "msg" keys
"""
if not logs:
logs: list[str] = []
options: dict = {}
error: str | None = None
live_in: str | None = None
is_premiere: bool = bool(entry.get("is_premiere", False))
release_in: str | None = None
if entry.get("release_timestamp"):
release_in = formatdate(entry.get("release_timestamp"), usegmt=True)
item.extras["release_in"] = release_in
# check if the video is live stream.
if "is_upcoming" == entry.get("live_status"):
if release_in:
live_in = release_in
item.extras["live_in"] = live_in
else:
error = f"No start time is set for {'premiere' if is_premiere else 'live stream'}."
else:
error = entry.get("msg")
LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
try:
_item: Download = await queue.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
err_msg: str = f"Removing {_item.info.name()} from history list."
LOG.warning(err_msg)
await queue.clear([_item.info._id], remove_file=False)
except KeyError:
pass
try:
_item: Download = await queue.queue.get(
key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url"))
)
err_msg: str = f"Item {_item.info.name()} is already in download queue."
LOG.info(err_msg)
return {"status": "error", "msg": err_msg}
except KeyError:
pass
live_status: list = ["is_live", "is_upcoming"]
is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
try:
download_dir: str = calc_download_path(base_path=queue.config.download_path, folder=item.folder)
except Exception as e:
LOG.exception(e)
return {"status": "error", "msg": str(e)}
for field in ("uploader", "channel", "thumbnail"):
if entry.get(field):
item.extras[field] = entry.get(field)
for key in entry:
if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
item.extras[key] = entry.get(key)
item.extras["duration"] = entry.get("duration", item.extras.get("duration"))
if not item.extras.get("live_in") and live_in:
item.extras["live_in"] = live_in
if not item.extras.get("is_premiere") and is_premiere:
item.extras["is_premiere"] = is_premiere
dl = ItemDTO(
id=str(entry.get("id")),
title=str(entry.get("title")),
description=str(entry.get("description", "")),
url=str(entry.get("webpage_url") or entry.get("url")),
preset=item.preset,
folder=item.folder,
download_dir=download_dir,
temp_dir=queue.config.temp_path,
cookies=item.cookies,
template=item.template if item.template else queue.config.output_template,
template_chapter=queue.config.output_template_chapter,
datetime=formatdate(time.time()),
error=error,
is_live=is_live,
live_in=live_in if live_in else item.extras.get("live_in", None),
options=options,
cli=item.cli,
auto_start=item.auto_start,
extras=merge_dict(item.extras, get_extras(entry)),
)
try:
dlInfo: Download = Download(info=dl, info_dict=entry if item.auto_start else None, logs=logs)
nEvent: str | None = None
nTitle: str | None = None
nMessage: str | None = None
nStore: str = "queue"
hasFormats: bool = len(entry.get("formats", [])) > 0 or entry.get("url")
text_logs: str = ""
if filtered_logs := extract_ytdlp_logs(logs):
text_logs = " " + ", ".join(filtered_logs)
if "is_upcoming" == entry.get("live_status"):
nEvent = Events.ITEM_MOVED
nStore = "history"
nTitle = "Upcoming Premiere" if is_premiere else "Upcoming Live Stream"
nMessage = f"{'Premiere video' if is_premiere else 'Stream'} '{dlInfo.info.title}' is not available yet. {text_logs}"
dlInfo.info.status = "not_live"
dlInfo.info.msg = nMessage.replace(f" '{dlInfo.info.title}'", "")
queue._notify.emit(
Events.LOG_INFO,
data={"preset": dlInfo.info.preset, "lowPriority": True},
title=nTitle,
message=nMessage,
)
itemDownload: Download = await queue.done.put(dlInfo)
elif not hasFormats:
ava: str = entry.get("availability", "public")
nTitle = "Download Error"
nMessage: str = f"No formats for '{dl.title}'."
nEvent = Events.ITEM_MOVED
nStore = "history"
if ava and ava not in ("public",):
nMessage += f" Availability is set for '{ava}'."
dlInfo.info.error = nMessage.replace(f" for '{dl.title}'.", ".") + text_logs
dlInfo.info.status = "error"
itemDownload = await queue.done.put(dlInfo)
queue._notify.emit(
Events.LOG_WARNING,
data={"preset": dlInfo.info.preset, "logs": text_logs},
title=nTitle,
message=nMessage,
)
elif is_premiere and queue.config.prevent_live_premiere:
nStore = "history"
nTitle = "Premiere Video"
dlInfo.info.error = "Premiering right now."
_requeue = True
if release_in:
try:
starts_in: datetime = str_to_dt(release_in)
starts_in: datetime = (
starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC)
)
starts_in = starts_in + timedelta(minutes=5, seconds=dl.extras.get("duration", 0))
dlInfo.info.error += f" Download will start at {starts_in.astimezone().isoformat()}."
_requeue = False
except Exception as e:
LOG.error(f"Failed to parse live_in date '{release_in}'. {e!s}")
dlInfo.info.error += f" Failed to parse live_in date '{release_in}'."
else:
dlInfo.info.error += f" Delaying download by '{300 + dl.extras.get('duration', 0)}' seconds."
nMessage = f"'{dlInfo.info.title}': '{dlInfo.info.error.strip()}'."
if _requeue:
nEvent = Events.ITEM_ADDED
itemDownload = await queue.queue.put(dlInfo)
if item.auto_start:
queue.pool.trigger_download()
else:
dlInfo.info.status = "not_live"
itemDownload = await queue.done.put(dlInfo)
nStore = "history"
nEvent = Events.ITEM_MOVED
nTitle = "Premiering right now"
queue._notify.emit(Events.LOG_INFO, data={"preset": dlInfo.info.preset}, title=nTitle, message=nMessage)
else:
nEvent = Events.ITEM_ADDED
nTitle = "Item Added"
nMessage = f"Item '{dlInfo.info.title}' has been added to the download queue."
itemDownload = await queue.queue.put(dlInfo)
if item.auto_start:
queue.pool.trigger_download()
else:
LOG.debug(f"Item {itemDownload.info.name()} is not set to auto-start.")
queue._notify.emit(
nEvent,
data={"to": nStore, "preset": itemDownload.info.preset, "item": itemDownload.info}
if Events.ITEM_MOVED == nEvent
else itemDownload.info,
title=nTitle,
message=nMessage,
)
return {"status": "ok"}
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to download item. '{e!s}'")
return {"status": "error", "msg": str(e)}

2
app/main.py
View file

@ -18,7 +18,7 @@ from app.library.BackgroundWorker import BackgroundWorker
from app.library.conditions import Conditions
from app.library.config import Config
from app.library.dl_fields import DLFields
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import DownloadQueue
from app.library.Events import EventBus, Events
from app.library.HttpAPI import HttpAPI
from app.library.HttpSocket import HttpSocket

2
app/routes/api/browser.py
View file

@ -9,7 +9,7 @@ from aiohttp.web import Request, Response
from app.library.cache import Cache
from app.library.config import Config
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import DownloadQueue
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.ffprobe import ffprobe

5
app/routes/api/history.py
View file

@ -7,8 +7,7 @@ from aiohttp.web import Request, Response
from app.library.config import Config
from app.library.DataStore import StoreType
from app.library.Download import Download
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import Download, DownloadQueue
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.ItemDTO import Item
@ -16,7 +15,7 @@ from app.library.Presets import Preset, Presets
from app.library.router import route
if TYPE_CHECKING:
from library.Download import Download
from library.downloads import Download
LOG: logging.Logger = logging.getLogger(__name__)

2
app/routes/api/ping.py
View file

@ -1,7 +1,7 @@
from aiohttp import web
from aiohttp.web import Response
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import DownloadQueue
from app.library.router import route

2
app/routes/api/system.py
View file

@ -7,7 +7,7 @@ from aiohttp.web import Request, Response
from aiohttp.web_runner import GracefulExit
from app.library.config import Config
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import DownloadQueue
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.router import route

2
app/routes/socket/connection.py
View file

@ -7,7 +7,7 @@ import socketio
from app.library.config import Config
from app.library.dl_fields import DLFields
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import DownloadQueue
from app.library.Events import EventBus, Events
from app.library.Presets import Presets
from app.library.router import RouteType, route

2
app/routes/socket/history.py
View file

@ -1,6 +1,6 @@
import logging
from app.library.DownloadQueue import DownloadQueue
from app.library.downloads import DownloadQueue
from app.library.Events import EventBus, Events
from app.library.ItemDTO import Item
from app.library.router import RouteType, route

2
app/tests/test_datastore_pagination.py
View file

@ -5,7 +5,7 @@ import pytest
import pytest_asyncio
from app.library.DataStore import DataStore, StoreType
from app.library.Download import Download
from app.library.downloads import Download
from app.library.ItemDTO import ItemDTO
from app.library.sqlite_store import SqliteStore

682
app/tests/test_download.py
View file

@ -1,10 +1,17 @@
import logging
import os
import signal
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, Mock, patch
import pytest
from app.library.Download import Download, NestedLogger, Terminator
from app.library.downloads import Download, NestedLogger, Terminator
from app.library.downloads.hooks import HookHandlers
from app.library.downloads.process_manager import ProcessManager
from app.library.downloads.status_tracker import StatusTracker
from app.library.downloads.temp_manager import TempManager
from app.library.ItemDTO import ItemDTO
@ -28,7 +35,7 @@ class DummyQueue:
def put(self, obj: Any) -> None:
self.items.append(obj)
def get(self) -> Any:
def get(self, timeout: float | None = None) -> Any:
if not self.items:
return None
return self.items.pop(0)
@ -39,7 +46,6 @@ class TestNestedLogger:
logger = logging.getLogger("nl_test")
logger.setLevel(logging.DEBUG)
cap = CaptureHandler()
# Remove existing handlers to avoid duplicates
for h in list(logger.handlers):
logger.removeHandler(h)
logger.addHandler(cap)
@ -49,20 +55,18 @@ class TestNestedLogger:
nl.debug("[download] progress")
nl.debug("[info] info message")
# Two DEBUG, one INFO
levels = [r.levelno for r in cap.records]
assert levels.count(logging.DEBUG) == 2
assert levels.count(logging.INFO) == 1
assert 2 == levels.count(logging.DEBUG), "Should have 2 debug messages"
assert 1 == levels.count(logging.INFO), "Should have 1 info message"
msgs = [r.getMessage() for r in cap.records]
assert "[debug]" not in msgs[0]
assert "[debug]" not in msgs[0], "[debug] prefix should be stripped"
assert msgs[1] == "[download] progress", "[download] prefix is not stripped by NestedLogger"
assert msgs[2] == "info message"
assert msgs[2] == "info message", "info message should have [info] prefix stripped"
class TestDownloadHooks:
@pytest.fixture(autouse=True)
def cfg_and_bus(self, monkeypatch: pytest.MonkeyPatch):
# Minimal Config stub
class Cfg:
debug = False
ytdlp_debug = False
@ -75,9 +79,8 @@ class TestDownloadHooks:
def get_instance():
return Cfg
monkeypatch.setattr("app.library.Download.Config", Cfg)
monkeypatch.setattr("app.library.downloads.core.Config", Cfg)
# EventBus.get_instance is used during __init__ and start, we don't hit start here
class EB:
@staticmethod
def get_instance():
@ -87,12 +90,12 @@ class TestDownloadHooks:
def emit(*_args, **_kwargs):
return None
monkeypatch.setattr("app.library.Download.EventBus", EB)
monkeypatch.setattr("app.library.downloads.core.EventBus", EB)
def test_progress_hook_filters_fields(self) -> None:
d = Download(make_item())
q = DummyQueue()
d.status_queue = q
hooks = HookHandlers(d.id, q, d.logger, d.debug)
payload = {
"tmpfilename": "t",
@ -106,12 +109,12 @@ class TestDownloadHooks:
"eta": 2,
"other": "x",
}
d._progress_hook(payload)
assert len(q.items) == 1
hooks.progress_hook(payload)
assert 1 == len(q.items), "Should have 1 item in queue"
ev = q.items[0]
assert ev["id"] == d.id
assert ev["action"] == "progress"
assert "other" not in ev, "ensure only whitelisted keys included"
assert ev["id"] == d.id, "Event should have correct download ID"
assert ev["action"] == "progress", "Action should be 'progress'"
assert "other" not in ev, "Non-whitelisted keys should not be included in event"
for k in (
"tmpfilename",
"filename",
@ -123,38 +126,15 @@ class TestDownloadHooks:
"speed",
"eta",
):
assert k in ev
def test_postprocessor_hook_movefiles_sets_final_name(self, tmp_path: Path) -> None:
d = Download(make_item())
q = DummyQueue()
d.status_queue = q
finaldir = tmp_path / "out"
finaldir.mkdir()
path = finaldir / "file.mp4"
# we don't need it to exist for this branch; hook doesn't check file existence
payload = {
"postprocessor": "MoveFiles",
"status": "finished",
"info_dict": {"__finaldir": str(finaldir), "filepath": str(path)},
}
d._postprocessor_hook(payload)
assert len(q.items) == 1
ev = q.items[0]
assert ev["action"] == "moved"
assert ev["status"] == "finished"
assert ev["final_name"].endswith("file.mp4")
assert k in ev, f"Key '{k}' should be in event"
def test_post_hooks_pushes_filename(self) -> None:
d = Download(make_item())
q = DummyQueue()
d.status_queue = q
d._post_hooks(None)
assert len(q.items) == 0
d._post_hooks("name.ext")
assert len(q.items) == 1
assert q.items[0]["filename"] == "name.ext"
hooks = HookHandlers(d.id, q, d.logger, d.debug)
hooks.post_hook("name.ext")
assert 1 == len(q.items), "Should have 1 item when filename is provided"
assert q.items[0]["final_name"] == "name.ext", "Filename should match"
class TestDownloadStale:
@ -172,7 +152,14 @@ class TestDownloadStale:
def get_instance():
return Cfg
monkeypatch.setattr("app.library.Download.Config", Cfg)
@staticmethod
def get_manager():
# Return a mock manager with Queue method
mock_manager = MagicMock()
mock_manager.Queue = MagicMock(return_value=DummyQueue())
return mock_manager
monkeypatch.setattr("app.library.downloads.core.Config", Cfg)
class EB:
@staticmethod
@ -183,203 +170,524 @@ class TestDownloadStale:
def emit(*_args, **_kwargs):
return None
monkeypatch.setattr("app.library.Download.EventBus", EB)
monkeypatch.setattr("app.library.downloads.core.EventBus", EB)
def test_is_stale_conditions(self, monkeypatch: pytest.MonkeyPatch) -> None:
d = Download(make_item())
# Auto-start disabled is never stale
d.info.auto_start = False
assert d.is_stale() is False
assert d.is_stale() is False, "Download with auto_start disabled should not be stale"
d.info.auto_start = True
# Not started yet -> not stale
d.started_time = 0
assert d.is_stale() is False
assert d.is_stale() is False, "Download that has not been started should not be stale"
# Less than 300 seconds elapsed -> not stale
d.started_time = 1000
monkeypatch.setattr("time.time", lambda: 1200)
assert d.is_stale() is False
assert d.is_stale() is False, "Download running for less than 300 seconds should not be stale"
# Not running after 300s -> stale
d.started_time = 0
d.started_time = 1000
monkeypatch.setattr("time.time", lambda: 1401)
monkeypatch.setattr("app.library.Download.Download.running", lambda _self: False)
d.info.status = "finished"
assert d.is_stale() is False, "Download with status 'finished' should not be stale regardless of process state"
d.info.status = "error"
assert d.is_stale() is False, "Download with status 'error' should not be stale regardless of process state"
d.info.status = "cancelled"
assert d.is_stale() is False, "Download with status 'cancelled' should not be stale regardless of process state"
d.info.status = "downloading"
assert d.is_stale() is False, (
"Download with status 'downloading' should not be stale regardless of process state"
)
d.info.status = "postprocessing"
assert d.is_stale() is False, (
"Download with status 'postprocessing' should not be stale regardless of process state"
)
d.info.status = "preparing"
assert d.is_stale() is True
assert d.is_stale() is True, (
"Download with status 'preparing' and no running process after 300s should be stale"
)
# Running but status stuck -> stale
monkeypatch.setattr("app.library.Download.Download.running", lambda _self: True)
d.info.status = "queued"
assert d.is_stale() is True
assert d.is_stale() is True, "Download with status 'queued' and no running process after 300s should be stale"
# Running and in allowed statuses -> not stale
for s in ("finished", "error", "cancelled", "downloading", "postprocessing"):
d.info.status = s
assert d.is_stale() is False
d.info.status = None
assert d.is_stale() is True, "Download with no status and no running process after 300s should be stale"
@pytest.mark.asyncio
async def test_started_time_is_set_in_main_process(self, monkeypatch: pytest.MonkeyPatch) -> None:
d = Download(make_item())
# Create a mock process
mock_proc = MagicMock()
mock_proc.join = MagicMock(return_value=0)
# Create a proper mock for create_task that consumes the coroutine
created_tasks = []
def mock_create_task(coro, **kwargs):
# Close the coroutine to avoid warning
coro.close()
task = MagicMock()
created_tasks.append(task)
return task
# Mock process manager to avoid actually starting a subprocess
with (
patch.object(d._process_manager, "create_process", return_value=mock_proc) as mock_create,
patch.object(d._process_manager, "start") as mock_start,
patch("asyncio.create_task", side_effect=mock_create_task) as mock_create_task_fn,
patch("asyncio.get_running_loop") as mock_loop,
):
# Set the mock proc on the process manager
d._process_manager.proc = mock_proc
# Mock the join to return immediately
async def mock_executor(*args):
return 0
mock_loop.return_value.run_in_executor = mock_executor
# Mock status tracker to prevent actual status updates
d._status_tracker = MagicMock()
d._status_tracker.final_update = True
assert d.started_time == 0, "started_time should be 0 before start() is called"
# Call start() - this should set started_time in the main process
await d.start()
# Verify started_time was set to a non-zero value
assert d.started_time > 0, "started_time should be set in main process after start() is called"
# Verify process was actually started
mock_create.assert_called_once()
mock_start.assert_called_once()
@pytest.mark.asyncio
async def test_start_initializes_and_emits(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
class Cfg:
debug = False
ytdlp_debug = False
max_workers = 1
temp_keep = False
temp_disabled = False
download_info_expires = 3600
class TestTempManager:
def test_create_temp_path_when_disabled(self) -> None:
info = make_item()
logger = logging.getLogger("test")
tm = TempManager(info, "/tmp", temp_disabled=True, temp_keep=False, logger=logger)
@staticmethod
def get_instance():
return Cfg
result = tm.create_temp_path()
assert result is None, "Should return None when temp_disabled is True"
assert tm.temp_path is None, "temp_path should remain None when disabled"
class _Mgr:
@staticmethod
def Queue(): # noqa: N802
return DummyQueue()
def test_create_temp_path_when_no_temp_dir(self) -> None:
info = make_item()
logger = logging.getLogger("test")
tm = TempManager(info, None, temp_disabled=False, temp_keep=False, logger=logger)
@staticmethod
def get_manager():
return Cfg._Mgr
result = tm.create_temp_path()
assert result is None, "Should return None when temp_dir is None"
assert tm.temp_path is None, "temp_path should remain None when no temp_dir"
monkeypatch.setattr("app.library.Download.Config", Cfg)
def test_create_temp_path_creates_directory(self, tmp_path: Path) -> None:
info = make_item(id="test123")
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
events = []
result = tm.create_temp_path()
assert result is not None, "Should return Path when enabled"
assert result.exists(), "Temporary directory should be created"
assert result.parent == tmp_path, "Temp directory should be created in temp_dir"
assert tm.temp_path == result, "temp_path should be set to created path"
class EB:
@staticmethod
def get_instance():
return EB
def test_create_temp_path_uses_consistent_hash(self, tmp_path: Path) -> None:
info = make_item(id="test123")
logger = logging.getLogger("test")
tm1 = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
tm2 = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
@staticmethod
def emit(event, data=None, **_kwargs):
events.append((event, data))
path1 = tm1.create_temp_path()
path2 = tm2.create_temp_path()
assert path1 == path2, "Same download ID should produce same temp path"
monkeypatch.setattr("app.library.Download.EventBus", EB)
def test_delete_temp_when_disabled(self, tmp_path: Path) -> None:
info = make_item()
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=True, temp_keep=False, logger=logger)
tm.temp_path = tmp_path / "test"
tm.temp_path.mkdir()
class FakeProc:
def __init__(self, name: str, target, *_args, **_kwargs):
self.name = name
self.target = target
self.started = False
tm.delete_temp()
assert tm.temp_path.exists(), "Should not delete when temp_disabled is True"
def start(self) -> None:
self.started = True
def test_delete_temp_when_temp_keep_enabled(self, tmp_path: Path) -> None:
info = make_item()
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=True, logger=logger)
tm.temp_path = tmp_path / "test"
tm.temp_path.mkdir()
def join(self) -> int:
return 0
tm.delete_temp()
assert tm.temp_path.exists(), "Should not delete when temp_keep is True"
monkeypatch.setattr("app.library.Download.multiprocessing.Process", FakeProc)
def test_delete_temp_when_no_temp_path(self) -> None:
info = make_item()
logger = logging.getLogger("test")
tm = TempManager(info, "/tmp", temp_disabled=False, temp_keep=False, logger=logger)
item = make_item(id="start1")
d = Download(item)
d.temp_dir = str(tmp_path)
d.download_dir = str(tmp_path)
tm.delete_temp()
await d.start()
def test_delete_temp_keeps_partial_download(self, tmp_path: Path) -> None:
info = make_item()
info.status = "downloading"
info.downloaded_bytes = 1000
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
tm.temp_path = tmp_path / "test"
tm.temp_path.mkdir()
assert d.info.status == "preparing"
assert isinstance(d.status_queue, DummyQueue)
assert d.temp_path is not None
assert Path(d.temp_path).exists()
assert len(events) >= 1
tm.delete_temp()
assert tm.temp_path.exists(), "Should keep temp dir for partial download"
def test_delete_temp_with_bypass(self, tmp_path: Path) -> None:
info = make_item()
info.status = "downloading"
info.downloaded_bytes = 1000
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
tm.temp_path = tmp_path / "test"
tm.temp_path.mkdir()
(tm.temp_path / "file.txt").write_text("test")
tm.delete_temp(by_pass=True)
assert tm.temp_path.exists(), "Directory should still exist with bypass"
assert not (tm.temp_path / "file.txt").exists(), "Contents should be deleted with bypass"
def test_delete_temp_finished_download(self, tmp_path: Path) -> None:
info = make_item()
info.status = "finished"
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
tm.temp_path = tmp_path / "test"
tm.temp_path.mkdir()
tm.delete_temp()
assert not tm.temp_path.exists(), "Should delete temp dir for finished download"
def test_delete_temp_refuses_to_delete_temp_root(self, tmp_path: Path) -> None:
info = make_item()
info.status = "finished"
logger = logging.getLogger("test")
tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger)
tm.temp_path = tmp_path
tm.delete_temp()
assert tm.temp_path.exists(), "Should refuse to delete temp root directory"
@pytest.mark.asyncio
async def test_progress_update_processes_statuses(monkeypatch: pytest.MonkeyPatch) -> None:
class Cfg:
debug = False
ytdlp_debug = False
max_workers = 1
temp_keep = False
temp_disabled = True
download_info_expires = 3600
class TestProcessManager:
def test_create_process(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
@staticmethod
def get_instance():
return Cfg
def dummy_target():
pass
monkeypatch.setattr("app.library.Download.Config", Cfg)
proc = pm.create_process(dummy_target)
assert proc is not None, "Should create a process"
assert pm.proc is proc, "Should store process reference"
assert "download-test-id" == proc.name, "Process name should include download ID"
class EB:
@staticmethod
def get_instance():
return EB
def test_started_returns_true_when_process_created(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
@staticmethod
def emit(*_a, **_kw):
return None
assert pm.started() is False, "Should return False when no process created"
monkeypatch.setattr("app.library.Download.EventBus", EB)
pm.create_process(lambda: None)
assert pm.started() is True, "Should return True after process created"
seen = []
def test_running_returns_false_when_no_process(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
async def spy(_self, status):
seen.append(status)
assert pm.running() is False, "Should return False when no process"
d = Download(make_item(id="p1"))
d.status_queue = DummyQueue()
monkeypatch.setattr(Download, "_process_status_update", spy, raising=False)
d.status_queue.put({"id": d.id, "status": "downloading"})
d.status_queue.put({"id": d.id, "status": "postprocessing"})
d.status_queue.put(Terminator())
def test_is_cancelled_returns_false_by_default(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
await d.progress_update()
assert pm.is_cancelled() is False, "Should return False by default"
assert [s["status"] for s in seen] == ["downloading", "postprocessing"]
def test_cancel_marks_as_cancelled(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
pm.proc = Mock()
pm.proc.is_alive = Mock(return_value=False)
result = pm.cancel()
assert pm.is_cancelled() is True, "Should mark as cancelled"
def test_cancel_returns_false_when_not_started(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
result = pm.cancel()
assert result is False, "Should return False when process not started"
assert pm.is_cancelled() is False, "Should not mark as cancelled when not started"
def test_kill_returns_false_when_not_running(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
result = pm.kill()
assert result is False, "Should return False when process not running"
def test_kill_sends_sigusr1_on_posix(self) -> None:
if "posix" != os.name:
pytest.skip("Test only runs on POSIX systems")
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
pm.proc = Mock()
pm.proc.pid = 12345
pm.proc.ident = 67890
pm.proc.is_alive = Mock(side_effect=[True, False, False])
with patch("app.library.downloads.process_manager.os.kill") as mock_kill:
result = pm.kill()
mock_kill.assert_called_once_with(12345, signal.SIGUSR1)
assert result is True, "Should return True when process killed successfully"
def test_kill_uses_shorter_timeout_for_live_streams(self) -> None:
logger = logging.getLogger("test")
pm_live = ProcessManager("test-id", is_live=True, logger=logger)
pm_regular = ProcessManager("test-id", is_live=False, logger=logger)
pm_live.proc = Mock()
pm_live.proc.pid = 12345
pm_live.proc.ident = 67890
pm_live.proc.is_alive = Mock(return_value=False)
pm_regular.proc = Mock()
pm_regular.proc.pid = 12346
pm_regular.proc.ident = 67891
pm_regular.proc.is_alive = Mock(return_value=False)
@pytest.mark.asyncio
async def test_close_returns_false_when_not_started(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
result = await pm.close()
assert result is False, "Should return False when process not started"
@pytest.mark.asyncio
async def test_close_returns_false_when_cancel_in_progress(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
pm.proc = Mock()
pm.cancel_in_progress = True
result = await pm.close()
assert result is False, "Should return False when cancellation already in progress"
@pytest.mark.asyncio
async def test_close_kills_and_joins_process(self) -> None:
logger = logging.getLogger("test")
pm = ProcessManager("test-id", is_live=False, logger=logger)
pm.proc = Mock()
pm.proc.ident = 12345
pm.proc.is_alive = Mock(side_effect=[False, False])
pm.proc.join = Mock()
pm.proc.close = Mock()
result = await pm.close()
assert result is True, "Should return True on successful close"
assert pm.proc is None, "Process reference should be cleared"
@pytest.mark.asyncio
async def test__process_status_update_finish_sets_fields(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
class Cfg:
debug = False
ytdlp_debug = False
max_workers = 1
temp_keep = False
temp_disabled = True
download_info_expires = 3600
class TestStatusTracker:
@pytest.fixture
def mock_config(self):
return {
"info": make_item(id="test-id"),
"download_id": "test-id",
"download_dir": "/downloads",
"temp_path": None,
"status_queue": DummyQueue(),
"logger": logging.getLogger("test"),
"debug": False,
}
@staticmethod
def get_instance():
return Cfg
def test_init_sets_attributes(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
assert st.id == "test-id", "Should set download ID"
assert st.info == mock_config["info"], "Should set info reference"
assert st.tmpfilename is None, "Should initialize tmpfilename as None"
assert st.final_update is False, "Should initialize final_update as False"
monkeypatch.setattr("app.library.Download.Config", Cfg)
@pytest.mark.asyncio
async def test_process_status_update_ignores_invalid_id(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {"id": "wrong-id", "status": "downloading"}
class EB:
@staticmethod
def get_instance():
return EB
await st.process_status_update(status)
assert st.info.status != "downloading", "Should not update status for wrong ID"
@staticmethod
def emit(*_a, **_kw):
return None
@pytest.mark.asyncio
async def test_process_status_update_ignores_short_status(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id"}
monkeypatch.setattr("app.library.Download.EventBus", EB)
await st.process_status_update(status)
class FF:
def has_video(self):
return True
@pytest.mark.asyncio
async def test_process_status_update_sets_status(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000}
def has_audio(self):
return True
await st.process_status_update(status)
assert st.info.status == "downloading", "Should update info status"
metadata = {"duration": "7.9"}
@pytest.mark.asyncio
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"}
async def fake_ffprobe(_p):
return FF()
await st.process_status_update(status)
assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename"
monkeypatch.setattr("app.library.Download.ffprobe", fake_ffprobe)
@pytest.mark.asyncio
async def test_process_status_update_calculates_percent(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {
"id": "test-id",
"status": "downloading",
"downloaded_bytes": 50,
"total_bytes": 100,
}
d = Download(make_item(id="f1"))
d.download_dir = str(tmp_path)
path = Path(tmp_path) / "file.bin"
path.write_bytes(b"x" * 10)
await st.process_status_update(status)
assert st.info.downloaded_bytes == 50, "Should set downloaded_bytes"
assert st.info.total_bytes == 100, "Should set total_bytes"
assert st.info.percent == 50.0, "Should calculate percent correctly"
await d._process_status_update({"id": d.id, "status": "finished", "final_name": str(path)})
@pytest.mark.asyncio
async def test_process_status_update_uses_estimated_total(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {
"id": "test-id",
"status": "downloading",
"downloaded_bytes": 30,
"total_bytes_estimate": 100,
}
assert d.info.file_size == 10
assert d.info.extras["is_video"] is True
assert d.info.extras["is_audio"] is True
assert d.info.extras["duration"] == int(7.9)
assert isinstance(d.info.datetime, str)
await st.process_status_update(status)
assert st.info.total_bytes == 100, "Should use total_bytes_estimate when total_bytes not available"
assert st.info.percent == 30.0, "Should calculate percent from estimate"
@pytest.mark.asyncio
async def test_process_status_update_handles_zero_division(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {
"id": "test-id",
"status": "downloading",
"downloaded_bytes": 50,
"total_bytes": 100,
}
await st.process_status_update(status)
assert st.info.percent == 50.0, "Should calculate percent correctly with valid total"
@pytest.mark.asyncio
async def test_process_status_update_sets_speed_and_eta(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60}
await st.process_status_update(status)
assert st.info.speed == 1024000, "Should set speed"
assert st.info.eta == 60, "Should set eta"
@pytest.mark.asyncio
async def test_process_status_update_sets_error(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "error", "error": "Download failed"}
await st.process_status_update(status)
assert st.info.status == "error", "Should set status to error"
assert st.info.error == "Download failed", "Should set error message"
@pytest.mark.asyncio
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None:
test_file = tmp_path / "test.mp4"
test_file.write_text("test content")
st = StatusTracker(**mock_config)
st.download_dir = str(tmp_path)
status = {"id": "test-id", "status": "finished", "final_name": str(test_file)}
await st.process_status_update(status)
assert st.final_update is True, "Should set final_update when final file exists"
assert st.info.filename == "test.mp4", "Should set relative filename"
@pytest.mark.asyncio
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None:
queue = DummyQueue()
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100})
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200})
queue.put(Terminator())
config = {**mock_config, "status_queue": queue}
st = StatusTracker(**config)
await st.drain_queue(max_iterations=10)
assert st.info.downloaded_bytes == 200, "Should process all queued updates"
@pytest.mark.asyncio
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None:
test_file = tmp_path / "test.mp4"
test_file.write_text("test content")
queue = DummyQueue()
queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)})
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999})
config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
st = StatusTracker(**config)
await st.drain_queue(max_iterations=10)
assert st.final_update is True, "Should stop draining after final update"
@pytest.mark.asyncio
async def test_drain_queue_handles_errors_gracefully(self, mock_config: dict) -> None:
queue = DummyQueue()
queue.put({"id": "test-id", "status": "downloading"})
queue.put(None)
config = {**mock_config, "status_queue": queue}
st = StatusTracker(**config)
await st.drain_queue(max_iterations=5)
def test_cancel_update_task_cancels_running_task(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
st.update_task = Mock()
st.update_task.done = Mock(return_value=False)
st.update_task.cancel = Mock()
st.cancel_update_task()
st.update_task.cancel.assert_called_once()
def test_cancel_update_task_handles_no_task(self, mock_config: dict) -> None:
st = StatusTracker(**mock_config)
st.cancel_update_task()
def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None:
queue = DummyQueue()
config = {**mock_config, "status_queue": queue}
st = StatusTracker(**config)
st.put_terminator()
assert 1 == len(queue.items), "Should add terminator to queue"
assert isinstance(queue.items[0], Terminator), "Should add Terminator instance"

377
app/tests/test_download_utils.py Normal file
View file

@ -0,0 +1,377 @@
import asyncio
import logging
import os
import time
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
from app.library.downloads.utils import (
BAD_LIVE_STREAM_OPTIONS,
DEBUG_MESSAGE_PREFIXES,
GENERIC_EXTRACTORS,
LIMITS,
YTDLP_PROGRESS_FIELDS,
create_debug_safe_dict,
get_extractor_limit,
handle_task_exception,
is_download_stale,
is_safe_to_delete_dir,
parse_extractor_limit,
safe_relative_path,
wait_for_process_with_timeout,
)
class TestConstants:
def test_generic_extractors_tuple(self) -> None:
assert isinstance(GENERIC_EXTRACTORS, tuple), "Should be a tuple"
assert "HTML5MediaEmbed" in GENERIC_EXTRACTORS, "Should contain HTML5MediaEmbed"
assert "generic" in GENERIC_EXTRACTORS, "Should contain generic"
def test_ytdlp_progress_fields_tuple(self) -> None:
assert isinstance(YTDLP_PROGRESS_FIELDS, tuple), "Should be a tuple"
assert "status" in YTDLP_PROGRESS_FIELDS, "Should contain status field"
assert "downloaded_bytes" in YTDLP_PROGRESS_FIELDS, "Should contain downloaded_bytes field"
def test_bad_live_stream_options_list(self) -> None:
assert isinstance(BAD_LIVE_STREAM_OPTIONS, list), "Should be a list"
assert "concurrent_fragment_downloads" in BAD_LIVE_STREAM_OPTIONS, "Should contain concurrent option"
def test_debug_message_prefixes_list(self) -> None:
assert isinstance(DEBUG_MESSAGE_PREFIXES, list), "Should be a list"
assert "[debug] " in DEBUG_MESSAGE_PREFIXES, "Should contain debug prefix"
class TestPathUtilities:
def test_safe_relative_path_success(self) -> None:
base = Path("/downloads")
file = Path("/downloads/video.mp4")
result = safe_relative_path(file, base)
assert "video.mp4" == result, "Should return relative path"
def test_safe_relative_path_nested(self) -> None:
base = Path("/downloads")
file = Path("/downloads/folder/subfolder/video.mp4")
result = safe_relative_path(file, base)
assert "folder/subfolder/video.mp4" == result, "Should return nested relative path"
def test_safe_relative_path_with_fallback(self) -> None:
base = Path("/wrong/path")
fallback = Path("/temp")
file = Path("/temp/video.mp4")
result = safe_relative_path(file, base, fallback)
assert "video.mp4" == result, "Should use fallback path"
def test_safe_relative_path_no_fallback_returns_absolute(self) -> None:
base = Path("/wrong/path")
file = Path("/downloads/video.mp4")
result = safe_relative_path(file, base)
assert "/downloads/video.mp4" == result, "Should return absolute path when both fail"
def test_safe_relative_path_fallback_also_fails(self) -> None:
base = Path("/wrong/path")
fallback = Path("/also/wrong")
file = Path("/downloads/video.mp4")
result = safe_relative_path(file, base, fallback)
assert "/downloads/video.mp4" == result, "Should return absolute when both fail"
def test_is_safe_to_delete_dir_root_protection(self) -> None:
path = Path("/tmp/downloads")
root = Path("/tmp/downloads")
result = is_safe_to_delete_dir(path, root)
assert result is False, "Should refuse to delete root directory"
def test_is_safe_to_delete_dir_safe_path(self) -> None:
path = Path("/tmp/downloads/subfolder")
root = Path("/tmp/downloads")
result = is_safe_to_delete_dir(path, root)
assert result is True, "Should allow deleting subdirectory"
def test_is_safe_to_delete_dir_string_comparison(self) -> None:
path = Path("/tmp/downloads")
root = "/tmp/downloads"
result = is_safe_to_delete_dir(path, root)
assert result is False, "Should handle string root path"
class TestProcessUtilities:
def test_wait_for_process_with_timeout_completes_immediately(self) -> None:
proc = Mock()
proc.is_alive = Mock(return_value=False)
result = wait_for_process_with_timeout(proc, timeout=1.0)
assert result is True, "Should return True when process terminates immediately"
def test_wait_for_process_with_timeout_completes_after_delay(self) -> None:
proc = Mock()
call_count = [0]
def is_alive_side_effect():
call_count[0] += 1
return call_count[0] < 3
proc.is_alive = Mock(side_effect=is_alive_side_effect)
with patch("time.sleep"):
result = wait_for_process_with_timeout(proc, timeout=1.0, check_interval=0.1)
assert result is True, "Should return True when process terminates during wait"
assert proc.is_alive.call_count >= 3, "Should check process multiple times"
def test_wait_for_process_with_timeout_expires(self) -> None:
proc = Mock()
proc.is_alive = Mock(return_value=True)
with patch("time.time") as mock_time, patch("time.sleep"):
mock_time.side_effect = [0, 0.5, 1.0, 1.5]
result = wait_for_process_with_timeout(proc, timeout=1.0, check_interval=0.1)
assert result is False, "Should return False when timeout reached"
def test_wait_for_process_with_custom_interval(self) -> None:
proc = Mock()
proc.is_alive = Mock(return_value=False)
result = wait_for_process_with_timeout(proc, timeout=5.0, check_interval=0.5)
assert result is True, "Should respect custom check interval"
class TestConfigUtilities:
def test_parse_extractor_limit_valid_env(self) -> None:
with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "3"}):
result = parse_extractor_limit("youtube", default_limit=5, max_workers=10)
assert 3 == result, "Should use environment variable value"
def test_parse_extractor_limit_env_exceeds_max(self) -> None:
with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "15"}):
result = parse_extractor_limit("youtube", default_limit=5, max_workers=10)
assert 10 == result, "Should cap at max_workers"
def test_parse_extractor_limit_invalid_env_not_digit(self) -> None:
logger = logging.getLogger("test")
with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "abc"}):
result = parse_extractor_limit("youtube", default_limit=5, max_workers=10, logger=logger)
assert 5 == result, "Should use default when env var is invalid"
def test_parse_extractor_limit_invalid_env_zero(self) -> None:
with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "0"}):
result = parse_extractor_limit("youtube", default_limit=5, max_workers=10)
assert 5 == result, "Should use default when env var is zero"
def test_parse_extractor_limit_no_env(self) -> None:
result = parse_extractor_limit("youtube", default_limit=5, max_workers=10)
assert 5 == result, "Should use default when no env var set"
def test_parse_extractor_limit_respects_max_on_default(self) -> None:
result = parse_extractor_limit("youtube", default_limit=15, max_workers=10)
assert 10 == result, "Should cap default at max_workers"
def test_parse_extractor_limit_logs_warning_on_invalid(self) -> None:
logger = Mock()
with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "invalid"}):
parse_extractor_limit("youtube", default_limit=5, max_workers=10, logger=logger)
logger.warning.assert_called_once()
assert "Invalid extractor limit" in logger.warning.call_args[0][0], "Should log warning"
def test_get_extractor_limit_creates_new(self) -> None:
LIMITS.clear()
logger = logging.getLogger("test")
semaphore = get_extractor_limit("youtube", max_workers=10, max_workers_per_extractor=5, logger=logger)
assert isinstance(semaphore, asyncio.Semaphore), "Should return semaphore"
assert "youtube" in LIMITS, "Should store in LIMITS dict"
def test_get_extractor_limit_reuses_existing(self) -> None:
LIMITS.clear()
logger = logging.getLogger("test")
sem1 = get_extractor_limit("youtube", max_workers=10, max_workers_per_extractor=5, logger=logger)
sem2 = get_extractor_limit("youtube", max_workers=10, max_workers_per_extractor=5, logger=logger)
assert sem1 is sem2, "Should reuse existing semaphore"
def test_get_extractor_limit_respects_env_var(self) -> None:
LIMITS.clear()
logger = logging.getLogger("test")
with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_TWITCH": "2"}):
semaphore = get_extractor_limit("twitch", max_workers=10, max_workers_per_extractor=5, logger=logger)
assert semaphore._value == 2, "Should respect environment variable"
class TestDataUtilities:
def test_create_debug_safe_dict_filters_formats(self) -> None:
data = {
"status": "downloading",
"filename": "video.mp4",
"info_dict": {"id": "123", "title": "Video", "formats": [{"format_id": "1"}], "description": "Long text"},
}
result = create_debug_safe_dict(data)
assert "downloading" == result["status"], "Should include status"
assert "video.mp4" == result["filename"], "Should include filename"
assert "formats" not in result["info_dict"], "Should exclude formats"
assert "description" not in result["info_dict"], "Should exclude description"
assert "123" == result["info_dict"]["id"], "Should include id"
def test_create_debug_safe_dict_custom_exclude(self) -> None:
data = {
"status": "downloading",
"filename": "video.mp4",
"info_dict": {"id": "123", "title": "Video", "custom_field": "value"},
}
result = create_debug_safe_dict(data, exclude_keys=["custom_field"])
assert "custom_field" not in result["info_dict"], "Should exclude custom field"
assert "123" == result["info_dict"]["id"], "Should include id"
def test_create_debug_safe_dict_filters_none_and_functions(self) -> None:
data = {
"status": "downloading",
"info_dict": {"id": "123", "none_value": None, "lambda_value": lambda: None, "title": "Video"},
}
result = create_debug_safe_dict(data)
assert "none_value" not in result["info_dict"], "Should filter None values"
assert "lambda_value" not in result["info_dict"], "Should filter lambda functions"
assert "Video" == result["info_dict"]["title"], "Should include regular values"
def test_create_debug_safe_dict_empty_info_dict(self) -> None:
data = {"status": "downloading", "filename": "video.mp4"}
result = create_debug_safe_dict(data)
assert {} == result["info_dict"], "Should handle missing info_dict"
class TestStateUtilities:
def test_is_download_stale_terminal_status_finished(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="finished", is_running=False, auto_start=True
)
assert result is False, "Finished downloads are never stale"
def test_is_download_stale_terminal_status_error(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="error", is_running=False, auto_start=True
)
assert result is False, "Error downloads are never stale"
def test_is_download_stale_terminal_status_cancelled(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="cancelled", is_running=False, auto_start=True
)
assert result is False, "Cancelled downloads are never stale"
def test_is_download_stale_terminal_status_downloading(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="downloading", is_running=False, auto_start=True
)
assert result is False, "Downloading status is never stale"
def test_is_download_stale_terminal_status_postprocessing(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="postprocessing", is_running=False, auto_start=True
)
assert result is False, "Postprocessing status is never stale"
def test_is_download_stale_not_auto_start(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="pending", is_running=False, auto_start=False
)
assert result is False, "Non-auto-start downloads are never stale"
def test_is_download_stale_still_running(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 500, current_status="pending", is_running=True, auto_start=True
)
assert result is False, "Running downloads are never stale"
def test_is_download_stale_not_enough_time(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 100,
current_status="pending",
is_running=False,
auto_start=True,
min_elapsed=300,
)
assert result is False, "Should not be stale before timeout"
def test_is_download_stale_timeout_reached(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 400,
current_status="pending",
is_running=False,
auto_start=True,
min_elapsed=300,
)
assert result is True, "Should be stale after timeout"
def test_is_download_stale_custom_timeout(self) -> None:
result = is_download_stale(
started_time=int(time.time()) - 150,
current_status="pending",
is_running=False,
auto_start=True,
min_elapsed=100,
)
assert result is True, "Should respect custom timeout"
class TestTaskExceptionHandling:
@pytest.mark.asyncio
async def test_handle_task_exception_ignores_cancelled(self) -> None:
logger = Mock()
async def cancelled_task():
raise asyncio.CancelledError()
task = asyncio.create_task(cancelled_task())
try:
await task
except asyncio.CancelledError:
pass
handle_task_exception(task, logger)
logger.error.assert_not_called()
@pytest.mark.asyncio
async def test_handle_task_exception_ignores_successful(self) -> None:
logger = Mock()
async def successful_task():
return "success"
task = asyncio.create_task(successful_task())
await task
handle_task_exception(task, logger)
logger.error.assert_not_called()
@pytest.mark.asyncio
async def test_handle_task_exception_logs_exception(self) -> None:
logger = Mock()
async def failing_task():
raise ValueError("Test error")
task = asyncio.create_task(failing_task(), name="test_task")
try:
await task
except ValueError:
pass
handle_task_exception(task, logger)
logger.error.assert_called_once()
error_msg = logger.error.call_args[0][0]
assert "test_task" in error_msg, "Should include task name"
assert "Test error" in error_msg, "Should include exception message"
@pytest.mark.asyncio
async def test_handle_task_exception_unknown_task_name(self) -> None:
logger = Mock()
async def failing_task():
raise RuntimeError("Unknown task error")
task = asyncio.create_task(failing_task())
try:
await task
except RuntimeError:
pass
handle_task_exception(task, logger)
logger.error.assert_called_once()
error_msg = logger.error.call_args[0][0]
assert "unknown_task" in error_msg or "Task" in error_msg, "Should handle unknown task name"

18
ui/app/components/DLFields.vue
View file

@ -132,13 +132,9 @@
</div>
</div>
<div class="column is-12">
<Message message_class="has-background-warning-90 has-text-dark" v-if="items.length === 0">
<span class="icon">
<i class="fas fa-exclamation-circle" />
</span>
<span>
No custom fields found, you can add new fields using the button above.
</span>
<Message class="is-warning" v-if="items.length === 0">
<span class="icon"><i class="fas fa-exclamation-circle" /></span>
<span>No custom fields found, you can add new fields using the button above.</span>
</Message>
</div>
</div>
@ -250,7 +246,6 @@ const addNewField = () => items.value.push({
const deleteField = (index: number) => items.value.splice(index, 1)
const validateItem = (item: DLField, index: number): boolean => {
const requiredFields = ['name', 'field', 'kind', 'description']
for (const field of requiredFields) {
@ -282,9 +277,10 @@ const validateItem = (item: DLField, index: number): boolean => {
const sortedDLFields = computed(() => items.value.sort((a, b) => (a.order || 0) - (b.order || 0)))
watch(() => config.ytdlp_options, newOptions => ytDlpOptions.value = newOptions
.filter(opt => !opt.ignored).flatMap(opt => opt.flags
.filter(flag => flag.startsWith('--'))
.map(flag => ({ value: flag, description: opt.description || '' }))),
.filter(opt => !opt.ignored)
.flatMap(
opt => opt.flags.filter(flag => flag.startsWith('--')
).map(flag => ({ value: flag, description: opt.description || '' }))),
{ immediate: true }
)
onMounted(async () => {

67
ui/app/components/DeprecatedNotice.vue
View file

@ -1,67 +0,0 @@
<template>
<Message :message_class="messageClass" :title="title" :icon="icon" :useClose="true" @close="() => dismissed = true"
v-if="!isDismissed">
<slot />
</Message>
</template>
<script setup lang="ts">
import { computed } from 'vue'
import { useStorage } from '@vueuse/core'
import Message from "~/components/Message.vue";
const props = withDefaults(defineProps<{
version: string
storageKey?: string
title?: string
icon?: string
tone?: 'warning' | 'danger' | 'info' | 'success'
}>(), {
storageKey: 'deprecated-notice',
title: 'Deprecated Feature',
icon: 'fas fa-exclamation-triangle',
tone: 'warning',
})
const config = useConfigStore()
const isDev = computed(() => 'development' === config.app?.app_env)
const storageKeyComputed = computed<string>(() => `${props.storageKey}:${props.version}`)
const dismissed = useStorage<boolean>(storageKeyComputed, false)
const isDismissed = computed(() => dismissed.value)
const messageClass = computed(() => {
switch (props.tone) {
case 'danger':
return 'is-danger has-background-danger-90 has-text-dark'
case 'info':
return 'is-info has-background-info-90 has-text-dark'
case 'success':
return 'is-success has-background-success-90 has-text-dark'
case 'warning':
default:
return 'is-warning has-background-warning-90 has-text-dark'
}
})
onMounted(() => {
if (!isDev.value) {
return
}
document.addEventListener('keydown', handle_event)
})
onBeforeUnmount(() => {
if (!isDev.value) {
return
}
document.removeEventListener('keydown', handle_event)
})
const handle_event = (e: KeyboardEvent) => {
if (e.ctrlKey && e.altKey && 'd' === e.key.toLowerCase()) {
e.preventDefault()
dismissed.value = !dismissed.value
}
}
</script>

85
ui/app/components/History.vue
View file

@ -98,32 +98,36 @@
</label>
</td>
<td class="is-text-overflow is-vcentered">
<div class="is-inline is-pulled-right"
v-if="item.extras?.duration || getPath(config.app.download_path, item)">
<div class="is-inline is-pulled-right" v-if="item.extras?.duration || show_popover">
<span class="tag is-info is-unselectable" v-if="item.extras?.duration">
{{ formatTime(item.extras.duration) }}
</span>
<span class="icon is-pointer" v-if="getPath(config.app.download_path, item)"
v-tooltip="`Path: ${getPath(config.app.download_path, item)}`">
<i class="fa-solid fa-folder-open" />
</span>
<Popover :showDelay="400" :maxWidth="450" v-if="show_popover">
<template #trigger>
<span class="icon is-pointer"><i class="fa-solid fa-info-circle" /></span>
</template>
<template #title>
<strong>
{{ item.title }}
<span class="tag is-info is-unselectable">{{ item.preset }}</span>
</strong>
</template>
<p v-if="getPath(config.app.download_path, item)">
<b>Path:</b> {{ getPath(config.app.download_path, item) }}
</p>
<hr
v-if="(showThumbnails && getImage(config.app.download_path, item, false)) || item.description" />
<img v-if="showThumbnails && getImage(config.app.download_path, item, false)"
:src="getImage(config.app.download_path, item, false)" class="card-image mt-2 mb-2" />
<p v-if="item.description">{{ item.description }}</p>
</Popover>
</div>
<div v-if="show_popover">
<div class="is-text-overflow">
<Popover :showDelay="400" :maxWidth="450">
<template #trigger>
<NuxtLink class="is-text-overflow" target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</template>
<template #title>
<strong>
{{ item.title }}
<span class="tag is-info is-unselectable">{{ item.preset }}</span>
</strong>
</template>
<img v-if="showThumbnails && getImage(config.app.download_path, item, false)"
:src="getImage(config.app.download_path, item, false)" class="mt-2 mb-2" />
<p v-if="item.description">{{ item.description }}</p>
</Popover>
<NuxtLink class="is-text-overflow" v-tooltip="`${item.preset}: ${item.title}`" target="_blank"
:href="item.url">
{{ item.title }}</NuxtLink>
</div>
</div>
<template v-else>
@ -241,16 +245,7 @@
:class="{ 'is-bordered-danger': item.status === 'error', 'is-bordered-info': item.live_in || item.is_live }">
<header class="card-header">
<div class="card-header-title is-text-overflow is-block">
<Popover :showDelay="400" :maxWidth="550" v-if="show_popover">
<template #trigger>
<NuxtLink class="is-text-overflow" target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</template>
<template #title><strong>{{ item.title }}</strong></template>
<p v-if="item.description">{{ item.description }}</p>
</Popover>
<template v-else>
<NuxtLink target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</template>
<NuxtLink target="_blank" v-tooltip="item.title" :href="item.url">{{ item.title }}</NuxtLink>
</div>
<div class="card-header-icon">
@ -261,10 +256,20 @@
</span>
</div>
<div class="control" v-if="getPath(config.app.download_path, item)">
<span class="icon" v-tooltip="`Path: ${getPath(config.app.download_path, item)}`">
<i class="fa-solid fa-folder-open" />
</span>
<div class="control" v-if="show_popover && getPath(config.app.download_path, item)">
<Popover :showDelay="400" :maxWidth="550" v-if="show_popover">
<template #trigger>
<span class="icon"><i class="fa-solid fa-info-circle" /></span>
</template>
<template #title><strong>{{ item.title }}</strong></template>
<p v-if="getPath(config.app.download_path, item)">
<b>Path:</b> {{ getPath(config.app.download_path, item) }}
</p>
<template v-if="item.description">
<hr>
<p>{{ item.description }}</p>
</template>
</Popover>
</div>
<div class="control">
@ -429,9 +434,9 @@
<div class="columns is-multiline" v-if="!hasItems && !paginationInfo.isLoading">
<div class="column is-12">
<Message message_class="is-warning" title="Filter results" icon="fas fa-search" :useClose="true"
@close="() => emitter('clear_search')" v-if="query" :newStyle="true">
<span class="is-block">No results found for '<span class="is-underlined is-bold">{{ query }}</span>'.</span>
<Message class="is-warning" title="Filter results" icon="fas fa-search" :useClose="true"
@close="() => emitter('clear_search')" v-if="query">
<span class="is-block">No results found for: <code>{{ query }}</code>.</span>
<hr>
<p>
You can search using any value shown in the item’s <code><span class="icon"><i
@ -447,9 +452,9 @@
<li><code>source_name:task_name</code> - items added by the specified task.</li>
</ul>
</Message>
<Message message_class="is-primary" title="No items" icon="fas fa-exclamation-triangle"
v-else-if="socket.isConnected" :new-style="true">
<p>Your download history is empty. Once queued downloads are completed, they will appear here.</p>
<Message v-else-if="socket.isConnected" class="is-primary" title="No items" icon="fas fa-exclamation-triangle"
:new-style="true">
<p>Download history is empty.</p>
</Message>
</div>
</div>

5
ui/app/components/Markdown.vue
View file

@ -71,8 +71,9 @@ code {
</div>
<div style="position: relative" v-if="!isLoading">
<Message v-if="error" message_class="has-background-warning-90 has-text-dark" title="Error"
icon="fas fa-exclamation" :message="error" />
<Message v-if="error" class="is-warning" title="Error" icon="fas fa-exclamation">
{{ error }}
</Message>
<div class="card" v-else>
<div class="card-body p-4">
<div class="content" v-html="content" />

25
ui/app/components/Message.vue
View file

@ -1,16 +1,13 @@
<template>
<div :class="[newStyle ? 'message' : 'notification', message_class]">
<button class="delete" @click="$emit('close')" v-if="!useToggle && useClose && !newStyle"></button>
<div class="message">
<div @click="$emit('toggle')" class="is-clickable is-pulled-right is-unselectable" v-if="useToggle">
<span class="icon">
<i class="fas" :class="{'fa-arrow-up':toggle,'fa-arrow-down':!toggle}"></i>
<i class="fas" :class="{ 'fa-arrow-up': toggle, 'fa-arrow-down': !toggle }"></i>
</span>
<span>{{ toggle ? 'Close' : 'Open' }}</span>
</div>
<div class="is-unselectable"
:class="{'is-clickable':useToggle, 'notification-title': !newStyle, 'message-header': newStyle}"
v-if="title || icon"
@click="true === useToggle ? $emit('toggle', toggle): null">
<div class="is-unselectable message-header" :class="{ 'is-clickable': useToggle, }" v-if="title || icon"
@click="true === useToggle ? $emit('toggle', toggle) : null">
<template v-if="icon">
<span class="icon-text">
<span class="icon"><i :class="icon"></i></span>
@ -18,12 +15,11 @@
</span>
</template>
<template v-else>{{ title }}</template>
<button class="delete" @click="$emit('close')" v-if="!useToggle && useClose && newStyle"/>
<button class="delete" @click="$emit('close')" v-if="!useToggle && useClose" />
</div>
<div class="content is-text-break" v-if="false === useToggle || toggle"
:class="{'notification-body': !newStyle, 'message-body': newStyle}">
<div class="content message-body is-text-break" v-if="false === useToggle || toggle" :class="body_class">
<template v-if="message">{{ message }}</template>
<slot/>
<slot />
</div>
</div>
</template>
@ -36,24 +32,21 @@ withDefaults(defineProps<{
icon?: string | null
/** Main message content */
message?: string | null
/** CSS class for the notification */
message_class?: string
/** If true, show toggle button */
useToggle?: boolean
/** Current toggle state */
toggle?: boolean
/** If true, show close button */
useClose?: boolean,
newStyle?: boolean
body_class?: string | null,
}>(), {
title: null,
icon: null,
message: null,
message_class: 'is-info',
useToggle: false,
toggle: false,
useClose: false,
newStyle: false
body_class: null,
})
defineEmits<{

2
ui/app/components/NewVersion.vue
View file

@ -1,5 +1,5 @@
<template>
<Message message_class="is-success has-background-success-90 has-text-dark">
<Message class="is-success">
<span class="icon"><i class="fas fa-info-circle" /></span>
<span>
A New WebUI Version installed, please

41
ui/app/components/Popover.vue
View file

@ -15,6 +15,11 @@
<div v-if="isOpen" class="popover-portal" ref="portal">
<div ref="popover" class="popover-card box" :class="placementClass" role="tooltip" :style="portalStyle"
@mouseenter="handleHoverEnter" @mouseleave="handleHoverLeave">
<button type="button" class="button is-bold is-fullwidth is-hidden-tablet" @click="closePopover">
<span class="icon"><i class="fas fa-times" /></span>
<span>Close</span>
</button>
<header v-if="hasTitleContent" class="popover-title mb-2">
<slot name="title">
<p class="title is-6 mb-1">{{ title }}</p>
@ -206,23 +211,47 @@ const updatePosition = () => {
const triggerRect = triggerRef.value.getBoundingClientRect()
const popoverRect = popover.value.getBoundingClientRect()
const offset = props.offset
const viewportWidth = window.innerWidth
const viewportHeight = window.innerHeight
let effectivePlacement = props.placement
if ('top' === props.placement) {
const spaceAbove = triggerRect.top - offset
if (spaceAbove < popoverRect.height) {
effectivePlacement = 'bottom'
}
} else if ('bottom' === props.placement) {
const spaceBelow = viewportHeight - triggerRect.bottom - offset
if (spaceBelow < popoverRect.height) {
effectivePlacement = 'top'
}
} else if ('left' === props.placement) {
const spaceLeft = triggerRect.left - offset
if (spaceLeft < popoverRect.width) {
effectivePlacement = 'right'
}
} else if ('right' === props.placement) {
const spaceRight = viewportWidth - triggerRect.right - offset
if (spaceRight < popoverRect.width) {
effectivePlacement = 'left'
}
}
let top = triggerRect.bottom + offset
let left = triggerRect.left + (triggerRect.width - popoverRect.width) / 2
if ('top' == props.placement) {
if ('top' === effectivePlacement) {
top = triggerRect.top - popoverRect.height - offset
} else if ('left' == props.placement) {
} else if ('left' === effectivePlacement) {
top = triggerRect.top + (triggerRect.height - popoverRect.height) / 2
left = triggerRect.left - popoverRect.width - offset
} else if ('right' == props.placement) {
} else if ('right' === effectivePlacement) {
top = triggerRect.top + (triggerRect.height - popoverRect.height) / 2
left = triggerRect.right + offset
}
const clamp = (value: number, min: number, max: number) => Math.min(Math.max(value, min), max)
const viewportWidth = window.innerWidth
const viewportHeight = window.innerHeight
const maxLeft = viewportWidth - popoverRect.width - 8
const maxTop = viewportHeight - popoverRect.height - 8
@ -284,7 +313,7 @@ onBeforeUnmount(() => {
min-width: 0;
}
.popover-trigger > * {
.popover-trigger>* {
max-width: 100%;
min-width: 0;
}

126
ui/app/components/Queue.vue
View file

@ -68,39 +68,34 @@
</label>
</td>
<td class="is-text-overflow is-vcentered">
<div class="is-inline is-pulled-right" v-if="item.downloaded_bytes || item.extras?.duration">
<div class="is-inline is-pulled-right" v-if="item.downloaded_bytes || show_popover">
<span class="tag" v-if="item.downloaded_bytes">{{ formatBytes(item.downloaded_bytes) }}</span>
<span class="tag is-info" v-if="item.extras?.duration">
{{ formatTime(item.extras.duration) }}
</span>
<span class="icon is-pointer" v-if="item.download_dir"
v-tooltip="`Path: ${getPath(config.app.download_path, item)}`">
<i class="fa-solid fa-folder-open" />
</span>
<Popover :showDelay="400" :maxWidth="450" v-if="show_popover">
<template #trigger>
<span class="icon is-pointer"><i class="fa-solid fa-info-circle" /></span>
</template>
<template #title>
<strong>
{{ item.title }}
<span class="tag is-info is-unselectable">{{ item.preset }}</span>
</strong>
</template>
<p v-if="item.extras?.duration">
<b>Duration</b>: {{ formatTime(item.extras.duration) }}
</p>
<p v-if="getPath(config.app.download_path, item)">
<b>Path:</b> {{ getPath(config.app.download_path, item) }}
</p>
<hr
v-if="(showThumbnails && getImage(config.app.download_path, item, false)) || item.description" />
<img v-if="showThumbnails && getImage(config.app.download_path, item, false)"
:src="getImage(config.app.download_path, item, false)" class="card-image mt-2 mb-2" />
<p v-if="item.description">{{ item.description }}</p>
</Popover>
</div>
<div v-if="show_popover">
<div class="is-text-overflow">
<Popover :showDelay="400" :maxWidth="450">
<template #trigger>
<NuxtLink class="is-text-overflow" target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</template>
<template #title>
<strong>
{{ item.title }}
<span class="tag is-info is-unselectable">{{ item.preset }}</span>
</strong>
</template>
<img v-if="showThumbnails && getImage(config.app.download_path, item)"
:src="getImage(config.app.download_path, item)" class="mt-2 mb-2" />
<p v-if="item.description">{{ item.description }}</p>
</Popover>
</div>
<div class="is-text-overflow" v-tooltip="`[${item.preset}] - ${item.title}`">
<NuxtLink target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</div>
<template v-else>
<div class="is-text-overflow" v-tooltip="`[${item.preset}] - ${item.title}`">
<NuxtLink target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</div>
</template>
</td>
<td class="has-text-centered is-text-overflow is-unselectable">
<span class="icon" :class="setIconColor(item)">
@ -176,16 +171,7 @@
<div class="card">
<header class="card-header">
<div class="card-header-title is-text-overflow is-block">
<Popover :showDelay="400" :maxWidth="550" v-if="show_popover">
<template #trigger>
<NuxtLink class="is-text-overflow" target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</template>
<template #title><strong>{{ item.title }}</strong></template>
<p v-if="item.description">{{ item.description }}</p>
</Popover>
<template v-else>
<NuxtLink target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
</template>
<NuxtLink target="_blank" v-tooltip="item.title" :href="item.url">{{ item.title }}</NuxtLink>
</div>
<div class="card-header-icon">
<div class="field is-grouped">
@ -194,11 +180,22 @@
{{ formatTime(item.extras.duration) }}
</span>
</div>
<div class="control" v-if="item.download_dir">
<span class="icon" v-tooltip="`Path: ${getPath(config.app.download_path, item)}`">
<i class="fa-solid fa-folder-open" />
</span>
</div>
<Popover :showDelay="400" :maxWidth="450" v-if="show_popover">
<template #trigger>
<span class="icon is-pointer"><i class="fa-solid fa-info-circle" /></span>
</template>
<template #title>
<strong>
{{ item.title }}
<span class="tag is-info is-unselectable">{{ item.preset }}</span>
</strong>
</template>
<p v-if="getPath(config.app.download_path, item)">
<b>Path:</b> {{ getPath(config.app.download_path, item) }}
</p>
<hr v-if="item.description" />
<p v-if="item.description">{{ item.description }}</p>
</Popover>
<div class="control">
<button @click="hideThumbnail = !hideThumbnail" v-if="thumbnails">
<span class="icon"><i class="fa-solid"
@ -252,8 +249,7 @@
<span v-tooltip="moment(item.datetime).format('MMMM Do YYYY, h:mm:ss a')" :data-datetime="item.datetime"
v-rtime="item.datetime" />
</div>
<div class="column is-half-mobile has-text-centered is-text-overflow is-unselectable"
v-if="item.downloaded_bytes" v-tooltip="`Saving to: ${makePath(item)}`">
<div class="column is-half-mobile has-text-centered is-unselectable" v-if="item.downloaded_bytes">
{{ formatBytes(item.downloaded_bytes) }}
</div>
@ -306,9 +302,9 @@
<div class="columns is-multiline" v-if="filteredItems.length < 1">
<div class="column is-12">
<Message message_class="is-warning" title="Filter results" :newStyle="true" icon="fas fa-search" :useClose="true"
<Message class="is-warning" title="Filter results" icon="fas fa-search" :useClose="true"
@close="() => emitter('clear_search')" v-if="query">
<span class="is-block">No results found for '<span class="is-underlined is-bold">{{ query }}</span>'.</span>
<span class="is-block">No results found for: <code>{{ query }}</code>.</span>
<hr>
<p>
You can search using any value shown in the item’s <code><span class="icon"><i
@ -324,9 +320,8 @@
<li><code>source_name:task_name</code> - items added by the specified task.</li>
</ul>
</Message>
<Message message_class="is-info" title="No items" icon="fas fa-exclamation-triangle" :useClose="false"
:newStyle="true" v-else>
<p>The download queue is empty.</p>
<Message v-else class="is-info" title="No items" icon="fas fa-exclamation-triangle" :useClose="false">
<p>Download queue is empty.</p>
</Message>
</div>
</div>
@ -452,6 +447,11 @@ const setStatus = (item: StoreItem): string => {
if ('downloading' === item.status && item.is_live) {
return 'Streaming'
}
if ('started' === item.status) {
return 'Starting'
}
if ('preparing' === item.status) {
return ag(item, 'extras.external_downloader') ? 'External-DL' : 'Preparing..'
}
@ -462,7 +462,7 @@ const setStatus = (item: StoreItem): string => {
}
const setIconColor = (item: StoreItem): string => {
if ('downloading' === item.status) {
if (['downloading', 'started'].includes(item.status as string)) {
return 'has-text-success'
}
if ('postprocessing' === item.status) {
@ -515,9 +515,18 @@ const updateProgress = (item: StoreItem): string => {
if (null === item.status && true === config.paused) {
return 'Global Pause'
}
if ('started' === item.status) {
return 'Starting'
}
if ('postprocessing' === item.status) {
if (item.postprocessor) {
return `PP Running: ${item.postprocessor}`
}
return 'Post-processors are running.'
}
if ('preparing' === item.status) {
return ag(item, 'extras.external_downloader') ? 'External downloader.' : 'Preparing'
}
@ -632,15 +641,4 @@ watch(embed_url, v => {
}
document.querySelector('body')?.setAttribute('style', `opacity: ${v ? 1 : bg_opacity.value}`)
})
const makePath = (item: StoreItem) => {
const parts = [
eTrim(item.download_dir, '/').replace(config.app.download_path, ''),
]
if (item?.filename) {
parts.push(eTrim(item.filename, '/'))
}
return '/' + sTrim(parts.filter(p => !!p).map(p => p.replace(/\\/g, '/').replace(/\/+/g, '/')).join('/'), '/')
}
</script>

13
ui/app/components/TaskDefinitionEditor.vue
View file

@ -85,15 +85,10 @@
</div>
</div>
<Message v-if="!guiSupported" message_class="is-warning">
<p>
<span>
<span class="icon"><i class="fa-solid fa-triangle-exclamation" /></span>
<span>This task definition uses features that cannot be represented with the visual editor. You can still
update it
via the advanced view.</span>
</span>
</p>
<Message v-if="!guiSupported" class="is-warning">
<span class="icon"><i class="fa-solid fa-triangle-exclamation" /></span>
<span>This task definition uses features that cannot be represented with the visual editor. You can still
update it via the advanced view.</span>
</Message>
<div v-if="'gui' === mode">

15
ui/app/components/TaskForm.vue
View file

@ -1,16 +1,15 @@
<template>
<main class="columns mt-2 is-multiline">
<div class="column is-12" v-if="form?.url && is_yt_handle(form.url)">
<Message title="Information" class="is-info is-background-info-80 has-text-dark" icon="fas fa-info-circle">
<span>You are using a YouTube link with <b>@handle</b> instead of <b>channel_id</b>. To activate RSS feed
support for URL click on the <NuxtLink @click="async () => form.url = await convert_url(form.url)"><b>Convert
URL</b></NuxtLink> link.</span>
<Message title="Information" class="is-info" icon="fas fa-info-circle">
You are using a YouTube link with <code>@handle</code> instead of <code>channel_id</code>. To activate RSS
feed support for URL click on the <b>Convert URL</b> link.
</Message>
</div>
<div class="column is-12" v-if="form?.url && is_generic_rss(form.url)">
<Message title="Information" class="is-info is-background-info-80 has-text-dark" icon="fas fa-info-circle">
<span>You are using a generic RSS/Atom feed URL. The task handler will automatically download new items found
in this feed.</span>
<Message title="Information" class="is-warning" icon="fas fa-info-circle">
You are using a generic RSS/Atom feed URL. The task handler will automatically download new items found
in this feed.
</Message>
</div>
<div class="column is-12">
@ -330,7 +329,7 @@
</div>
<div class="column is-12">
<Message class="is-info" :newStyle="true">
<Message class="is-info">
<span>
<ul>
<li><strong>Tasks:</strong> requires <code>--download-archive</code> in

5
ui/app/components/TaskInspect.vue
View file

@ -86,7 +86,7 @@ code {
</div>
</form>
<Message v-if="loading" class="has-background-info-90 has-text-dark mt-5">
<Message v-if="loading" class="is-info">
<p>
<span class="icon-text">
<span class="icon"><i class="fas fa-spinner fa-spin" /></span>
@ -96,7 +96,7 @@ code {
</Message>
<div v-if="response" class="mt-4">
<Message v-if="response.error" message_class="has-background-danger-90 has-text-dark" title="Error"
<Message v-if="response.error" class="is-danger" title="Error"
icon="fas fa-exclamation-triangle">
<p>{{ response.error }}</p>
<p v-if="response.message">{{ response.message }}</p>
@ -109,7 +109,6 @@ code {
</div>
</template>
<script setup lang="ts">
import { ref, watch } from 'vue'
import { request } from '~/utils'

2
ui/app/layouts/default.vue
View file

@ -132,7 +132,7 @@
<div>
<NuxtLoadingIndicator />
<NuxtPage v-if="config.is_loaded" :isLoading="loadingImage" @reload_bg="() => loadImage(true)" />
<Message v-if="!config.is_loaded" class="mt-5" :newStyle="true" title="Loading Configuration"
<Message v-if="!config.is_loaded" class="is-info mt-5" title="Loading Configuration"
icon="fas fa-spinner fa-spin">
<p>This usually takes less than a second.
<span v-if="!socket.isConnected" class="mt-2">

19
ui/app/pages/browser/[...slug].vue
View file

@ -105,7 +105,8 @@
</div>
</div>
<div class="columns is-mobile is-multiline is-justify-content-flex-end" v-if="config.app.browser_control_enabled && items && items.length > 0">
<div class="columns is-mobile is-multiline is-justify-content-flex-end"
v-if="config.app.browser_control_enabled && items && items.length > 0">
<div class="column is-narrow">
<button type="button" class="button" @click="masterSelectAll = !masterSelectAll"
:class="{ 'has-text-primary': !masterSelectAll, 'has-text-danger': masterSelectAll }"
@ -341,8 +342,8 @@
</template>
<div class="column is-12" v-if="search && filteredItems.length < 1">
<Message message_class="has-background-warning-90 has-text-dark" title="No results" icon="fas fa-filter"
:useClose="true" @close="() => search = ''" v-if="search">
<Message class="is-warning" title="No results" icon="fas fa-filter" :useClose="true"
@close="() => search = ''" v-if="search">
<p class="is-block">
No results found for '<span class="is-underlined is-bold">{{ search }}</span>'.
</p>
@ -350,13 +351,12 @@
</div>
<div class="column is-12" v-if="!items || items.length < 1">
<Message title="Loading content" class="has-background-info-90 has-text-dark" icon="fas fa-refresh fa-spin"
<Message title="Loading content" class="is-info" icon="fas fa-refresh fa-spin"
v-if="isLoading">
Loading file browser contents...
</Message>
<Message v-else title="No Content" class="is-background-warning-80 has-text-dark"
icon="fas fa-exclamation-circle">
No content found in this directory.
<Message v-else title="No Content" class="is-warning" icon="fas fa-exclamation-circle">
Directory is empty.
</Message>
</div>
<div class="column is-12" v-if="!config.app.browser_control_enabled">
@ -446,13 +446,10 @@ const filteredItems = computed<FileItem[]>(() => {
}
const searchLower = search.value.toLowerCase()
return sortedItems(items.value.filter((item: FileItem) => {
return item.name.toLowerCase().includes(searchLower)
}))
return sortedItems(items.value.filter((item: FileItem) => deepIncludes(item, searchLower, new WeakSet())))
})
const sortedItems = (items: FileItem[]): FileItem[] => {
if (!items || items.length < 1) {
return []
}

41
ui/app/pages/changelog.vue
View file

@ -47,8 +47,9 @@ hr {
<div class="columns is-multiline" v-if="isLoading">
<div class="column is-12">
<Message message_class="has-background-info-90 has-text-dark" title="Loading" icon="fas fa-spinner fa-spin"
message="Loading data. Please wait..." />
<Message class="is-info" title="Loading" icon="fas fa-spinner fa-spin">
Loading data. Please wait...
</Message>
</div>
</div>
@ -92,8 +93,8 @@ hr {
<div class="columns is-multiline" v-if="!filteredLogs || filteredLogs.length < 1">
<div class="column is-12">
<Message title="No Results" class="is-background-warning-80 has-text-dark" icon="fas fa-search"
v-if="query" :useClose="true" @close="query = ''">
<Message title="No Results" class="is-warning" icon="fas fa-search" v-if="query"
:useClose="true" @close="query = ''">
<p>No changelog entries found for the query: <strong>{{ query }}</strong>.</p>
<p>Please try a different search term.</p>
</Message>
@ -135,29 +136,21 @@ const filteredLogs = computed<changelogs>(() => {
const q = query.value?.toLowerCase()
if (!q) return logs.value
return logs.value
.map(log => {
// Check if tag matches
const tagMatches = log.tag.toLowerCase().includes(q)
return logs.value.map(log => {
const tagMatches = log.tag.toLowerCase().includes(q)
// Filter commits that match the query
const filteredCommits = log.commits?.filter(commit =>
commit.message.toLowerCase().includes(q) ||
commit.author.toLowerCase().includes(q) ||
commit.full_sha.toLowerCase().includes(q)
) ?? []
const filteredCommits = log.commits?.filter(commit =>
commit.message.toLowerCase().includes(q) ||
commit.author.toLowerCase().includes(q) ||
commit.full_sha.toLowerCase().includes(q)
) ?? []
// Return log only if tag matches OR if there are matching commits
if (tagMatches || filteredCommits.length > 0) {
return {
...log,
commits: tagMatches ? log.commits : filteredCommits
}
}
if (tagMatches || filteredCommits.length > 0) {
return { ...log, commits: tagMatches ? log.commits : filteredCommits }
}
return null
})
.filter((log): log is changeset => log !== null)
return null
}).filter((log): log is changeset => log !== null)
})
const loadContent = async () => {

382
ui/app/pages/conditions.vue
View file

@ -16,6 +16,19 @@
</span>
<div class="is-pulled-right" v-if="!toggleForm">
<div class="field is-grouped">
<p class="control has-icons-left" v-if="toggleFilter && items && items.length > 0">
<input type="search" v-model.lazy="query" class="input" id="filter"
placeholder="Filter displayed content">
<span class="icon is-left"><i class="fas fa-filter" /></span>
</p>
<p class="control" v-if="items && items.length > 0">
<button class="button is-danger is-light" @click="toggleFilter = !toggleFilter">
<span class="icon"><i class="fas fa-filter" /></span>
<span v-if="!isMobile">Filter</span>
</button>
</p>
<p class="control">
<button class="button is-primary" @click="resetForm(false); toggleForm = !toggleForm;">
<span class="icon"><i class="fas fa-add" /></span>
@ -44,7 +57,7 @@
</div>
<div class="is-hidden-mobile" v-if="!toggleForm">
<span class="subtitle">
Run yt-dlp custom match filter on returned info. and apply cli arguments if matched.
Run yt-dlp custom match filter on returned info. and apply options.
</span>
</div>
</div>
@ -53,192 +66,202 @@
<ConditionForm :addInProgress="addInProgress" :reference="itemRef" :item="item as ConditionItem"
@cancel="resetForm(true)" @submit="updateItem" />
</div>
</div>
<div class="column is-12" v-if="!toggleForm">
<div class="columns is-multiline" v-if="items.length > 0">
<template v-if="'list' === display_style">
<div class="column is-12">
<div class="table-container">
<table class="table is-striped is-hoverable is-fullwidth is-bordered"
style="min-width: 850px; table-layout: fixed;">
<thead>
<tr class="has-text-centered is-unselectable">
<th width="80%">
<div class="columns is-multiline" v-if="!isLoading && !toggleForm && (filteredItems && filteredItems.length > 0)">
<div class="column is-12" v-if="'list' === display_style">
<div class="table-container">
<table class="table is-striped is-hoverable is-fullwidth is-bordered"
style="min-width: 850px; table-layout: fixed;">
<thead>
<tr class="has-text-centered is-unselectable">
<th width="80%">
<span class="icon"><i class="fa-solid fa-filter" /></span>
<span>Condition</span>
</th>
<th width="20%">
<span class="icon"><i class="fa-solid fa-gear" /></span>
<span>Actions</span>
</th>
</tr>
</thead>
<tbody>
<tr v-for="cond in filteredItems" :key="cond.id">
<td class="is-vcentered">
<div class="is-text-overflow is-bold">
{{ cond.name }}
</div>
<div class="is-unselectable">
<span class="icon-text is-clickable" @click="toggleEnabled(cond)"
v-tooltip="'Click to ' + (cond.enabled !== false ? 'disable' : 'enable') + ' condition'">
<span class="icon">
<i class="fa-solid fa-power-off"
:class="{ 'has-text-success': cond.enabled !== false, 'has-text-danger': cond.enabled === false }" />
</span>
<span>{{ cond.enabled !== false ? 'Enabled' : 'Disabled' }}</span>
</span>
&nbsp;
<Popover :maxWidth="450">
<template #trigger>
<span class="is-clickable">
<span class="icon"> <i class="fa-solid fa-info-circle" /></span>
<span>Show Details</span>
</span>
</template>
<template #title><strong>Condition Details</strong></template>
<div v-if="cond.filter">
<span class="icon"><i class="fa-solid fa-filter" /></span>
<span>Condition</span>
</th>
<th width="20%">
<span class="icon"><i class="fa-solid fa-gear" /></span>
<span>Actions</span>
</th>
</tr>
</thead>
<tbody>
<tr v-for="cond in items" :key="cond.id">
<td class="is-vcentered">
<div class="is-text-overflow is-bold">
{{ cond.name }}
</div>
<div class="is-unselectable">
<span class="icon-text is-clickable" @click="toggleEnabled(cond)"
v-tooltip="'Click to ' + (cond.enabled !== false ? 'disable' : 'enable') + ' condition'">
<span class="icon">
<i class="fa-solid fa-power-off"
:class="{ 'has-text-success': cond.enabled !== false, 'has-text-danger': cond.enabled === false }" />
</span>
<span>{{ cond.enabled !== false ? 'Enabled' : 'Disabled' }}</span>
</span>
&nbsp;
<Popover :maxWidth="450">
<template #trigger>
<span class="is-clickable">
<span class="icon"> <i class="fa-solid fa-info-circle" /></span>
<span>Show Details</span>
</span>
</template>
<template #title><strong>Condition Details</strong></template>
<div v-if="cond.filter">
<span class="icon"><i class="fa-solid fa-filter" /></span>
<code>{{ cond.filter }}</code>
</div>
<div v-if="cond.cli">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<code>{{ cond.cli }}</code>
</div>
<span v-if="cond.extras && Object.keys(cond.extras).length > 0">
<template v-for="(value, key) in cond.extras" :key="key">
<div>
<span class="icon"><i class="fa-solid fa-list" /></span>
<code>{{ key }}: {{ value }}</code>
</div>
</template>
</span>
</Popover>
<template v-if="cond.priority > 0">
&nbsp;
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-sort-numeric-down" /></span>
<span>Priority: {{ cond.priority }}</span>
</span>
</template>
</div>
</td>
<td class="is-vcentered is-items-center">
<div class="field is-grouped is-grouped-centered">
<div class="control">
<button class="button is-info is-small is-fullwidth" @click="exportItem(cond)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
<span v-if="!isMobile">Export</span>
</button>
</div>
<div class="control">
<button class="button is-warning is-small is-fullwidth" @click="editItem(cond)">
<span class="icon"><i class="fa-solid fa-edit" /></span>
<span v-if="!isMobile">Edit</span>
</button>
</div>
<div class="control">
<button class="button is-danger is-small is-fullwidth" @click="deleteItem(cond)">
<span class="icon"><i class="fa-solid fa-trash" /></span>
<span v-if="!isMobile">Delete</span>
</button>
</div>
</div>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</template>
<template v-else>
<div class="column is-6" v-for="cond in items" :key="cond.id">
<div class="card is-flex is-full-height is-flex-direction-column">
<header class="card-header">
<div class="card-header-title is-text-overflow is-block" v-text="cond.name" />
<div class="card-header-icon">
<div class="field is-grouped">
<div class="control" v-if="cond.priority > 0">
<span class="tag is-dark">
<span class="icon"><i class="fa-solid fa-sort-numeric-down" /></span>
<span v-text="cond.priority" />
</span>
<code>{{ cond.filter }}</code>
</div>
<div class="control" @click="toggleEnabled(cond)">
<span class="icon" :class="cond.enabled ? 'has-text-success' : 'has-text-danger'"
v-tooltip="`Condition is ${cond.enabled !== false ? 'enabled' : 'disabled'}. Click to toggle.`">
<i class="fa-solid fa-power-off" />
</span>
</div>
<div class="control">
<a class="has-text-info" v-tooltip="'Export item'" @click.prevent="exportItem(cond)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</a>
<div v-if="cond.cli">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<code>{{ cond.cli }}</code>
</div>
<span v-if="cond.extras && Object.keys(cond.extras).length > 0">
<template v-for="(value, key) in cond.extras" :key="key">
<div>
<span class="icon"><i class="fa-solid fa-list" /></span>
<code>{{ key }}: {{ value }}</code>
</div>
</template>
</span>
</Popover>
<template v-if="cond.priority > 0">
&nbsp;
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-sort-numeric-down" /></span>
<span>Priority: {{ cond.priority }}</span>
</span>
</template>
</div>
</td>
<td class="is-vcentered is-items-center">
<div class="field is-grouped is-grouped-centered">
<div class="control">
<button class="button is-info is-small is-fullwidth" @click="exportItem(cond)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
<span v-if="!isMobile">Export</span>
</button>
</div>
<div class="control">
<button class="button is-warning is-small is-fullwidth" @click="editItem(cond)">
<span class="icon"><i class="fa-solid fa-edit" /></span>
<span v-if="!isMobile">Edit</span>
</button>
</div>
<div class="control">
<button class="button is-danger is-small is-fullwidth" @click="deleteItem(cond)">
<span class="icon"><i class="fa-solid fa-trash" /></span>
<span v-if="!isMobile">Delete</span>
</button>
</div>
</div>
</header>
<div class="card-content is-flex-grow-1">
<div class="content">
<p class="is-text-overflow">
<span class="icon"><i class="fa-solid fa-filter" /></span>
<span v-text="cond.filter" />
</p>
<p class="is-text-overflow" v-if="cond.cli">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span>{{ cond.cli }}</span>
</p>
<p class="is-text-overflow" v-if="cond.extras && Object.keys(cond.extras).length > 0">
<span class="icon"><i class="fa-solid fa-list" /></span>
<span>Extras:
<span v-for="(value, key) in cond.extras" :key="key" class="tag is-info mr-2">
<b>{{ key }}</b>: {{ value }}
</span>
</span>
</p>
<p class="is-clickable" :class="{ 'is-text-overflow': !isExpanded(cond.id, 'description') }"
v-if="cond.description" @click="toggleExpand(cond.id, 'description')">
<span class="icon"><i class="fa-solid fa-comment" /></span>
<span>{{ cond.description }}</span>
</p>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<template v-else>
<div class="column is-6" v-for="cond in filteredItems" :key="cond.id">
<div class="card is-flex is-full-height is-flex-direction-column">
<header class="card-header">
<div class="card-header-title is-text-overflow is-block" v-text="cond.name" />
<div class="card-header-icon">
<div class="field is-grouped">
<div class="control" v-if="cond.priority > 0">
<span class="tag is-dark">
<span class="icon"><i class="fa-solid fa-sort-numeric-down" /></span>
<span v-text="cond.priority" />
</span>
</div>
</div>
<div class="card-footer mt-auto">
<div class="card-footer-item">
<button class="button is-warning is-fullwidth" @click="editItem(cond)">
<span class="icon"><i class="fa-solid fa-edit" /></span>
<span>Edit</span>
</button>
<div class="control" @click="toggleEnabled(cond)">
<span class="icon" :class="cond.enabled ? 'has-text-success' : 'has-text-danger'"
v-tooltip="`Condition is ${cond.enabled !== false ? 'enabled' : 'disabled'}. Click to toggle.`">
<i class="fa-solid fa-power-off" />
</span>
</div>
<div class="card-footer-item">
<button class="button is-danger is-fullwidth" @click="deleteItem(cond)">
<span class="icon"><i class="fa-solid fa-trash" /></span>
<span>Delete</span>
</button>
<div class="control">
<a class="has-text-info" v-tooltip="'Export item'" @click.prevent="exportItem(cond)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</a>
</div>
</div>
</div>
</header>
<div class="card-content is-flex-grow-1">
<div class="content">
<p class="is-text-overflow">
<span class="icon"><i class="fa-solid fa-filter" /></span>
<span v-text="cond.filter" />
</p>
<p class="is-text-overflow" v-if="cond.cli">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span>{{ cond.cli }}</span>
</p>
<p class="is-text-overflow" v-if="cond.extras && Object.keys(cond.extras).length > 0">
<span class="icon"><i class="fa-solid fa-list" /></span>
<span>Extras:
<span v-for="(value, key) in cond.extras" :key="key" class="tag is-info mr-2">
<b>{{ key }}</b>: {{ value }}
</span>
</span>
</p>
<p class="is-clickable" :class="{ 'is-text-overflow': !isExpanded(cond.id, 'description') }"
v-if="cond.description" @click="toggleExpand(cond.id, 'description')">
<span class="icon"><i class="fa-solid fa-comment" /></span>
<span>{{ cond.description }}</span>
</p>
</div>
</div>
</template>
<div class="card-footer mt-auto">
<div class="card-footer-item">
<button class="button is-warning is-fullwidth" @click="editItem(cond)">
<span class="icon"><i class="fa-solid fa-edit" /></span>
<span>Edit</span>
</button>
</div>
<div class="card-footer-item">
<button class="button is-danger is-fullwidth" @click="deleteItem(cond)">
<span class="icon"><i class="fa-solid fa-trash" /></span>
<span>Delete</span>
</button>
</div>
</div>
</div>
</div>
<Message title="No items" message="There are no custom conditions defined."
class="is-background-warning-80 has-text-dark" icon="fas fa-exclamation-circle"
v-if="!items || items.length < 1" />
</template>
</div>
<div class="columns is-multiline" v-if="!toggleForm && (isLoading || !filteredItems || filteredItems.length < 1)">
<div class="column is-12">
<Message v-if="isLoading" class="is-info" title="Loading" icon="fas fa-spinner fa-spin">
Loading data. Please wait...
</Message>
<Message title="No Results" class="is-warning" icon="fas fa-search" v-else-if="query" :useClose="true"
@close="query = ''">
<p>No results found for the query: <code>{{ query }}</code>.</p>
<p>Please try a different search term.</p>
</Message>
<Message v-else title="No items" class="is-warning" icon="fas fa-exclamation-circle">
There are no custom defined conditions yet. Click the <span class="icon"><i class="fas fa-add" /></span>
<strong>New Condition</strong> button to add your first condition.
</Message>
</div>
</div>
<div class="column is-12" v-if="items && items.length > 0 && !toggleForm">
<div class="message is-info">
<div class="message-body content pl-0">
<div class="columns is-multiline" v-if="filteredItems && filteredItems.length > 0 && !toggleForm">
<div class="column is-12">
<Message class="is-info" :body_class="'pl-0'">
<ul>
<li>Filtering is based on yt-dlp’s <code>--match-filter</code> logic. Any expression that works with yt-dlp
will also work here, including the same boolean operators. We added extended support for the
<code>OR</code> ( <code>||</code> ) operator, which yt-dlp does not natively support. This allows you to combine
multiple conditions more flexibly.
<li>Filtering is based on yt-dlp’s <code>--match-filter</code> logic. Any expression that works with
yt-dlp will also work here, including the same boolean operators. We added extended support for the
<code>OR</code> ( <code>||</code> ) operator, which yt-dlp does not natively support. This allows you to
combine multiple conditions more flexibly.
</li>
<li>
The primary use case for this feature is to apply custom cli arguments to specific returned info.
@ -254,7 +277,7 @@
filter matches or not.
</li>
</ul>
</div>
</Message>
</div>
</div>
</div>
@ -280,9 +303,18 @@ const toggleForm = ref(false)
const isLoading = ref(false)
const initialLoad = ref(true)
const addInProgress = ref(false)
const query = ref<string>('')
const toggleFilter = ref(false)
const remove_keys = ['raw', 'toggle_description']
const expandedItems = ref<Record<string, Set<string>>>({})
const filteredItems = computed<ConditionItemWithUI[]>(() => {
const q = query.value?.toLowerCase();
if (!q) return items.value;
return items.value.filter((item: ConditionItemWithUI) => deepIncludes(item, q, new WeakSet()));
});
const toggleExpand = (itemId: string | undefined, field: string) => {
if (!itemId) return
@ -309,6 +341,12 @@ watch(() => socket.isConnected, async () => {
}
})
watch(toggleFilter, (val) => {
if (!val) {
query.value = ''
}
})
const reloadContent = async (fromMounted = false): Promise<void> => {
try {

2
ui/app/pages/console.vue
View file

@ -85,7 +85,7 @@
</p>
</header>
<div v-show="!isHistoryCollapsed" class="card-content p-2">
<Message :newStyle="true" class="is-info" v-if="commandHistory.length < 1">
<Message class="is-info" v-if="commandHistory.length < 1">
Commands history is empty.
</Message>
<div class="table-container" v-if="commandHistory.length > 0">

204
ui/app/pages/notifications.vue
View file

@ -16,6 +16,19 @@
</span>
<div class="is-pulled-right" v-if="!toggleForm">
<div class="field is-grouped">
<p class="control has-icons-left" v-if="toggleFilter && notifications && notifications.length > 0">
<input type="search" v-model.lazy="query" class="input" id="filter"
placeholder="Filter displayed content">
<span class="icon is-left"><i class="fas fa-filter" /></span>
</p>
<p class="control" v-if="notifications && notifications.length > 0">
<button class="button is-danger is-light" @click="toggleFilter = !toggleFilter">
<span class="icon"><i class="fas fa-filter" /></span>
<span v-if="!isMobile">Filter</span>
</button>
</p>
<p class="control">
<button class="button is-primary" @click="resetForm(false); toggleForm = true"
v-tooltip="'Add new notification target.'">
@ -25,7 +38,7 @@
</p>
<p class="control" v-if="notifications.length > 0">
<button class="button is-warning" @click="sendTest" v-tooltip="'Send test notification.'"
:class="{ 'is-loading': isLoading }" :disabled="isLoading">
:class="{ 'is-loading': sendingTest }" :disabled="!notifications.length || sendingTest">
<span class="icon"><i class="fas fa-paper-plane" /></span>
<span v-if="!isMobile">Send Test</span>
</button>
@ -62,9 +75,11 @@
<NotificationForm :addInProgress="addInProgress" :reference="targetRef as string" :item="target"
@cancel="resetForm(true);" @submit="updateItem" :allowedEvents="allowedEvents" />
</div>
</div>
<div class="column is-12" v-if="!toggleForm && notifications && notifications.length > 0">
<template v-if="'list' === display_style">
<div class="columns is-multiline" v-if="!isLoading && !toggleForm && filteredTargets && filteredTargets.length > 0">
<template v-if="'list' === display_style">
<div class="column is-12">
<div class="table-container">
<table class="table is-striped is-hoverable is-fullwidth is-bordered"
style="min-width: 850px; table-layout: fixed;">
@ -81,7 +96,7 @@
</tr>
</thead>
<tbody>
<tr v-for="item in notifications" :key="item.id">
<tr v-for="item in filteredTargets" :key="item.id">
<td class="is-text-overflow is-vcentered">
<div class="is-bold">
{{ item.request.method.toUpperCase() }}({{ ucFirst(item.request.type) }}) @
@ -133,89 +148,99 @@
</tbody>
</table>
</div>
</template>
<template v-else>
<div class="columns is-multiline" v-if="notifications && notifications.length > 0">
<div class="column is-6" v-for="item in notifications" :key="item.id">
<div class="card is-flex is-full-height is-flex-direction-column">
<header class="card-header">
<div class="card-header-title is-text-overflow is-block">
{{ item.request.method.toUpperCase() }}({{ ucFirst(item.request.type) }}) @
<NuxtLink target="_blank" :href="item.request.url">{{ item.name }}</NuxtLink>
</div>
</template>
<template v-else>
<div class="column is-6 is-12-mobile" v-for="item in filteredTargets" :key="item.id">
<div class="card is-flex is-full-height is-flex-direction-column">
<header class="card-header">
<div class="card-header-title is-text-overflow is-block">
{{ item.request.method.toUpperCase() }}({{ ucFirst(item.request.type) }}) @
<NuxtLink target="_blank" :href="item.request.url">{{ item.name }}</NuxtLink>
</div>
<div class="card-header-icon">
<div class="field is-grouped">
<div class="control" @click="toggleEnabled(item)">
<span class="icon" :class="item.enabled ? 'has-text-success' : 'has-text-danger'"
v-tooltip="`Notification is ${item.enabled !== false ? 'enabled' : 'disabled'}. Click to toggle.`">
<i class="fa-solid fa-power-off" />
</span>
</div>
<div class="card-header-icon">
<div class="field is-grouped">
<div class="control" @click="toggleEnabled(item)">
<span class="icon" :class="item.enabled ? 'has-text-success' : 'has-text-danger'"
v-tooltip="`Notification is ${item.enabled !== false ? 'enabled' : 'disabled'}. Click to toggle.`">
<i class="fa-solid fa-power-off" />
</span>
</div>
<div class="control">
<a class="has-text-info" v-tooltip="'Export target.'" @click.prevent="exportItem(item)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</a>
</div>
<div class="control">
<button @click="item.raw = !item.raw">
<span class="icon"><i class="fa-solid"
:class="{ 'fa-arrow-down': !item?.raw, 'fa-arrow-up': item?.raw }" /></span>
</button>
</div>
</div>
<div class="control">
<a class="has-text-info" v-tooltip="'Export target.'" @click.prevent="exportItem(item)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</a>
</div>
</header>
<div class="card-content is-flex-grow-1">
<div class="content">
<p>
<span class="icon"><i class="fa-solid fa-list-ul" /></span>
<span>On: {{ join_events(item.on) }}</span>
</p>
<p>
<span class="icon"><i class="fa-solid fa-sliders" /></span>
<span>Presets: {{ join_presets(item.presets) }}</span>
</p>
<p v-if="item.request?.headers && item.request.headers.length > 0">
<span class="icon"><i class="fa-solid fa-heading" /></span>
<span>Headers: {{item.request.headers.map(h => h.key).join(', ')}}</span>
</p>
</div>
</div>
<div class="card-content" v-if="item?.raw">
<div class="content">
<pre><code>{{ JSON.stringify(cleanObject(item, ['raw']), null, 2) }}</code></pre>
</div>
</div>
<div class="card-footer mt-auto">
<div class="card-footer-item">
<button class="button is-warning is-fullwidth" @click="editItem(item);">
<span class="icon"><i class="fa-solid fa-edit" /></span>
<span>Edit</span>
</button>
</div>
<div class="card-footer-item">
<button class="button is-danger is-fullwidth" @click="deleteItem(item)">
<span class="icon"><i class="fa-solid fa-trash" /></span>
<span>Delete</span>
<div class="control">
<button @click="item.raw = !item.raw">
<span class="icon"><i class="fa-solid"
:class="{ 'fa-arrow-down': !item?.raw, 'fa-arrow-up': item?.raw }" /></span>
</button>
</div>
</div>
</div>
</header>
<div class="card-content is-flex-grow-1">
<div class="content">
<p>
<span class="icon"><i class="fa-solid fa-list-ul" /></span>
<span>On: {{ join_events(item.on) }}</span>
</p>
<p>
<span class="icon"><i class="fa-solid fa-sliders" /></span>
<span>Presets: {{ join_presets(item.presets) }}</span>
</p>
<p v-if="item.request?.headers && item.request.headers.length > 0">
<span class="icon"><i class="fa-solid fa-heading" /></span>
<span>Headers: {{item.request.headers.map(h => h.key).join(', ')}}</span>
</p>
</div>
</div>
<div class="card-content" v-if="item?.raw">
<div class="content">
<pre><code>{{ JSON.stringify(cleanObject(item, ['raw']), null, 2) }}</code></pre>
</div>
</div>
<div class="card-footer mt-auto">
<div class="card-footer-item">
<button class="button is-warning is-fullwidth" @click="editItem(item);">
<span class="icon"><i class="fa-solid fa-edit" /></span>
<span>Edit</span>
</button>
</div>
<div class="card-footer-item">
<button class="button is-danger is-fullwidth" @click="deleteItem(item)">
<span class="icon"><i class="fa-solid fa-trash" /></span>
<span>Delete</span>
</button>
</div>
</div>
</div>
</template>
</div>
</div>
</template>
</div>
<div class="column is-12" v-if="!toggleForm && (!notifications || notifications.length < 1)">
<Message title="No Endpoints" class="is-background-warning-80 has-text-dark" icon="fas fa-exclamation-circle">
There are no notifications endpoints configured to receive web notifications.
<div class="columns is-multiline"
v-if="!toggleForm && (isLoading || !filteredTargets || filteredTargets.length < 1)">
<div class="column is-12">
<Message v-if="isLoading" class="is-info" title="Loading" icon="fas fa-spinner fa-spin">
Loading data. Please wait...
</Message>
<Message title="No Results" class="is-warning" icon="fas fa-search" v-else-if="query" :useClose="true"
@close="query = ''">
<p>No results found for the query: <code>{{ query }}</code>.</p>
<p>Please try a different search term.</p>
</Message>
<Message v-else title="No targets" class="is-warning" icon="fas fa-exclamation-circle">
No notification targets found. Click on the <span class="icon"><i class="fas fa-add" /></span> <strong>New
Notification</strong> button to add your first notification target.
</Message>
</div>
</div>
<div class="column is-12" v-if="notifications && notifications.length > 0 && !toggleForm">
<div class="message is-info">
<div class="message-body content pl-0">
<div class="columns is-multiline" v-if="!toggleForm && filteredTargets && filteredTargets.length > 0">
<div class="column is-12">
<Message class="is-info" :body_class="'pl-0'">
<ul>
<li>
When you export notification target, We remove <code>Authorization</code> header key by default,
@ -227,16 +252,19 @@
<code>...&data_key=json_string</code>, only the <code>data</code> field will be JSON encoded.
The other keys <code>id</code>, <code>event</code> and <code>created_at</code> will be sent as they are.
</li>
<li>We also send two special headers <code>X-Event-ID</code> and <code>X-Event</code> with the request.</li>
<li>We also send two special headers <code>X-Event-ID</code> and <code>X-Event</code> with the request.
</li>
<li>
If you have selected specific presets or events, this will take priority, For example, if you limited the
target to <code>default</code> preset and selected <code>ALL</code> events, only events that reference the
If you have selected specific presets or events, this will take priority, For example, if you limited
the
target to <code>default</code> preset and selected <code>ALL</code> events, only events that reference
the
<code>default</code> preset will be sent to that target. Like wise, if you have limited both events and
presets, then ONLY events that satisfy both conditions will be sent to that target. Only the
<code>test</code> events can bypass these conditions.
</li>
</ul>
</div>
</Message>
</div>
</div>
</div>
@ -272,6 +300,22 @@ const toggleForm = ref(false)
const isLoading = ref(false)
const initialLoad = ref(true)
const addInProgress = ref(false)
const sendingTest = ref(false)
const query = ref<string>('')
const toggleFilter = ref(false)
const filteredTargets = computed<notificationItemWithUI[]>(() => {
const q = query.value?.toLowerCase();
if (!q) return notifications.value;
return notifications.value.filter((item: notificationItemWithUI) => deepIncludes(item, q, new WeakSet()));
});
watch(toggleFilter, (val) => {
if (!val) {
query.value = ''
}
})
watch(() => socket.isConnected, async () => {
if (socket.isConnected && initialLoad.value) {
@ -413,7 +457,7 @@ const sendTest = async () => {
}
try {
isLoading.value = true
sendingTest.value = true
const response = await request('/api/notifications/test', { method: 'POST' })
if (200 !== response.status) {
@ -427,7 +471,7 @@ const sendTest = async () => {
console.error(e)
toast.error(`Failed to send test notification. ${e.message}`)
} finally {
isLoading.value = false
sendingTest.value = false
}
}

67
ui/app/pages/presets.vue
View file

@ -16,6 +16,19 @@
</span>
<div class="is-pulled-right" v-if="!toggleForm">
<div class="field is-grouped">
<p class="control has-icons-left" v-if="toggleFilter && presets && presets.length > 0">
<input type="search" v-model.lazy="query" class="input" id="filter"
placeholder="Filter displayed content">
<span class="icon is-left"><i class="fas fa-filter" /></span>
</p>
<p class="control" v-if="presets && presets.length > 0">
<button class="button is-danger is-light" @click="toggleFilter = !toggleFilter">
<span class="icon"><i class="fas fa-filter" /></span>
<span v-if="!isMobile">Filter</span>
</button>
</p>
<p class="control">
<button class="button is-primary" @click="resetForm(false); toggleForm = !toggleForm;"
v-tooltip.bottom="'Toggle add form'">
@ -35,6 +48,7 @@
</span>
</button>
</p>
<p class="control">
<button class="button is-info" @click="reloadContent()" :class="{ 'is-loading': isLoading }"
:disabled="isLoading" v-if="presets && presets.length > 0">
@ -59,7 +73,7 @@
</div>
<template v-if="!toggleForm">
<div class="columns is-multiline" v-if="presetsNoDefault && presetsNoDefault.length > 0">
<div class="columns is-multiline" v-if="!isLoading && presetsNoDefault && presetsNoDefault.length > 0">
<template v-if="'list' === display_style">
<div class="column is-12">
<div class="table-container">
@ -72,7 +86,7 @@
</tr>
</thead>
<tbody>
<tr v-for="item in presetsNoDefault" :key="item.id">
<tr v-for="item in filteredPresets" :key="item.id">
<td class="is-text-overflow is-vcentered">
<div class="is-text-overflow is-bold">
{{ item.name }}
@ -122,7 +136,7 @@
</template>
<template v-else>
<div class="column is-6" v-for="item in presetsNoDefault" :key="item.id">
<div class="column is-6" v-for="item in filteredPresets" :key="item.id">
<div class="card is-flex is-full-height is-flex-direction-column">
<header class="card-header">
<div class="card-header-title is-block is-clickable"
@ -204,24 +218,30 @@
</template>
</div>
<div class="columns is-multiline" v-if="!presets || presets.length < 1">
<div class="columns is-multiline"
v-if="!toggleForm && (isLoading || !filteredPresets || filteredPresets.length < 1)">
<div class="column is-12">
<Message message_class="has-background-info-90 has-text-dark" title="Loading" icon="fas fa-spinner fa-spin"
message="Loading data. Please wait..." v-if="isLoading" />
<Message title="No presets" message="There are no custom defined presets."
class="is-background-warning-80 has-text-dark" icon="fas fa-exclamation-circle" v-else />
<Message v-if="isLoading" class="is-info" title="Loading" icon="fas fa-spinner fa-spin">
Loading data. Please wait...
</Message>
<Message title="No Results" class="is-warning" icon="fas fa-search" v-else-if="query" :useClose="true"
@close="query = ''">
<p>No results found for the query: <code>{{ query }}</code>.</p>
<p>Please try a different search term.</p>
</Message>
<Message v-else title="No presets" class="is-warning" icon="fas fa-exclamation-circle">
There are no custom defined presets.
</Message>
</div>
</div>
<div class="columns is-multiline" v-if="presets && presets.length > 0">
<div class="columns is-multiline" v-if="!query && presets && presets.length > 0">
<div class="column is-12">
<div class="message is-info">
<p class="message-body">
<span class="icon"> <i class="fas fa-info-circle" /></span>
When you export preset, it doesn't include the <strong>cookies</strong> field contents for security
reasons.
</p>
</div>
<Message class="is-info">
<span class="icon"><i class="fas fa-info-circle" /></span>
When you <b>export</b> preset, it doesn't include the <code>cookies</code> field contents for security
reasons.
</Message>
</div>
</div>
</template>
@ -243,6 +263,9 @@ const box = useConfirm()
const display_style = useStorage<string>('preset_display_style', 'cards')
const isMobile = useMediaQuery({ maxWidth: 1024 })
const query = ref<string>('')
const toggleFilter = ref(false)
const presets = ref<PresetWithUI[]>([])
const preset = ref<Partial<Preset>>({})
const presetRef = ref<string | null>('')
@ -255,6 +278,12 @@ const expandedItems = ref<Record<string, Set<string>>>({})
const presetsNoDefault = computed(() => presets.value.filter((t) => !t.default))
const filteredPresets = computed<PresetWithUI[]>(() => {
const q = query.value?.toLowerCase();
if (!q) return presetsNoDefault.value;
return presetsNoDefault.value.filter((item: PresetWithUI) => deepIncludes(item, q, new WeakSet()));
});
const toggleExpand = (itemId: string | undefined, field: string) => {
if (!itemId) return
@ -281,6 +310,12 @@ watch(() => socket.isConnected, async () => {
}
})
watch(toggleFilter, (val) => {
if (!val) {
query.value = ''
}
})
const reloadContent = async (fromMounted = false) => {
try {
isLoading.value = true

16
ui/app/pages/task_definitions.vue
View file

@ -80,7 +80,8 @@
</td>
<td class="is-vcentered has-text-centered">{{ definition.priority }}</td>
<td class="is-vcentered has-text-centered">
<span class="has-tooltip" :date-datetime="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
<span class="has-tooltip"
:date-datetime="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-tooltip="moment.unix(definition.updated_at).format('YYYY-M-DD H:mm Z')"
v-rtime="definition.updated_at" />
</td>
@ -170,11 +171,14 @@
<div class="columns is-multiline" v-if="!definitions.length">
<div class="column is-12">
<Message message_class="has-background-info-90 has-text-dark" title="Loading" icon="fas fa-spinner fa-spin"
message="Loading data. Please wait..." v-if="isLoading" />
<Message title="No definitions" message="No task definitions are configured yet. Create one to get started."
class="is-background-warning-80 has-text-dark" icon="fas fa-exclamation-circle"
v-if="!isLoading && !isEditorOpen" />
<Message v-if="isLoading" class="is-info" title="Loading" icon="fas fa-spinner fa-spin">
Loading data. Please wait...
</Message>
<Message v-if="!isLoading && !isEditorOpen" title="No definitions" class="is-warning"
icon="fas fa-exclamation-circle">
There are no task definitions. Click the <span class="icon"><i class="fas fa-add" /></span> <strong>New
Definition</strong> button to create your first task definition.
</Message>
</div>
</div>

29
ui/app/pages/tasks.vue
View file

@ -73,7 +73,7 @@
</div>
<div class="columns is-multiline is-mobile is-justify-content-flex-end"
v-if="!toggleForm && filteredTasks && filteredTasks.length > 0">
v-if="!isLoading && !toggleForm && filteredTasks && filteredTasks.length > 0">
<div class="column is-narrow">
<button type="button" class="button" @click="masterSelectAll = !masterSelectAll"
:class="{ 'has-text-primary': !masterSelectAll, 'has-text-danger': masterSelectAll }">
@ -407,29 +407,26 @@
</div>
</div>
<div class="columns is-multiline" v-if="!filteredTasks || filteredTasks.length < 1">
<div class="columns is-multiline" v-if="!toggleForm && (isLoading || !filteredTasks || filteredTasks.length < 1)">
<div class="column is-12">
<Message :newStyle="true" message_class="is-info" v-if="isLoading">
<p><span class="icon"><i class="fas fa-spinner fa-spin" /></span> Loading data. Please wait...</p>
<Message class="is-info" title="Loading" icon="fas fa-spinner fa-spin" v-if="isLoading">
Loading data. Please wait...
</Message>
<Message title="No Results" message_class="is-warning" icon="fas fa-search" v-else-if="query" :useClose="true"
@close="query = ''" :newStyle="true">
<p>No results found for the query: <strong>{{ query }}</strong>.</p>
<Message title="No Results" class="is-warning" icon="fas fa-search" v-else-if="query" :useClose="true"
@close="query = ''">
<p>No results found for the query: <code>{{ query }}</code>.</p>
<p>Please try a different search term.</p>
</Message>
<Message message_class="is-warning" :newStyle="true" v-else>
<p>
<span class="icon"><i class="fa-solid fa-info-circle" /></span>
<span>No tasks are defined.</span>
</p>
<Message class="is-warning" icon="fa-solid fa-info-circle" title="No tasks." v-else>
There are no tasks defined yet. Click the <span class="icon"><i class="fas fa-add" /></span> <strong>New
Task</strong> button to create your first automated download task.
</Message>
</div>
</div>
<div class="columns is-multiline" v-if="!toggleForm && tasks && tasks.length > 0">
<div class="column is-12">
<Message :newStyle="true" message_class="is-info">
<Message class="is-info">
<ul>
<li class="has-text-danger">
<span class="icon">
@ -611,9 +608,7 @@ const filteredTasks = computed<task_item[]>(() => {
const q = query.value?.toLowerCase();
if (!q) return tasks.value;
return tasks.value.filter(
task => Object.values(task).some(value => typeof value === 'string' && value.toLowerCase().includes(q))
);
return tasks.value.filter(task => deepIncludes(task, q, new WeakSet()));
});
const reloadContent = async (fromMounted: boolean = false) => {

4
ui/app/types/store.d.ts vendored
View file

@ -1,4 +1,4 @@
type ItemStatus = 'finished' | 'preparing' | 'error' | 'cancelled' | 'downloading' | 'postprocessing' | 'not_live' | 'skip' | null;
type ItemStatus = 'started' | 'finished' | 'preparing' | 'error' | 'cancelled' | 'downloading' | 'postprocessing' | 'not_live' | 'skip' | null;
type SideCar = {
file: string
@ -106,6 +106,8 @@ type StoreItem = {
is_archived?: boolean
/** Item archive ID */
archive_id?: string | null
/** Postprocessor running for the item if available */
postprocessor?: string | null
}
export type { ItemStatus, StoreItem }