diff --git a/FAQ.md b/FAQ.md index e1e2eec0..dd5813a0 100644 --- a/FAQ.md +++ b/FAQ.md @@ -3,53 +3,54 @@ Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file. -| Environment Variable | Description | Default | -| ------------------------------ | ------------------------------------------------------------------ | --------------------- | -| TZ | The timezone to use for the application | `(not_set)` | -| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | -| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | -| YTP_INSTANCE_TITLE | The title of the instance | `(not_set)` | -| YTP_FILE_LOGGING | Whether to log to file | `false` | -| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | -| YTP_MAX_WORKERS | The maximum number of workers to use for downloading | `20` | -| YTP_MAX_WORKERS_PER_EXTRACTOR | The maximum number of concurrent downloads per extractor | `2` | -| YTP_AUTH_USERNAME | Username for basic authentication | `(not_set)` | -| YTP_AUTH_PASSWORD | Password for basic authentication | `(not_set)` | -| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | -| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | -| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | -| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | -| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | -| YTP_HOST | Which IP address to bind to | `0.0.0.0` | -| YTP_PORT | Which port to bind to | `8081` | -| YTP_LOG_LEVEL | Log level | `info` | -| YTP_STREAMER_VCODEC | The video encoding codec, default to GPU and fallback to software | `""` | -| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | -| YTP_VAAPI_DEVICE | The VAAPI device to use for hardware acceleration. | `/dev/dri/renderD128` | -| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | -| YTP_DEBUG | Whether to turn on debug mode | `false` | -| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | -| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | -| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | -| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` | -| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | -| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` | -| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | -| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | -| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` | -| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | -| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | -| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | -| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | -| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | -| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | -| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | -| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` | -| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | -| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | -| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | -| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | +| Environment Variable | Description | Default | +| ------------------------------ | ------------------------------------------------------------------- | --------------------- | +| TZ | The timezone to use for the application | `(not_set)` | +| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | +| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | +| YTP_INSTANCE_TITLE | The title of the instance | `(not_set)` | +| YTP_FILE_LOGGING | Whether to log to file | `false` | +| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | +| YTP_MAX_WORKERS | The maximum number of workers to use for downloading | `20` | +| YTP_MAX_WORKERS_PER_EXTRACTOR | The maximum number of concurrent downloads per extractor | `2` | +| YTP_AUTH_USERNAME | Username for basic authentication | `(not_set)` | +| YTP_AUTH_PASSWORD | Password for basic authentication | `(not_set)` | +| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | +| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | +| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | +| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | +| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | +| YTP_HOST | Which IP address to bind to | `0.0.0.0` | +| YTP_PORT | Which port to bind to | `8081` | +| YTP_LOG_LEVEL | Log level | `info` | +| YTP_STREAMER_VCODEC | The video encoding codec, default to GPU and fallback to software | `""` | +| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | +| YTP_VAAPI_DEVICE | The VAAPI device to use for hardware acceleration. | `/dev/dri/renderD128` | +| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | +| YTP_DEBUG | Whether to turn on debug mode | `false` | +| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | +| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | +| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | +| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` | +| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | +| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` | +| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | +| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` | +| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | +| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | +| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | +| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | +| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | +| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | +| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` | +| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | +| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | +| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | +| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | +| YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` | > [!NOTE] > To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_`. diff --git a/app/library/Tasks.py b/app/library/Tasks.py index 349c38d8..056eae21 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -3,6 +3,7 @@ import inspect import json import logging import pkgutil +import random import time import uuid from dataclasses import asdict, dataclass, field @@ -696,6 +697,8 @@ class HandleTask: def _dispatcher(self): s: dict[list[str]] = {"h": [], "d": [], "u": [], "f": []} + handler_groups: dict[str, list[tuple[Task, type]]] = {} + for task in self._tasks.get_all(): if not task.enabled or not task.handler_enabled: s["d"].append(task.name) @@ -712,19 +715,53 @@ class HandleTask: s["u"].append(task.name) continue - coro = self.dispatch(task, handler=handler) - t = asyncio.create_task(coro, name=f"task-{task.id}") - t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) + handler_name = handler.__name__ + if handler_name not in handler_groups: + handler_groups[handler_name] = [] + handler_groups[handler_name].append((task, handler)) s["h"].append(task.name) except Exception as e: LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.") s["f"].append(task.name) + for tasks_with_handlers in handler_groups.values(): + for idx, (task, handler) in enumerate(tasks_with_handlers): + try: + t = asyncio.create_task( + coro=self._dispatch( + task, + handler, + delay=0.0 if 0 == idx else random.uniform(1.0, self._config.task_handler_random_delay), + ), + name=f"task-{task.id}", + ) + t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) + except Exception as e: + LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.") + if len(self._tasks.get_all()) > 0: LOG.info( f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}." ) + async def _dispatch(self, task: Task, handler: type, delay: float) -> TaskResult | TaskFailure | None: + """ + Dispatch a task after a random delay to avoid rate limiting. + + Args: + task (Task): The task to dispatch. + handler (type): The handler to use. + delay (float): The delay in seconds before dispatching. + + Returns: + TaskResult|TaskFailure|None: The dispatch result. + + """ + if delay > 0: + LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.") + await asyncio.sleep(delay) + return await self.dispatch(task, handler=handler) + def _handle_exception(self, fut: asyncio.Task, task: Task) -> None: if fut.cancelled(): return diff --git a/app/library/config.py b/app/library/config.py index 6fca0d6a..368fa8dd 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -201,6 +201,9 @@ class Config(metaclass=Singleton): default_pagination: int = 50 """The default number of items per page for pagination.""" + task_handler_random_delay: float = 60.0 + """The maximum random delay in seconds before starting a task handler.""" + pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", @@ -265,6 +268,9 @@ class Config(metaclass=Singleton): ) "The variables that are booleans." + _float_vars: tuple = ("task_handler_random_delay",) + "The variables that are floats." + _frontend_vars: tuple = ( "download_path", "keep_archive", @@ -365,6 +371,9 @@ class Config(metaclass=Singleton): if k in self._int_vars: setattr(self, k, int(v)) + if k in self._float_vars: + setattr(self, k, float(v)) + if isinstance(self.pictures_backends, str) and self.pictures_backends: self.pictures_backends = self.pictures_backends.split(",") diff --git a/app/routes/api/images.py b/app/routes/api/images.py index c7e4e389..aa73cb17 100644 --- a/app/routes/api/images.py +++ b/app/routes/api/images.py @@ -120,7 +120,7 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp try: IS_REQUESTING_BACKGROUND = True - backend = random.choice(config.pictures_backends) # noqa: S311 + backend = random.choice(config.pictures_backends) CACHE_KEY_BING = "random_background_bing" CACHE_KEY = "random_background" diff --git a/pyproject.toml b/pyproject.toml index dd6482b9..baff0ead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,6 +168,7 @@ ignore = [ "TRY004", # Like it's our choice to use ValuesError :| "PT011", "RUF001", # We like unicode chars. + "S311", # Not used for cryptography ] # Allow fix for all enabled rules (when `--fix`) is provided.