From 8a3673bc072630084b5b51c001efb72da8ed5c9d Mon Sep 17 00:00:00 2001 From: arabcoders Date: Wed, 18 Jun 2025 19:58:58 +0300 Subject: [PATCH] added playlist items concurrency support. --- README.md | 79 ++++++++++++++++++------------------ app/library/DownloadQueue.py | 50 +++++++++++++++++++++++ app/library/config.py | 4 ++ 3 files changed, 94 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 5fbb95c7..11670832 100644 --- a/README.md +++ b/README.md @@ -274,43 +274,44 @@ If you feel like donating and appreciate my work, you can do so by donating to c Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file. -| Environment Variable | Description | Default | -| ------------------------- | ------------------------------------------------------------------ | ---------------------------------- | -| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | -| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | -| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | -| YTP_FILE_LOGGING | Whether to log to file | `false` | -| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | -| YTP_MAX_WORKERS | How many works to use for downloads | `1` | -| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | -| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | -| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | -| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | -| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | -| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | -| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | -| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | -| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_HOST | Which IP address to bind to | `0.0.0.0` | -| YTP_PORT | Which port to bind to | `8081` | -| YTP_LOG_LEVEL | Log level | `info` | -| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | -| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | -| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | -| YTP_DEBUG | Whether to turn on debug mode | `false` | -| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | -| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` | -| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | -| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | -| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` | -| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | -| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | -| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | -| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | -| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` | -| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | -| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | -| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | -| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | -| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer. | `*15 */1 * * *` | +| Environment Variable | Description | Default | +| ------------------------------ | ------------------------------------------------------------------ | ---------------------------------- | +| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | +| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | +| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | +| YTP_FILE_LOGGING | Whether to log to file | `false` | +| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | +| YTP_MAX_WORKERS | How many works to use for downloads | `1` | +| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | +| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | +| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | +| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | +| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | +| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | +| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | +| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | +| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_HOST | Which IP address to bind to | `0.0.0.0` | +| YTP_PORT | Which port to bind to | `8081` | +| YTP_LOG_LEVEL | Log level | `info` | +| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | +| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | +| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | +| YTP_DEBUG | Whether to turn on debug mode | `false` | +| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | +| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` | +| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | +| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | +| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` | +| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | +| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | +| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | +| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | +| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` | +| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | +| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | +| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer. | `*15 */1 * * *` | +| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time. | `1` | diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index b1a13585..f885e46b 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -202,6 +202,54 @@ class DownloadQueue(metaclass=Singleton): LOG.error(f"Failed to cancel downloads. {e!s}") async def _process_playlist(self, entry: dict, item: Item, already=None): + if 1 == self.config.playlist_items_concurrency: + return await self._process_playlist_old(entry=entry, item=item, already=already) + + LOG.info(f"Playlist '{entry.get('id')}: {entry.get('title')}' processing.") + entries = entry.get("entries", []) + playlistCount = int(entry.get("playlist_count", len(entries))) + results = [] + + semaphore = asyncio.Semaphore(self.config.playlist_items_concurrency) + + async def process_entry(i, etr): + extras = { + "playlist": entry.get("id"), + "playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i), + "playlist_autonumber": i, + } + + for property in ("id", "title", "uploader", "uploader_id"): + if property in entry: + extras[f"playlist_{property}"] = entry.get(property) + + LOG.debug(f"Processing entry {i}/{playlistCount} - ID: {etr.get('id')} - Title: {etr.get('title')}") + + if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""): + extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg" + + async with semaphore: + return await self.add( + item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras), + already=already, + ) + + tasks = [process_entry(i, etr) for i, etr in enumerate(entries, start=1)] + results = await asyncio.gather(*tasks) + + LOG.info( + f"Playlist '{entry.get('id')}: {entry.get('title')}' processing completed with '{len(results)}' entries." + ) + + if any("error" == res["status"] for res in results): + return { + "status": "error", + "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res), + } + + return {"status": "ok"} + + async def _process_playlist_old(self, entry: dict, item: Item, already=None): LOG.info(f"Playlist '{entry.get('id')}: {entry.get('title')}' processing.") entries = entry.get("entries", []) playlistCount = int(entry.get("playlist_count", len(entries))) @@ -221,6 +269,8 @@ class DownloadQueue(metaclass=Singleton): if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""): extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg" + LOG.debug(f"Processing entry {i}/{playlistCount} - ID: {etr.get('id')} - Title: {etr.get('title')}") + results.append( await self.add( item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras), diff --git a/app/library/config.py b/app/library/config.py index cc269215..919d8d95 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -167,6 +167,9 @@ class Config: prevent_live_premiere: bool = False """Prevent downloading of the initial premiere live broadcast.""" + playlist_items_concurrency: int = 1 + """The number of concurrent playlist items to be processed at same time.""" + pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", @@ -199,6 +202,7 @@ class Config: "socket_timeout", "extract_info_timeout", "debugpy_port", + "playlist_items_concurrency", ) "The variables that are integers."