added playlist items concurrency support.
This commit is contained in:
arabcoders
parent
872d85c458
commit
8a3673bc07
3 changed files with 94 additions and 39 deletions
79
README.md
79
README.md
|
|
@ -274,43 +274,44 @@ If you feel like donating and appreciate my work, you can do so by donating to c
|
|||
|
||||
Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file.
|
||||
|
||||
| Environment Variable | Description | Default |
|
||||
| ------------------------- | ------------------------------------------------------------------ | ---------------------------------- |
|
||||
| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` |
|
||||
| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` |
|
||||
| YTP_INSTANCE_TITLE | The title of the instance | `empty string` |
|
||||
| YTP_FILE_LOGGING | Whether to log to file | `false` |
|
||||
| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` |
|
||||
| YTP_MAX_WORKERS | How many works to use for downloads | `1` |
|
||||
| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` |
|
||||
| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` |
|
||||
| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` |
|
||||
| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` |
|
||||
| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` |
|
||||
| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` |
|
||||
| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` |
|
||||
| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` |
|
||||
| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
|
||||
| YTP_HOST | Which IP address to bind to | `0.0.0.0` |
|
||||
| YTP_PORT | Which port to bind to | `8081` |
|
||||
| YTP_LOG_LEVEL | Log level | `info` |
|
||||
| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` |
|
||||
| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` |
|
||||
| YTP_ACCESS_LOG | Whether to log access to the web server | `true` |
|
||||
| YTP_DEBUG | Whether to turn on debug mode | `false` |
|
||||
| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` |
|
||||
| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` |
|
||||
| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` |
|
||||
| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` |
|
||||
| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` |
|
||||
| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` |
|
||||
| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` |
|
||||
| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` |
|
||||
| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` |
|
||||
| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` |
|
||||
| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` |
|
||||
| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` |
|
||||
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
|
||||
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
|
||||
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer. | `*15 */1 * * *` |
|
||||
| Environment Variable | Description | Default |
|
||||
| ------------------------------ | ------------------------------------------------------------------ | ---------------------------------- |
|
||||
| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` |
|
||||
| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` |
|
||||
| YTP_INSTANCE_TITLE | The title of the instance | `empty string` |
|
||||
| YTP_FILE_LOGGING | Whether to log to file | `false` |
|
||||
| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` |
|
||||
| YTP_MAX_WORKERS | How many works to use for downloads | `1` |
|
||||
| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` |
|
||||
| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` |
|
||||
| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` |
|
||||
| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` |
|
||||
| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` |
|
||||
| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` |
|
||||
| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` |
|
||||
| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` |
|
||||
| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
|
||||
| YTP_HOST | Which IP address to bind to | `0.0.0.0` |
|
||||
| YTP_PORT | Which port to bind to | `8081` |
|
||||
| YTP_LOG_LEVEL | Log level | `info` |
|
||||
| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` |
|
||||
| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` |
|
||||
| YTP_ACCESS_LOG | Whether to log access to the web server | `true` |
|
||||
| YTP_DEBUG | Whether to turn on debug mode | `false` |
|
||||
| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` |
|
||||
| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` |
|
||||
| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` |
|
||||
| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` |
|
||||
| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` |
|
||||
| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` |
|
||||
| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` |
|
||||
| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` |
|
||||
| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` |
|
||||
| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` |
|
||||
| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` |
|
||||
| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` |
|
||||
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
|
||||
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
|
||||
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer. | `*15 */1 * * *` |
|
||||
| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time. | `1` |
|
||||
|
||||
|
|
|
|||
|
|
@ -202,6 +202,54 @@ class DownloadQueue(metaclass=Singleton):
|
|||
LOG.error(f"Failed to cancel downloads. {e!s}")
|
||||
|
||||
async def _process_playlist(self, entry: dict, item: Item, already=None):
|
||||
if 1 == self.config.playlist_items_concurrency:
|
||||
return await self._process_playlist_old(entry=entry, item=item, already=already)
|
||||
|
||||
LOG.info(f"Playlist '{entry.get('id')}: {entry.get('title')}' processing.")
|
||||
entries = entry.get("entries", [])
|
||||
playlistCount = int(entry.get("playlist_count", len(entries)))
|
||||
results = []
|
||||
|
||||
semaphore = asyncio.Semaphore(self.config.playlist_items_concurrency)
|
||||
|
||||
async def process_entry(i, etr):
|
||||
extras = {
|
||||
"playlist": entry.get("id"),
|
||||
"playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i),
|
||||
"playlist_autonumber": i,
|
||||
}
|
||||
|
||||
for property in ("id", "title", "uploader", "uploader_id"):
|
||||
if property in entry:
|
||||
extras[f"playlist_{property}"] = entry.get(property)
|
||||
|
||||
LOG.debug(f"Processing entry {i}/{playlistCount} - ID: {etr.get('id')} - Title: {etr.get('title')}")
|
||||
|
||||
if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
|
||||
extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
|
||||
|
||||
async with semaphore:
|
||||
return await self.add(
|
||||
item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras),
|
||||
already=already,
|
||||
)
|
||||
|
||||
tasks = [process_entry(i, etr) for i, etr in enumerate(entries, start=1)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
LOG.info(
|
||||
f"Playlist '{entry.get('id')}: {entry.get('title')}' processing completed with '{len(results)}' entries."
|
||||
)
|
||||
|
||||
if any("error" == res["status"] for res in results):
|
||||
return {
|
||||
"status": "error",
|
||||
"msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
|
||||
}
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
async def _process_playlist_old(self, entry: dict, item: Item, already=None):
|
||||
LOG.info(f"Playlist '{entry.get('id')}: {entry.get('title')}' processing.")
|
||||
entries = entry.get("entries", [])
|
||||
playlistCount = int(entry.get("playlist_count", len(entries)))
|
||||
|
|
@ -221,6 +269,8 @@ class DownloadQueue(metaclass=Singleton):
|
|||
if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
|
||||
extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
|
||||
|
||||
LOG.debug(f"Processing entry {i}/{playlistCount} - ID: {etr.get('id')} - Title: {etr.get('title')}")
|
||||
|
||||
results.append(
|
||||
await self.add(
|
||||
item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras),
|
||||
|
|
|
|||
|
|
@ -167,6 +167,9 @@ class Config:
|
|||
prevent_live_premiere: bool = False
|
||||
"""Prevent downloading of the initial premiere live broadcast."""
|
||||
|
||||
playlist_items_concurrency: int = 1
|
||||
"""The number of concurrent playlist items to be processed at same time."""
|
||||
|
||||
pictures_backends: list[str] = [
|
||||
"https://unsplash.it/1920/1080?random",
|
||||
"https://picsum.photos/1920/1080",
|
||||
|
|
@ -199,6 +202,7 @@ class Config:
|
|||
"socket_timeout",
|
||||
"extract_info_timeout",
|
||||
"debugpy_port",
|
||||
"playlist_items_concurrency",
|
||||
)
|
||||
"The variables that are integers."
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue