From a4827202d8faa32c8ac7edc3262cc8bfcade634e Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Fri, 24 Jan 2025 23:13:53 +0300 Subject: [PATCH] overhauled the tasks page to support adding/editing/deleting tasks from the WebUI directly instead of relying on tasks.json file. --- README.md | 46 +----- app/library/Emitter.py | 16 +- app/library/HttpAPI.py | 72 ++++++++- app/library/config.py | 71 +++++++++ app/main.py | 119 +++++++++++--- ui/components/Message.vue | 67 ++++++++ ui/components/NewDownload.vue | 18 +-- ui/components/TaskForm.vue | 275 +++++++++++++++++++++++++++++++++ ui/layouts/default.vue | 2 +- ui/pages/tasks.vue | 283 ++++++++++++++++++++++++++++------ ui/stores/SocketStore.js | 10 ++ 11 files changed, 854 insertions(+), 125 deletions(-) create mode 100644 ui/components/Message.vue create mode 100644 ui/components/TaskForm.vue diff --git a/README.md b/README.md index aa451dee..d7db6187 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,16 @@ Web GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp) with playlist & channel s YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since then it went under heavy changes, and it supports many new features. # YTPTube Features. +* Multi-downloads support. +* Handle live streams. +* Schedule Channels or Playlists to be downloaded automatically at a specific time. +* Support per link `yt-dlp JSON config or cli options`, `cookies` & `output format`. +* Queue multiple URLs separated by comma. * A built in video player that can play any video file regardless of the format. **With support for sidecar external subtitles**. * New `/api/add_batch` endpoint that allow multiple links to be sent. * Completely redesigned the frontend UI. * Switched out of binary file storage in favor of SQLite. -* Handle live streams. -* Support per link `yt-dlp JSON config or cli options`, `cookies` & `output format`. -* Tasks Runner. It allow you to queue channels for downloading using simple `json` file. * Webhook sender. It allow you to add webhook endpoints that receive events related to downloads using simple `json` file. -* Multi-downloads support. -* Queue multiple URLs separated by comma. * Basic Authentication support. * Support for curl_cffi, see [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation) * Support for both advanced and basic mode for WebUI. @@ -238,42 +238,6 @@ The `config/ytdlp.json`, is a json file which can be used to alter the default ` } ``` -### tasks.json File - -The `config/tasks.json`, is a json file, which can be used to queue URLs for downloading, it's mainly useful if you follow specific channels and you want it downloaded automatically, The schema for the file is as the following, Only the `URL` key is required. - -```json5 -[ - { - // (URL: string) **REQUIRED**, URL to the content. - "url": "", - // (Name: string) Optional field. Mainly used for logging. If omitted, random GUID will be shown. - "name": "My super secret channel", - // (Timer: string) Optional field. Using regular cronjob timer, if the field is omitted, it will run every hour in random minute. - "timer": "1 */1 * * *", - // (yt-dlp cookies: object) Optional field. A JSON cookies exported by flagCookies. - "ytdlp_cookies": {}, - // (yt-dlp config: object) Optional field. A JSON yt-dlp config. - "ytdlp_config": {}, - // (Output Template: string) Optional field. A File output format, - "output_template": "", - // (Folder: string) Optional field. Where to store the downloads relative to the main download path. - "folder":"", - // (preset: string) Optional field. The default preset to use for the download. if omitted, it will use the default preset. - "preset": "" - }, - { - // (URL: string) **REQUIRED**, URL to the content. - "url": "https://..." // This is valid config, it will queue the channel for downloading every hour at random minute. - }, - ... -] -``` - -The task runner is doing what you are doing when you click the add button on the WebGUI, this just fancy way to automate that. - -**WARNING**: We strongly advice turning on `YTP_KEEP_ARCHIVE` option. Otherwise, you will keep re-downloading the items, and you will eventually get banned from the source or or you will waste space, bandwidth re-downloading content over and over. - ### webhooks.json File The `config/webhooks.json`, is a json file, which can be used to add webhook endpoints that would receive events related to the downloads. diff --git a/app/library/Emitter.py b/app/library/Emitter.py index e22b0893..c130e523 100644 --- a/app/library/Emitter.py +++ b/app/library/Emitter.py @@ -9,9 +9,9 @@ class Emitter: This class is used to emit events to the registered emitters. """ - emitters: list[callable] = [] # type: ignore + emitters: list[callable] = [] # type: ignore - def add_emitter(self, emitter: callable): # type: ignore + def add_emitter(self, emitter: callable): # type: ignore """ Add an emitter to the list of emitters. @@ -44,6 +44,18 @@ class Emitter: async def warning(self, message: str, **kwargs): await self.emit("error", message, **kwargs) + async def info(self, message: str, data: dict = {}, **kwargs): + msg = {"type": "info", "message": message, "data": {}} + if data: + msg.update({"data": data}) + await self.emit("log_info", msg, **kwargs) + + async def success(self, message: str, data: dict = {}, **kwargs): + msg = {"type": "success", "message": message, "data": {}} + if data: + msg.update({"data": data}) + await self.emit("log_success", msg, **kwargs) + async def emit(self, event: str, data, **kwargs): tasks = [] diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index fd3ea578..1368d839 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -4,9 +4,11 @@ import hmac import json import logging import os +import random import time from datetime import datetime from pathlib import Path +import uuid import httpx import magic @@ -40,7 +42,7 @@ class HttpAPI(common): ".ico": "image/x-icon", } - def __init__(self, queue: DownloadQueue, emitter: Emitter, encoder: Encoder): + def __init__(self, queue: DownloadQueue, emitter: Emitter, encoder: Encoder, load_tasks: callable): super().__init__(queue=queue, encoder=encoder) self.rootPath = str(Path(__file__).parent.parent.parent) @@ -52,6 +54,7 @@ class HttpAPI(common): self.emitter = emitter self.queue = queue self.cache = Cache() + self.load_tasks = load_tasks def route(method: str, path: str): """ @@ -275,15 +278,68 @@ class HttpAPI(common): async def tasks(self, _: Request) -> Response: tasks_file: str = os.path.join(self.config.config_path, "tasks.json") - if not os.path.exists(tasks_file): - return web.json_response({"error": "No tasks defined."}, status=web.HTTPNotFound.status_code) + if os.path.exists(tasks_file): + try: + with open(tasks_file, "r") as f: + tasks = json.load(f) + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to load tasks."}, status=web.HTTPInternalServerError.status_code + ) + else: + tasks = [] + + return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=self.encoder.encode) + + @route("PUT", "api/tasks") + async def add_tasks(self, request: Request) -> Response: + tasks_file: str = os.path.join(self.config.config_path, "tasks.json") + + post = await request.json() + if not isinstance(post, list): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + tasks: list = [] + + for item in post: + if not isinstance(item, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + if not item.get("url"): + return web.json_response( + {"error": "url is required.", "data": post}, status=web.HTTPBadRequest.status_code + ) + + if not item.get("id", None): + item["id"] = str(uuid.uuid4()) + + if not item.get("timer", None): + item["timer"] = f"{random.randint(1,59)} */1 * * *" + + tasks.append(item) try: - with open(tasks_file, "r") as f: - tasks = json.load(f) + with open(tasks_file, "w") as f: + json.dump(tasks, f, indent=4) except Exception as e: LOG.exception(e) - return web.json_response({"error": "Failed to load tasks."}, status=web.HTTPInternalServerError.status_code) + return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code) + + try: + self.load_tasks() + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to reload tasks.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=self.encoder.encode) @@ -655,6 +711,10 @@ class HttpAPI(common): async def add_cors(self, _: Request) -> Response: return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code) + @route("OPTIONS", "api/tasks") + async def cors_add_tasks(self, _: Request) -> Response: + return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code) + @route("OPTIONS", "api/yt-dlp/convert") async def cors_ytdlp_convert(self, _: Request) -> Response: return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code) diff --git a/app/library/config.py b/app/library/config.py index eb1db1ab..9f437531 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -18,72 +18,137 @@ from .version import APP_VERSION class Config: config_path: str = "." + """The path to the configuration directory.""" + download_path: str = "." + """The path to the download directory.""" + temp_path: str = "/tmp" + """The path to the temporary directory.""" + temp_keep: bool = False + """Keep temporary files after the download is complete.""" url_host: str = "" + """The host to bind the server to.""" url_prefix: str = "" + """The prefix to use for the server URL.""" url_socketio: str = "{url_prefix}socket.io" + """The URL to use for the socket.io server.""" output_template: str = "%(title)s.%(ext)s" + """The output template to use for the downloaded files.""" + output_template_chapter: str = "%(title)s - %(section_number)s %(section_title)s.%(ext)s" + """The output template to use for the downloaded files with chapters.""" keep_archive: bool = True + """Keep the download archive file.""" ytdl_debug: bool = False + """Enable yt-dlp debugging.""" + allow_manifestless: bool = False + """Allow downloading videos without manifest.""" host: str = "0.0.0.0" + """The host to bind the server to.""" + port: int = 8081 + """The port to bind the server to.""" log_level: str = "info" + """The log level to use for the application.""" + max_workers: int = 1 + """The maximum number of workers to use for downloading.""" streamer_vcodec: str = "libx264" + """The video codec to use for streaming.""" streamer_acodec: str = "aac" + """The audio codec to use for streaming.""" auth_username: str | None = None + """The username to use for basic authentication.""" + auth_password: str | None = None + """The password to use for basic authentication.""" remove_files: bool = False + """Remove downloaded files when removing the record.""" access_log: bool = True + """Enable access logging.""" debug: bool = False + """Enable debugging.""" + debugpy_port: int = 5678 + """The port to use for the debugpy server.""" socket_timeout: int = 30 + """The socket timeout to use for yt-dlp.""" + extract_info_timeout: int = 70 + """The timeout to use for extracting video information.""" db_file: str = "{config_path}/ytptube.db" + """The path to the database file.""" + manual_archive: str = "{config_path}/archive.manual.log" + """The path to the manual archive file.""" + ui_update_title: bool = True + """Update the title of the browser tab with the current status.""" + pip_packages: str = "" + """The pip packages to install.""" + pip_ignore_updates: bool = False + """Ignore pip package updates.""" # immutable config vars. version: str = APP_VERSION + "The version of the application." + __instance = None + "The instance of the class." + ytdl_options: dict = {} + "The options to use for yt-dlp." + tasks: list = [] + "The tasks to run." + new_version_available: bool = False + "A new version of the application is available." + ytdlp_version: str = YTDLP_VERSION + "The version of yt-dlp." + started: int = 0 + "The time the application was started." + ignore_ui: bool = False + "Ignore the UI and run the application in the background." + basic_mode: bool = False + "Run the frontend in basic mode." presets: list = [ {"name": "default", "format": "default"}, ] + "The presets to use for downloading." default_preset: str = "default" + "The default preset to use when no preset is specified." _manual_vars: tuple = ( "temp_path", "config_path", "download_path", ) + "The variables that are set manually." _immutable: tuple = ( "version", @@ -95,6 +160,7 @@ class Config: "started", "presets", ) + "The variables that are immutable." _int_vars: tuple = ( "port", @@ -103,6 +169,8 @@ class Config: "extract_info_timeout", "debugpy_port", ) + "The variables that are integers." + _boolean_vars: tuple = ( "keep_archive", "ytdl_debug", @@ -116,6 +184,7 @@ class Config: "pip_ignore_updates", "basic_mode", ) + "The variables that are booleans." _frontend_vars: tuple = ( "download_path", @@ -132,8 +201,10 @@ class Config: "basic_mode", "default_preset", ) + "The variables that are relevant to the frontend." _manager: SyncManager | None = None + "The manager instance." @staticmethod def get_instance(): diff --git a/app/main.py b/app/main.py index c9403686..b4d019ec 100644 --- a/app/main.py +++ b/app/main.py @@ -1,17 +1,21 @@ #!/usr/bin/env python3 import asyncio +import json import logging import os import random import sqlite3 from datetime import datetime from pathlib import Path +import time +from typing import TypedDict import caribou import magic -from aiocron import crontab +from aiocron import crontab, Cron from aiohttp import web +from library.Utils import load_file from library.config import Config from library.DownloadQueue import DownloadQueue from library.Emitter import Emitter @@ -25,11 +29,17 @@ LOG = logging.getLogger("app") MIME = magic.Magic(mime=True) +class job_item(TypedDict): + name: str + job: Cron + + class Main: config: Config app: web.Application http: HttpAPI socket: HttpSocket + cron: list[job_item] = [] def __init__(self): self.config = Config.get_instance() @@ -51,17 +61,17 @@ class Main: connection.row_factory = sqlite3.Row connection.execute("PRAGMA journal_mode=wal") - emitter = Emitter() + self.emitter = Emitter() - queue = DownloadQueue(emitter=emitter, connection=connection) + queue = DownloadQueue(emitter=self.emitter, connection=connection) self.app.on_startup.append(lambda _: queue.initialize()) - self.http = HttpAPI(queue=queue, emitter=emitter, encoder=self.encoder) - self.socket = HttpSocket(queue=queue, emitter=emitter, encoder=self.encoder) + self.http = HttpAPI(queue=queue, emitter=self.emitter, encoder=self.encoder, load_tasks=self.load_tasks) + self.socket = HttpSocket(queue=queue, emitter=self.emitter, encoder=self.encoder) WebhookFile = os.path.join(self.config.config_path, "webhooks.json") if os.path.exists(WebhookFile): - emitter.add_emitter(Webhooks(WebhookFile).emit) + self.emitter.add_emitter(Webhooks(WebhookFile).emit) def checkFolders(self) -> None: try: @@ -105,38 +115,101 @@ class Main: try: taskName = task.get("name", task.get("url")) timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + started = time.time() + url = task.get("url") + if not url: + LOG.error(f"Invalid task '{task}'. No URL found.") + return + + preset: str = str(task.get("preset", self.config.default_preset)) + folder: str = str(task.get("folder")) if task.get("folder") else "" + ytdlp_cookies: str = str(task.get("ytdlp_cookies")) if task.get("ytdlp_cookies") else "" + output_template: str = str(task.get("output_template")) if task.get("output_template") else "" + + ytdlp_config = task.get("ytdlp_config") + if isinstance(ytdlp_config, str) and ytdlp_config: + try: + ytdlp_config = json.loads(ytdlp_config) + except Exception as e: + LOG.error(f"Failed to parse json yt-dlp config for '{taskName}'. {str(e)}") + return + + await self.emitter.info(f"Started 'Task: {taskName}' at '{timeNow}'.") LOG.info(f"Started 'Task: {taskName}' at '{timeNow}'.") await self.socket.add( - url=task.get("url"), - preset=task.get("preset", "default"), - folder=task.get("folder"), - ytdlp_cookies=task.get("ytdlp_cookies"), - ytdlp_config=task.get("ytdlp_config"), - output_template=task.get("output_template"), + url=url, + preset=preset, + folder=folder, + ytdlp_cookies=ytdlp_cookies, + ytdlp_config=ytdlp_config, + output_template=output_template, ) timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - LOG.info(f"Completed 'Task: {taskName}' at '{timeNow}'.") + + ended = time.time() + LOG.info(f"Completed 'Task: {taskName}' at '{timeNow}' ") + + await self.emitter.success(f"Completed 'Task: {taskName}' '{ended - started:.2f}' seconds.") except Exception as e: timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S") LOG.error(f"Failed 'Task: {taskName}' at '{timeNow}'. Error message '{str(e)}'.") + await self.emitter.error(f"Failed 'Task: {taskName}' at '{timeNow}'. Error message '{str(e)}'.") def load_tasks(self): - for task in self.config.tasks: + tasksFile = os.path.join(self.config.config_path, "tasks.json") + if not os.path.exists(tasksFile): + return + + LOG.info(f"Loading tasks from '{tasksFile}'.") + try: + (tasks, status, error) = load_file(tasksFile, list) + if not status: + LOG.error(f"Could not load tasks file from '{tasksFile}'. '{error}'.") + return + except Exception: + pass + + for job in self.cron: + try: + LOG.info(f"Stopping job '{job['name']}'.") + job["job"].stop() + except Exception as e: + LOG.error(f"Failed to stop job. Error message '{str(e)}'.") + LOG.exception(e) + + self.cron.clear() + + if not tasks or len(tasks) < 1: + LOG.warning(f"No tasks found in '{tasksFile}'.") + return + + loop = asyncio.get_event_loop() + for task in tasks: if not task.get("url"): LOG.warning(f"Invalid task '{task}'. No URL found.") continue - cron_timer: str = task.get("timer", f"{random.randint(1,59)} */1 * * *") + try: + cron_timer: str = task.get("timer", f"{random.randint(1,59)} */1 * * *") + self.cron.append( + job_item( + name=task.get("name", "??"), + job=crontab( + spec=cron_timer, + func=self.cron_runner, + args=(task,), + start=True, + loop=loop, + ), + ) + ) - crontab( - spec=cron_timer, - func=self.cron_runner, - args=(task,), - start=True, - loop=asyncio.get_event_loop(), - ) + LOG.info(f"Queued 'Task: {task.get('name','??')}' to be executed every '{cron_timer}'.") + except Exception as e: + LOG.error(f"Failed to add 'Task: {task.get('name', '??')}'. Error message '{str(e)}'.") + LOG.exception(e) - LOG.info(f"Added 'Task: {task.get('name', task.get('url'))}' to be executed every '{cron_timer}'.") + self.config.tasks = tasks def start(self): self.socket.attach(self.app) diff --git a/ui/components/Message.vue b/ui/components/Message.vue new file mode 100644 index 00000000..4209ce2d --- /dev/null +++ b/ui/components/Message.vue @@ -0,0 +1,67 @@ + + + diff --git a/ui/components/NewDownload.vue b/ui/components/NewDownload.vue index b6f26be9..0c6933ed 100644 --- a/ui/components/NewDownload.vue +++ b/ui/components/NewDownload.vue @@ -66,9 +66,9 @@ - - All output template naming options can be found at this page. + + All output template naming options can be found at this page. @@ -82,10 +82,10 @@ - + Some config fields are ignored like cookiefile, path, and output_format etc. - Available option can be found at + Available option can be found at this page. Warning: Use with caution some of those options can break yt-dlp or the frontend. @@ -100,9 +100,9 @@ - - Use something like flagCookies to extract cookies as JSON string. + + Use + flagCookies to extract cookies as JSON string. diff --git a/ui/components/TaskForm.vue b/ui/components/TaskForm.vue new file mode 100644 index 00000000..c8863e1d --- /dev/null +++ b/ui/components/TaskForm.vue @@ -0,0 +1,275 @@ + + + diff --git a/ui/layouts/default.vue b/ui/layouts/default.vue index e7a5f9d4..67e4a88a 100644 --- a/ui/layouts/default.vue +++ b/ui/layouts/default.vue @@ -42,7 +42,7 @@ -
+
diff --git a/ui/pages/tasks.vue b/ui/pages/tasks.vue index f9331b1a..4e7a1724 100644 --- a/ui/pages/tasks.vue +++ b/ui/pages/tasks.vue @@ -6,55 +6,114 @@ table.is-fixed { div.table-container { overflow: hidden; } + +div.is-centered { + justify-content: center; +}