diff --git a/README.md b/README.md index 95853cae..cc944ec1 100644 --- a/README.md +++ b/README.md @@ -274,41 +274,42 @@ If you feel like donating and appreciate my work, you can do so by donating to c Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file. -| Environment Variable | Description | Default | -| ------------------------ | ---------------------------------------------------------------- | ---------------------------------- | -| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | -| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | -| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | -| YTP_FILE_LOGGING | Whether to log to file | `false` | -| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | -| YTP_MAX_WORKERS | How many works to use for downloads | `1` | -| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | -| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | -| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | -| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | -| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | -| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | -| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | -| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | -| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_HOST | Which IP address to bind to | `0.0.0.0` | -| YTP_PORT | Which port to bind to | `8081` | -| YTP_LOG_LEVEL | Log level | `info` | -| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | -| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | -| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | -| YTP_DEBUG | Whether to turn on debug mode | `false` | -| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | -| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` | -| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | -| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | -| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` | -| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | -| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | -| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | -| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | -| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` | -| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | -| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | -| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| Environment Variable | Description | Default | +| ------------------------- | ------------------------------------------------------------------ | ---------------------------------- | +| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | +| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | +| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | +| YTP_FILE_LOGGING | Whether to log to file | `false` | +| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | +| YTP_MAX_WORKERS | How many works to use for downloads | `1` | +| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | +| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | +| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | +| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | +| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | +| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | +| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | +| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | +| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_HOST | Which IP address to bind to | `0.0.0.0` | +| YTP_PORT | Which port to bind to | `8081` | +| YTP_LOG_LEVEL | Log level | `info` | +| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | +| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | +| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | +| YTP_DEBUG | Whether to turn on debug mode | `false` | +| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | +| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` | +| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | +| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | +| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` | +| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | +| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | +| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | +| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | +| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` | +| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | +| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 5c3ec100..721b02ae 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -18,6 +18,7 @@ from .config import Config from .DataStore import DataStore from .Download import Download from .Events import EventBus, Events +from .Events import info as event_info from .Events import warning as event_warning from .ItemDTO import Item, ItemDTO from .Presets import Presets @@ -63,6 +64,12 @@ class DownloadQueue(metaclass=Singleton): _instance = None """Instance of the DownloadQueue.""" + queue: DataStore + """DataStore for the download queue.""" + + done: DataStore + """DataStore for the completed downloads.""" + def __init__(self, connection: Connection, config: Config | None = None): DownloadQueue._instance = self @@ -233,6 +240,7 @@ class DownloadQueue(metaclass=Singleton): options: dict = {} error: str | None = None live_in: str | None = None + is_premiere: bool = bool(entry.get("is_premiere", False)) # check if the video is live stream. if "live_status" in entry and "is_upcoming" == entry.get("live_status"): @@ -240,7 +248,7 @@ class DownloadQueue(metaclass=Singleton): live_in = formatdate(entry.get("release_timestamp"), usegmt=True) item.extras.update({"live_in": live_in}) else: - error = "Live stream not yet started. And no date is set." + error = f"No start time is set for {'premiere' if is_premiere else 'live stream'}." else: error = entry.get("msg") @@ -268,7 +276,7 @@ class DownloadQueue(metaclass=Singleton): is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) try: - download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder) + download_dir: str = calc_download_path(base_path=self.config.download_path, folder=item.folder) except Exception as e: LOG.exception(e) return {"status": "error", "msg": str(e)} @@ -281,6 +289,14 @@ class DownloadQueue(metaclass=Singleton): if isinstance(key, str) and key.startswith("playlist") and entry.get(key): item.extras[key] = entry.get(key) + item.extras["duration"] = entry.get("duration", item.extras.get("duration")) + + if not item.extras.get("live_in") and live_in: + item.extras["live_in"] = live_in + + if not item.extras.get("is_premiere") and is_premiere: + item.extras["is_premiere"] = is_premiere + dl = ItemDTO( id=str(entry.get("id")), title=str(entry.get("title")), @@ -304,18 +320,20 @@ class DownloadQueue(metaclass=Singleton): try: dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs) - text_logs = "" + text_logs: str = "" if filtered_logs := extract_ytdlp_logs(logs): text_logs = f" Logs: {', '.join(filtered_logs)}" if live_in or "is_upcoming" == entry.get("live_status"): NotifyEvent = Events.COMPLETED dlInfo.info.status = "not_live" - dlInfo.info.msg = "Stream is not live yet." + text_logs - itemDownload = self.done.put(dlInfo) + dlInfo.info.msg = ( + f"{'Premiere video' if is_premiere else 'Live Stream' } is not available yet." + text_logs + ) + itemDownload: Download = self.done.put(dlInfo) elif len(entry.get("formats", [])) < 1: - availability = entry.get("availability", "public") - msg = "No formats found." + availability: str = entry.get("availability", "public") + msg: str = "No formats found." if availability and availability not in ("public",): msg += f" Availability is set for '{availability}'." @@ -323,7 +341,27 @@ class DownloadQueue(metaclass=Singleton): dlInfo.info.status = "error" itemDownload = self.done.put(dlInfo) NotifyEvent = Events.COMPLETED - await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg)) + await self._notify.emit(Events.LOG_WARNING, data=event_warning(f"No formats found for '{dl.title}'.")) + elif is_premiere and self.config.prevent_live_premiere: + dlInfo.info.error = ( + f"Premiering right now. Delaying download by '{300+dl.extras.get('duration',0)}' seconds." + ) + _live_in = live_in or item.extras.get("live_in", None) + if _live_in: + starts_in = parsedate_to_datetime(live_in) + starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) + starts_in = starts_in + timedelta(minutes=5, seconds=dl.extras.get("duration", 0)) + dlInfo.info.error += f" Starts in {starts_in.isoformat()}." + + dlInfo.info.status = "not_live" + itemDownload = self.done.put(dlInfo) + NotifyEvent = Events.COMPLETED + await self._notify.emit( + Events.LOG_INFO, + data=event_info( + f"'{dl.title}' is premiering. Download delayed by '{300+dl.extras.get('duration')}'s." + ), + ) else: NotifyEvent = Events.ADDED itemDownload = self.queue.put(dlInfo) @@ -464,12 +502,10 @@ class DownloadQueue(metaclass=Singleton): if not entry: return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)} - if not item.requeued: - condition = Conditions.get_instance().match(info=entry) - if condition is not None: - already.pop() - LOG.info(f"Condition '{condition}' matched for '{item.url}'.") - return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) + if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): + already.pop() + LOG.info(f"Condition '{condition.name}' matched for '{item.url}'.") + return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) end_time = time.perf_counter() - started LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.") @@ -566,7 +602,7 @@ class DownloadQueue(metaclass=Singleton): removed_files = 0 filename: str = "" - LOG.info( + LOG.debug( f"{remove_file=} {itemRef} - Removing local files: {self.config.remove_files}, {item.info.status=}" ) @@ -827,32 +863,44 @@ class DownloadQueue(metaclass=Singleton): if self.is_paused() or self.done.empty(): return - LOG.debug("Checking for live stream items in the history queue.") + LOG.debug("Checking history queue for queued live stream links.") time_now = datetime.now(tz=UTC) - status = ["not_live", "is_upcoming", "is_live"] + status: list[str] = ["not_live", "is_upcoming", "is_live"] for id, item in list(self.done.items()): if item.info.status not in status: continue - item_ref = f"{id=} {item.info.id=} {item.info.title=}" + item_ref: str = f"{id=} {item.info.id=} {item.info.title=}" if not item.is_live: LOG.debug(f"Item '{item_ref}' is not a live stream.") continue - if not item.info.live_in: + live_in: str | None = item.info.live_in or item.info.extras.get("live_in", None) + if not live_in: LOG.debug(f"Item '{item_ref}' marked as live stream, but no date is set.") continue - starts_in = parsedate_to_datetime(item.info.live_in) + starts_in = parsedate_to_datetime(live_in) starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) if time_now < (starts_in + timedelta(minutes=1)): LOG.debug(f"Item '{item_ref}' is not yet live. will start in '{dt_delta(starts_in-time_now)}'.") continue + duration: int | None = item.info.extras.get("duration", None) + is_premiere: bool = item.info.extras.get("is_premiere", False) + + if is_premiere and duration and self.config.prevent_live_premiere: + premiere_ends: datetime = starts_in + timedelta(minutes=5, seconds=duration) + if time_now < premiere_ends: + LOG.debug( + f"Item '{item_ref}' is premiering and download is delayed by '{300+duration}' seconds. Will start at '{premiere_ends.isoformat()}'" + ) + continue + LOG.info(f"Re-queuing item '{item_ref} {item.info.extras=}' for download.") try: @@ -862,17 +910,18 @@ class DownloadQueue(metaclass=Singleton): continue try: - info = item.info - new_queue = Item( - url=info.url, - preset=info.preset, - folder=info.folder, - cookies=info.cookies, - template=info.template, - cli=item.info.cli, - extras=item.info.extras, + await self.add( + item=Item( + url=item.info.url, + preset=item.info.preset, + folder=item.info.folder, + cookies=item.info.cookies, + template=item.info.template, + cli=item.info.cli, + extras=item.info.extras, + ) ) - await self.add(item=new_queue) except Exception as e: - LOG.error(f"Failed to re-queue item '{item_ref}'. {e!s}") + self.done.put(item) LOG.exception(e) + LOG.error(f"Failed to re-queue item '{item_ref}'. {e!s}") diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index d11b693a..27f4a241 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -1,140 +1,43 @@ -import asyncio import base64 -import functools import hmac -import json +import inspect import logging -import random -import time -import uuid from collections.abc import Awaitable from datetime import UTC, datetime, timedelta from pathlib import Path -from urllib.parse import unquote_plus, urlparse +from typing import Any import anyio -import httpx -import magic from aiohttp import web from aiohttp.web import Request, RequestHandler, Response -from library.ag_utils import ag -from yt_dlp.cookies import LenientSimpleCookie from .cache import Cache -from .common import Common -from .conditions import Condition, Conditions from .config import Config from .DownloadQueue import DownloadQueue from .encoder import Encoder -from .Events import EventBus, Events, message +from .Events import EventBus from .ffprobe import ffprobe -from .ItemDTO import Item -from .M3u8 import M3u8 -from .Notifications import Notification, NotificationEvents -from .Playlist import Playlist -from .Presets import Preset, Presets -from .Segments import Segments -from .Subtitle import Subtitle -from .Tasks import Task, Tasks -from .Utils import ( - REMOVE_KEYS, - StreamingError, - arg_converter, - decrypt_data, - encrypt_data, - extract_info, - get_file, - get_file_sidecar, - get_files, - get_mime_type, - init_class, - read_logfile, - validate_url, - validate_uuid, -) -from .YTDLPOpts import YTDLPOpts +from .router import RouteType, get_routes +from .Utils import decrypt_data, encrypt_data, get_file, get_mime_type, load_modules -LOG = logging.getLogger("http_api") -MIME = magic.Magic(mime=True) +LOG: logging.Logger = logging.getLogger("http_api") -class HttpAPI(Common): - _static_holder: dict = {} - """Holds loaded static assets.""" +class HttpAPI: + def __init__(self, root_path: Path, queue: DownloadQueue): + self.queue: DownloadQueue = queue or DownloadQueue.get_instance() + self.encoder: Encoder = Encoder() + self.config: Config = Config.get_instance() + self._notify: EventBus = EventBus.get_instance() - _ext_to_mime: dict = { - ".html": "text/html", - ".css": "text/css", - ".js": "application/javascript", - ".json": "application/json", - ".ico": "image/x-icon", - } - """Map ext to mimetype""" - - _frontend_routes: list[str] = [ - "/console", - "/presets", - "/tasks", - "/notifications", - "/changelog", - "/logs", - "/conditions", - "/browser", - "/browser/{path:.*}", - ] - """Frontend routes to be preloaded""" - - def __init__( - self, - root_path: str, - queue: DownloadQueue | None = None, - encoder: Encoder | None = None, - config: Config | None = None, - ): - self.queue = queue or DownloadQueue.get_instance() - self.encoder = encoder or Encoder() - self.config = config or Config.get_instance() - self._notify = EventBus.get_instance() - - self.rootPath = root_path - self.routes = web.RouteTableDef() + self.rootPath: Path = root_path self.cache = Cache() self.app: web.Application | None = None - self.isRequestingBackground = False - - super().__init__(queue=self.queue, encoder=self.encoder, config=self.config) - - @staticmethod - def route(method: str, path: str, name: str | None = None) -> Awaitable: - """ - Decorator to mark a method as an HTTP route handler. - - Args: - method (str): The HTTP method. - path (str): The path to the route. - name (str | None): The name of the route (optional). - - Returns: - Awaitable: The decorated function. - - """ - - def decorator(func): - @functools.wraps(func) - async def wrapper(*args, **kwargs): - return await func(*args, **kwargs) - - wrapper._http_method = method.upper() - wrapper._http_path = path - wrapper._http_name = name - return wrapper - - return decorator async def on_shutdown(self, _: web.Application): pass - def attach(self, app: web.Application) -> "HttpAPI": + def attach(self, app: web.Application): """ Attach the routes to the application. @@ -152,6 +55,7 @@ class HttpAPI(Common): app=app, base_path=self.config.base_path.rstrip("/"), download_path=self.config.download_path, + this=self, ) ) @@ -176,114 +80,8 @@ class HttpAPI(Common): LOG.exception(e) app.on_shutdown.append(self.on_shutdown) - return self - async def _static_file(self, req: Request) -> Response: - """ - Preload static files from the ui/exported folder. - - Args: - req (Request): The request object. - - Returns: - Response: The response object. - - """ - path = req.path - - if "/" != self.config.base_path and path.rstrip("/") == self.config.base_path.rstrip("/"): - path: str = f"{self.config.base_path.rstrip('/')}/index.html" - - if path not in self._static_holder: - for k in self._static_holder: - if path.startswith(k): - path = k - break - else: - return web.json_response( - {"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code - ) - - item: dict = self._static_holder[path] - - return web.Response( - body=item.get("content"), - headers={ - "Pragma": "public", - "Cache-Control": "public, max-age=31536000", - "Content-Type": item.get("content_type"), - "X-Via": "memory" if not item.get("file") else "disk", - }, - status=web.HTTPOk.status_code, - ) - - def _preload_static(self, app: web.Application) -> "HttpAPI": - """ - Preload static files from the ui/exported folder. - - Args: - app (web.Application): The application to attach the routes to. - - Returns: - HttpAPI: The instance of the HttpAPI. - - """ - staticDir = (Path(self.rootPath) / "ui" / "exported").absolute() - if not staticDir.exists(): - staticDir = (Path(self.rootPath).parent / "ui" / "exported").absolute() - if not staticDir.exists(): - msg = f"Could not find the frontend UI static assets. '{staticDir}'." - raise ValueError(msg) - - preloaded = 0 - - base_path: str = str(Path(self.config.base_path).as_posix()).rstrip("/") - - for file in staticDir.rglob("*.*"): - if ".map" == file.suffix: - continue - - urlPath: str = f"{base_path}/{str(file.as_posix()).replace(f'{staticDir.as_posix()!s}/', '')}" - - with open(file, "rb") as f: - content = f.read() - - contentType = self._ext_to_mime.get(file.suffix, MIME.from_file(file)) - - self._static_holder[urlPath] = {"content": content, "content_type": contentType} - LOG.debug(f"Preloading static '{urlPath}'.") - app.router.add_get(urlPath, self._static_file) - preloaded += 1 - - if "/index.html" in self._static_holder: - for path in self._frontend_routes: - path: str = f"{base_path}/{path.lstrip('/')}" - self._static_holder[path] = self._static_holder["/index.html"] - app.router.add_get(path, self._static_file) - if "{" not in path: - self._static_holder[path + "/"] = self._static_holder["/index.html"] - app.router.add_get(path + "/", self._static_file) - LOG.debug(f"Preloading static route '{path}'.") - preloaded += 1 - - if "/" != self.config.base_path: - LOG.debug(f"adding base_path folder '{base_path}' to routes.") - app.router.add_get(base_path, self._static_file, name="_base_path") - app.router.add_get(f"{base_path}/", self._static_file, name="_base_path_slash") - - if preloaded < 1: - message = f"Failed to find any static files in '{staticDir}'." - if self.config.ignore_ui: - LOG.warning(message) - return self - - raise ValueError(message) - - LOG.info(f"Preloaded '{preloaded}' static files.") - - return self - - def add_routes(self, app: web.Application) -> "HttpAPI": + def add_routes(self, app: web.Application): """ Add the routes to the application. @@ -297,43 +95,33 @@ class HttpAPI(Common): registered_options: list = [] base_path: str = self.config.base_path.rstrip("/") + from app.routes.api._static import preload_static - for attr_name in dir(self): - method = getattr(self, attr_name) - if hasattr(method, "_http_method") and hasattr(method, "_http_path"): - http_path = method._http_path - if http_path.startswith("/"): - http_path = method._http_path[1:] + load_modules(self.rootPath, self.rootPath / "routes" / "api") + preload_static(self.rootPath, self.config) - opts = {} - if hasattr(method, "_http_name") and method._http_name: - opts["name"] = method._http_name + async def options_handler(_: Request) -> Response: + return web.Response(status=204) - LOG.debug(f"Adding API route {method._http_method} {base_path}/{http_path}' {opts}.") - self.routes.route(method._http_method, f"{base_path}/{http_path}", **opts)(method) + for route in get_routes(RouteType.HTTP).values(): + routePath: str = f"/{route.path.lstrip('/')}" - if http_path in registered_options: - continue + if self.config.base_path == route.path: + pass + elif "" == base_path or not routePath.rstrip("/").startswith(base_path.rstrip("/")): + route.path = f"{base_path}/{route.path.lstrip('/')}" - async def options_handler(_: Request) -> Response: - return web.Response(status=204) + LOG.debug(f"Add ({route.name}) {route.method}: {route.path}.") - if "name" in opts: - opts["name"] = f"{opts['name']}_options" + app.router.add_route(route.method, route.path, handler=route.handler, name=route.name) - self.routes.route("OPTIONS", f"{base_path}/{http_path}", **opts)(options_handler) - registered_options.append(http_path) + if route.path in registered_options: + continue - self.routes.static(f"{base_path}/api/download/", self.config.download_path, name="download_static") - self._preload_static(app) + app.router.add_route("OPTIONS", route.path, handler=options_handler, name=f"{route.name}_opts") + registered_options.append(route.path) - try: - app.add_routes(self.routes) - except ValueError as e: - if "ui/exported" in str(e): - msg = f"Could not find the frontend UI static assets. '{e}'." - raise RuntimeError(msg) from e - raise + app.router.add_static(f"{base_path}/api/download/", self.config.download_path, name="download_static") @staticmethod def basic_auth(username: str, password: str) -> Awaitable: @@ -422,7 +210,7 @@ class HttpAPI(Common): return middleware_handler @staticmethod - def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable: + def middle_wares(app: web.Application, base_path: str, download_path: str, this: "HttpAPI") -> Awaitable: @web.middleware async def middleware_handler(request: Request, handler: RequestHandler) -> Response: static_path = str(app.router["download_static"].url_for(filename="")) @@ -443,8 +231,33 @@ class HttpAPI(Common): }, ) + kwargs: dict[str, Any] = { + "request": request, + "queue": this.queue, + "encoder": this.encoder, + "config": this.config, + "notify": this._notify, + "cache": this.cache, + "app": app, + "http_api": this, + "root_path": this.rootPath, + } + try: - response = await handler(request) + sig = inspect.signature(handler) + expected_args = sig.parameters.keys() + + try: + if 1 == len(expected_args) and "request" in expected_args: + response = await handler(request) + else: + filtered = {k: v for k, v in kwargs.items() if k in expected_args} + response = await handler(**filtered) + except TypeError as te: + if "missing 1 required positional argument" in str(te) and "request" in str(te): + response = await handler(request) + else: + raise except web.HTTPException as e: return web.json_response(data={"error": str(e)}, status=e.status_code) except Exception as e: @@ -457,15 +270,21 @@ class HttpAPI(Common): contentType: str | None = response.headers.get("content-type", None) if contentType and "/" != base_path and contentType.startswith("text/html"): rewrite_path: str = base_path.rstrip("/") - content: str = ( - response.body.decode("utf-8") - .replace('', f'') - .replace('baseURL:""', f'baseURL:"{rewrite_path}/"') + async with await anyio.open_file(response._path, "rb") as f: + content = await f.read() + content: str = ( + content.decode("utf-8") + .replace('', f'') + .replace('baseURL:""', f'baseURL:"{rewrite_path}/"') + ) + + return web.Response( + body=content.encode("utf-8"), + status=response.status, + headers=response.headers, ) - response.body = content.encode("utf-8") - - if isinstance(response, web.FileResponse): + if request.path.startswith(static_path) and isinstance(response, web.FileResponse): try: ff_info = await ffprobe(response._path) mime_type = get_mime_type(ff_info.get("metadata", {}), response._path) @@ -476,1608 +295,3 @@ class HttpAPI(Common): return response return middleware_handler - - @route("GET", "api/ping", "ping") - async def ping(self, _: Request) -> Response: - """ - Ping the server. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - await self.queue.test() - return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code) - - @route("POST", "api/yt-dlp/convert", "yt_dlp_convert") - async def yt_dlp_convert(self, request: Request) -> Response: - """ - Convert the yt-dlp args to a dict. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - post = await request.json() - args: str | None = post.get("args") - - if not args: - return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code) - - try: - response = {"opts": {}, "output_template": None, "download_path": None} - - data = arg_converter(args, dumps=True) - - if "outtmpl" in data and "default" in data["outtmpl"]: - response["output_template"] = data["outtmpl"]["default"] - - if "paths" in data and "home" in data["paths"]: - response["download_path"] = data["paths"]["home"] - - if "format" in data: - response["format"] = data["format"] - - bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} - removed_options = [] - - for key in data: - if key in bad_options.items(): - removed_options.append(bad_options[key]) - continue - if not key.startswith("_"): - response["opts"][key] = data[key] - - if len(removed_options) > 0: - response["removed_options"] = removed_options - - return web.json_response(data=response, status=web.HTTPOk.status_code) - except Exception as e: - err = str(e).strip() - err = err.split("\n")[-1] if "\n" in err else err - err = err.replace("main.py: error: ", "").strip().capitalize() - return web.json_response( - data={"error": f"Failed to parse command options for yt-dlp. '{err}'."}, - status=web.HTTPBadRequest.status_code, - ) - - @route("GET", "api/yt-dlp/url/info", "get_info") - async def get_info(self, request: Request) -> Response: - """ - Get the video info. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - url = request.query.get("url") - if not url: - return web.json_response(data={"error": "URL is required."}, status=web.HTTPBadRequest.status_code) - - try: - validate_url(url) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - preset = request.query.get("preset") - if preset: - exists = Presets.get_instance().get(preset) - if not exists: - return web.json_response( - data={"status": False, "message": f"Preset '{preset}' does not exist."}, - status=web.HTTPBadRequest.status_code, - ) - else: - preset = self.config.default_preset - - try: - key = self.cache.hash(url + str(preset)) - - if self.cache.has(key) and not request.query.get("force", False): - data = self.cache.get(key) - data["_cached"] = { - "status": "hit", - "key": key, - "ttl": data.get("_cached", {}).get("ttl", 300), - "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(), - "expires": data.get("_cached", {}).get("expires", time.time() + 300), - } - return web.Response(body=json.dumps(data, indent=4), status=web.HTTPOk.status_code) - - opts = {} - - if ytdlp_proxy := self.config.get_ytdlp_args().get("proxy", None): - opts["proxy"] = ytdlp_proxy - - ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset).add(opts).get_all() - - data = extract_info( - config=ytdlp_opts, - url=url, - debug=False, - no_archive=True, - follow_redirect=True, - sanitize_info=True, - ) - - if "formats" in data: - for index, item in enumerate(data["formats"]): - if "cookies" in item and len(item["cookies"]) > 0: - cookies = [f"{c.key}={c.value}" for c in LenientSimpleCookie(item["cookies"]).values()] - if len(cookies) > 0: - data["formats"][index]["h_cookies"] = "; ".join(cookies) - data["formats"][index]["h_cookies"] = data["formats"][index]["h_cookies"].strip() - - data["_cached"] = { - "status": "miss", - "key": key, - "ttl": 300, - "ttl_left": 300, - "expires": time.time() + 300, - } - - self.cache.set(key=key, value=data, ttl=300) - - return web.Response(body=json.dumps(data, indent=4), status=web.HTTPOk.status_code) - except Exception as e: - LOG.exception(e) - LOG.error(f"Error encountered while getting video info for '{url}'. '{e!s}'.") - return web.json_response( - data={ - "error": "failed to get video info.", - "message": str(e), - "formats": [], - }, - status=web.HTTPInternalServerError.status_code, - ) - - @route("GET", "api/yt-dlp/archive/recheck", "archive_recheck") - async def archive_recheck(self, _) -> Response: - """ - Recheck the manual archive entries. - - Args: - _ (Request): The request object. - - Returns: - Response: The response object - - """ - manual_archive = self.config.manual_archive - if not manual_archive: - return web.json_response( - data={"error": "Manual archive is not enabled."}, status=web.HTTPNotFound.status_code - ) - - manual_archive = Path(manual_archive) - if not manual_archive.exists(): - return web.json_response( - data={"error": "Manual archive file not found.", "file": manual_archive}, - status=web.HTTPNotFound.status_code, - ) - - tasks = [] - response = [] - - def info_wrapper(id: str, url: str) -> tuple[str, dict]: - try: - return ( - id, - extract_info( - config={ - "proxy": self.config.get_ytdlp_args().get("proxy", None), - "simulate": True, - "dump_single_json": True, - }, - url=url, - no_archive=True, - ), - ) - except Exception as e: - return (id, {"error": str(e)}) - - async with await anyio.open_file(manual_archive) as f: - # line format is "youtube ID - at: ISO8601" - async for line in f: - line = line.strip() - - if not line or not line.startswith("youtube"): - continue - - id = line.split(" ")[1].strip() - - if not id: - continue - - url = f"https://www.youtube.com/watch?v={id}" - key = self.cache.hash(id) - - if self.cache.has(key): - data = self.cache.get(key) - response.append({id: bool(data.get("id", None)) if isinstance(data, dict) else False}) - continue - - tasks.append( - asyncio.get_event_loop().run_in_executor(None, lambda i=id, url=url: info_wrapper(id=i, url=url)) - ) - - if len(tasks) > 0: - results = await asyncio.gather(*tasks) - for data in results: - if not data: - continue - - id, info = data - self.cache.set(key=self.cache.hash(id), value=info, ttl=3600 * 6) - response.append({id: bool(data.get("id", None)) if isinstance(data, dict) else False}) - - return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("GET", "api/history/add", "quick_add") - async def quick_add(self, request: Request) -> Response: - """ - Add a URL to the download queue. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - url: str | None = request.query.get("url") - if not url: - return web.json_response(data={"error": "url param is required."}, status=web.HTTPBadRequest.status_code) - - data = { - "url": url, - } - - preset = request.query.get("preset") - if preset: - exists = Presets.get_instance().get(preset) - if not exists: - return web.json_response( - data={"status": False, "message": f"Preset '{preset}' does not exist."}, - status=web.HTTPBadRequest.status_code, - ) - data["preset"] = preset - - try: - status = await self.add(item=Item.format(data)) - except ValueError as e: - return web.json_response(data={"status": False, "message": str(e)}, status=web.HTTPBadRequest.status_code) - - return web.json_response( - data={"status": status.get("status") == "ok", "message": status.get("msg", "URL added")}, - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("POST", "api/history", "history_item_add") - async def history_item_add(self, request: Request) -> Response: - """ - Add a URL to the download queue. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - data = await request.json() - - if isinstance(data, dict): - data = [data] - - items = [] - for item in data: - try: - items.append(Item.format(item)) - except ValueError as e: - return web.json_response(data={"error": str(e), "data": item}, status=web.HTTPBadRequest.status_code) - - status = await asyncio.wait_for( - fut=asyncio.gather(*[self.add(item=item) for item in items]), - timeout=None, - ) - response = [] - for i, item in enumerate(items): - response.append({"item": item, "status": status[i].get("status") == "ok", "msg": status[i].get("msg")}) - - return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("GET", "api/logs", "logs") - async def logs(self, request: Request) -> Response: - """ - Get recent logs - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - if not self.config.file_logging: - return web.json_response( - data={"error": "File logging is not enabled."}, status=web.HTTPNotFound.status_code - ) - - offset = int(request.query.get("offset", 0)) - limit = int(request.query.get("limit", 100)) - if limit < 1 or limit > 150: - limit = 50 - - logs_data = await read_logfile( - file=Path(self.config.config_path) / "logs" / "app.log", - offset=offset, - limit=limit, - ) - return web.json_response( - data={ - "logs": logs_data["logs"], - "offset": offset, - "limit": limit, - "next_offset": logs_data["next_offset"], - "end_is_reached": logs_data["end_is_reached"], - }, - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("GET", "api/conditions", "conditions") - async def conditions(self, _: Request) -> Response: - """ - Get the conditions - - Args: - _ (Request): The request object. - - Returns: - Response: The response object. - - """ - return web.json_response( - data=Conditions.get_instance().get_all(), - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("PUT", "api/conditions", "conditions_add") - async def conditions_add(self, request: Request) -> Response: - """ - Save Conditions. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - items: list = [] - - cls = Conditions.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("name"): - return web.json_response( - {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("filter"): - return web.json_response( - {"error": "filter is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("cli"): - return web.json_response( - {"error": "command options for yt-dlp is required.", "data": item}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - cls.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate condition '{item.get('name')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - items.append(init_class(Condition, item)) - try: - items = cls.save(items=items).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save conditions.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - await self._notify.emit(Events.CONDITIONS_UPDATE, data=items) - return web.json_response(data=items, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("POST", "api/conditions/test", "conditions_test") - async def conditions_test(self, request: Request) -> Response: - """ - Test condition against URL. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - params = await request.json() - - if not isinstance(params, dict): - return web.json_response( - {"error": "Invalid request body expecting dict."}, - status=web.HTTPBadRequest.status_code, - ) - - url = params.get("url") - if not url: - return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) - - cond = params.get("condition") - if not cond: - return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code) - - try: - preset = params.get("preset", self.config.default_preset) - key = self.cache.hash(url + str(preset)) - if not self.cache.has(key): - opts = {} - if ytdlp_proxy := self.config.get_ytdlp_args().get("proxy", None): - opts["proxy"] = ytdlp_proxy - ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset).add(opts).get_all() - - data = extract_info( - config=ytdlp_opts, - url=url, - debug=False, - no_archive=True, - follow_redirect=True, - sanitize_info=True, - ) - if not data: - return web.json_response( - data={"error": f"Failed to extract info from '{url!s}'."}, - status=web.HTTPBadRequest.status_code, - ) - self.cache.set(key=key, value=data, ttl=600) - else: - data = self.cache.get(key) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": f"Failed to extract video info. '{e!s}'"}, - status=web.HTTPInternalServerError.status_code, - ) - - try: - from yt_dlp.utils import match_str - - status = match_str(cond, data) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": str(e)}, - status=web.HTTPBadRequest.status_code, - ) - - return web.json_response( - data={"status": status, "condition": cond, "data": data}, - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("GET", "api/presets", "presets") - async def presets(self, request: Request) -> Response: - """ - Get the presets. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - data = Presets.get_instance().get_all() - filter_fields = request.query.get("filter", None) - - if filter_fields: - fields = [field.strip() for field in filter_fields.split(",")] - data = [{key: value for key, value in preset.__dict__.items() if key in fields} for preset in data] - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("PUT", "api/presets", "presets_add") - async def presets_add(self, request: Request) -> Response: - """ - Add presets. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - presets: list = [] - - cls = Presets.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("name"): - return web.json_response( - {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - cls.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate preset '{item.get('name')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - presets.append(Preset(**item)) - try: - presets = cls.save(items=presets).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save presets.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - await self._notify.emit(Events.PRESETS_UPDATE, data=presets) - return web.json_response(data=presets, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("GET", "api/tasks", "tasks") - async def tasks(self, _: Request) -> Response: - """ - Get the tasks. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - return web.json_response( - data=Tasks.get_instance().get_all(), status=web.HTTPOk.status_code, dumps=self.encoder.encode - ) - - @route("PUT", "api/tasks", "tasks_add") - async def tasks_add(self, request: Request) -> Response: - """ - Add tasks to the queue. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - tasks: list = [] - - ins = Tasks.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("url"): - return web.json_response( - {"error": "url is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - if not item.get("template", None): - item["template"] = "" - - if not item.get("cli", None): - item["cli"] = "" - - try: - Tasks.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate task '{item.get('name')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - tasks.append(Task(**item)) - - try: - tasks = ins.save(tasks=tasks).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save tasks.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("DELETE", "api/history", "history_delete") - async def history_delete(self, request: Request) -> Response: - """ - Delete an item from the queue. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - data = await request.json() - ids = data.get("ids") - where = data.get("where") - if not ids or where not in ["queue", "done"]: - return web.json_response( - data={"error": "ids and where are required."}, status=web.HTTPBadRequest.status_code - ) - - remove_file: bool = bool(data.get("remove_file", True)) - - return web.json_response( - data=await (self.queue.cancel(ids) if where == "queue" else self.queue.clear(ids, remove_file=remove_file)), - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("POST", "api/history/{id}", "history_item_update") - async def history_item_update(self, request: Request) -> Response: - """ - Update an item in the history. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - id: str = request.match_info.get("id") - if not id: - return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) - - item = self.queue.done.get_by_id(id) - if not item: - return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) - - post = await request.json() - if not post: - return web.json_response(data={"error": "no data provided."}, status=web.HTTPBadRequest.status_code) - - updated = False - - for k, v in post.items(): - if not hasattr(item.info, k): - continue - - if getattr(item.info, k) == v: - continue - - updated = True - setattr(item.info, k, v) - LOG.debug(f"Updated '{k}' to '{v}' for '{item.info.id}'") - - if updated: - self.queue.done.put(item) - await self._notify.emit(Events.UPDATE, data=item.info) - - return web.json_response( - data=item.info, - status=web.HTTPOk.status_code if updated else web.HTTPNotModified.status_code, - dumps=self.encoder.encode, - ) - - @route("GET", "api/history", "history") - async def history(self, _: Request) -> Response: - """ - Get the history. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - data: dict = {"queue": [], "history": []} - q = self.queue.get() - - data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) - data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist") - async def playlist(self, request: Request) -> Response: - """ - Get the playlist. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - file: str = request.match_info.get("file") - - if not file: - return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) - - base_path: str = self.config.base_path.rstrip("/") - - try: - realFile, status = get_file(download_path=self.config.download_path, file=file) - if web.HTTPFound.status_code == status: - return Response( - status=web.HTTPFound.status_code, - headers={ - "Location": str( - self.app.router["playlist"].url_for( - file=str(realFile).replace(self.config.download_path, "").strip("/") - ) - ), - }, - ) - - if web.HTTPNotFound.status_code == status: - return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) - - return web.Response( - text=await Playlist(download_path=Path(self.config.download_path), url=f"{base_path}/").make( - file=realFile - ), - headers={ - "Content-Type": "application/x-mpegURL", - "Cache-Control": "no-cache", - "Access-Control-Max-Age": "300", - }, - status=web.HTTPOk.status_code, - ) - except StreamingError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code) - - @route("GET", "api/player/m3u8/{mode}/{file:.*}.m3u8", "m3u8") - async def m3u8(self, request: Request) -> Response: - """ - Get the m3u8 file. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - file: str = request.match_info.get("file") - mode: str = request.match_info.get("mode") - - if mode not in ["video", "subtitle"]: - return web.json_response( - data={"error": "Only video and subtitle modes are supported."}, status=web.HTTPBadRequest.status_code - ) - - if not file: - return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) - - duration = request.query.get("duration", None) - - if "subtitle" in mode: - if not duration: - return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code) - - duration = float(duration) - - base_path: str = self.config.base_path.rstrip("/") - - try: - cls = M3u8(download_path=Path(self.config.download_path), url=f"{base_path}/") - - realFile, status = get_file(download_path=self.config.download_path, file=file) - if web.HTTPFound.status_code == status: - return Response( - status=status, - headers={ - "Location": str( - self.app.router["m3u8"].url_for( - mode=mode, - file=str(realFile).replace(self.config.download_path, "").strip("/"), - ) - ), - }, - ) - - if web.HTTPNotFound.status_code == status: - return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) - - if "subtitle" in mode: - text = await cls.make_subtitle(file=realFile, duration=duration) - else: - text = await cls.make_stream(file=realFile) - except StreamingError as e: - LOG.exception(e) - return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code) - - return web.Response( - text=text, - headers={ - "Content-Type": "application/x-mpegURL", - "Cache-Control": "no-cache", - "Access-Control-Max-Age": "300", - }, - status=web.HTTPOk.status_code, - ) - - @route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments") - async def segments(self, request: Request) -> Response: - """ - Get the segments. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - file: str = request.match_info.get("file") - segment: int = request.match_info.get("segment") - sd: int = request.query.get("sd") - vc: int = int(request.query.get("vc", 0)) - ac: int = int(request.query.get("ac", 0)) - - if not file: - return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) - - if not segment: - return web.json_response(data={"error": "segment id is required."}, status=web.HTTPBadRequest.status_code) - - realFile, status = get_file(download_path=self.config.download_path, file=file) - if web.HTTPFound.status_code == status: - return Response( - status=status, - headers={ - "Location": str( - self.app.router["segments"].url_for( - segment=segment, - file=str(realFile).replace(self.config.download_path, "").strip("/"), - ) - ), - }, - ) - - if web.HTTPNotFound.status_code == status: - return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) - - mtime = realFile.stat().st_mtime - - if request.if_modified_since and request.if_modified_since.timestamp() == mtime: - lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()) - return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) - - resp = web.StreamResponse( - status=web.HTTPOk.status_code, - headers={ - "Content-Type": "video/mpegts", - "X-Accel-Buffering": "no", - "Access-Control-Allow-Origin": "*", - "Pragma": "public", - "Cache-Control": f"public, max-age={time.time() + 31536000}", - "Last-Modified": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple() - ), - "Expires": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple() - ), - }, - ) - - await resp.prepare(request) - - await Segments( - download_path=self.config.download_path, - index=int(segment), - duration=float(f"{float(sd if sd else M3u8.duration):.6f}"), - vconvert=vc == 1, - aconvert=ac == 1, - ).stream(realFile, resp) - - return resp - - @route("GET", "api/player/subtitle/{file:.*}.vtt", "subtitles") - async def subtitles(self, request: Request) -> Response: - """ - Get the subtitles. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - file: str = request.match_info.get("file") - - if not file: - return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) - - realFile, status = get_file(download_path=self.config.download_path, file=file) - if web.HTTPFound.status_code == status: - return Response( - status=status, - headers={ - "Location": str( - self.app.router["subtitles"].url_for( - file=str(realFile).replace(self.config.download_path, "").strip("/") - ) - ), - }, - ) - - if web.HTTPNotFound.status_code == status: - return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) - - mtime = realFile.stat().st_mtime - - if request.if_modified_since and request.if_modified_since.timestamp() == mtime: - lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()) - return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) - - return web.Response( - body=await Subtitle().make(file=realFile), - headers={ - "Content-Type": "text/vtt; charset=UTF-8", - "X-Accel-Buffering": "no", - "Access-Control-Allow-Origin": "*", - "Pragma": "public", - "Cache-Control": f"public, max-age={time.time() + 31536000}", - "Last-Modified": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple() - ), - "Expires": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple() - ), - }, - status=web.HTTPOk.status_code, - ) - - @route("GET", "/", "index") - async def index(self, _: Request) -> Response: - """ - Get the index file. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - if "/index.html" not in self._static_holder: - LOG.error("Static frontend files not found.") - return web.json_response( - data={"error": "File not found.", "file": "/index.html"}, status=web.HTTPNotFound.status_code - ) - - data = self._static_holder["/index.html"] - return web.Response( - body=data.get("content"), - content_type=data.get("content_type"), - charset="utf-8", - status=web.HTTPOk.status_code, - ) - - @route("GET", "api/thumbnail", "get_thumbnail") - async def get_thumbnail(self, request: Request) -> Response: - """ - Get the thumbnail. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - url = request.query.get("url") - if not url: - return web.json_response(data={"error": "URL is required."}, status=web.HTTPForbidden.status_code) - - try: - validate_url(url) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPForbidden.status_code) - - try: - ytdlp_args = self.config.get_ytdlp_args() - opts = { - "proxy": ytdlp_args.get("proxy", None), - "headers": { - "User-Agent": ytdlp_args.get( - "user_agent", request.headers.get("User-Agent", f"YTPTube/{self.config.version}") - ), - }, - } - - async with httpx.AsyncClient(**opts) as client: - LOG.debug(f"Fetching thumbnail from '{url}'.") - response = await client.request(method="GET", url=url) - return web.Response( - body=response.content, - headers={ - "Content-Type": response.headers.get("Content-Type"), - "Pragma": "public", - "Access-Control-Allow-Origin": "*", - "Cache-Control": f"public, max-age={time.time() + 31536000}", - "Expires": time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", - datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple(), - ), - }, - ) - except Exception as e: - LOG.error(f"Error fetching thumbnail from '{url}'. '{e}'.") - return web.json_response( - data={"error": "failed to retrieve the thumbnail."}, status=web.HTTPInternalServerError.status_code - ) - - @route("GET", "api/random/background", "get_background") - async def get_background(self, request: Request) -> Response: - """ - Get random background. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - backend = None - - if self.isRequestingBackground: - return web.Response(status=web.HTTPTooManyRequests.status_code) - - try: - self.isRequestingBackground = True - backend = random.choice(self.config.pictures_backends) # noqa: S311 - CACHE_KEY_BING = "random_background_bing" - CACHE_KEY = "random_background" - - if self.cache.has(CACHE_KEY) and not request.query.get("force", False): - data = await self.cache.aget(CACHE_KEY) - return web.Response( - body=data.get("content"), - headers={ - "X-Cache": "HIT", - "X-Cache-TTL": str(await self.cache.attl(CACHE_KEY)), - "X-Image-Via": data.get("backend"), - **data.get("headers"), - }, - ) - - ytdlp_args = self.config.get_ytdlp_args() - opts = { - "proxy": ytdlp_args.get("proxy", None), - "headers": { - "User-Agent": ytdlp_args.get("user_agent", f"YTPTube/{self.config.version}"), - }, - } - - async with httpx.AsyncClient(**opts) as client: - if backend.startswith("https://www.bing.com/HPImageArchive.aspx"): - if not self.cache.has(CACHE_KEY_BING): - response = await client.request(method="GET", url=backend) - if response.status_code != web.HTTPOk.status_code: - return web.json_response( - data={"error": "failed to retrieve the random background image."}, - status=web.HTTPInternalServerError.status_code, - ) - - img_url: str | None = ag(response.json(), "images.0.url") - if not img_url: - return web.json_response( - data={"error": "failed to retrieve the random background image."}, - status=web.HTTPInternalServerError.status_code, - ) - - backend = f"https://www.bing.com{img_url}" - await self.cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24) - else: - backend: str = await self.cache.aget(CACHE_KEY_BING) - - LOG.debug(f"Requesting random picture from '{backend!s}'.") - - response = await client.request(method="GET", url=backend, follow_redirects=True) - - if response.status_code != web.HTTPOk.status_code: - return web.json_response( - data={"error": "failed to retrieve the random background image."}, - status=web.HTTPInternalServerError.status_code, - ) - - data = { - "content": response.content, - "backend": urlparse(backend).netloc, - "headers": { - "Content-Type": response.headers.get("Content-Type", "image/jpeg"), - "Content-Length": str(len(response.content)), - }, - } - - await self.cache.aset(key=CACHE_KEY, value=data, ttl=3600) - - LOG.debug(f"Random background image from '{backend!s}' cached.") - - return web.Response( - body=data.get("content"), - headers={ - "X-Cache": "MISS", - "X-Cache-TTL": "3600", - "X-Image-Via": data.get("backend"), - **data.get("headers"), - }, - ) - except Exception as e: - LOG.error(f"Failed to request random background image from '{backend!s}'.'. '{e!s}'.") - return web.json_response( - data={"error": "failed to retrieve the random background image."}, - status=web.HTTPInternalServerError.status_code, - ) - finally: - self.isRequestingBackground = False - - @route("GET", "api/file/ffprobe/{file:.*}", "ffprobe") - async def get_ffprobe(self, request: Request) -> Response: - """ - Get the ffprobe data. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - file: str = request.match_info.get("file") - if not file: - return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) - - try: - realFile, status = get_file(download_path=self.config.download_path, file=file) - if web.HTTPFound.status_code == status: - return Response( - status=web.HTTPFound.status_code, - headers={ - "Location": str( - self.app.router["ffprobe"].url_for( - file=str(realFile.relative_to(self.config.download_path).as_posix()).strip("/") - ) - ), - }, - ) - - if web.HTTPNotFound.status_code == status: - return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) - - return web.json_response( - data=await ffprobe(realFile), status=web.HTTPOk.status_code, dumps=self.encoder.encode - ) - except Exception as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code) - - @route("GET", "api/file/info/{file:.*}", "file_info") - async def get_file_info(self, request: Request) -> Response: - """ - Get file info - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - file: str = request.match_info.get("file") - if not file: - return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) - - try: - realFile, status = get_file(download_path=self.config.download_path, file=file) - if web.HTTPFound.status_code == status: - return Response( - status=web.HTTPFound.status_code, - headers={ - "Location": str( - self.app.router["file_info"].url_for( - file=str(realFile.relative_to(self.config.download_path).as_posix()).strip("/") - ) - ), - }, - ) - - if web.HTTPNotFound.status_code == status: - return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) - - ff_info = await ffprobe(realFile) - - response = { - "title": realFile.stem, - "ffprobe": ff_info, - "mimetype": get_mime_type(ff_info.get("metadata", {}), realFile), - "sidecar": get_file_sidecar(realFile), - } - - for key in response["sidecar"]: - for i, f in enumerate(response["sidecar"][key]): - response["sidecar"][key][i]["file"] = str( - realFile.with_name(f["file"].name).relative_to(self.config.download_path) - ).strip("/") - - return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - except Exception as e: - LOG.exception(e) - return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code) - - @route("GET", "api/notifications", "notifications") - async def notifications(self, _: Request) -> Response: - """ - Get the notification targets. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - return web.json_response( - data={ - "notifications": Notification.get_instance().get_targets(), - "allowedTypes": list(NotificationEvents.get_events().values()), - }, - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("PUT", "api/notifications", "notification_add") - async def notification_add(self, request: Request) -> Response: - """ - Add notification targets. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - post = await request.json() - if not isinstance(post, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - targets: list = [] - - ins = Notification.get_instance() - for item in post: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("id", None) or validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - Notification.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Invalid notification target settings. {e!s}", "data": item}, - status=web.HTTPBadRequest.status_code, - ) - - targets.append(ins.make_target(item)) - - try: - ins.save(targets=targets) - ins.load() - except Exception as e: - LOG.exception(e) - return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code) - - data = {"notifications": targets, "allowedTypes": list(NotificationEvents.get_events().values())} - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("POST", "api/notifications/test", "notification_test") - async def notification_test(self, _: Request) -> Response: - """ - Test the notification. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - data = message("test", "This is a test notification.") - - await self._notify.emit(Events.TEST, data=data) - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - - @route("GET", "api/file/browser/{path:.*}", "file_browser") - async def file_browser(self, request: Request) -> Response: - """ - Get the file browser. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - if not self.config.browser_enabled: - return web.json_response(data={"error": "File browser is disabled."}, status=web.HTTPForbidden.status_code) - - req_path: str = request.match_info.get("path") - req_path: str = "/" if not req_path else unquote_plus(req_path) - - test: Path = Path(self.config.download_path).joinpath(req_path) - if not test.exists(): - return web.json_response( - data={"error": f"path '{req_path}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - if not test.is_dir() and not test.is_symlink(): - return web.json_response( - data={"error": f"path '{req_path}' is not a directory."}, status=web.HTTPBadRequest.status_code - ) - - try: - return web.json_response( - data={ - "path": req_path, - "contents": get_files(base_path=Path(self.config.download_path), dir=req_path), - }, - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - except OSError as e: - LOG.exception(e) - return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code) - - @route("GET", "api/dev/loop") - async def debug_asyncio(self, _: Request) -> Response: - if not self.config.is_dev(): - return web.json_response( - data={"error": "This endpoint is only available in development mode."}, - status=web.HTTPForbidden.status_code, - ) - - import traceback - - tasks = [] - for task in asyncio.all_tasks(): - task_info = {"task": str(task), "stack": []} - for frame in task.get_stack(): - formatted = traceback.format_stack(f=frame) - task_info["stack"].extend(formatted) - - tasks.append(task_info) - - return web.json_response( - data={ - "total_tasks": len(tasks), - "loop": str(asyncio.get_event_loop()), - "tasks": tasks, - }, - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, - ) - - @route("GET", "api/dev/workers", "pool_list") - async def pool_list(self, _) -> Response: - """ - Get the workers status. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - if not self.config.is_dev(): - return web.json_response( - {"error": "This endpoint is only available in development mode."}, - status=web.HTTPNotFound.status_code, - ) - - if self.queue.pool is None: - return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) - - status = self.queue.pool.get_workers_status() - - data = [] - - for worker in status: - worker_status = status.get(worker) - data.append( - { - "id": worker, - "data": {"status": "Waiting for download."} if worker_status is None else worker_status, - } - ) - - return web.json_response( - data={ - "open": self.queue.pool.has_open_workers(), - "count": self.queue.pool.get_available_workers(), - "workers": data, - }, - status=web.HTTPOk.status_code, - dumps=lambda obj: json.dumps(obj, default=lambda o: f"<>"), - ) - - @route("POST", "api/dev/workers", "pool_start") - async def pool_restart(self, _) -> Response: - """ - Restart the workers pool. - - Args: - _: The request object. - - Returns: - Response: The response object. - - """ - if not self.config.is_dev(): - return web.json_response( - {"error": "This endpoint is only available in development mode."}, - status=web.HTTPNotFound.status_code, - ) - - if self.queue.pool is None: - return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) - - self.queue.pool.start() - - return web.json_response({"message": "Workers pool being restarted."}, status=web.HTTPOk.status_code) - - @route("PATCH", "api/dev/workers/{id}", "worker_restart") - async def worker_restart(self, request: Request) -> Response: - """ - Restart a worker. - - Args: - request (Request): The request object. - - Returns: - Response: The response object - - """ - if not self.config.is_dev(): - return web.json_response( - {"error": "This endpoint is only available in development mode."}, - status=web.HTTPNotFound.status_code, - ) - - worker_id: str = request.match_info.get("id") - if not worker_id: - return web.json_response({"error": "worker id is required."}, status=web.HTTPBadRequest.status_code) - - if self.queue.pool is None: - return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) - - status = await self.queue.pool.restart(worker_id, "requested by user.") - - return web.json_response({"status": "restarted" if status else "in_error_state"}, status=web.HTTPOk.status_code) - - @route("DELETE", "api/dev/workers/{id}", "worker_stop") - async def worker_stop(self, request: Request) -> Response: - """ - Stop a worker. - - Args: - request (Request): The request object. - - Returns: - Response: The response object. - - """ - if not self.config.is_dev(): - return web.json_response( - {"error": "This endpoint is only available in development mode."}, - status=web.HTTPNotFound.status_code, - ) - - worker_id: str = request.match_info.get("id") - if not worker_id: - raise web.HTTPBadRequest(text="worker id is required.") - - if self.queue.pool is None: - return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) - - status = await self.queue.pool.stop(worker_id, "requested by user.") - - return web.json_response({"status": "stopped" if status else "in_error_state"}, status=web.HTTPOk.status_code) diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index 9724f905..6975bcd7 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -1,30 +1,24 @@ -import asyncio -import errno import functools +import inspect import logging -import os -import shlex -import time -from datetime import UTC, datetime from pathlib import Path -import anyio import socketio from aiohttp import web -from .common import Common +from app.library.router import RouteType, get_routes +from app.library.Utils import load_modules + from .config import Config from .DownloadQueue import DownloadQueue from .encoder import Encoder -from .Events import Event, EventBus, Events, error +from .Events import Event, EventBus, Events from .ItemDTO import Item -from .Presets import Presets -from .Utils import is_downloaded, tail_log -LOG = logging.getLogger("socket_api") +LOG: logging.Logger = logging.getLogger("socket_api") -class HttpSocket(Common): +class HttpSocket: """ This class is used to handle WebSocket events. """ @@ -32,15 +26,11 @@ class HttpSocket(Common): config: Config sio: socketio.AsyncServer queue: DownloadQueue - - subscribers: dict[str, list[str]] = {} - """Event subscriber list.""" - - log_task = None - """Task to tail the log file.""" + di_context: dict[str, object] = {} def __init__( self, + root_path: Path, queue: DownloadQueue | None = None, encoder: Encoder | None = None, config: Config | None = None, @@ -55,20 +45,28 @@ class HttpSocket(Common): async_handlers=True, async_mode="aiohttp", cors_allowed_origins=[], - transports=["websocket"], + transports=["websocket", "polling"], logger=self.config.debug, engineio_logger=self.config.debug, ping_interval=10, ping_timeout=5, ) encoder = encoder or Encoder() + self.rootPath = root_path def emit(e: Event, _, **kwargs): return self.sio.emit(event=e.event, data=encoder.encode(e.data), **kwargs) - self._notify.subscribe("frontend", emit, f"{__class__.__name__}.emit") + self.di_context = { + "config": self.config, + "queue": self.queue, + "sio": self.sio, + "encoder": encoder, + "notify": self._notify, + "root_path": self.rootPath, + } - super().__init__(queue=queue, encoder=encoder, config=config) + self._notify.subscribe("frontend", emit, f"{__class__.__name__}.emit") @staticmethod def ws_event(func): # type: ignore @@ -93,347 +91,38 @@ class HttpSocket(Common): LOG.debug("Socket server shutdown complete.") def attach(self, app: web.Application): + app.on_shutdown.append(self.on_shutdown) + self.sio.attach(app, socketio_path=f"{self.config.base_path.rstrip('/')}/socket.io") - - for attr_name in dir(self): - method = getattr(self, attr_name) - if hasattr(method, "_ws_event") and self.sio: - self.sio.on(method._ws_event)(method) # type: ignore - self._notify.subscribe( Events.ADD_URL, - lambda data, _, **kwargs: self.add(item=Item.format(data.data)), # noqa: ARG005 + lambda data, _, **kwargs: self.queue.add(item=Item.format(data.data)), # noqa: ARG005 f"{__class__.__name__}.add", ) - # register the shutdown event. - app.on_shutdown.append(self.on_shutdown) + load_modules(self.rootPath, self.rootPath / "routes" / "socket") - @ws_event - async def cli_post(self, sid: str, data): - if not self.config.console_enabled: - await self._notify.emit(Events.ERROR, data=error("Console is disabled."), to=sid) - return - - if not data: - await self._notify.emit(Events.CLI_CLOSE, data={"exitcode": 0}, to=sid) - return - - try: - LOG.info(f"Cli command from client '{sid}'. '{data}'") - - args = ["yt-dlp", *shlex.split(data)] - _env = os.environ.copy() - _env.update( - { - "TERM": "xterm-256color", - "LANG": "en_US.UTF-8", - "SHELL": "/bin/bash", - "LC_ALL": "en_US.UTF-8", - "PWD": self.config.download_path, - "FORCE_COLOR": "1", - "PYTHONUNBUFFERED": "1", - } + for route in get_routes(RouteType.SOCKET).values(): + LOG.debug( + f"Add ({route.name}) {route.method.value if isinstance(route.method,RouteType) else route.method}: {route.path}." ) + self.sio.on(route.path)(HttpSocket._injector(route.handler, route.path, self.di_context)) - try: - import pty + @staticmethod + def _injector(func, event: str, container: dict): + sig: inspect.Signature = inspect.signature(func) - master_fd, slave_fd = pty.openpty() - stdin_arg = asyncio.subprocess.DEVNULL - stdout_arg = stderr_arg = slave_fd - use_pty = True - except ImportError: - use_pty = False - master_fd = slave_fd = None - stdin_arg = asyncio.subprocess.DEVNULL - stdout_arg = asyncio.subprocess.PIPE - stderr_arg = asyncio.subprocess.STDOUT + async def wrapper(sid, data, **kwargs): + args = {} - proc = await asyncio.create_subprocess_exec( - *args, - cwd=self.config.download_path, - stdin=stdin_arg, - stdout=stdout_arg, - stderr=stderr_arg, - env=_env, - ) + merged = {**container, "sid": sid, "data": data, "event_name": event} + if isinstance(kwargs, dict): + merged.update(kwargs) - if use_pty: - try: - os.close(slave_fd) - except Exception as e: - LOG.error(f"Error closing PTY. '{e!s}'.") + for name in sig.parameters: + if name in merged: + args[name] = merged[name] - async def reader(sid: str): - if use_pty is False: - assert proc.stdout is not None - async for raw_line in proc.stdout: - line = raw_line.rstrip(b"\n") - await self._notify.emit( - Events.CLI_OUTPUT, - data={"type": "stdout", "line": line.decode("utf-8", errors="replace")}, - to=sid, - ) - return + return await func(**args) - loop = asyncio.get_running_loop() - buffer = b"" - while True: - try: - chunk = await loop.run_in_executor(None, lambda: os.read(master_fd, 1024)) - except OSError as e: - if e.errno == errno.EIO: - break - raise - - if not chunk: - # No more output - if buffer: - # Emit any remaining partial line - await self._notify.emit( - Events.CLI_OUTPUT, - data={"type": "stdout", "line": buffer.decode("utf-8", errors="replace")}, - to=sid, - ) - break - buffer += chunk - *lines, buffer = buffer.split(b"\n") - for line in lines: - await self._notify.emit( - Events.CLI_OUTPUT, - data={"type": "stdout", "line": line.decode("utf-8", errors="replace")}, - to=sid, - ) - try: - os.close(master_fd) - except Exception as e: - LOG.error(f"Error closing PTY. '{e!s}'.") - - # Start reading output from PTY - read_task = asyncio.create_task(reader(sid=sid)) - - # Wait until process finishes - returncode = await proc.wait() - - # Ensure reading is done - await read_task - - await self._notify.emit(Events.CLI_CLOSE, data={"exitcode": returncode}, to=sid) - except Exception as e: - LOG.error(f"CLI execute exception was thrown for client '{sid}'.") - LOG.exception(e) - await self._notify.emit(Events.CLI_OUTPUT, data={"type": "stderr", "line": str(e)}, to=sid) - await self._notify.emit(Events.CLI_CLOSE, data={"exitcode": -1}, to=sid) - - @ws_event - async def add_url(self, sid: str, data: dict): - url: str | None = data.get("url") - - if not url: - await self._notify.emit(Events.ERROR, data=error("No URL provided.", data={"unlock": True}), to=sid) - return - - try: - await self._notify.emit( - event=Events.STATUS, - data=await self.add(item=Item.format(data)), - to=sid, - ) - except ValueError as e: - LOG.exception(e) - await self._notify.emit(Events.ERROR, data=error(str(e)), to=sid) - return - - @ws_event - async def item_cancel(self, sid: str, id: str): - if not id: - await self._notify.emit(Events.ERROR, data=error("Invalid request."), to=sid) - return - - status: dict[str, str] = {} - status = await self.queue.cancel([id]) - status.update({"identifier": id}) - - await self._notify.emit(Events.ITEM_CANCEL, data=status) - - @ws_event - async def item_delete(self, sid: str, data: dict): - if not data: - await self._notify.emit(Events.ERROR, data=error("Invalid request."), to=sid) - return - - id: str | None = data.get("id") - if not id: - await self._notify.emit(Events.ERROR, data=error("Invalid request."), to=sid) - return - - status: dict[str, str] = {} - status = await self.queue.clear([id], remove_file=bool(data.get("remove_file", False))) - status.update({"identifier": id}) - - await self._notify.emit(Events.ITEM_DELETE, data=status) - - @ws_event - async def archive_item(self, _: str, data: dict): - if not isinstance(data, dict) or "url" not in data: - return - - from .YTDLPOpts import YTDLPOpts - - params = YTDLPOpts.get_instance() - - if "preset" in data and isinstance(data["preset"], str): - params.preset(name=data["preset"]) - - if "cli" in data and isinstance(data["cli"], str) and len(data["cli"]) > 1: - params.add_cli(data["cli"], from_user=True) - - params = params.get_all() - - file: str = params.get("download_archive", None) - - if not file: - return - - exists, idDict = is_downloaded(file, data["url"]) - if exists or "archive_id" not in idDict or idDict["archive_id"] is None: - return - - async with await anyio.open_file(file, "a") as f: - await f.write(f"{idDict['archive_id']}\n") - - manual_archive = self.config.manual_archive - if manual_archive: - manual_archive = Path(manual_archive) - - if not manual_archive.exists(): - manual_archive.touch(exist_ok=True) - - previouslyArchived = False - async with await anyio.open_file(manual_archive) as f: - async for line in f: - if idDict["archive_id"] in line: - previouslyArchived = True - break - - if not previouslyArchived: - async with await anyio.open_file(manual_archive, "a") as f: - await f.write(f"{idDict['archive_id']} - at: {datetime.now(UTC).isoformat()}\n") - LOG.info(f"Archiving url '{data['url']}' with id '{idDict['archive_id']}'.") - else: - LOG.info(f"URL '{data['url']}' with id '{idDict['archive_id']}' already archived.") - - @ws_event - async def connect(self, sid: str, _=None): - data = { - **self.queue.get(), - "config": self.config.frontend(), - "presets": Presets.get_instance().get_all(), - "paused": self.queue.is_paused(), - } - - data["folders"] = [folder.name for folder in Path(self.config.download_path).iterdir() if folder.is_dir()] - - await self._notify.emit(Events.INITIAL_DATA, data=data, to=sid) - - @ws_event - async def pause(self, *_, **__): - self.queue.pause() - await self._notify.emit(Events.PAUSED, data={"paused": True, "at": time.time()}) - - @ws_event - async def resume(self, *_, **__): - self.queue.resume() - await self._notify.emit(Events.PAUSED, data={"paused": False, "at": time.time()}) - - @ws_event - async def subscribe(self, sid: str, event: str): - """ - Subscribe to a specific event. - - Args: - sid (str): The session ID of the client. - event (str): The event to subscribe to. - - """ - if not isinstance(event, str): - await self._notify.emit(Events.ERROR, data=error("Invalid event."), to=sid) - return - - if event not in self.subscribers: - self.subscribers[event] = [] - - if sid not in self.subscribers[event]: - self.subscribers[event].append(sid) - LOG.debug(f"Client '{sid}' subscribed to event '{event}'.") - await self.sio.emit(Events.SUBSCRIBED, data={"event": event}, to=sid) - - async def emit_logs(data: dict): - await self.subscribe_emit(event=event, data=data) - - if "log_lines" == event and self.log_task is None: - log_file = Path(self.config.config_path) / "logs" / "app.log" - LOG.debug(f"Starting tailing '{log_file!s}'.") - self.log_task = asyncio.create_task( - tail_log( - file=log_file, - emitter=emit_logs, - ), - name="tail_log", - ) - - @ws_event - async def unsubscribe(self, sid: str, event: str): - """ - Unsubscribe from a specific event. - - Args: - sid (str): The session ID of the client. - event (str): The event to unsubscribe from. - - """ - if event not in self.subscribers: - return - - if sid not in self.subscribers[event]: - return - - self.subscribers[event].remove(sid) - await self.sio.emit(Events.UNSUBSCRIBED, data={"event": event}, to=sid) - - LOG.debug(f"Client '{sid}' unsubscribed from event '{event}'.") - - if "log_lines" != event or not self.log_task or "log_lines" not in self.subscribers: - return - - if len(self.subscribers["log_lines"]) < 1: - try: - LOG.debug("Stopping log tailing task.") - self.log_task.cancel() - self.log_task = None - except asyncio.CancelledError: - pass - - @ws_event - async def disconnect(self, sid: str, reason: str): - """ - Handle client disconnection. - - Args: - sid (str): The session ID of the client. - reason (str): The reason for disconnection. - - """ - LOG.debug(f"Client '{sid}' disconnected. {reason}") - - for event in self.subscribers: - if sid in self.subscribers[event]: - await self.unsubscribe(sid=sid, event=event) - - async def subscribe_emit(self, event: str, data: dict): - if event not in self.subscribers or len(self.subscribers[event]) < 1: - return - - for sid in self.subscribers[event]: - await self.sio.emit(event=event, data=data, to=sid) + return wrapper diff --git a/app/library/Presets.py b/app/library/Presets.py index dbb044ff..6ef5e2c6 100644 --- a/app/library/Presets.py +++ b/app/library/Presets.py @@ -175,13 +175,14 @@ class Presets(metaclass=Singleton): Presets: The current instance. """ + has: int = len(self._items) self.clear() if not self._file.exists() or self._file.stat().st_size < 10: return self try: - LOG.info(f"Loading '{self._file}'.") + LOG.info(f"{'Reloading' if has else 'Loading'} '{self._file}'.") presets: dict = json.loads(self._file.read_text()) except Exception as e: LOG.error(f"Failed to parse '{self._file}'. '{e}'.") diff --git a/app/library/Scheduler.py b/app/library/Scheduler.py index 2baac1eb..e26d49fe 100644 --- a/app/library/Scheduler.py +++ b/app/library/Scheduler.py @@ -103,7 +103,7 @@ class Scheduler(metaclass=Singleton): self._jobs[job_id] = job - LOG.debug(f"Added job '{job_id}' to the schedule.") + LOG.debug(f"Added '{job_id}' to the scheduler.") return job_id diff --git a/app/library/Tasks.py b/app/library/Tasks.py index 71666845..8981fc0b 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -17,7 +17,7 @@ from .Scheduler import Scheduler from .Singleton import Singleton from .Utils import init_class -LOG = logging.getLogger("tasks") +LOG: logging.Logger = logging.getLogger("tasks") @dataclass(kw_only=True) @@ -124,13 +124,14 @@ class Tasks(metaclass=Singleton): Tasks: The current instance. """ + has_tasks: bool = len(self._tasks) > 0 self.clear() if not self._file.exists() or self._file.stat().st_size < 1: return self try: - LOG.info(f"Loading '{self._file}'.") + LOG.info(f"{'Reloading' if has_tasks else 'Loading'} '{self._file}'.") tasks = json.loads(self._file.read_text()) except Exception as e: LOG.error(f"Error loading '{self._file}'. '{e!s}'.") diff --git a/app/library/Utils.py b/app/library/Utils.py index c65cf851..11376d44 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -17,10 +17,11 @@ from typing import TypeVar import yt_dlp from Crypto.Cipher import AES +from yt_dlp.utils import match_str from .LogWrapper import LogWrapper -LOG = logging.getLogger("Utils") +LOG: logging.Logger = logging.getLogger("Utils") REMOVE_KEYS: list = [ { @@ -59,7 +60,8 @@ FILES_TYPE: list = [ {"rx": re.compile(r"\.(nfo|json|jpg|torrent|\.info\.json)$", re.IGNORECASE), "type": "metadata"}, ] -DATETIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?") +TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c") +DT_PATTERN: re.Pattern[str] = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?") T = TypeVar("T") @@ -204,6 +206,10 @@ def extract_info( if not data: return data + data["is_premiere"] = match_str("media_type=video & duration & is_live", data) + if not data["is_premiere"]: + data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status") + return yt_dlp.YoutubeDL.sanitize_info(data) if sanitize_info else data @@ -277,6 +283,41 @@ def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, s return (False, idDict) +def remove_from_archive(archive_file: str | Path, url: str) -> bool: + """ + Remove the downloaded video record from the archive file. + + Args: + archive_file (str): Archive file path. + url (str): URL to check and remove. + + Returns: + bool: True if the record removed, False otherwise. + + """ + if not url or not archive_file: + return False + + archive_path: Path = Path(archive_file) if not isinstance(archive_file, Path) else archive_file + if not archive_path.exists(): + return False + + idDict = get_archive_id(url=url) + archive_id: str | None = idDict.get("archive_id") + + if not archive_id: + return False + + lines: list[str] = archive_path.read_text(encoding="utf-8").splitlines() + new_lines: list[str] = [line for line in lines if archive_id not in line] + + if len(lines) == len(new_lines): + return False + + archive_path.write_text("\n".join(new_lines) + "\n", encoding="utf-8") + return True + + def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: """ Load a JSON or JSON5 file and return the contents as a dictionary @@ -803,7 +844,8 @@ def get_files(base_path: Path | str, dir: str | None = None): if file.name.startswith(".") or file.name.startswith("_"): continue - if file.is_symlink(): + is_symlink: bool = file.is_symlink() + if is_symlink: try: test: Path = file.resolve() test.relative_to(base_path) @@ -831,9 +873,10 @@ def get_files(base_path: Path | str, dir: str | None = None): content_type = "download" stat = file.stat() + contents.append( { - "type": "file" if file.is_file() else "dir", + "type": "file" if file.is_file() else "link" if is_symlink else "dir", "content_type": content_type, "name": file.name, "path": str(file.relative_to(base_path)).strip("/"), @@ -843,6 +886,7 @@ def get_files(base_path: Path | str, dir: str | None = None): "ctime": datetime.fromtimestamp(stat.st_ctime, tz=UTC).isoformat(), "is_dir": file.is_dir(), "is_file": file.is_file(), + "is_symlink": is_symlink, } ) @@ -960,7 +1004,7 @@ async def read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict: for line in lines[-(offset + limit) : -offset] if offset else lines[-limit:]: line_bytes = line if isinstance(line, bytes) else line.encode() msg = line.decode(errors="replace") - dt_match = DATETIME_PATTERN.match(msg) + dt_match = DT_PATTERN.match(msg) result.append( { "id": sha256(line_bytes).hexdigest(), @@ -1002,7 +1046,7 @@ async def tail_log(file: Path, emitter: callable, sleep_time: float = 0.5): continue msg = line.decode(errors="replace") - dt_match = DATETIME_PATTERN.match(msg) + dt_match = DT_PATTERN.match(msg) await emitter( { @@ -1198,3 +1242,52 @@ def init_class(cls: type[T], data: dict) -> T: from dataclasses import fields return cls(**{k: v for k, v in data.items() if k in {f.name for f in fields(cls)}}) + + +def load_modules(root_path: Path, directory: Path): + """ + Load all modules from a given directory relative to the root path. + + Args: + root_path (Path): The root path of the application. + directory (Path): The directory from which to load modules. + + """ + import importlib + import pkgutil + + package_name: str = str(directory.relative_to(root_path).as_posix()).replace("/", ".") + + LOG.debug(f"Loading routes from '{directory}' with package name '{package_name}'.") + + for _, name, _ in pkgutil.iter_modules([directory]): + full_name: str = f"{package_name}.{name}" + if name.startswith("_"): + continue + try: + LOG.debug(f"Loading module '{full_name}'.") + importlib.import_module(full_name) + except ImportError as e: + LOG.error(f"Failed to import module '{full_name}': {e}") + + +def parse_tags(text: str) -> tuple[str, dict[str, str | bool]]: + """ + Parse tags from a string formatted with %{tag_name[:value]}c. + + Args: + text (str): The input string containing tags. + + Returns: + tuple[str, dict[str, str | bool]]: A tuple containing the string with tags removed and a dictionary of tags. + + """ + tags: dict[str, str | bool] = {} + + def replacer(match: re.Match) -> str: + name = match.group(1) + value = match.group(2) + tags[name] = value if value is not None else True + return "" + + return TAG_REGEX.sub(replacer, text).strip(), tags diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py index 2954165a..a7a8a43f 100644 --- a/app/library/YTDLPOpts.py +++ b/app/library/YTDLPOpts.py @@ -2,7 +2,7 @@ import logging from pathlib import Path from .config import Config -from .Presets import Presets, Preset +from .Presets import Preset, Presets from .Singleton import Singleton from .Utils import REMOVE_KEYS, arg_converter, calc_download_path, load_cookies, merge_dict diff --git a/app/library/common.py b/app/library/common.py deleted file mode 100644 index f8ff5fda..00000000 --- a/app/library/common.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging - -from .config import Config -from .DownloadQueue import DownloadQueue -from .encoder import Encoder -from .ItemDTO import Item - -LOG = logging.getLogger("common") - - -class Common: - """ - This class is used to share common methods between the socket and the API gateways. - """ - - def __init__( - self, - queue: DownloadQueue | None = None, - encoder: Encoder | None = None, - config: Config | None = None, - ): - super().__init__() - self.queue = queue or DownloadQueue.get_instance() - self.encoder = encoder or Encoder() - - config = config or Config.get_instance() - self.default_preset = config.default_preset - - async def add(self, item: Item) -> dict[str, str]: - """ - Add an item to the download queue. - - Args: - item (Item): The item to be added to the queue. - - Returns: - dict[str, str]: The status of the download. - { "status": "text" } - - """ - return await self.queue.add(item=item) diff --git a/app/library/config.py b/app/library/config.py index bf3ab7da..5337a7eb 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -94,6 +94,9 @@ class Config: db_file: str = "{config_path}/ytptube.db" """The path to the database file.""" + archive_file: str = "{config_path}/archive.log" + """The path to the download archive file.""" + manual_archive: str = "{config_path}/archive.manual.log" """The path to the manual archive file.""" @@ -158,6 +161,9 @@ class Config: is_native: bool = False "Is the application running in webview." + prevent_live_premiere: bool = False + """Prevent downloading of the initial premiere live broadcast.""" + pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", @@ -209,6 +215,7 @@ class Config: "console_enabled", "browser_enabled", "ytdlp_auto_update", + "prevent_premiere_live", ) "The variables that are booleans." @@ -367,7 +374,7 @@ class Config: self._ytdlp_cli_mutable += f"\n--socket-timeout {self.socket_timeout}" if self.keep_archive: - archive_file: Path = Path(self.config_path) / "archive.log" + archive_file: Path = Path(self.archive_file) if not archive_file.exists(): LOG.info(f"Creating archive file '{archive_file}'.") archive_file.touch(exist_ok=True) diff --git a/app/library/router.py b/app/library/router.py new file mode 100644 index 00000000..8572b27a --- /dev/null +++ b/app/library/router.py @@ -0,0 +1,149 @@ +import logging +import re +from collections.abc import Awaitable +from enum import Enum +from functools import wraps + +LOG: logging.Logger = logging.getLogger(__name__) + + +# make a enum for route types +class RouteType(str, Enum): + HTTP = "http" + SOCKET = "socket" + + @classmethod + def all(cls) -> list[str]: + return [member.value for member in cls] + + +class Route: + """ + A class to represent an route. + + Attributes: + method (str): The HTTP method (GET, POST, etc.). + path (str): The path for the route. + name (str): The name of the route. + handler (Awaitable): The function that handles the route. + + """ + + def __init__(self, method: str, path: str, name: str, handler: Awaitable): + self.method: str = method.upper() + self.path: str = path + self.name: str = name + self.handler: Awaitable = handler + + +ROUTES: dict[str, dict[str, Route]] = {} + + +def make_route_name(method: str, path: str) -> str: + method = method.lower() + path = path.strip("/") + + segments: list = [] + for part in path.split("/"): + part = re.sub(r"[^\w]", "_", part) # remove invalid chars + if not part: + part = "part" + elif part[0].isdigit(): + part = f"p_{part}" + segments.append(part) + + return f"{method}:" + ".".join(segments or ["root"]) + + +def route(method: RouteType | str, path: str, name: str | None = None, **kwargs) -> Awaitable: + """ + Decorator to mark a method as an HTTP route handler. + + Args: + method (RouteType|str): The HTTP method. + path (str): The path to the route. + name (str): The name of the route. + kwargs: Additional keyword arguments. + + Returns: + Awaitable: The decorated function. + + """ + if not name: + name = make_route_name(method, path) + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + return await func(*args, **kwargs) + + route_type: str = RouteType.SOCKET if RouteType.SOCKET == method else RouteType.HTTP + if route_type not in ROUTES: + ROUTES[route_type] = {} + + ROUTES[route_type][name] = Route(method=method.upper(), path=path, name=name, handler=wrapper) + if "http" == route_type and path.endswith("/") and "/" != path and not kwargs.get("no_slash", False): + ROUTES[route_type][f"{name}_no_slash"] = Route( + method=method.upper(), path=path[:-1], name=f"{name}_no_slash", handler=wrapper + ) + + return wrapper + + return decorator + + +def add_route(method: RouteType | str, path: str, handler: Awaitable, name: str | None = None, **kwargs): + """ + Decorator to mark a method as an HTTP route handler. + + Args: + method (RouteType|str): The HTTP method. + path (str): The path to the route. + name (str): The name of the route. + handler (Awaitable): The function that handles the route. + kwargs: Additional keyword arguments. + + """ + if not name: + name = make_route_name(method, path) + + route_type: str = RouteType.SOCKET if RouteType.SOCKET == method else RouteType.HTTP + + if route_type not in ROUTES: + ROUTES[route_type] = {} + + ROUTES[route_type][name] = Route(method=method.upper(), path=path, name=name, handler=handler) + + if "http" == route_type and path.endswith("/") and "/" != path and not kwargs.get("no_slash", False): + ROUTES[route_type][f"{name}_no_slash"] = Route( + method=method.upper(), path=path[:-1], name=f"{name}_no_slash", handler=handler + ) + + +def get_route(route_type: RouteType, name: str) -> dict[str, Route] | None: + """ + Get the route information by name. + + Args: + route_type (RouteType): The type of the route (e.g., RouteType.HTTP, RouteType.SOCKET). + name (str): The name of the route. + + Returns: + dict: The route information, or None if not found. + + """ + return ROUTES.get(route_type, {}).get(name, None) + + +def get_routes(route_type: RouteType) -> dict[str, Route]: + """ + Get all registered routes. + + Args: + route_type (RouteType): The type of the route (e.g., RouteType.HTTP, RouteType.SOCKET). + + Returns: + dict[str, dict]: A dictionary of all registered routes. + + """ + return ROUTES.get(route_type, {}) diff --git a/app/main.py b/app/main.py index 1b646740..42f91235 100644 --- a/app/main.py +++ b/app/main.py @@ -1,24 +1,31 @@ #!/usr/bin/env python3 +import sys +from pathlib import Path + +APP_ROOT = str((Path(__file__).parent / "..").resolve()) +if APP_ROOT not in sys.path: + sys.path.insert(0, APP_ROOT) + import asyncio import logging import sqlite3 -import sys from pathlib import Path import caribou import magic from aiohttp import web -from library.conditions import Conditions -from library.config import Config -from library.DownloadQueue import DownloadQueue -from library.Events import EventBus, Events -from library.HttpAPI import HttpAPI -from library.HttpSocket import HttpSocket -from library.Notifications import Notification -from library.Presets import Presets -from library.Scheduler import Scheduler -from library.Tasks import Tasks + +from app.library.conditions import Conditions +from app.library.config import Config +from app.library.DownloadQueue import DownloadQueue +from app.library.Events import EventBus, Events +from app.library.HttpAPI import HttpAPI +from app.library.HttpSocket import HttpSocket +from app.library.Notifications import Notification +from app.library.Presets import Presets +from app.library.Scheduler import Scheduler +from app.library.Tasks import Tasks LOG = logging.getLogger("app") MIME = magic.Magic(mime=True) @@ -58,7 +65,7 @@ class Main: self._queue = DownloadQueue(connection=connection) self._http = HttpAPI(root_path=ROOT_PATH, queue=self._queue) - self._socket = HttpSocket(queue=self._queue) + self._socket = HttpSocket(root_path=ROOT_PATH, queue=self._queue) self._app.on_cleanup.append(_close_connection) @@ -117,7 +124,7 @@ class Main: HTTP_LOGGER = None if self._config.access_log: - from library.HttpAPI import LOG as HTTP_LOGGER + from app.library.HttpAPI import LOG as HTTP_LOGGER HTTP_LOGGER.addFilter(lambda record: f"GET {self._app.router['ping'].url_for()}" not in record.getMessage()) diff --git a/app/native.py b/app/native.py index 120b2b0a..9aa302af 100644 --- a/app/native.py +++ b/app/native.py @@ -1,9 +1,17 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +APP_ROOT = str((Path(__file__).parent / "..").resolve()) +if APP_ROOT not in sys.path: + sys.path.insert(0, APP_ROOT) + + import json import os import queue import socket import threading -from pathlib import Path ready = threading.Event() exception_holder = queue.Queue() diff --git a/app/routes/api/_static.py b/app/routes/api/_static.py new file mode 100644 index 00000000..816a2505 --- /dev/null +++ b/app/routes/api/_static.py @@ -0,0 +1,135 @@ +import logging +from pathlib import Path + +import magic +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.config import Config +from app.library.router import add_route + +MIME = magic.Magic(mime=True) +LOG: logging.Logger = logging.getLogger(__name__) + +STATIC_FILES: dict[str, dict] = {} + +EXT_TO_MIME: dict = { + ".html": "text/html", + ".css": "text/css", + ".js": "application/javascript", + ".json": "application/json", + ".ico": "image/x-icon", +} + +FRONTEND_ROUTES: list[str] = [ + "/console/", + "/presets/", + "/tasks/", + "/notifications/", + "/changelog/", + "/logs/", + "/conditions/", + "/browser/", + "/browser/{path:.*}", +] + + +async def serve_static_file(request: Request, config: Config) -> Response: + """ + Preload static files from the ui/exported folder. + + Args: + request (Request): The request object. + config (Config): The configuration instance. + + Returns: + Response: The response object. + + """ + path: str = request.path + + if "/" != config.base_path and path.startswith(config.base_path): + path = path.replace(config.base_path[:-1], "", 1) + + if path not in STATIC_FILES: + for k in STATIC_FILES: + if path.startswith(k): + path = k + break + else: + return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code) + + item: dict = STATIC_FILES[path] + + return web.FileResponse( + path=item["file"], + headers={ + "Pragma": "public", + "Cache-Control": "public, max-age=31536000", + "Content-Type": item.get("content_type"), + }, + status=web.HTTPOk.status_code, + ) + + +def preload_static(root_path: Path, config: Config) -> None: + """ + Preload static files from the ui/exported folder. + + Args: + root_path (Path): The root path of the application. + config (Config): The configuration instance. + + """ + global STATIC_FILES # noqa: PLW0602 + + static_dir: Path = (root_path / "ui" / "exported").absolute() + if not static_dir.exists(): + static_dir = (root_path.parent / "ui" / "exported").absolute() + if not static_dir.exists(): + msg: str = f"Could not find the frontend UI static assets. '{static_dir}'." + raise ValueError(msg) + + preloaded = 0 + + for file in static_dir.rglob("*.*"): + if ".map" == file.suffix: + continue + + uri_path: str = f"/{file.relative_to(static_dir).as_posix()!s}" + # uri_path: str = f"/{str(file.as_posix()).replace(f'{static_dir.as_posix()!s}/', '')}" + contentType = EXT_TO_MIME.get(file.suffix) + if not contentType: + contentType = MIME.from_file(file) + + STATIC_FILES[uri_path] = { + "uri": uri_path, + "content_type": contentType, + "file": file, + } + + add_route(method="GET", path=uri_path, handler=serve_static_file) + preloaded += 1 + + if "/index.html" in STATIC_FILES: + for path in FRONTEND_ROUTES: + STATIC_FILES[path] = STATIC_FILES["/index.html"] + STATIC_FILES[path.lstrip("/")] = STATIC_FILES["/index.html"] + add_route(method="GET", path=path, handler=serve_static_file) + LOG.debug(f"Preloading frontend static route '{path}'.") + preloaded += 1 + + # Add main app route + STATIC_FILES["/"] = STATIC_FILES["/index.html"] + STATIC_FILES[config.base_path] = STATIC_FILES["/index.html"] + add_route(method="GET", path=config.base_path, handler=serve_static_file, name="index") + + if preloaded < 1: + message = f"Failed to find any static files in '{static_dir}'." + if config.ignore_ui: + LOG.warning(message) + return + + raise ValueError(message) + + LOG.info(f"Loaded '{preloaded}' static files.") diff --git a/app/routes/api/archive.py b/app/routes/api/archive.py new file mode 100644 index 00000000..cbd42f23 --- /dev/null +++ b/app/routes/api/archive.py @@ -0,0 +1,168 @@ +import logging +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +import anyio +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.config import Config +from app.library.Download import Download +from app.library.DownloadQueue import DownloadQueue +from app.library.router import route +from app.library.Utils import is_downloaded, remove_from_archive +from app.library.YTDLPOpts import YTDLPOpts + +if TYPE_CHECKING: + from library.Download import Download + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("DELETE", r"api/archive/{id}", "archive.remove") +async def archive_remove(request: Request, queue: DownloadQueue, config: Config) -> Response: + """ + Remove an item from the archive. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + config (Config): The configuration instance. + + Returns: + Response: The response object. + + """ + item = None + try: + data: dict | None = await request.json() + except Exception: + data = {} + + title: str = "" + + url: str | None = data.get("url", None) if data else None + + if not url: + id: str = request.match_info.get("id") + if not id: + return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) + + try: + item: Download | None = queue.done.get_by_id(id) + if not item: + return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code) + + url = item.info.url + title = f" '{item.info.title}'" + except KeyError: + return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code) + + if config.manual_archive: + remove_from_archive(archive_file=Path(config.manual_archive), url=url) + + archive_file: Path | None = Path(config.archive_file) if config.keep_archive else None + if item: + params: YTDLPOpts = YTDLPOpts.get_instance().preset(name=item.info.preset) + if item.info.cli: + params.add_cli(item.info.cli, from_user=True) + + params = params.get_all() + if user_file := params.get("download_archive", None): + archive_file = Path(user_file) + + if not archive_file: + return web.json_response( + data={ + "error": "Archive file is not configured." if not config.keep_archive else "Archive file is not set." + }, + status=web.HTTPBadRequest.status_code, + ) + + if not archive_file.exists(): + return web.json_response( + data={"error": f"Archive file '{archive_file}' does not exist."}, + status=web.HTTPNotFound.status_code, + ) + + if not remove_from_archive(archive_file=archive_file, url=url): + return web.json_response( + data={"error": f"item{title} not found in '{archive_file}' archive."}, + status=web.HTTPNotFound.status_code, + ) + + return web.json_response( + data={"message": f"item{title} removed from '{archive_file}' archive."}, + status=web.HTTPOk.status_code, + ) + + +@route("POST", r"api/archive/{id}", "archive.item") +async def archive_item(request: Request, queue: DownloadQueue, config: Config): + """ + Manually mark an item as archived. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + config (Config): The configuration instance. + + Returns: + Response: The response object. + + """ + id: str = request.match_info.get("id") + + if not id: + return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) + + try: + item: Download | None = queue.done.get_by_id(id) + if not item: + return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) + except KeyError: + return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) + + if config.manual_archive: + manual_archive = Path(config.manual_archive) + if manual_archive.exists(): + exists, idDict = is_downloaded(manual_archive, item.info.url) + if exists is False and idDict.get("archive_id"): + async with await anyio.open_file(manual_archive, "a") as f: + await f.write(f"{idDict['archive_id']} - at: {datetime.now(UTC).isoformat()}\n") + + params: YTDLPOpts = YTDLPOpts.get_instance().preset(name=item.info.preset) + if item.info.cli: + params.add_cli(item.info.cli, from_user=True) + + params = params.get_all() + + user_file: str | None = params.get("download_archive", None) + archive_file: Path = Path(user_file) if user_file else Path(config.archive_file) + if not archive_file.exists(): + return web.json_response( + data={"error": f"Archive file '{archive_file}' does not exist."}, + status=web.HTTPNotFound.status_code, + ) + + exists, idDict = is_downloaded(archive_file, item.info.url) + item_id: str | None = idDict.get("archive_id") + if not item_id: + return web.json_response( + data={"error": "item does not have an archive ID."}, status=web.HTTPBadRequest.status_code + ) + + if exists is True: + return web.json_response( + data={"error": f"item '{item_id}' already archived in file '{archive_file}'."}, + status=web.HTTPConflict.status_code, + ) + + async with await anyio.open_file(archive_file, "a") as f: + await f.write(f"{item_id}\n") + + return web.json_response( + data={"message": f"item '{item_id}' archived in file '{archive_file}'."}, + status=web.HTTPOk.status_code, + ) diff --git a/app/routes/api/browser.py b/app/routes/api/browser.py new file mode 100644 index 00000000..4a4d0ef0 --- /dev/null +++ b/app/routes/api/browser.py @@ -0,0 +1,157 @@ +import logging +from pathlib import Path +from urllib.parse import unquote_plus + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.ffprobe import ffprobe +from app.library.router import route +from app.library.Utils import get_file, get_file_sidecar, get_files, get_mime_type + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/file/ffprobe/{file:.*}", "ffprobe") +async def get_ffprobe(request: Request, config: Config, encoder: Encoder, app: web.Application) -> Response: + """ + Get the ffprobe data. + + Args: + request (Request): The request object. + config (Config): The configuration object. + encoder (Encoder): The encoder object. + app (web.Application): The aiohttp application object. + + Returns: + Response: The response object. + + """ + file: str = request.match_info.get("file") + if not file: + return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) + + try: + realFile, status = get_file(download_path=config.download_path, file=file) + if web.HTTPFound.status_code == status: + return Response( + status=web.HTTPFound.status_code, + headers={ + "Location": str( + app.router["ffprobe"].url_for( + file=str(realFile.relative_to(config.download_path).as_posix()).strip("/") + ) + ), + }, + ) + + if web.HTTPNotFound.status_code == status: + return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) + + return web.json_response(data=await ffprobe(realFile), status=web.HTTPOk.status_code, dumps=encoder.encode) + except Exception as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code) + + +@route("GET", "api/file/info/{file:.*}", "file_info") +async def get_file_info(request: Request, config: Config, encoder: Encoder, app: web.Application) -> Response: + """ + Get file info + + Args: + request (Request): The request object. + config (Config): The configuration object. + encoder (Encoder): The encoder object. + app (web.Application): The aiohttp application object. + + Returns: + Response: The response object. + + """ + file: str = request.match_info.get("file") + if not file: + return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) + + try: + realFile, status = get_file(download_path=config.download_path, file=file) + if web.HTTPFound.status_code == status: + return Response( + status=web.HTTPFound.status_code, + headers={ + "Location": str( + app.router["file_info"].url_for( + file=str(realFile.relative_to(config.download_path).as_posix()).strip("/") + ) + ), + }, + ) + + if web.HTTPNotFound.status_code == status: + return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) + + ff_info = await ffprobe(realFile) + + response = { + "title": realFile.stem, + "ffprobe": ff_info, + "mimetype": get_mime_type(ff_info.get("metadata", {}), realFile), + "sidecar": get_file_sidecar(realFile), + } + + for key in response["sidecar"]: + for i, f in enumerate(response["sidecar"][key]): + response["sidecar"][key][i]["file"] = str( + realFile.with_name(f["file"].name).relative_to(config.download_path) + ).strip("/") + + return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=encoder.encode) + except Exception as e: + LOG.exception(e) + return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code) + + +@route("GET", "api/file/browser/{path:.*}", "file_browser") +async def file_browser(request: Request, config: Config, encoder: Encoder) -> Response: + """ + Get the file browser. + + Args: + request (Request): The request object. + config (Config): The configuration object. + encoder (Encoder): The encoder object. + + Returns: + Response: The response object. + + """ + if not config.browser_enabled: + return web.json_response(data={"error": "File browser is disabled."}, status=web.HTTPForbidden.status_code) + + req_path: str = request.match_info.get("path") + req_path: str = "/" if not req_path else unquote_plus(req_path) + + test: Path = Path(config.download_path).joinpath(req_path) + if not test.exists(): + return web.json_response( + data={"error": f"path '{req_path}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + if not test.is_dir() and not test.is_symlink(): + return web.json_response( + data={"error": f"path '{req_path}' is not a directory."}, status=web.HTTPBadRequest.status_code + ) + + try: + return web.json_response( + data={ + "path": req_path, + "contents": get_files(base_path=Path(config.download_path), dir=req_path), + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + except OSError as e: + LOG.exception(e) + return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code) diff --git a/app/routes/api/conditions.py b/app/routes/api/conditions.py new file mode 100644 index 00000000..9895cd45 --- /dev/null +++ b/app/routes/api/conditions.py @@ -0,0 +1,191 @@ +import logging +import uuid + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.cache import Cache +from app.library.conditions import Condition, Conditions +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route +from app.library.Utils import extract_info, init_class, validate_uuid +from app.library.YTDLPOpts import YTDLPOpts + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/conditions/", "conditions_list") +async def conditions_list(encoder: Encoder) -> Response: + """ + Get the conditions + + Args: + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + return web.json_response( + data=Conditions.get_instance().get_all(), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", "api/conditions/", "conditions_add") +async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: + """ + Save Conditions. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object + + """ + data = await request.json() + + if not isinstance(data, list): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + items: list = [] + + cls = Conditions.get_instance() + + for item in data: + if not isinstance(item, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + if not item.get("name"): + return web.json_response( + {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code + ) + + if not item.get("filter"): + return web.json_response( + {"error": "filter is required.", "data": item}, status=web.HTTPBadRequest.status_code + ) + + if not item.get("cli"): + return web.json_response( + {"error": "command options for yt-dlp is required.", "data": item}, + status=web.HTTPBadRequest.status_code, + ) + + if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): + item["id"] = str(uuid.uuid4()) + + try: + cls.validate(item) + except ValueError as e: + return web.json_response( + {"error": f"Failed to validate condition '{item.get('name')}'. '{e!s}'"}, + status=web.HTTPBadRequest.status_code, + ) + + items.append(init_class(Condition, item)) + + try: + items = cls.save(items=items).load().get_all() + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to save conditions.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + await notify.emit(Events.CONDITIONS_UPDATE, data=items) + return web.json_response(data=items, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/conditions/test/", "conditions_test") +async def conditions_test(request: Request, encoder: Encoder, cache: Cache, config: Config) -> Response: + """ + Test condition against URL. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + cache (Cache): The cache instance. + config (Config): The configuration instance. + + Returns: + Response: The response object + + """ + params = await request.json() + + if not isinstance(params, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + url: str | None = params.get("url") + if not url: + return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) + + cond: str | None = params.get("condition") + if not cond: + return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code) + + try: + preset: str = params.get("preset", config.default_preset) + key: str = cache.hash(url + str(preset)) + if not cache.has(key): + opts = {} + if ytdlp_proxy := config.get_ytdlp_args().get("proxy", None): + opts["proxy"] = ytdlp_proxy + ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset).add(opts).get_all() + + data = extract_info( + config=ytdlp_opts, + url=url, + debug=False, + no_archive=True, + follow_redirect=True, + sanitize_info=True, + ) + if not data: + return web.json_response( + data={"error": f"Failed to extract info from '{url!s}'."}, + status=web.HTTPBadRequest.status_code, + ) + cache.set(key=key, value=data, ttl=600) + else: + data = cache.get(key) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": f"Failed to extract video info. '{e!s}'"}, + status=web.HTTPInternalServerError.status_code, + ) + + try: + from yt_dlp.utils import match_str + + status = match_str(cond, data) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": str(e)}, + status=web.HTTPBadRequest.status_code, + ) + + return web.json_response( + data={"status": status, "condition": cond, "data": data}, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) diff --git a/app/routes/api/dev.py b/app/routes/api/dev.py new file mode 100644 index 00000000..06dc4e81 --- /dev/null +++ b/app/routes/api/dev.py @@ -0,0 +1,180 @@ +import asyncio +import json +import logging + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library import DownloadQueue +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.router import route + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/dev/loop/", "debug_loop") +async def debug_asyncio(config: Config, encoder: Encoder) -> Response: + if not config.is_dev(): + return web.json_response( + data={"error": "This endpoint is only available in development mode."}, + status=web.HTTPForbidden.status_code, + ) + + import traceback + + tasks = [] + for task in asyncio.all_tasks(): + task_info = {"task": str(task), "stack": []} + for frame in task.get_stack(): + formatted = traceback.format_stack(f=frame) + task_info["stack"].extend(formatted) + + tasks.append(task_info) + + return web.json_response( + data={ + "total_tasks": len(tasks), + "loop": str(asyncio.get_event_loop()), + "tasks": tasks, + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("GET", "api/dev/workers/", "pool_list") +async def pool_list(config: Config, queue: DownloadQueue) -> Response: + """ + Get the workers status. + + Args: + config (Config): The configuration object. + queue (DownloadQueue): The download queue object. + + Returns: + Response: The response object. + + """ + if not config.is_dev(): + return web.json_response( + {"error": "This endpoint is only available in development mode."}, + status=web.HTTPNotFound.status_code, + ) + + if queue.pool is None: + return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) + + status = queue.pool.get_workers_status() + + data = [] + + for worker in status: + worker_status = status.get(worker) + data.append( + { + "id": worker, + "data": {"status": "Waiting for download."} if worker_status is None else worker_status, + } + ) + + return web.json_response( + data={ + "open": queue.pool.has_open_workers(), + "count": queue.pool.get_available_workers(), + "workers": data, + }, + status=web.HTTPOk.status_code, + dumps=lambda obj: json.dumps(obj, default=lambda o: f"<>"), + ) + + +@route("POST", "api/dev/workers/", "pool_start") +async def pool_restart(config: Config, queue: DownloadQueue) -> Response: + """ + Restart the workers pool. + + Args: + config (Config): The configuration object. + queue (DownloadQueue): The download queue object. + + Returns: + Response: The response object. + + """ + if not config.is_dev(): + return web.json_response( + {"error": "This endpoint is only available in development mode."}, + status=web.HTTPNotFound.status_code, + ) + + if queue.pool is None: + return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) + + queue.pool.start() + + return web.json_response({"message": "Workers pool being restarted."}, status=web.HTTPOk.status_code) + + +@route("PATCH", "api/dev/workers/{id}", "worker_restart") +async def worker_restart(request: Request, config: Config, queue: DownloadQueue) -> Response: + """ + Restart a worker. + + Args: + request (Request): The request object. + config (Config): The configuration object. + queue (DownloadQueue): The download queue object. + + Returns: + Response: The response object + + """ + if not config.is_dev(): + return web.json_response( + {"error": "This endpoint is only available in development mode."}, + status=web.HTTPNotFound.status_code, + ) + + worker_id: str = request.match_info.get("id") + if not worker_id: + return web.json_response({"error": "worker id is required."}, status=web.HTTPBadRequest.status_code) + + if queue.pool is None: + return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) + + status = await queue.pool.restart(worker_id, "requested by user.") + + return web.json_response({"status": "restarted" if status else "in_error_state"}, status=web.HTTPOk.status_code) + + +@route("DELETE", "api/dev/workers/{id}", "worker_stop") +async def worker_stop(request: Request, config: Config, queue: DownloadQueue) -> Response: + """ + Stop a worker. + + Args: + request (Request): The request object. + config (Config): The configuration object. + queue (DownloadQueue): The download queue object. + + Returns: + Response: The response object. + + """ + if not config.is_dev(): + return web.json_response( + {"error": "This endpoint is only available in development mode."}, + status=web.HTTPNotFound.status_code, + ) + + worker_id: str = request.match_info.get("id") + if not worker_id: + raise web.HTTPBadRequest(text="worker id is required.") + + if queue.pool is None: + return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) + + status = await queue.pool.stop(worker_id, "requested by user.") + + return web.json_response({"status": "stopped" if status else "in_error_state"}, status=web.HTTPOk.status_code) diff --git a/app/routes/api/history.py b/app/routes/api/history.py new file mode 100644 index 00000000..6765c24b --- /dev/null +++ b/app/routes/api/history.py @@ -0,0 +1,201 @@ +import asyncio +import logging +from typing import TYPE_CHECKING + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.DownloadQueue import DownloadQueue +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.ItemDTO import Item +from app.library.Presets import Preset, Presets +from app.library.router import route + +if TYPE_CHECKING: + from library.Download import Download + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", r"api/history/", "items_list") +async def items_list(queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Get the history. + + Args: + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data: dict = {"queue": [], "history": []} + q = queue.get() + + data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) + data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) + + return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", "api/history/", "item_delete") +async def item_delete(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Delete an item from the queue. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data = await request.json() + ids = data.get("ids") + where = data.get("where") + if not ids or where not in ["queue", "done"]: + return web.json_response(data={"error": "ids and where are required."}, status=web.HTTPBadRequest.status_code) + + remove_file: bool = bool(data.get("remove_file", True)) + + return web.json_response( + data=await (queue.cancel(ids) if where == "queue" else queue.clear(ids, remove_file=remove_file)), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/history/{id}", "item_update") +async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder, notify: EventBus) -> Response: + """ + Update an item in the history. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object. + + """ + id: str = request.match_info.get("id") + if not id: + return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) + + item: Download | None = queue.done.get_by_id(id) + if not item: + return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) + + post = await request.json() + if not post: + return web.json_response(data={"error": "no data provided."}, status=web.HTTPBadRequest.status_code) + + updated = False + + for k, v in post.items(): + if not hasattr(item.info, k): + continue + + if getattr(item.info, k) == v: + continue + + updated = True + setattr(item.info, k, v) + LOG.debug(f"Updated '{k}' to '{v}' for '{item.info.id}'") + + if updated: + queue.done.put(item) + await notify.emit(Events.UPDATE, data=item.info) + + return web.json_response( + data=item.info, + status=web.HTTPOk.status_code if updated else web.HTTPNotModified.status_code, + dumps=encoder.encode, + ) + + +@route("GET", "api/history/add/", "item_add") +async def item_add(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Add a URL to the download queue. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object + + """ + url: str | None = request.query.get("url") + if not url: + return web.json_response(data={"error": "url param is required."}, status=web.HTTPBadRequest.status_code) + + data: dict[str, str] = {"url": url} + + preset: str | None = request.query.get("preset") + if preset: + exists: Preset | None = Presets.get_instance().get(preset) + if not exists: + return web.json_response( + data={"status": False, "message": f"Preset '{preset}' does not exist."}, + status=web.HTTPBadRequest.status_code, + ) + data["preset"] = preset + + try: + status: dict[str, str] = await queue.add(item=Item.format(data)) + except ValueError as e: + return web.json_response(data={"status": False, "message": str(e)}, status=web.HTTPBadRequest.status_code) + + return web.json_response( + data={"status": status.get("status") == "ok", "message": status.get("msg", "URL added")}, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/history/", "items_add") +async def items_add(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Add a URL to the download queue. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data = await request.json() + + if isinstance(data, dict): + data = [data] + + items = [] + for item in data: + try: + items.append(Item.format(item)) + except ValueError as e: + return web.json_response(data={"error": str(e), "data": item}, status=web.HTTPBadRequest.status_code) + + status: list = await asyncio.wait_for( + fut=asyncio.gather(*[queue.add(item=item) for item in items]), + timeout=None, + ) + + response: list = [] + + for i, item in enumerate(items): + response.append({"item": item, "status": "ok" == status[i].get("status"), "msg": status[i].get("msg")}) + + return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/routes/api/images.py b/app/routes/api/images.py new file mode 100644 index 00000000..ac794814 --- /dev/null +++ b/app/routes/api/images.py @@ -0,0 +1,186 @@ +import logging +import random +import time +from datetime import UTC, datetime +from urllib.parse import urlparse + +import httpx +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.ag_utils import ag +from app.library.cache import Cache +from app.library.config import Config +from app.library.router import route +from app.library.Utils import validate_url + +LOG: logging.Logger = logging.getLogger(__name__) + +IS_REQUESTING_BACKGROUND: bool = False + + +@route("GET", "api/thumbnail/", "get_thumbnail") +async def get_thumbnail(request: Request, config: Config) -> Response: + """ + Get the thumbnail. + + Args: + request (Request): The request object. + config (Config): The configuration object. + + Returns: + Response: The response object. + + """ + url = request.query.get("url") + if not url: + return web.json_response(data={"error": "URL is required."}, status=web.HTTPForbidden.status_code) + + try: + validate_url(url) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPForbidden.status_code) + + try: + ytdlp_args = config.get_ytdlp_args() + opts = { + "proxy": ytdlp_args.get("proxy", None), + "headers": { + "User-Agent": ytdlp_args.get( + "user_agent", request.headers.get("User-Agent", f"YTPTube/{config.version}") + ), + }, + } + + async with httpx.AsyncClient(**opts) as client: + LOG.debug(f"Fetching thumbnail from '{url}'.") + response = await client.request(method="GET", url=url) + return web.Response( + body=response.content, + headers={ + "Content-Type": response.headers.get("Content-Type"), + "Pragma": "public", + "Access-Control-Allow-Origin": "*", + "Cache-Control": f"public, max-age={time.time() + 31536000}", + "Expires": time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", + datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple(), + ), + }, + ) + except Exception as e: + LOG.error(f"Error fetching thumbnail from '{url}'. '{e}'.") + return web.json_response( + data={"error": "failed to retrieve the thumbnail."}, status=web.HTTPInternalServerError.status_code + ) + + +@route("GET", "api/random/background/", "get_background") +async def get_background(request: Request, config: Config, cache: Cache) -> Response: + """ + Get random background. + + Args: + request (Request): The request object. + config (Config): The configuration object. + cache (Cache): The cache object. + + Returns: + Response: The response object. + + """ + global IS_REQUESTING_BACKGROUND # noqa: PLW0603 + + backend = None + + if IS_REQUESTING_BACKGROUND: + return web.Response(status=web.HTTPTooManyRequests.status_code) + + try: + IS_REQUESTING_BACKGROUND = True + backend = random.choice(config.pictures_backends) # noqa: S311 + CACHE_KEY_BING = "random_background_bing" + CACHE_KEY = "random_background" + + if cache.has(CACHE_KEY) and not request.query.get("force", False): + data = await cache.aget(CACHE_KEY) + return web.Response( + body=data.get("content"), + headers={ + "X-Cache": "HIT", + "X-Cache-TTL": str(await cache.attl(CACHE_KEY)), + "X-Image-Via": data.get("backend"), + **data.get("headers"), + }, + ) + + ytdlp_args = config.get_ytdlp_args() + opts = { + "proxy": ytdlp_args.get("proxy", None), + "headers": { + "User-Agent": ytdlp_args.get("user_agent", f"YTPTube/{config.version}"), + }, + } + + async with httpx.AsyncClient(**opts) as client: + if backend.startswith("https://www.bing.com/HPImageArchive.aspx"): + if not cache.has(CACHE_KEY_BING): + response = await client.request(method="GET", url=backend) + if response.status_code != web.HTTPOk.status_code: + return web.json_response( + data={"error": "failed to retrieve the random background image."}, + status=web.HTTPInternalServerError.status_code, + ) + + img_url: str | None = ag(response.json(), "images.0.url") + if not img_url: + return web.json_response( + data={"error": "failed to retrieve the random background image."}, + status=web.HTTPInternalServerError.status_code, + ) + + backend = f"https://www.bing.com{img_url}" + await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24) + else: + backend: str = await cache.aget(CACHE_KEY_BING) + + LOG.debug(f"Requesting random picture from '{backend!s}'.") + + response = await client.request(method="GET", url=backend, follow_redirects=True) + + if response.status_code != web.HTTPOk.status_code: + return web.json_response( + data={"error": "failed to retrieve the random background image."}, + status=web.HTTPInternalServerError.status_code, + ) + + data = { + "content": response.content, + "backend": urlparse(backend).netloc, + "headers": { + "Content-Type": response.headers.get("Content-Type", "image/jpeg"), + "Content-Length": str(len(response.content)), + }, + } + + await cache.aset(key=CACHE_KEY, value=data, ttl=3600) + + LOG.debug(f"Random background image from '{backend!s}' cached.") + + return web.Response( + body=data.get("content"), + headers={ + "X-Cache": "MISS", + "X-Cache-TTL": "3600", + "X-Image-Via": data.get("backend"), + **data.get("headers"), + }, + ) + except Exception as e: + LOG.error(f"Failed to request random background image from '{backend!s}'.'. '{e!s}'.") + return web.json_response( + data={"error": "failed to retrieve the random background image."}, + status=web.HTTPInternalServerError.status_code, + ) + finally: + IS_REQUESTING_BACKGROUND = False diff --git a/app/routes/api/logs.py b/app/routes/api/logs.py new file mode 100644 index 00000000..6e00a23b --- /dev/null +++ b/app/routes/api/logs.py @@ -0,0 +1,50 @@ +from pathlib import Path + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.router import route +from app.library.Utils import read_logfile + + +@route("GET", "api/logs/", "logs") +async def logs(request: Request, config: Config, encoder: Encoder) -> Response: + """ + Get recent logs + + Args: + request (Request): The request object. + config (Config): The configuration instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + if not config.file_logging: + return web.json_response(data={"error": "File logging is not enabled."}, status=web.HTTPNotFound.status_code) + + offset = int(request.query.get("offset", 0)) + limit = int(request.query.get("limit", 100)) + if limit < 1 or limit > 150: + limit = 50 + + logs_data = await read_logfile( + file=Path(config.config_path) / "logs" / "app.log", + offset=offset, + limit=limit, + ) + + return web.json_response( + data={ + "logs": logs_data["logs"], + "offset": offset, + "limit": limit, + "next_offset": logs_data["next_offset"], + "end_is_reached": logs_data["end_is_reached"], + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) diff --git a/app/routes/api/notifications.py b/app/routes/api/notifications.py new file mode 100644 index 00000000..9d912002 --- /dev/null +++ b/app/routes/api/notifications.py @@ -0,0 +1,111 @@ +import logging +import uuid + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events, message +from app.library.Notifications import Notification, NotificationEvents +from app.library.router import route +from app.library.Utils import validate_uuid + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/notifications/", "notifications_list") +async def notifications_list(encoder: Encoder) -> Response: + """ + Get the notification targets. + + Args: + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + return web.json_response( + data={ + "notifications": Notification.get_instance().get_targets(), + "allowedTypes": list(NotificationEvents.get_events().values()), + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", "api/notifications/", "notification_add") +async def notification_add(request: Request, encoder: Encoder) -> Response: + """ + Add notification targets. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object. + + """ + post = await request.json() + if not isinstance(post, list): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + targets: list = [] + + ins: Notification = Notification.get_instance() + for item in post: + if not isinstance(item, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + if not item.get("id", None) or validate_uuid(item.get("id"), version=4): + item["id"] = str(uuid.uuid4()) + + try: + Notification.validate(item) + except ValueError as e: + return web.json_response( + {"error": f"Invalid notification target settings. {e!s}", "data": item}, + status=web.HTTPBadRequest.status_code, + ) + + targets.append(ins.make_target(item)) + + try: + ins.save(targets=targets) + ins.load() + except Exception as e: + LOG.exception(e) + return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code) + + data = {"notifications": targets, "allowedTypes": list(NotificationEvents.get_events().values())} + + return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/notifications/test/", "notification_test") +async def notification_test(encoder: Encoder, notify: EventBus) -> Response: + """ + Test the notification. + + Args: + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object. + + """ + data = message("test", "This is a test notification.") + + await notify.emit(Events.TEST, data=data) + + return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/routes/api/ping.py b/app/routes/api/ping.py new file mode 100644 index 00000000..a0da11bc --- /dev/null +++ b/app/routes/api/ping.py @@ -0,0 +1,19 @@ +from aiohttp import web +from aiohttp.web import Response + +from app.library.DownloadQueue import DownloadQueue +from app.library.router import route + + +@route("GET", "api/ping/", "ping") +async def ping(queue: DownloadQueue) -> Response: + """ + Ping the server. + + Returns: + Response: The response object. + + """ + await queue.queue.test() + + return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code) diff --git a/app/routes/api/player.py b/app/routes/api/player.py new file mode 100644 index 00000000..ee6355b6 --- /dev/null +++ b/app/routes/api/player.py @@ -0,0 +1,278 @@ +import logging +import time +from datetime import UTC, datetime +from pathlib import Path + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.config import Config +from app.library.M3u8 import M3u8 +from app.library.Playlist import Playlist +from app.library.router import route +from app.library.Segments import Segments +from app.library.Subtitle import Subtitle +from app.library.Utils import StreamingError, get_file + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist_create") +async def playlist_create(request: Request, config: Config, app: web.Application) -> Response: + """ + Get the playlist. + + Args: + request (Request): The request object. + config (Config): The configuration instance. + app (web.Application): The aiohttp application instance. + + Returns: + Response: The response object. + + """ + file: str = request.match_info.get("file") + + if not file: + return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) + + base_path: str = config.base_path.rstrip("/") + + try: + realFile, status = get_file(download_path=config.download_path, file=file) + if web.HTTPFound.status_code == status: + return Response( + status=web.HTTPFound.status_code, + headers={ + "Location": str( + app.router["playlist_create"].url_for( + file=str(realFile).replace(config.download_path, "").strip("/") + ) + ), + }, + ) + + if web.HTTPNotFound.status_code == status: + return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) + + return web.Response( + text=await Playlist(download_path=Path(config.download_path), url=f"{base_path}/").make(file=realFile), + headers={ + "Content-Type": "application/x-mpegURL", + "Cache-Control": "no-cache", + "Access-Control-Max-Age": "300", + }, + status=web.HTTPOk.status_code, + ) + except StreamingError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code) + + +@route("GET", "api/player/m3u8/{mode}/{file:.*}.m3u8", "m3u8_create") +async def m3u8_create(request: Request, config: Config, app: web.Application) -> Response: + """ + Get the m3u8 file. + + Args: + request (Request): The request object. + config (Config): The configuration instance. + app (web.Application): The aiohttp application instance. + + Returns: + Response: The response object. + + """ + file: str = request.match_info.get("file") + mode: str = request.match_info.get("mode") + + if mode not in ["video", "subtitle"]: + return web.json_response( + data={"error": "Only video and subtitle modes are supported."}, status=web.HTTPBadRequest.status_code + ) + + if not file: + return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) + + duration = request.query.get("duration", None) + + if "subtitle" in mode: + if not duration: + return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code) + + duration = float(duration) + + base_path: str = config.base_path.rstrip("/") + + try: + cls = M3u8(download_path=Path(config.download_path), url=f"{base_path}/") + + realFile, status = get_file(download_path=config.download_path, file=file) + if web.HTTPFound.status_code == status: + return Response( + status=status, + headers={ + "Location": str( + app.router["m3u8_create"].url_for( + mode=mode, file=str(realFile).replace(config.download_path, "").strip("/") + ) + ), + }, + ) + + if web.HTTPNotFound.status_code == status: + return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) + + if "subtitle" in mode: + text = await cls.make_subtitle(file=realFile, duration=duration) + else: + text = await cls.make_stream(file=realFile) + except StreamingError as e: + LOG.exception(e) + return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code) + + return web.Response( + text=text, + headers={ + "Content-Type": "application/x-mpegURL", + "Cache-Control": "no-cache", + "Access-Control-Max-Age": "300", + }, + status=web.HTTPOk.status_code, + ) + + +@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream") +async def segments_stream(request: Request, config: Config, app: web.Application) -> Response: + """ + Get the segments. + + Args: + request (Request): The request object. + config (Config): The configuration instance. + app (web.Application): The aiohttp application instance. + + Returns: + Response: The response object. + + """ + file: str = request.match_info.get("file") + segment: int = request.match_info.get("segment") + sd: int = request.query.get("sd") + vc: int = int(request.query.get("vc", 0)) + ac: int = int(request.query.get("ac", 0)) + + if not file: + return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) + + if not segment: + return web.json_response(data={"error": "segment id is required."}, status=web.HTTPBadRequest.status_code) + + realFile, status = get_file(download_path=config.download_path, file=file) + if web.HTTPFound.status_code == status: + return Response( + status=status, + headers={ + "Location": str( + app.router["segments"].url_for( + segment=segment, + file=str(realFile).replace(config.download_path, "").strip("/"), + ) + ), + }, + ) + + if web.HTTPNotFound.status_code == status: + return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) + + mtime = realFile.stat().st_mtime + + if request.if_modified_since and request.if_modified_since.timestamp() == mtime: + lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()) + return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) + + resp = web.StreamResponse( + status=web.HTTPOk.status_code, + headers={ + "Content-Type": "video/mpegts", + "X-Accel-Buffering": "no", + "Access-Control-Allow-Origin": "*", + "Pragma": "public", + "Cache-Control": f"public, max-age={time.time() + 31536000}", + "Last-Modified": time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple() + ), + "Expires": time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple() + ), + }, + ) + + await resp.prepare(request) + + await Segments( + download_path=config.download_path, + index=int(segment), + duration=float(f"{float(sd if sd else M3u8.duration):.6f}"), + vconvert=vc == 1, + aconvert=ac == 1, + ).stream(realFile, resp) + + return resp + + +@route("GET", "api/player/subtitle/{file:.*}.vtt", "subtitles_get") +async def subtitles_get(request: Request, config: Config, app: web.Application) -> Response: + """ + Get the subtitles. + + Args: + request (Request): The request object. + config (Config): The configuration instance. + app (web.Application): The aiohttp application instance. + + Returns: + Response: The response object. + + """ + file: str = request.match_info.get("file") + + if not file: + return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) + + realFile, status = get_file(download_path=config.download_path, file=file) + if web.HTTPFound.status_code == status: + return Response( + status=status, + headers={ + "Location": str( + app.router["subtitles_get"].url_for(file=str(realFile).replace(config.download_path, "").strip("/")) + ), + }, + ) + + if web.HTTPNotFound.status_code == status: + return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) + + mtime = realFile.stat().st_mtime + + if request.if_modified_since and request.if_modified_since.timestamp() == mtime: + lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()) + return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) + + return web.Response( + body=await Subtitle().make(file=realFile), + headers={ + "Content-Type": "text/vtt; charset=UTF-8", + "X-Accel-Buffering": "no", + "Access-Control-Allow-Origin": "*", + "Pragma": "public", + "Cache-Control": f"public, max-age={time.time() + 31536000}", + "Last-Modified": time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple() + ), + "Expires": time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple() + ), + }, + status=web.HTTPOk.status_code, + ) diff --git a/app/routes/api/presets.py b/app/routes/api/presets.py new file mode 100644 index 00000000..cd4bcdb8 --- /dev/null +++ b/app/routes/api/presets.py @@ -0,0 +1,100 @@ +import logging +import uuid + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.Presets import Preset, Presets +from app.library.router import route +from app.library.Utils import init_class, validate_uuid + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/presets/", "presets") +async def presets(request: Request, encoder: Encoder) -> Response: + """ + Get the presets. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data: list[Preset] = Presets.get_instance().get_all() + filter_fields: str | None = request.query.get("filter", None) + + if filter_fields: + fields: list[str] = [field.strip() for field in filter_fields.split(",")] + data = [{key: value for key, value in preset.serialize().items() if key in fields} for preset in data] + + return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("PUT", "api/presets/", "presets_add") +async def presets_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: + """ + Add presets. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object + + """ + data = await request.json() + + if not isinstance(data, list): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + presets: list = [] + + cls = Presets.get_instance() + + for item in data: + if not isinstance(item, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + if not item.get("name"): + return web.json_response( + {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code + ) + + if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): + item["id"] = str(uuid.uuid4()) + + try: + cls.validate(item) + except ValueError as e: + return web.json_response( + {"error": f"Failed to validate preset '{item.get('name')}'. '{e!s}'"}, + status=web.HTTPBadRequest.status_code, + ) + + presets.append(init_class(Preset, item)) + + try: + presets = cls.save(items=presets).load().get_all() + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to save presets.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + await notify.emit(Events.PRESETS_UPDATE, data=presets) + return web.json_response(data=presets, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/routes/api/tasks.py b/app/routes/api/tasks.py new file mode 100644 index 00000000..c326433b --- /dev/null +++ b/app/routes/api/tasks.py @@ -0,0 +1,93 @@ +import logging +import uuid + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.encoder import Encoder +from app.library.router import route +from app.library.Tasks import Task, Tasks +from app.library.Utils import init_class, validate_uuid + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/tasks/", "tasks") +async def tasks(encoder: Encoder) -> Response: + """ + Get the tasks. + + Args: + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + return web.json_response(data=Tasks.get_instance().get_all(), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("PUT", "api/tasks/", "tasks_add") +async def tasks_add(request: Request, encoder: Encoder) -> Response: + """ + Add tasks to the queue. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object + + """ + data = await request.json() + + if not isinstance(data, list): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + tasks: list = [] + + ins = Tasks.get_instance() + + for item in data: + if not isinstance(item, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + if not item.get("url"): + return web.json_response({"error": "url is required.", "data": item}, status=web.HTTPBadRequest.status_code) + + if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): + item["id"] = str(uuid.uuid4()) + + if not item.get("template", None): + item["template"] = "" + + if not item.get("cli", None): + item["cli"] = "" + + try: + Tasks.validate(item) + except ValueError as e: + return web.json_response( + {"error": f"Failed to validate task '{item.get('name')}'. '{e!s}'"}, + status=web.HTTPBadRequest.status_code, + ) + + tasks.append(init_class(Task, item)) + + try: + tasks = ins.save(tasks=tasks).load().get_all() + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to save tasks.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/routes/api/yt_dlp.py b/app/routes/api/yt_dlp.py new file mode 100644 index 00000000..a735b3a4 --- /dev/null +++ b/app/routes/api/yt_dlp.py @@ -0,0 +1,258 @@ +import asyncio +import json +import logging +import time +from collections import OrderedDict +from pathlib import Path +from typing import Any + +import anyio +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.cache import Cache +from app.library.config import Config +from app.library.Presets import Preset, Presets +from app.library.router import route +from app.library.Utils import REMOVE_KEYS, arg_converter, extract_info, validate_url +from app.library.YTDLPOpts import YTDLPOpts + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("POST", "api/yt-dlp/convert/", "convert") +async def convert(request: Request) -> Response: + """ + Convert the yt-dlp args to a dict. + + Args: + request (Request): The request object. + + Returns: + Response: The response object. + + """ + post = await request.json() + args: str | None = post.get("args") + + if not args: + return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code) + + try: + response = {"opts": {}, "output_template": None, "download_path": None} + + data = arg_converter(args, dumps=True) + + if "outtmpl" in data and "default" in data["outtmpl"]: + response["output_template"] = data["outtmpl"]["default"] + + if "paths" in data and "home" in data["paths"]: + response["download_path"] = data["paths"]["home"] + + if "format" in data: + response["format"] = data["format"] + + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] + + for key in data: + if key in bad_options.items(): + removed_options.append(bad_options[key]) + continue + if not key.startswith("_"): + response["opts"][key] = data[key] + + if len(removed_options) > 0: + response["removed_options"] = removed_options + + return web.json_response(data=response, status=web.HTTPOk.status_code) + except Exception as e: + err = str(e).strip() + err = err.split("\n")[-1] if "\n" in err else err + err = err.replace("main.py: error: ", "").strip().capitalize() + return web.json_response( + data={"error": f"Failed to parse command options for yt-dlp. '{err}'."}, + status=web.HTTPBadRequest.status_code, + ) + + +@route("GET", "api/yt-dlp/url/info/", "get_info") +async def get_info(request: Request, cache: Cache) -> Response: + """ + Get the video info. + + Args: + request (Request): The request object. + cache (Cache): The cache instance. + + Returns: + Response: The response object + + """ + url: str | None = request.query.get("url") + if not url: + return web.json_response(data={"error": "URL is required."}, status=web.HTTPBadRequest.status_code) + + try: + validate_url(url) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + + config = Config.get_instance() + + preset = request.query.get("preset") + if preset: + exists: Preset | None = Presets.get_instance().get(preset) + if not exists: + return web.json_response( + data={"status": False, "message": f"Preset '{preset}' does not exist."}, + status=web.HTTPBadRequest.status_code, + ) + else: + preset: str = config.default_preset + + try: + key: str = cache.hash(f"{preset}:{url}") + + if cache.has(key) and not request.query.get("force", False): + data: Any | None = cache.get(key) + data["_cached"] = { + "status": "hit", + "key": key, + "ttl": data.get("_cached", {}).get("ttl", 300), + "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(), + "expires": data.get("_cached", {}).get("expires", time.time() + 300), + } + return web.Response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) + + opts: dict = {} + + if ytdlp_proxy := config.get_ytdlp_args().get("proxy", None): + opts["proxy"] = ytdlp_proxy + + ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset).add(opts).get_all() + + data = extract_info( + config=ytdlp_opts, + url=url, + debug=False, + no_archive=True, + follow_redirect=True, + sanitize_info=True, + ) + + if "formats" in data: + from yt_dlp.cookies import LenientSimpleCookie + + for index, item in enumerate(data["formats"]): + if "cookies" in item and len(item["cookies"]) > 0: + cookies: list[str] = [f"{c.key}={c.value}" for c in LenientSimpleCookie(item["cookies"]).values()] + if len(cookies) > 0: + data["formats"][index]["h_cookies"] = "; ".join(cookies) + data["formats"][index]["h_cookies"] = data["formats"][index]["h_cookies"].strip() + + data["_cached"] = { + "status": "miss", + "key": key, + "ttl": 300, + "ttl_left": 300, + "expires": time.time() + 300, + } + + data = OrderedDict(sorted(data.items(), key=lambda item: len(str(item[1])))) + + cache.set(key=key, value=data, ttl=300) + + return web.Response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) + except Exception as e: + LOG.exception(e) + LOG.error(f"Error encountered while getting video info for '{url}'. '{e!s}'.") + return web.json_response( + data={ + "error": "failed to get video info.", + "message": str(e), + "formats": [], + }, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("GET", "api/yt-dlp/archive/recheck/", "archive_recheck") +async def archive_recheck(cache: Cache) -> Response: + """ + Recheck the manual archive entries. + + Args: + cache (Cache): The cache instance. + + Returns: + Response: The response object + + """ + config: Config = Config.get_instance() + if not config.manual_archive: + return web.json_response(data={"error": "Manual archive is not enabled."}, status=web.HTTPNotFound.status_code) + + manual_archive = Path(config.manual_archive) + if not manual_archive.exists(): + return web.json_response( + data={"error": "Manual archive file not found.", "file": manual_archive}, + status=web.HTTPNotFound.status_code, + ) + + tasks: list = [] + response: list = [] + + def info_wrapper(id: str, url: str) -> tuple[str, dict]: + try: + return ( + id, + extract_info( + config={ + "proxy": config.get_ytdlp_args().get("proxy", None), + "simulate": True, + "dump_single_json": True, + }, + url=url, + no_archive=True, + ), + ) + except Exception as e: + return (id, {"error": str(e)}) + + async with await anyio.open_file(manual_archive) as f: + # line format is "youtube ID - at: ISO8601" + async for line in f: + line = line.strip() + + if not line or not line.startswith("youtube"): + continue + + id = line.split(" ")[1].strip() + + if not id: + continue + + url = f"https://www.youtube.com/watch?v={id}" + key = cache.hash(id) + + if cache.has(key): + data = cache.get(key) + response.append({id: bool(data.get("id", None)) if isinstance(data, dict) else False}) + continue + + tasks.append( + asyncio.get_event_loop().run_in_executor(None, lambda i=id, url=url: info_wrapper(id=i, url=url)) + ) + + if len(tasks) > 0: + results = await asyncio.gather(*tasks) + for data in results: + if not data: + continue + + id, info = data + cache.set(key=cache.hash(id), value=info, ttl=3600 * 6) + response.append({id: bool(data.get("id", None)) if isinstance(data, dict) else False}) + + return web.json_response(data=response, status=web.HTTPOk.status_code) diff --git a/app/routes/socket/connection.py b/app/routes/socket/connection.py new file mode 100644 index 00000000..d1d49f9e --- /dev/null +++ b/app/routes/socket/connection.py @@ -0,0 +1,126 @@ +import asyncio +import logging +from pathlib import Path + +import socketio + +from app.library.config import Config +from app.library.DownloadQueue import DownloadQueue +from app.library.Events import EventBus, Events, error +from app.library.Presets import Presets +from app.library.router import RouteType, route +from app.library.Utils import tail_log + +LOG: logging.Logger = logging.getLogger(__name__) + + +class _Data: + subscribers: dict[str, list[str]] = {} + log_task: asyncio.Task | None = None + + +@route(RouteType.SOCKET, "connect", "socket_connect") +async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: str): + data = { + **queue.get(), + "config": config.frontend(), + "presets": Presets.get_instance().get_all(), + "paused": queue.is_paused(), + } + + data["folders"] = [folder.name for folder in Path(config.download_path).iterdir() if folder.is_dir()] + + await notify.emit(Events.INITIAL_DATA, data=data, to=sid) + + +@route(RouteType.SOCKET, "disconnect", "socket_disconnect") +async def disconnect(sid: str, data: str = None): + """ + Handle client disconnection. + + Args: + sid (str): The session ID of the client. + data (str): The reason for disconnection. + + """ + LOG.debug(f"Client '{sid}' disconnected. {data}") + + for event in _Data.subscribers: + if sid in _Data.subscribers[event]: + await unsubscribe(sid=sid, data=event) + + +@route(RouteType.SOCKET, "subscribe", "socket_subscribe") +async def subscribe(config: Config, notify: EventBus, sio: socketio.AsyncServer, sid: str, data: str): + """ + Subscribe to a specific event. + + Args: + config (Config): The configuration instance. + notify (EventBus): The event bus to use for notifications. + sio (socketio.AsyncServer): The Socket.IO server instance. + sid (str): The session ID of the client. + data (str): The event to subscribe to. + + """ + if not isinstance(data, str): + await notify.emit(Events.ERROR, data=error("Invalid event."), to=sid) + return + + if data not in _Data.subscribers: + _Data.subscribers[data] = [] + + if sid not in _Data.subscribers[data]: + _Data.subscribers[data].append(sid) + LOG.debug(f"Client '{sid}' subscribed to event '{data}'.") + await sio.emit(Events.SUBSCRIBED, data={"event": data}, to=sid) + + async def emit_logs(data: dict): + await subscribe_emit(sio=sio, event=data, data=data) + + if "log_lines" == data and _Data.log_task is None: + log_file = Path(config.config_path) / "logs" / "app.log" + LOG.debug(f"Starting tailing '{log_file!s}'.") + _Data.log_task = asyncio.create_task(tail_log(file=log_file, emitter=emit_logs), name="tail_log") + + +@route(RouteType.SOCKET, "unsubscribe", "socket_unsubscribe") +async def unsubscribe(sio: socketio.AsyncServer, sid: str, data: str): + """ + Unsubscribe from a specific event. + + Args: + sio (socketio.AsyncServer): The Socket.IO server instance. + sid (str): The session ID of the client. + data (str): The event to unsubscribe from. + + """ + if data not in _Data.subscribers: + return + + if sid not in _Data.subscribers[data]: + return + + _Data.subscribers[data].remove(sid) + await sio.emit(Events.UNSUBSCRIBED, data={"event": data}, to=sid) + + LOG.debug(f"Client '{sid}' unsubscribed from event '{data}'.") + + if "log_lines" != data or not _Data.log_task or "log_lines" not in _Data.subscribers: + return + + if len(_Data.subscribers["log_lines"]) < 1: + try: + LOG.debug("Stopping log tailing task.") + _Data.log_task.cancel() + _Data.log_task = None + except asyncio.CancelledError: + pass + + +async def subscribe_emit(sio: socketio.AsyncServer, event: str, data: dict): + if event not in _Data.subscribers or len(_Data.subscribers[event]) < 1: + return + + for sid in _Data.subscribers[event]: + await sio.emit(event=event, data=data, to=sid) diff --git a/app/routes/socket/history.py b/app/routes/socket/history.py new file mode 100644 index 00000000..2a16be2f --- /dev/null +++ b/app/routes/socket/history.py @@ -0,0 +1,130 @@ +import logging +import time +from datetime import UTC, datetime +from pathlib import Path + +import anyio + +from app.library.config import Config +from app.library.DownloadQueue import DownloadQueue +from app.library.Events import EventBus, Events, error +from app.library.ItemDTO import Item +from app.library.router import RouteType, route +from app.library.Utils import is_downloaded +from app.library.YTDLPOpts import YTDLPOpts + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route(RouteType.SOCKET, "pause", "pause_downloads") +async def pause(notify: EventBus, queue: DownloadQueue): + queue.pause() + await notify.emit(Events.PAUSED, data={"paused": True, "at": time.time()}) + + +@route(RouteType.SOCKET, "resume", "resume_downloads") +async def resume(notify: EventBus, queue: DownloadQueue): + queue.resume() + await notify.emit(Events.PAUSED, data={"paused": False, "at": time.time()}) + + +@route(RouteType.SOCKET, "add_url", "add_url") +async def add_url(queue: DownloadQueue, notify: EventBus, sid: str, data: dict): + url: str | None = data.get("url") + + if not url: + await notify.emit(Events.ERROR, data=error("No URL provided.", data={"unlock": True}), to=sid) + return + + try: + await notify.emit( + event=Events.STATUS, + data=await queue.add(item=Item.format(data)), + to=sid, + ) + except ValueError as e: + LOG.exception(e) + await notify.emit(Events.ERROR, data=error(str(e)), to=sid) + return + + +@route(RouteType.SOCKET, "item_cancel", "item_cancel") +async def item_cancel(queue: DownloadQueue, notify: EventBus, sid: str, data: str): + if not data: + await notify.emit(Events.ERROR, data=error("Invalid request."), to=sid) + return + + status: dict[str, str] = {} + status = await queue.cancel([data]) + status.update({"identifier": data}) + + await notify.emit(Events.ITEM_CANCEL, data=status) + + +@route(RouteType.SOCKET, "item_delete", "item_delete") +async def item_delete(queue: DownloadQueue, notify: EventBus, sid: str, data: dict): + if not data: + await notify.emit(Events.ERROR, data=error("Invalid request."), to=sid) + return + + id: str | None = data.get("id") + if not id: + await notify.emit(Events.ERROR, data=error("Invalid request."), to=sid) + return + + status: dict[str, str] = {} + status = await queue.clear([id], remove_file=bool(data.get("remove_file", False))) + status.update({"identifier": id}) + + await notify.emit(Events.ITEM_DELETE, data=status) + + +@route(RouteType.SOCKET, "archive_item", "archive_item") +async def archive_item(config: Config, data: dict): + if not isinstance(data, dict) or "url" not in data: + return + + params: YTDLPOpts = YTDLPOpts.get_instance() + + if "preset" in data and isinstance(data["preset"], str): + params.preset(name=data["preset"]) + + if "cli" in data and isinstance(data["cli"], str) and len(data["cli"]) > 1: + params.add_cli(data["cli"], from_user=True) + + params = params.get_all() + + file: str = params.get("download_archive", None) + + if not file: + return + + exists, idDict = is_downloaded(file, data["url"]) + if exists or "archive_id" not in idDict or idDict["archive_id"] is None: + return + + async with await anyio.open_file(file, "a") as f: + await f.write(f"{idDict['archive_id']}\n") + + manual_archive: str = config.manual_archive + if not manual_archive: + return + + manual_archive = Path(manual_archive) + + if not manual_archive.exists(): + manual_archive.touch(exist_ok=True) + + previouslyArchived = False + async with await anyio.open_file(manual_archive) as f: + async for line in f: + if idDict["archive_id"] in line: + previouslyArchived = True + break + + if not previouslyArchived: + async with await anyio.open_file(manual_archive, "a") as f: + await f.write(f"{idDict['archive_id']} - at: {datetime.now(UTC).isoformat()}\n") + LOG.info(f"Archiving url '{data['url']}' with id '{idDict['archive_id']}'.") + else: + LOG.info(f"URL '{data['url']}' with id '{idDict['archive_id']}' already archived.") diff --git a/app/routes/socket/terminal.py b/app/routes/socket/terminal.py new file mode 100644 index 00000000..911223a0 --- /dev/null +++ b/app/routes/socket/terminal.py @@ -0,0 +1,133 @@ +import logging +from typing import TYPE_CHECKING + +from app.library.config import Config +from app.library.Events import EventBus, Events, error +from app.library.router import RouteType, route + +if TYPE_CHECKING: + from asyncio import Task + from asyncio.events import AbstractEventLoop + from asyncio.subprocess import Process + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route(RouteType.SOCKET, "cli_post", "socket_cli_post") +async def cli_post(config: Config, notify: EventBus, sid: str, data: str): + if not config.console_enabled: + await notify.emit(Events.ERROR, data=error("Console is disabled."), to=sid) + return + + if not data: + await notify.emit(Events.CLI_CLOSE, data={"exitcode": 0}, to=sid) + return + + import asyncio + import errno + import os + import shlex + + try: + LOG.info(f"Cli command from client '{sid}'. '{data}'") + + args: list[str] = ["yt-dlp", *shlex.split(data)] + _env: dict[str, str] = os.environ.copy() + _env.update( + { + "TERM": "xterm-256color", + "LANG": "en_US.UTF-8", + "SHELL": "/bin/bash", + "LC_ALL": "en_US.UTF-8", + "PWD": config.download_path, + "FORCE_COLOR": "1", + "PYTHONUNBUFFERED": "1", + } + ) + + try: + import pty + + master_fd, slave_fd = pty.openpty() + stdin_arg = asyncio.subprocess.DEVNULL + stdout_arg = stderr_arg = slave_fd + use_pty = True + except ImportError: + use_pty = False + master_fd = slave_fd = None + stdin_arg = asyncio.subprocess.DEVNULL + stdout_arg = asyncio.subprocess.PIPE + stderr_arg = asyncio.subprocess.STDOUT + + proc: Process = await asyncio.create_subprocess_exec( + *args, + cwd=config.download_path, + stdin=stdin_arg, + stdout=stdout_arg, + stderr=stderr_arg, + env=_env, + ) + + if use_pty: + try: + os.close(slave_fd) + except Exception as e: + LOG.error(f"Error closing PTY. '{e!s}'.") + + async def reader(sid: str): + if use_pty is False: + assert proc.stdout is not None + async for raw_line in proc.stdout: + line = raw_line.rstrip(b"\n") + await notify.emit( + Events.CLI_OUTPUT, + data={"type": "stdout", "line": line.decode("utf-8", errors="replace")}, + to=sid, + ) + return + + loop: AbstractEventLoop = asyncio.get_running_loop() + buffer: bytes = b"" + while True: + try: + chunk: bytes = await loop.run_in_executor(None, lambda: os.read(master_fd, 1024)) + except OSError as e: + if e.errno == errno.EIO: + break + raise + + if not chunk: + if buffer: + await notify.emit( + Events.CLI_OUTPUT, + data={"type": "stdout", "line": buffer.decode("utf-8", errors="replace")}, + to=sid, + ) + break + + buffer += chunk + *lines, buffer = buffer.split(b"\n") + + for line in lines: + await notify.emit( + Events.CLI_OUTPUT, + data={"type": "stdout", "line": line.decode("utf-8", errors="replace")}, + to=sid, + ) + try: + os.close(master_fd) + except Exception as e: + LOG.error(f"Error closing PTY. '{e!s}'.") + + read_task: Task = asyncio.create_task(reader(sid=sid)) + + returncode: int = await proc.wait() + + await read_task + + await notify.emit(Events.CLI_CLOSE, data={"exitcode": returncode}, to=sid) + except Exception as e: + LOG.error(f"CLI execute exception was thrown for client '{sid}'.") + LOG.exception(e) + await notify.emit(Events.CLI_OUTPUT, data={"type": "stderr", "line": str(e)}, to=sid) + await notify.emit(Events.CLI_CLOSE, data={"exitcode": -1}, to=sid) diff --git a/app/upgrader.py b/app/upgrader.py index 0eade685..cb353122 100644 --- a/app/upgrader.py +++ b/app/upgrader.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +import sys +from pathlib import Path + +APP_ROOT = str((Path(__file__).parent / "..").resolve()) +if APP_ROOT not in sys.path: + sys.path.insert(0, APP_ROOT) import logging import os diff --git a/ui/@types/logs.ts b/ui/@types/logs.ts new file mode 100644 index 00000000..04f94c8a --- /dev/null +++ b/ui/@types/logs.ts @@ -0,0 +1,8 @@ +type log_line = { + id: number, + line: string, + datetime?: string, +} + + +export type { log_line }; diff --git a/ui/@types/tasks.ts b/ui/@types/tasks.ts new file mode 100644 index 00000000..c1a25d40 --- /dev/null +++ b/ui/@types/tasks.ts @@ -0,0 +1,17 @@ +type task_item = { + id: string, + name: string, + url: string, + preset?: string, + folder?: string, + template?: string, + cli?: string, + timer?: string, + in_progress?: boolean, +} + +type exported_task = task_item & { _type: string, _version: string } + +type error_response = { error: string } + +export type { task_item, exported_task, error_response }; diff --git a/ui/components/ConfirmDialog.vue b/ui/components/ConfirmDialog.vue new file mode 100644 index 00000000..19998c26 --- /dev/null +++ b/ui/components/ConfirmDialog.vue @@ -0,0 +1,98 @@ + + + diff --git a/ui/components/Dropdown.vue b/ui/components/Dropdown.vue new file mode 100644 index 00000000..9add139f --- /dev/null +++ b/ui/components/Dropdown.vue @@ -0,0 +1,109 @@ + + + + + diff --git a/ui/components/EmbedPlayer.vue b/ui/components/EmbedPlayer.vue index ce22b216..2f6afe04 100644 --- a/ui/components/EmbedPlayer.vue +++ b/ui/components/EmbedPlayer.vue @@ -7,6 +7,7 @@ width: 80vw; } + - diff --git a/ui/components/GetInfo.vue b/ui/components/GetInfo.vue index 5135dd2c..8624f49d 100644 --- a/ui/components/GetInfo.vue +++ b/ui/components/GetInfo.vue @@ -1,47 +1,50 @@ - diff --git a/ui/components/History.vue b/ui/components/History.vue index 71d74be3..42c97606 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -76,7 +76,7 @@
-
+
@@ -106,7 +106,12 @@ :id="'checkbox-' + item._id" :value="item._id"> - @@ -202,12 +228,16 @@ v-for="item in sortCompleted" :key="item._id">
-
+
- {{ item.title }} + {{ item.title }}
+ + {{ formatTime(item.extras.duration) }} + + @@ -305,14 +335,7 @@
-
- - - - Archive - - -
+
@@ -323,12 +346,36 @@
- + + + + + + + +
@@ -336,23 +383,13 @@ -
-

- - - - - No records. - -

-

- - -

+ +
+ + {{ formatTime(item.extras.duration) }} + +
@@ -164,30 +169,51 @@
-
- -
-
- -
+
+ + + + Information + + + + + + + + +
-
- {{ formatBytes(item.downloaded_bytes) }} +
+ {{ formatBytes(item.downloaded_bytes) }} + + {{ formatTime(item.extras.duration) }} +
{{ item.title }}
+ + {{ formatTime(item.extras.duration) }} + + @@ -218,24 +225,6 @@
-
-

- - - - - No queued items. - -

-

- - -