diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index e8f06278..101fd9a8 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -4,7 +4,6 @@ import functools import hmac import json import logging -import os import random import time import uuid @@ -18,6 +17,7 @@ import httpx import magic from aiohttp import web from aiohttp.web import Request, RequestHandler, Response +from library.ag_utils import ag from yt_dlp.cookies import LenientSimpleCookie from .cache import Cache @@ -227,46 +227,48 @@ class HttpAPI(Common): HttpAPI: The instance of the HttpAPI. """ - staticDir = Path(os.path.join(self.rootPath, "ui", "exported")).absolute() + staticDir = (Path(self.rootPath) / "ui" / "exported").absolute() if not staticDir.exists(): - staticDir = Path(os.path.join(self.rootPath, "..", "ui", "exported")).absolute() + staticDir = (Path(self.rootPath).parent / "ui" / "exported").absolute() if not staticDir.exists(): msg = f"Could not find the frontend UI static assets. '{staticDir}'." raise ValueError(msg) - staticDir = str(staticDir.as_posix()) preloaded = 0 - base_path: str = self.config.base_path.rstrip("/") + base_path: str = str(Path(self.config.base_path).as_posix()).rstrip("/") - for root, _, files in os.walk(staticDir): - for file in files: - if file.endswith(".map"): - continue + for file in staticDir.rglob("*.*"): + if ".map" == file.suffix: + continue - file = str(Path(os.path.join(root, file)).as_posix()) - urlPath: str = f"{base_path}/{file.replace(f'{staticDir}/', '')}" + urlPath: str = f"{base_path}/{str(file.as_posix()).replace(f'{staticDir.as_posix()!s}/', '')}" - with open(file, "rb") as f: - content = f.read() + with open(file, "rb") as f: + content = f.read() - contentType = self._ext_to_mime.get(os.path.splitext(file)[1], MIME.from_file(file)) + contentType = self._ext_to_mime.get(file.suffix, MIME.from_file(file)) - self._static_holder[urlPath] = {"content": content, "content_type": contentType} - LOG.debug(f"Preloading '{urlPath}'.") - app.router.add_get(urlPath, self._static_file) + self._static_holder[urlPath] = {"content": content, "content_type": contentType} + LOG.debug(f"Preloading static '{urlPath}'.") + app.router.add_get(urlPath, self._static_file) + preloaded += 1 + + if "/index.html" in self._static_holder: + for path in self._frontend_routes: + path: str = f"{base_path}/{path.lstrip('/')}" + self._static_holder[path] = self._static_holder["/index.html"] + app.router.add_get(path, self._static_file) + if "{" not in path: + self._static_holder[path + "/"] = self._static_holder["/index.html"] + app.router.add_get(path + "/", self._static_file) + LOG.debug(f"Preloading static route '{path}'.") preloaded += 1 - if urlPath.endswith("/index.html"): - for path in self._frontend_routes: - path: str = f"{base_path}/{path.lstrip('/')}" - self._static_holder[path] = {"content": content, "content_type": contentType} - app.router.add_get(path, self._static_file) - if "{" not in path: - self._static_holder[path + "/"] = {"content": content, "content_type": contentType} - app.router.add_get(path + "/", self._static_file) - LOG.debug(f"Preloading '{path}'.") - preloaded += 1 + if "/" != self.config.base_path: + LOG.debug(f"adding base_path folder '{base_path}' to routes.") + app.router.add_get(base_path, self._static_file, name="_base_path") + app.router.add_get(f"{base_path}/", self._static_file, name="_base_path_slash") if preloaded < 1: message = f"Failed to find any static files in '{staticDir}'." @@ -278,13 +280,6 @@ class HttpAPI(Common): LOG.info(f"Preloaded '{preloaded}' static files.") - if "/" != self.config.base_path: - LOG.debug(f"adding base_path folder '{self.config.base_path}' to routes.") - app.router.add_get(self.config.base_path.rstrip("/"), self._static_file, name="base_path_static") - app.router.add_get( - f"{self.config.base_path.rstrip('/')}/", self._static_file, name="base_path_static_slash" - ) - return self def add_routes(self, app: web.Application) -> "HttpAPI": @@ -313,7 +308,7 @@ class HttpAPI(Common): if hasattr(method, "_http_name") and method._http_name: opts["name"] = method._http_name - LOG.debug(f"Registering route {method._http_method} {base_path}/{http_path}' {opts}.") + LOG.debug(f"Adding API route {method._http_method} {base_path}/{http_path}' {opts}.") self.routes.route(method._http_method, f"{base_path}/{http_path}", **opts)(method) if http_path in registered_options: @@ -662,7 +657,8 @@ class HttpAPI(Common): data={"error": "Manual archive is not enabled."}, status=web.HTTPNotFound.status_code ) - if not os.path.exists(manual_archive): + manual_archive = Path(manual_archive) + if not manual_archive.exists(): return web.json_response( data={"error": "Manual archive file not found.", "file": manual_archive}, status=web.HTTPNotFound.status_code, @@ -823,7 +819,7 @@ class HttpAPI(Common): limit = 50 logs_data = await read_logfile( - file=os.path.join(self.config.config_path, "logs", "app.log"), + file=Path(self.config.config_path) / "logs" / "app.log", offset=offset, limit=limit, ) @@ -1714,6 +1710,7 @@ class HttpAPI(Common): try: backend = random.choice(self.config.pictures_backends) # noqa: S311 + CACHE_KEY_BING = "random_background_bing" CACHE_KEY = "random_background" if self.cache.has(CACHE_KEY) and not request.query.get("force", False): @@ -1737,6 +1734,29 @@ class HttpAPI(Common): } async with httpx.AsyncClient(**opts) as client: + if backend.startswith("https://www.bing.com/HPImageArchive.aspx"): + if not self.cache.has(CACHE_KEY_BING): + response = await client.request(method="GET", url=backend) + if response.status_code != web.HTTPOk.status_code: + return web.json_response( + data={"error": "failed to retrieve the random background image."}, + status=web.HTTPInternalServerError.status_code, + ) + + img_url: str | None = ag(response.json(), "images.0.url") + if not img_url: + return web.json_response( + data={"error": "failed to retrieve the random background image."}, + status=web.HTTPInternalServerError.status_code, + ) + + backend = f"https://www.bing.com{img_url}" + await self.cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24) + else: + backend: str = await self.cache.aget(CACHE_KEY_BING) + + LOG.debug(f"Requesting random picture from '{backend!s}'.") + response = await client.request(method="GET", url=backend, follow_redirects=True) if response.status_code != web.HTTPOk.status_code: @@ -1796,7 +1816,7 @@ class HttpAPI(Common): headers={ "Location": str( self.app.router["ffprobe"].url_for( - file=str(realFile).replace(self.config.download_path, "").strip("/") + file=str(realFile.relative_to(self.config.download_path).as_posix()).strip("/") ) ), }, @@ -1835,7 +1855,7 @@ class HttpAPI(Common): headers={ "Location": str( self.app.router["file_info"].url_for( - file=str(realFile).replace(self.config.download_path, "").strip("/") + file=str(realFile.relative_to(self.config.download_path).as_posix()).strip("/") ) ), }, @@ -1847,7 +1867,7 @@ class HttpAPI(Common): ff_info = await ffprobe(realFile) response = { - "title": str(Path(realFile).stem), + "title": realFile.stem, "ffprobe": ff_info, "mimetype": get_mime_type(ff_info.get("metadata", {}), realFile), "sidecar": get_file_sidecar(realFile), @@ -1855,11 +1875,9 @@ class HttpAPI(Common): for key in response["sidecar"]: for i, f in enumerate(response["sidecar"][key]): - response["sidecar"][key][i]["file"] = ( - str(Path(realFile).with_name(Path(f["file"]).name)) - .replace(self.config.download_path, "") - .strip("/") - ) + response["sidecar"][key][i]["file"] = str( + realFile.with_name(f["file"].name).relative_to(self.config.download_path) + ).strip("/") return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=self.encoder.encode) except Exception as e: @@ -1930,9 +1948,6 @@ class HttpAPI(Common): targets.append(ins.make_target(item)) try: - if len(targets) < 1: - ins.clear() - ins.save(targets=targets) ins.load() except Exception as e: @@ -1976,20 +1991,25 @@ class HttpAPI(Common): if not self.config.browser_enabled: return web.json_response(data={"error": "File browser is disabled."}, status=web.HTTPForbidden.status_code) - path = request.match_info.get("path") - path = "/" if not path else unquote_plus(path) + req_path: str = request.match_info.get("path") + req_path: str = "/" if not req_path else unquote_plus(req_path) - test = os.path.realpath(os.path.join(self.config.download_path, path)) - if not os.path.exists(test): + test: Path = Path(self.config.download_path).joinpath(req_path) + if not test.exists(): return web.json_response( - data={"error": f"path '{path}' does not exist."}, status=web.HTTPNotFound.status_code + data={"error": f"path '{req_path}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + if not test.is_dir() and not test.is_symlink(): + return web.json_response( + data={"error": f"path '{req_path}' is not a directory."}, status=web.HTTPBadRequest.status_code ) try: return web.json_response( data={ - "path": path, - "contents": get_files(base_path=self.config.download_path, dir=path), + "path": req_path, + "contents": get_files(base_path=Path(self.config.download_path), dir=req_path), }, status=web.HTTPOk.status_code, dumps=self.encoder.encode, diff --git a/app/library/Utils.py b/app/library/Utils.py index 01385fad..98d68ba8 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -5,7 +5,6 @@ import ipaddress import json import logging import os -import pathlib import re import shlex import socket @@ -13,6 +12,7 @@ import uuid from datetime import UTC, datetime, timedelta from functools import lru_cache from http.cookiejar import MozillaCookieJar +from pathlib import Path import yt_dlp from Crypto.Cipher import AES @@ -70,7 +70,7 @@ class FileLogFormatter(logging.Formatter): return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") -def calc_download_path(base_path: str, folder: str | None = None, create_path: bool = True) -> str: +def calc_download_path(base_path: str | Path, folder: str | None = None, create_path: bool = True) -> str: """ Calculates download path and prevents folder traversal. @@ -83,19 +83,28 @@ def calc_download_path(base_path: str, folder: str | None = None, create_path: b Download path with base folder factored in. """ + if not isinstance(base_path, Path): + base_path = Path(base_path) + if not folder: - return base_path + return str(base_path) if folder.startswith("/"): folder = folder[1:] - realBasePath = pathlib.Path(base_path).absolute() - download_path = pathlib.Path(realBasePath / folder).absolute() + realBasePath = base_path.resolve() + download_path = Path(realBasePath).joinpath(folder).resolve(strict=False) if not str(download_path).startswith(str(realBasePath)): msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".' raise Exception(msg) + try: + download_path.relative_to(realBasePath) + except ValueError as e: + msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".' + raise Exception(msg) from e + if not download_path.is_dir() and create_path: download_path.mkdir(parents=True, exist_ok=True) @@ -248,7 +257,10 @@ def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, s """ idDict = {"id": None, "ie_key": None, "archive_id": None} - if not url or not archive_file or not os.path.exists(archive_file): + if not url or not archive_file: + return (False, idDict) + + if not Path(archive_file).exists(): return (False, idDict) idDict = get_archive_id(url=url) @@ -301,13 +313,13 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: return ({}, False, f"{e}") -def check_id(file: pathlib.Path) -> bool | str: +def check_id(file: Path) -> bool | str: """ Check if we are able to get an id from the file name. if so check if any video file with the same id exists. Args: - file (pathlib.Path): File to check. + file (Path): File to check. Returns: bool|str: False if no file found, else the file path. @@ -477,12 +489,12 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool: return False -def get_file_sidecar(file: pathlib.Path) -> list[dict]: +def get_file_sidecar(file: Path) -> list[dict]: """ Get sidecar files for the given file. Args: - file (pathlib.Path): The video file. + file (Path): The video file. Returns: list: List of sidecar files. @@ -531,7 +543,7 @@ def get_file_sidecar(file: pathlib.Path) -> list[dict]: def get_possible_images(dir: str) -> list[dict]: images = [] - path_loc = pathlib.Path(dir, "test.jpg") + path_loc = Path(dir, "test.jpg") for filename in ["poster", "thumbnail", "artwork", "cover", "fanart"]: for ext in [".jpg", ".jpeg", ".png", ".webp"]: @@ -542,7 +554,7 @@ def get_possible_images(dir: str) -> list[dict]: return images -def get_mime_type(metadata: dict, file_path: pathlib.Path) -> str: +def get_mime_type(metadata: dict, file_path: Path) -> str: """ Determine the correct MIME type for a video file based on ffprobe metadata. @@ -587,32 +599,35 @@ def get_mime_type(metadata: dict, file_path: pathlib.Path) -> str: return "application/octet-stream" -def get_file(download_path: str, file: str | pathlib.Path) -> tuple[pathlib.Path, int]: +def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]: """ Get the real file path. Args: - download_path (str): Base download path. - file (str|pathlib.Path): File path. + download_path (str|Path): Base download path. + file (str|Path): File path. Returns: - pathlib.Path: Real file path. + Path: Real file path. int: http status code. """ - file_path: str = os.path.normpath(os.path.join(download_path, str(file))) - if not file_path.startswith(download_path): - return (pathlib.Path(file_path), 404) + if not isinstance(download_path, Path): + download_path = Path(download_path) - realFile: str = pathlib.Path(calc_download_path(base_path=str(download_path), folder=str(file), create_path=False)) - if realFile.exists(): - return (realFile, 200) + try: + realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False)) + if realFile.exists(): + return (realFile, 200) + except Exception as e: + LOG.error(f"Error calculating download path. {e!s}") + return (Path(file), 404) possibleFile = check_id(file=realFile) if not possibleFile: return (realFile, 404) - return (pathlib.Path(possibleFile), 302) + return (Path(possibleFile), 302) def encrypt_data(data: str, key: bytes) -> str: @@ -736,12 +751,12 @@ def get( return data -def get_files(base_path: str, dir: str | None = None): +def get_files(base_path: Path | str, dir: str | None = None): """ Get directory contents. Args: - base_path (str): Base download path. + base_path (Path|str): Base download path. dir (str): Directory to check. Returns: @@ -751,31 +766,33 @@ def get_files(base_path: str, dir: str | None = None): OSError: If the directory is invalid or not a directory. """ + if not isinstance(base_path, Path): + base_path = Path(base_path) + + base_path = base_path.resolve() + + dir_path = base_path if dir and dir != "/": - path = os.path.normpath(os.path.join(base_path, str(dir))) - if not path.startswith(base_path): - msg = f"Invalid path: '{dir}' - '{path}' - must be inside '{base_path}'." - raise OSError(msg) - dir_path = os.path.realpath(path) - else: - dir_path = base_path + dir_path: Path = base_path.joinpath(dir) - dir_path = pathlib.Path(dir_path) - if dir_path.is_symlink(): - try: - dir_path = dir_path.resolve() - except OSError as e: - LOG.warning(f"Skipping broken symlink: {dir_path} - {e}") - return [] + try: + dir_path = dir_path.resolve() + except OSError as e: + LOG.warning(f"Failed to resolve '{dir}' - {e}") + return [] - if not str(dir_path).startswith(base_path): - msg = f"Invalid path: '{dir_path}' - must be inside '{base_path}'." - LOG.warning(msg) + try: + dir_path.relative_to(base_path) + except ValueError: + LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.") + return [] + + if not str(dir_path).startswith(str(base_path)): + LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.") return [] if not dir_path.is_dir(): - msg = f"Invalid path: '{dir_path}' - must be a directory." - LOG.warning(msg) + LOG.warning(f"Invalid path: '{dir_path}' - must be a directory.") return [] contents: list = [] @@ -785,11 +802,14 @@ def get_files(base_path: str, dir: str | None = None): if file.is_symlink(): try: - test: pathlib.Path = file.resolve() - if not str(test).startswith(base_path): - msg = f"Invalid symlink: '{file}' - must resolve inside '{base_path}'." - LOG.warning(msg) + test: Path = file.resolve() + test.relative_to(base_path) + if not str(test).startswith(str(base_path)): + LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.") continue + except ValueError: + LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.") + continue except OSError: LOG.warning(f"Skipping broken symlink: {file}") continue @@ -813,7 +833,7 @@ def get_files(base_path: str, dir: str | None = None): "type": "file" if file.is_file() else "dir", "content_type": content_type, "name": file.name, - "path": str(file).replace(base_path, "").strip("/"), + "path": str(file.relative_to(base_path)).strip("/"), "size": stat.st_size, "mime": get_mime_type({}, file) if file.is_file() else "directory", "mtime": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(), @@ -883,12 +903,12 @@ def strip_newline(string: str) -> str: return res.strip() if res else "" -async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict: +async def read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict: """ Read a log file and return a set of log lines along with pagination metadata. Args: - file (str): The log file path. + file (Path): The log file path. offset (int): Number of lines to skip from the end (newer entries). limit (int): Number of lines to return. @@ -903,7 +923,7 @@ async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict: from anyio import open_file - if not pathlib.Path(file).exists(): + if not file.exists(): return {"logs": [], "next_offset": None, "end_is_reached": True} result = [] @@ -951,7 +971,7 @@ async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict: return {"logs": [], "next_offset": None, "end_is_reached": True} -async def tail_log(file: pathlib.Path, emitter: callable, sleep_time: float = 0.5): +async def tail_log(file: Path, emitter: callable, sleep_time: float = 0.5): """ Continuously read a log file and emit new lines. @@ -1133,7 +1153,7 @@ def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) return list(dict.fromkeys(matched)) -def delete_dir(dir: pathlib.Path) -> bool: +def delete_dir(dir: Path) -> bool: """ Delete a directory and all its contents. diff --git a/app/library/config.py b/app/library/config.py index a1f55346..1753c696 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -11,7 +11,7 @@ from pathlib import Path import coloredlogs from dotenv import load_dotenv -from .Utils import FileLogFormatter, arg_converter, load_cookies +from .Utils import FileLogFormatter, arg_converter from .version import APP_VERSION @@ -158,7 +158,7 @@ class Config: pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", - "https://placedog.net/1920/1080", + "https://www.bing.com/HPImageArchive.aspx?format=js&idx=0&n=1&mkt=en-US", ] "The list of picture backends to use for the background." @@ -371,23 +371,6 @@ class Config: LOG.info(f"keep archive option is enabled. Using archive file '{archive_file}'.") self._ytdlp_cli_mutable += f"\n--download-archive {archive_file.as_posix()!s}" - if cookies_file := ytdl_options.get("cookiefile", None): - cookies_file = Path(cookies_file) - if cookies_file.exists() and cookies_file.stat().st_size > 2: - LOG.info(f"Using cookies from '{cookies_file}'.") - load_cookies(cookies_file) - self._ytdlp_cli_mutable += f"\n--cookies {cookies_file.as_posix()!s}" - else: - LOG.warning(f"Invalid cookie file '{cookies_file}' specified.") - ytdl_options.pop("cookiefile", None) - - if not ytdl_options.get("cookiefile", None): - cookies_file: Path = Path(self.config_path) / "cookies.txt" - if cookies_file.exists() and cookies_file.stat().st_size > 2: - LOG.info(f"Using cookies from '{cookies_file}'.") - load_cookies(cookies_file) - self._ytdlp_cli_mutable += f"\n--cookies {cookies_file.as_posix()!s}" - if self.temp_keep: LOG.info("Keep temp files option is enabled.") @@ -469,12 +452,6 @@ class Config: ytdlp_args = self.get_ytdlp_args() - data["has_cookies"] = False - - if cookie := ytdlp_args.get("cookiefile", None): - cookie_file = Path(cookie) - data["has_cookies"] = cookie_file.exists() and cookie_file.stat().st_size > 2 - if not data.get("keep_archive", False) and ytdlp_args.get("download_archive", None): data["keep_archive"] = True diff --git a/ui/stores/ConfigStore.js b/ui/stores/ConfigStore.js index 1512808b..93887f2a 100644 --- a/ui/stores/ConfigStore.js +++ b/ui/stores/ConfigStore.js @@ -11,7 +11,6 @@ const CONFIG_KEYS = { ytdlp_version: '', max_workers: 1, version: '', - has_cookies: false, basic_mode: true, default_preset: 'default', instance_title: null,