diff --git a/app/library/DataStore.py b/app/library/DataStore.py index 83dcff5c..b38fa890 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -110,6 +110,10 @@ class DataStore: return None + async def test(self) -> bool: + await self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() + return True + def _updateStoreItem(self, type: str, item: ItemDTO) -> None: sqlStatement = """ INSERT INTO "history" ("id", "type", "url", "data") diff --git a/app/library/Download.py b/app/library/Download.py index 12615c96..99777f61 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -8,7 +8,7 @@ import shutil import yt_dlp import hashlib from .AsyncPool import Terminator -from .Utils import get_format, get_opts, jsonCookie, mergeConfig +from .Utils import get_opts, jsonCookie, mergeConfig from .ItemDTO import ItemDTO from .config import Config from .Emitter import Emitter @@ -26,7 +26,6 @@ class Download: temp_dir: str = None output_template: str = None output_template_chapter: str = None - format: str = None ytdl_opts: dict = None info: ItemDTO = None default_ytdl_opts: dict = None @@ -61,7 +60,7 @@ class Download: "Fields to be extracted from yt-dlp progress hook." tempKeep: bool = False - "Keep temp directory after download." + "Keep temp folder after download." def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False): config = Config.get_instance() @@ -70,8 +69,8 @@ class Download: self.temp_dir = info.temp_dir self.output_template_chapter = info.output_template_chapter self.output_template = info.output_template - self.format = config.ytdl_options.get('format', get_format(info.format, info.quality)) - self.ytdl_opts = get_opts(info.format, info.quality, info.ytdlp_config if info.ytdlp_config else {}) + self.preset = info.preset + self.ytdl_opts = get_opts(self.preset, info.ytdlp_config if info.ytdlp_config else {}) self.info = info self.id = info._id self.default_ytdl_opts = config.ytdl_options @@ -110,7 +109,6 @@ class Download: try: params: dict = { 'color': 'no_color', - 'format': self.format, 'paths': { 'home': self.download_dir, 'temp': self.tempPath, @@ -121,12 +119,14 @@ class Download: }, 'noprogress': True, 'break_on_existing': True, - 'ignoreerrors': False, 'progress_hooks': [self._progress_hook], 'postprocessor_hooks': [self._postprocessor_hook], **mergeConfig(self.default_ytdl_opts, self.ytdl_opts), } + if 'format' not in params and self.default_ytdl_opts.get('format', None): + params['format'] = self.default_ytdl_opts['format'] + params['ignoreerrors'] = False if self.debug: @@ -158,10 +158,10 @@ class Download: if hasDeletedOptions: LOG.warning( - f'Live stream detected for [{self.info.title}], The following opts [{deletedOpts=}] have been deleted which are known to cause issues with live stream and post stream manifestless mode.') + f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode.") LOG.info( - f'Downloading {os.getpid()=} id="{self.info.id}" title="{self.info.title}" format="{self.format}" fq="{self.info.format}/{self.info.quality}".') + f'Downloading pid: {os.getpid()} id="{self.info.id}" title="{self.info.title}" preset="{self.preset}".') cls = yt_dlp.YoutubeDL(params=params) @@ -292,7 +292,7 @@ class Download: if self.info.status != 'finished' and self.is_live: LOG.warning( - f'Keeping live temp directory [{self.tempPath}], as the reported status is not finished [{self.info.status}].') + f'Keeping live temp folder [{self.tempPath}], as the reported status is not finished [{self.info.status}].') return if not os.path.exists(self.tempPath): @@ -300,10 +300,10 @@ class Download: if self.tempPath == self.temp_dir: LOG.warning( - f'Attempted to delete video temp directory: {self.tempPath}, but it is the same as main temp directory.') + f'Attempted to delete video temp folder: {self.tempPath}, but it is the same as main temp folder.') return - LOG.info(f'Deleting Temp directory: {self.tempPath}') + LOG.info(f'Deleting Temp folder: {self.tempPath}') shutil.rmtree(self.tempPath, ignore_errors=True) async def progress_update(self): @@ -341,10 +341,6 @@ class Download: if os.path.exists(status.get('filename')): self.info.file_size = os.path.getsize(status.get('filename')) - # Set correct file extension for thumbnails - if self.info.format == 'thumbnail': - self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) - self.info.status = status.get('status', self.info.status) self.info.msg = status.get('msg') @@ -363,7 +359,7 @@ class Download: self.info.speed = status.get('speed') self.info.eta = status.get('eta') - if self.info.status == 'finished' and 'filename' in status and self.info.format != 'thumbnail': + if self.info.status == 'finished' and 'filename' in status: try: self.info.file_size = os.path.getsize(status.get('filename')) except FileNotFoundError: diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index e8d038df..7f1b53e2 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -10,7 +10,7 @@ from .config import Config from .Download import Download from .ItemDTO import ItemDTO from .DataStore import DataStore -from .Utils import calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig +from .Utils import ag, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig from .AsyncPool import AsyncPool from .Emitter import Emitter @@ -26,19 +26,17 @@ class DownloadQueue: done: DataStore = None event: asyncio.Event = None pool: AsyncPool = None - connection: Connection = None def __init__(self, emitter: Emitter, connection: Connection): self.config = Config.get_instance() self.emitter = emitter - self.connection = connection - self.done = DataStore(type=TYPE_DONE, connection=self.connection) - self.queue = DataStore(type=TYPE_QUEUE, connection=self.connection) + self.done = DataStore(type=TYPE_DONE, connection=connection) + self.queue = DataStore(type=TYPE_QUEUE, connection=connection) self.done.load() self.queue.load() async def test(self) -> bool: - self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() + await self.done.test() return True async def initialize(self): @@ -53,8 +51,7 @@ class DownloadQueue: async def __add_entry( self, entry: dict, - quality: str, - format: str, + preset: str, folder: str, ytdlp_config: dict = {}, ytdlp_cookies: str = '', @@ -74,9 +71,9 @@ class DownloadQueue: entries = entry['entries'] playlist_index_digits = len(str(len(entries))) results = [] - for index, etr in enumerate(entries, start=1): + for i, etr in enumerate(entries, start=1): etr['playlist'] = entry.get('id') - etr['playlist_index'] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index) + etr['playlist_index'] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(i) for property in ('id', 'title', 'uploader', 'uploader_id'): if property in entry: etr[f'playlist_{property}'] = entry.get(property) @@ -84,8 +81,7 @@ class DownloadQueue: results.append( await self.__add_entry( entry=etr, - quality=quality, - format=format, + preset=preset, folder=folder, ytdlp_config=ytdlp_config, ytdlp_cookies=ytdlp_cookies, @@ -108,19 +104,19 @@ class DownloadQueue: else: error = entry.get('msg', None) - LOG.debug(f"entry: {entry.get('id', None)} - {entry.get('webpage_url', None)} - {entry.get('url', None)}") + LOG.debug( + f"Entry id '{entry.get('id', None)}' url '{entry.get('webpage_url', None)} - {entry.get('url', None)}'." + ) if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry.get('url')): item = self.done.get(key=entry['id'], url=entry.get('webpage_url') or entry['url']) - - LOG.debug(f'Item [{item.info.title}] already downloaded. Removing from history.') - + LOG.warning(f"Item '{item.info.id}' - '{item.info.title}' already downloaded. Removing from history.") await self.clear([item.info._id]) try: item = self.queue.get(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url')) if item is not None: - err_message = f'Item [{item.info.id}: {item.info.title}] already in download queue.' + err_message = f"Item ID '{item.info.id}' - '{item.info.title}' already in download queue." LOG.info(err_message) return {'status': 'error', 'msg': err_message} except KeyError: @@ -142,8 +138,8 @@ class DownloadQueue: id=entry.get('id'), title=entry.get('title'), url=entry.get('webpage_url') or entry.get('url'), - quality=quality, - format=format, + preset=preset, + thumbnail=entry.get('thumbnail', None), folder=folder, download_dir=download_dir, temp_dir=self.config.temp_path, @@ -170,7 +166,7 @@ class DownloadQueue: NotifyEvent = 'completed' elif self.config.allow_manifestless is False and is_manifestless is True: dlInfo.info.status = 'error' - dlInfo.info.error = 'Video is in Post-Live Manifestless mode.' + dlInfo.info.error = 'Video is in post-live manifestless mode.' itemDownload = self.done.put(dlInfo) NotifyEvent = 'completed' else: @@ -188,8 +184,7 @@ class DownloadQueue: elif eventType.startswith('url'): return await self.add( url=entry.get('url'), - quality=quality, - format=format, + preset=preset, folder=folder, ytdlp_config=ytdlp_config, ytdlp_cookies=ytdlp_cookies, @@ -205,8 +200,7 @@ class DownloadQueue: async def add( self, url: str, - quality: str, - format: str, + preset: str, folder: str, ytdlp_config: dict = {}, ytdlp_cookies: str = '', @@ -215,7 +209,8 @@ class DownloadQueue: ): ytdlp_config = ytdlp_config if ytdlp_config else {} - LOG.info(f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}') + LOG.info(f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset=}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'.") + if isinstance(ytdlp_config, str): try: ytdlp_config = json.loads(ytdlp_config) @@ -226,7 +221,7 @@ class DownloadQueue: already = set() if already is None else already if url in already: - LOG.info('recursion detected, skipping.') + LOG.warning(f"Recursion detected with url '{url}' skipping.") return {'status': 'ok'} already.add(url) @@ -234,7 +229,7 @@ class DownloadQueue: try: downloaded, id_dict = self.isDownloaded(url) if downloaded is True: - message = f"[ { id_dict.get('id') } ]: has been downloaded already." + message = f"This url with ID '{id_dict.get('id')}' has been downloaded already and recorded in archive." LOG.info(message) return {'status': 'error', 'msg': message} @@ -254,7 +249,8 @@ class DownloadQueue: if not entry: return {'status': 'error', 'msg': 'Unable to extract info check logs.'} - LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}') + LOG.debug( + f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'.") except yt_dlp.utils.ExistingVideoReached as exc: LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}') return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'} @@ -267,8 +263,7 @@ class DownloadQueue: return await self.__add_entry( entry=entry, - quality=quality, - format=format, + preset=preset, folder=folder, ytdlp_config=ytdlp_config, ytdlp_cookies=ytdlp_cookies, @@ -277,8 +272,8 @@ class DownloadQueue: ) async def cancel(self, ids: list[str]) -> dict[str, str]: - status: dict[str, str] = {"status": "ok"} + for id in ids: try: item = self.queue.get(key=id) @@ -400,7 +395,8 @@ class DownloadQueue: await self.__downloadFile(id, entry) async def __downloadFile(self, id: str, entry: Download): - LOG.info(f'Queuing {id=} - {entry.info.title=} - {entry.info.url=} - {entry.info.folder=}.') + LOG.info( + f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'folder: {entry.info.folder}'.") try: await entry.start(self.emitter) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 3244a2cf..034bb0d8 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -1,11 +1,11 @@ -import asyncio import base64 from datetime import datetime import functools import json import os -import random import time + +import httpx from .config import Config from .DownloadQueue import DownloadQueue from aiohttp import web @@ -14,9 +14,7 @@ from .Playlist import Playlist from .M3u8 import M3u8 from .Segments import Segments from .Subtitle import Subtitle -import socketio import logging -import sqlite3 import magic from .common import common from pathlib import Path @@ -30,16 +28,11 @@ MIME = magic.Magic(mime=True) class HttpAPI(common): config: Config = None encoder: Encoder = None - app: web.Application = None - sio: socketio.AsyncServer = None routes: web.RouteTableDef = None - connection: sqlite3.Connection = None queue: DownloadQueue = None - loop: asyncio.AbstractEventLoop = None rootPath: str = None staticHolder: dict = {} - extToMime: dict = { '.html': 'text/html', '.css': 'text/css', @@ -54,11 +47,9 @@ class HttpAPI(common): self.rootPath = str(Path(__file__).parent.parent.parent) self.config = Config.get_instance() - self.loop = asyncio.get_event_loop() - self.encoder = encoder - self.routes = web.RouteTableDef() + self.encoder = encoder self.emitter = emitter self.queue = queue @@ -138,10 +129,13 @@ class HttpAPI(common): self.routes.route(method._http_method, self.config.url_prefix + http_path)(method) async def on_prepare(request: Request, response: Response): + if 'Server' in response.headers: + del response.headers['Server'] + if 'Origin' in request.headers: response.headers['Access-Control-Allow-Origin'] = request.headers['Origin'] response.headers['Access-Control-Allow-Headers'] = 'Content-Type' - response.headers['Access-Control-Allow-Methods'] = 'PUT, POST, DELETE' + response.headers['Access-Control-Allow-Methods'] = 'PATCH, PUT, POST, DELETE' if self.config.url_prefix != '/': self.routes.route('GET', '/')(lambda _: web.HTTPFound(self.config.url_prefix)) @@ -194,12 +188,11 @@ class HttpAPI(common): post = await request.json() url: str = post.get('url') - quality: str = post.get('quality') + preset: str = post.get('preset', 'default') if not url: raise web.HTTPBadRequest() - format: str = post.get('format') folder: str = post.get('folder') ytdlp_cookies: str = post.get('ytdlp_cookies') ytdlp_config: dict = post.get('ytdlp_config') @@ -209,8 +202,7 @@ class HttpAPI(common): status = await self.add( url=url, - quality=quality, - format=format, + preset=preset, folder=folder, ytdlp_cookies=ytdlp_cookies, ytdlp_config=ytdlp_config, @@ -251,8 +243,7 @@ class HttpAPI(common): status[item.get('url')] = await self.add( url=item.get('url'), - quality=item.get('quality'), - format=item.get('format'), + preset=item.get('preset', 'default'), folder=item.get('folder'), ytdlp_cookies=item.get('ytdlp_cookies'), ytdlp_config=item.get('ytdlp_config'), @@ -519,9 +510,42 @@ class HttpAPI(common): @route('GET', '/') async def index(self, _) -> Response: + if '/index.html' not in self.staticHolder: + LOG.error("Static frontend files not found.") + return web.json_response({"error": "File not found.", "file": '/index.html'}, status=404) + data = self.staticHolder['/index.html'] return web.Response( body=data.get('content'), content_type=data.get('content_type'), charset='utf-8', status=web.HTTPOk.status_code) + + @route('GET', '/thumbnail') + async def get_thumbnail(self, request: Request) -> dict: + url = request.query.get('url') + if not url: + return web.json_response({"error": "URL is required."}, status=400) + + try: + opts = { + 'proxy': self.config.ytdl_options.get('proxy', None), + 'headers': { + 'User-Agent': self.config.ytdl_options.get( + 'user_agent', + request.headers.get('User-Agent', f"YTPTube/{self.config.version}")), + }, + } + async with httpx.AsyncClient(**opts) as client: + LOG.info(f"Fetching thumbnail from '{url}'.") + response = await client.request(method='GET', url=url) + return web.Response(body=response.content, headers={ + 'Content-Type': response.headers.get('Content-Type'), + 'Pragma': 'public', + 'Access-Control-Allow-Origin': '*', + 'Cache-Control': f"public, max-age={time.time() + 31536000}", + 'Expires': time.strftime('%a, %d %b %Y %H:%M:%S GMT', datetime.fromtimestamp(time.time() + 31536000).timetuple()), + }) + except Exception as e: + LOG.error(f"Error fetching thumbnail from '{url}'. '{e}'") + return web.json_response({"error": str(e)}, status=500) diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index c9e31131..ef0d457b 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -8,7 +8,6 @@ import shlex from aiohttp import web import socketio import logging -import sqlite3 from .config import Config from .DownloadQueue import DownloadQueue from .common import common @@ -24,9 +23,7 @@ class HttpSocket(common): This class is used to handle WebSocket events. """ config: Config = None - app: web.Application = None sio: socketio.AsyncServer = None - connection: sqlite3.Connection = None def __init__(self, queue: DownloadQueue, emitter: Emitter, encoder: Encoder): super().__init__(queue=queue, encoder=encoder) @@ -149,13 +146,12 @@ class HttpSocket(common): @ws_event async def add_url(self, sid: str, data: dict): url: str = data.get('url') - quality: str = data.get('quality') if not url: self.emitter.warning('No URL provided.', to=sid) return - format: str = data.get('format') + preset: str = data.get('preset', 'default') folder: str = data.get('folder') ytdlp_cookies: str = data.get('ytdlp_cookies') ytdlp_config: dict = data.get('ytdlp_config') @@ -165,8 +161,7 @@ class HttpSocket(common): status = await self.add( url=url, - quality=quality, - format=format, + preset=preset, folder=folder, ytdlp_cookies=ytdlp_cookies, ytdlp_config=ytdlp_config, @@ -234,18 +229,17 @@ class HttpSocket(common): @ws_event async def connect(self, sid: str, _=None): - data: dict = { + data = { **self.queue.get(), "config": self.config.frontend(), "tasks": self.config.tasks, - "presets": [], - "directories": [], + "presets": self.config.presets, } - # get directory listing - dlDir: str = self.config.download_path - data['directories'] = [ - name for name in os.listdir(dlDir) if os.path.isdir(os.path.join(dlDir, name)) + # get download folder listing + downloadPath: str = self.config.download_path + data['folders'] = [ + name for name in os.listdir(downloadPath) if os.path.isdir(os.path.join(downloadPath, name)) ] await self.emitter.emit('initial_data', data, to=sid) diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index 5b07fbb6..afdff6f2 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -13,8 +13,9 @@ class ItemDTO: id: str title: str url: str - quality: str - format: str + quality: str = None + format: str = None + thumbnail: str = None preset: str = "default" folder: str download_dir: str = None diff --git a/app/library/Utils.py b/app/library/Utils.py index 4bebf3c5..e7b41755 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -6,99 +6,56 @@ import os import pathlib import re from typing import Any +import uuid import yt_dlp LOG = logging.getLogger('Utils') -AUDIO_FORMATS: tuple[str] = ('m4a', 'mp3', 'opus', 'wav',) IGNORED_KEYS: tuple[str] = ('cookiefile', 'paths', 'outtmpl', 'progress_hooks', 'postprocessor_hooks',) - YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None -def get_format(format: str, quality: str) -> str: +def get_opts(preset: str, ytdl_opts: dict) -> dict: """ - Returns format for download + Returns ytdlp options download options Args: - format (str): format selected - quality (str): quality selected - - Raises: - Exception: unknown quality, unknown format - - Returns: - dl_format: Formatted download string - """ - format = format or "any" - - if format.startswith("custom:"): - return format[7:] - - if format == "thumbnail": - # Quality is irrelevant in this case since we skip the download - return "bestaudio/best" - - if format in AUDIO_FORMATS: - # Audio quality needs to be set post-download, set in opts - return "bestaudio/best" - - if format in ('mp4', 'any'): - if quality == 'audio': - return "bestaudio/best" - - videoFormat, audioFormat = ('[ext=mp4]', '[ext=m4a]') if format == "mp4" else ('', '') - - videoResolution = f'[height<={quality}]' if quality != 'best' else '' - - videoCombo = videoResolution + videoFormat - - return f'bestvideo{videoCombo}+bestaudio{audioFormat}/best{videoCombo}' - - raise Exception(f"Unknown format {format}") - - -def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict: - """ - Returns extra download options - Mostly postprocessing options - - Args: - format (str): format selected - quality (str): quality of format selected (needed for some formats) + preset (str): the name of the preset selected. ytdl_opts (dict): current options selected Returns: - ytdl_opts: Extra options + ytdl extra options """ opts = copy.deepcopy(ytdl_opts) - if not opts: - opts: dict = { - "postprocessors": [], - } - postprocessors = [] + if 'default' == preset: + return opts - if format in AUDIO_FORMATS: - postprocessors.append({ - "key": "FFmpegExtractAudio", - "preferredcodec": format, - "preferredquality": 0 if quality == "best" else quality, - }) + from .config import Config + presets = Config.get_instance().presets - # Audio formats without thumbnail - if format not in ("wav") and "writethumbnail" not in opts: - opts["writethumbnail"] = True - postprocessors.append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}) - postprocessors.append({"key": "FFmpegMetadata"}) - postprocessors.append({"key": "EmbedThumbnail"}) + if preset not in presets: + LOG.error(f"Preset '{preset}' is not defined in the presets.") + return opts - if format == "thumbnail": - opts["skip_download"] = True - opts["writethumbnail"] = True - postprocessors.append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}) + preset_opts = presets[preset] + + opts['format'] = preset_opts.get('format') + + if 'postprocessors' in preset_opts: + if 'postprocessors' not in opts: + opts['postprocessors'] = [] + + opts['postprocessors'].extend(preset_opts['postprocessors']) + + if 'args' in preset_opts: + for ignored_key in IGNORED_KEYS: + for key, value in preset_opts['args'].items(): + if key == ignored_key: + continue + + opts[key] = value - opts["postprocessors"] = postprocessors + (opts["postprocessors"] if "postprocessors" in opts else []) return opts @@ -116,10 +73,10 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None) -> (Any | dict[str, Any] | N def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) -> str: - """Calculates download path and prevents directory traversal. + """Calculates download path and prevents folder traversal. Returns: - Download path with base directory factored in. + Download path with base folder factored in. """ if not folder: return basePath @@ -128,7 +85,7 @@ def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) download_path = os.path.realpath(os.path.join(basePath, folder)) if not download_path.startswith(realBasePath): - raise Exception(f'Folder "{folder}" must resolve inside the base download directory "{realBasePath}".') + raise Exception(f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".') if not os.path.isdir(download_path) and createPath: os.makedirs(download_path, exist_ok=True) @@ -348,3 +305,57 @@ def checkId(basePath: str, file: pathlib.Path) -> bool | str: return f.absolute() return False + + +def get_value(value): + return value() if callable(value) else value + + +def ag(array: dict | list, path: list[str | int] | str | int, default: Any = None, separator: str = '.') -> Any: + """ + dict/array getter: Retrieve a value from a nested dict or object using a path. + + :param array_or_object: dict-like or object from which to retrieve values + :param path: string, list, or None. Represents the path to retrieve: + - If None or empty string, returns the entire structure. + - If list, tries each path and returns the first found. + - If string, navigates through nested dict keys separated by `separator`. + :param default: Value (or callable) returned if nothing is found. + :param separator: Separator for nested paths in strings. + :return: The found value or the default if not found. + """ + + if path is None or path == '': + return array + + if not isinstance(array, dict): + try: + array = vars(array) + except TypeError: + array = dict(array) + + if isinstance(path, list): + randomValue = str(uuid.uuid4()) + for key in path: + val = ag(array, key, randomValue, separator) + if val != randomValue: + return val + + return get_value(default) + + if path in array and array[path] is not None: + return array[path] + + if separator not in path: + return array.get(path, get_value(default)) + + keys = path.split(separator) + current = array + + for segment in keys: + if isinstance(current, dict) and segment in current: + current = current[segment] + else: + return get_value(default) + + return current diff --git a/app/library/Webhooks.py b/app/library/Webhooks.py index 17117eb2..6b586f50 100644 --- a/app/library/Webhooks.py +++ b/app/library/Webhooks.py @@ -7,6 +7,7 @@ from .version import APP_VERSION LOG = logging.getLogger('Webhooks') + class Webhooks: targets: list[dict] = [] allowed_events: tuple = ('added', 'completed', 'error', 'not_live',) @@ -17,10 +18,10 @@ class Webhooks: def load(self, file: str): try: - if os.path.getsize(file) < 2: + if os.path.getsize(file) < 3: raise Exception(f'file is empty.') - LOG.info(f'Loading webhooks from {file}') + LOG.info(f"Loading webhooks from '{file}'.") from .Utils import load_file (target, status, error) = load_file(file, list) if not status: @@ -28,7 +29,7 @@ class Webhooks: self.targets = target except Exception as e: - LOG.error(f'Error loading webhooks from {file}: {e}') + LOG.error(f"Error loading webhooks from '{file}'. '{e}'") pass async def send(self, event: str, item: ItemDTO) -> list[dict]: @@ -53,7 +54,7 @@ class Webhooks: from .config import Config req: dict = target.get('request') try: - LOG.info(f"Sending {event=} {item.id=} to [{target.get('name')}]") + LOG.info(f"Sending event '{event}' id '{item.id}' to '{target.get('name')}'.") async with httpx.AsyncClient() as client: request_type = req.get('type', 'json') @@ -84,14 +85,15 @@ class Webhooks: 'text': response.text } - msg = f"[{target.get('name')}] Response to [{event=} {item.id=}] [status: {response.status_code}]." + msg = f"Webhook target '{target.get('name')}' Responded to event '{event}' id '{item.id}' with status '{response.status_code}'." if Config.get_instance().debug and respData.get('text'): - msg += f" [Body: {respData.get('text','??')}]" + msg += f" body '{respData.get('text','??')}'." LOG.info(msg) return respData except Exception as e: + LOG.error(f"Error sending event '{event}' id '{item.id}' to '{target.get('name')}'. '{e}'") return { 'url': req.get('url'), 'status': 500, diff --git a/app/library/common.py b/app/library/common.py index 66ada752..5715ab0e 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -17,16 +17,8 @@ class common(): self.queue = queue self.encoder = encoder - async def add( - self, - url: str, - quality: str, - format: str, - folder: str, - ytdlp_cookies: str, - ytdlp_config: dict, - output_template: str - ) -> dict[str, str]: + async def add(self, url: str, preset: str, folder: str, ytdlp_cookies: str, ytdlp_config: dict, + output_template: str) -> dict[str, str]: if ytdlp_config is None: ytdlp_config = {} @@ -35,8 +27,7 @@ class common(): status = await self.queue.add( url=url, - format=format if format else 'any', - quality=quality if quality else 'best', + preset=preset if preset else 'default', folder=folder, ytdlp_cookies=ytdlp_cookies, ytdlp_config=ytdlp_config, diff --git a/app/library/config.py b/app/library/config.py index 62215ab0..6abc5815 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -65,11 +65,50 @@ class Config: ytdlp_version: str = YTDLP_VERSION tasks: list = [] + presets: list = [ + { + 'name': 'Default - Use default yt-dlp format', + 'format': 'default', + 'postprocessors': [], + 'args': {} + }, + # { + # 'name': 'Audio - Best audio [MP3]', + # 'format': 'bestaudio', + # 'postprocessors': [ + # {'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': 'best'}, + # {"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}, + # {'key': 'FFmpegMetadata'}, + # {'key': 'EmbedThumbnail'}, + # ], + # 'args': { + # 'writethumbnail': True + # } + # }, + { + 'name': 'Video - 1080p h264/acc', + 'format': 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]', + 'args': { + 'format_sort': [ + 'vcodec:h264' + ], + }, + }, + { + 'name': 'Video - 720p h264/acc', + 'format': 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]', + 'args': { + 'format_sort': [ + 'vcodec:h264' + ], + }, + }, + ] _manual_vars: tuple = ('temp_path', 'config_path', 'download_path',) _immutable: tuple = ( 'version', '__instance', 'ytdl_options', 'tasks', - 'new_version_available', 'ytdlp_version', 'started', + 'new_version_available', 'ytdlp_version', 'started', 'presets', ) _int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout', 'debugpy_port',) @@ -77,7 +116,7 @@ class Config: _frontend_vars: tuple = ( 'download_path', 'keep_archive', 'output_template', - 'ytdlp_version', 'url_host', 'url_prefix', + 'ytdlp_version', 'version', 'url_host', 'started', 'url_prefix', ) @staticmethod @@ -192,6 +231,34 @@ class Config: except Exception as e: pass + presetsFile = os.path.join(self.config_path, 'presets.json') + if os.path.exists(presetsFile) and os.path.getsize(presetsFile) > 0: + LOG.info(f"Loading extra presets from '{presetsFile}'.") + try: + (presets, status, error) = load_file(presetsFile, list) + if not status: + LOG.error(f"Could not load presets file from '{presetsFile}'. '{error}'.") + sys.exit(1) + + for preset in presets: + if 'name' not in preset: + LOG.error(f"Missing 'name' key in preset '{preset}'.") + continue + + if 'format' not in preset: + LOG.error(f"Missing 'format' key in preset '{preset}'.") + continue + + if 'args' not in preset: + preset['args'] = {} + + if 'postprocessors' not in preset: + preset['postprocessors'] = [] + + self.presets.append(preset) + except Exception as e: + pass + self.ytdl_options['socket_timeout'] = self.socket_timeout if self.keep_archive: diff --git a/app/library/encoder.py b/app/library/encoder.py index b4e48393..b17f6a91 100644 --- a/app/library/encoder.py +++ b/app/library/encoder.py @@ -8,8 +8,8 @@ class Encoder(json.JSONEncoder): will call the __dict__ method of an object if it exists. """ - def default(self, obj): - if isinstance(obj, object) and hasattr(obj, '__dict__'): - return obj.__dict__ + def default(self, o): + if isinstance(o, object) and hasattr(o, '__dict__'): + return o.__dict__ - return json.JSONEncoder.default(self, obj) + return json.JSONEncoder.default(self, o) diff --git a/app/library/version.py b/app/library/version.py index 646ec0bd..8a7718b5 100644 --- a/app/library/version.py +++ b/app/library/version.py @@ -1 +1,6 @@ +""" +Version of the application, +This file is updated by the CI/CD pipeline, do not edit it manually. +""" APP_VERSION = "dev-master" +"Application version" diff --git a/app/main.py b/app/main.py index fc110dad..16971048 100644 --- a/app/main.py +++ b/app/main.py @@ -35,7 +35,7 @@ class main: self.app = web.Application() self.encoder = Encoder() - self.checkDirectories() + self.checkFolders() caribou.upgrade(self.config.db_file, os.path.join(self.rootPath, "migrations")) connection = sqlite3.connect(database=self.config.db_file, isolation_level=None) @@ -54,7 +54,7 @@ class main: if os.path.exists(WebhookFile): emitter.add_emitter(Webhooks(WebhookFile).emit) - def checkDirectories(self) -> None: + def checkFolders(self) -> None: try: LOG.debug(f"Checking download folder at '{self.config.download_path}'.") if not os.path.exists(self.config.download_path): @@ -101,8 +101,7 @@ class main: LOG.info(f"Started 'Task: {taskName}' at '{timeNow}'.") await self.socket.add( url=task.get("url"), - quality=task.get("quality", "best"), - format=task.get("format", "any"), + preset=task.get("preset", "default"), folder=task.get("folder"), ytdlp_cookies=task.get("ytdlp_cookies"), ytdlp_config=task.get("ytdlp_config"), diff --git a/ui/components/FloatingImage.vue b/ui/components/FloatingImage.vue new file mode 100644 index 00000000..41106f36 --- /dev/null +++ b/ui/components/FloatingImage.vue @@ -0,0 +1,112 @@ + + + diff --git a/ui/components/History.vue b/ui/components/History.vue index 01b0f7c9..96abca3d 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -79,20 +79,44 @@
+
- - {{ item.title }} - - {{ item.title }} + {{ item.title }}
+
- + + - + + +
+
+
+
+ + +
+
@@ -106,7 +130,7 @@
{{ item.msg }}
-
+
-
+
{{ moment(item.datetime).fromNow() }}
-
+
{{ moment(item.live_in).fromNow() }}
-
+
{{ formatBytes(item.file_size) }}
@@ -176,14 +201,13 @@
- +
@@ -209,6 +233,15 @@

+ +
+
+
+ +
+ +
@@ -225,8 +258,17 @@ const socket = useSocketStore() const selectedElms = ref([]); const masterSelectAll = ref(false); const showCompleted = useStorage('showCompleted', true) +const hideThumbnail = useStorage('hideThumbnail', false) const direction = useStorage('sortCompleted', 'desc') +const video_link = ref('') +const video_title = ref('') + +const playVideo = item => { + video_link.value = makeDownload(config, item, 'm3u8'); + video_title.value = item.title; +} + watch(masterSelectAll, (value) => { for (const key in stateStore.history) { const element = stateStore.history[key]; diff --git a/ui/components/NewDownload.vue b/ui/components/NewDownload.vue index 651eac11..2a2c920c 100644 --- a/ui/components/NewDownload.vue +++ b/ui/components/NewDownload.vue @@ -29,13 +29,13 @@
-
+
- Download Path + Save in
+ :disabled="!socket.isConnected" list="folders">
@@ -64,11 +64,11 @@
+ placeholder="Uses default output format naming if empty.">
- - All format options can be found at this page. + + All output format naming options can be found at this page.
@@ -82,11 +82,12 @@
- + Some config fields are ignored like cookiefile, path, and output_format etc. - Available option can be found at this - page. + Available option can be found at + this page. Warning: Use with caution some of those options can break yt-dlp or the + frontend. @@ -99,9 +100,9 @@ - - Use something like flagCookies - to extract cookies as JSON string. + + Use something like flagCookies to extract cookies as JSON string. @@ -121,8 +122,8 @@ - -