From 643688bf97a7c0bd0df7288bbbd8308a8108ec2b Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Mon, 10 Mar 2025 03:56:10 +0300 Subject: [PATCH 1/6] support sending presets with api/yt-dlp/url/info --- app/library/HttpAPI.py | 33 ++++++++++++++++++++++++++------- app/library/Utils.py | 5 +++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 5cf0e176..3e8096a7 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -47,6 +47,7 @@ from .Utils import ( validate_url, validate_uuid, ) +from .YTDLPOpts import YTDLPOpts LOG = logging.getLogger("http_api") MIME = magic.Magic(mime=True) @@ -471,10 +472,21 @@ class HttpAPI(Common): except ValueError as e: return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - try: - key = self.cache.hash(url) + preset = request.query.get("preset") + if preset: + exists = Presets.get_instance().get(name=preset) + if not exists: + return web.json_response( + data={"status": False, "message": f"Preset '{preset}' does not exist."}, + status=web.HTTPBadRequest.status_code, + ) + else: + preset = self.config.default_preset - if self.cache.has(key): + try: + key = self.cache.hash(url+str(preset)) + + if self.cache.has(key) and not request.query.get("force"): data = self.cache.get(key) data["_cached"] = { "key": key, @@ -484,12 +496,19 @@ class HttpAPI(Common): } return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) - opts = { - "proxy": self.config.ytdl_options.get("proxy", None), - } + opts = {} + + if self.config.ytdl_options.get("proxy", None): + opts["proxy"] = self.config.ytdl_options.get("proxy", None) + + data = get_video_info( + url=url, + ytdlp_opts=YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all(), + no_archive=True, + ) - data = get_video_info(url=url, ytdlp_opts=opts, no_archive=True) self.cache.set(key=self.cache.hash(url), value=data, ttl=300) + data["_cached"] = { "key": self.cache.hash(url), "ttl": 300, diff --git a/app/library/Utils.py b/app/library/Utils.py index dc73d4aa..cd3e1ead 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -58,6 +58,11 @@ def get_video_info(url: str, ytdlp_opts: dict | None = None, no_archive: bool = if no_archive and "download_archive" in params: del params["download_archive"] + # Remove keys that are not needed for info extraction. + keys_to_remove = [key for key in params if str(key).startswith("write") or key in ["postprocessors"]] + for key in keys_to_remove: + params.pop(key, None) + return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) From 72bac24f46e897b8e75f3632f6bcfc8ac1227768 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Mon, 10 Mar 2025 23:12:08 +0300 Subject: [PATCH 2/6] minor updates to extract_info --- app/library/HttpAPI.py | 31 +++++++++++++----------- app/library/Utils.py | 54 +++++++++++++----------------------------- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 3e8096a7..f3a53d23 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -40,10 +40,10 @@ from .Utils import ( arg_converter, decrypt_data, encrypt_data, + extract_info, get_file, get_mime_type, get_sidecar_subtitles, - get_video_info, validate_url, validate_uuid, ) @@ -484,11 +484,12 @@ class HttpAPI(Common): preset = self.config.default_preset try: - key = self.cache.hash(url+str(preset)) + key = self.cache.hash(url + str(preset)) - if self.cache.has(key) and not request.query.get("force"): + if self.cache.has(key) and not request.query.get("force", False): data = self.cache.get(key) data["_cached"] = { + "status": "hit", "key": key, "ttl": data.get("_cached", {}).get("ttl", 300), "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(), @@ -501,16 +502,19 @@ class HttpAPI(Common): if self.config.ytdl_options.get("proxy", None): opts["proxy"] = self.config.ytdl_options.get("proxy", None) - data = get_video_info( + data = extract_info( + config=YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all(), url=url, - ytdlp_opts=YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all(), + debug=False, no_archive=True, + follow_redirect=True, ) - self.cache.set(key=self.cache.hash(url), value=data, ttl=300) + self.cache.set(key=key, value=data, ttl=300) data["_cached"] = { - "key": self.cache.hash(url), + "status": "miss", + "key": key, "ttl": 300, "ttl_left": 300, "expires": time.time() + 300, @@ -524,6 +528,7 @@ class HttpAPI(Common): data={ "error": "failed to get video info.", "message": str(e), + "formats": [], }, status=web.HTTPInternalServerError.status_code, ) @@ -555,17 +560,17 @@ class HttpAPI(Common): tasks = [] response = [] - def get_video_info_wrapper(id: str, url: str) -> tuple[str, dict]: + def info_wrapper(id: str, url: str) -> tuple[str, dict]: try: return ( id, - get_video_info( - url=url, - ytdlp_opts={ + extract_info( + config={ "proxy": self.config.ytdl_options.get("proxy", None), "simulate": True, "dump_single_json": True, }, + url=url, no_archive=True, ), ) @@ -594,9 +599,7 @@ class HttpAPI(Common): continue tasks.append( - asyncio.get_event_loop().run_in_executor( - None, lambda id=id, url=url: get_video_info_wrapper(id=id, url=url) - ) + asyncio.get_event_loop().run_in_executor(None, lambda id=id, url=url: info_wrapper(id=id, url=url)) ) if len(tasks) > 0: diff --git a/app/library/Utils.py b/app/library/Utils.py index cd3e1ead..4bc4515f 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -30,42 +30,6 @@ class StreamingError(Exception): """Raised when an error occurs during streaming.""" -def get_video_info(url: str, ytdlp_opts: dict | None = None, no_archive: bool = True) -> Any | dict[str, Any] | None: - """ - Extracts video information from the given URL. - - Args: - url (str): URL to extract information from. - ytdlp_opts (dict): Additional options to pass to yt-dlp. - no_archive (bool): Do not use download archive. - - Returns: - dict: Video information. - - """ - params: dict = { - "quiet": True, - "color": "no_color", - "extract_flat": True, - "ignoreerrors": True, - "skip_download": True, - "ignore_no_formats_error": True, - } - - if ytdlp_opts: - params = {**params, **ytdlp_opts} - - if no_archive and "download_archive" in params: - del params["download_archive"] - - # Remove keys that are not needed for info extraction. - keys_to_remove = [key for key in params if str(key).startswith("write") or key in ["postprocessors"]] - for key in keys_to_remove: - params.pop(key, None) - - return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) - - def calc_download_path(base_path: str, folder: str | None = None, create_path: bool = True) -> str: """ Calculates download path and prevents folder traversal. @@ -98,7 +62,13 @@ def calc_download_path(base_path: str, folder: str | None = None, create_path: b return download_path -def extract_info(config: dict, url: str, debug: bool = False) -> dict: +def extract_info( + config: dict, + url: str, + debug: bool = False, + no_archive: bool = False, + follow_redirect: bool = False, +) -> dict: """ Extracts video information from the given URL. @@ -149,7 +119,15 @@ def extract_info(config: dict, url: str, debug: bool = False) -> dict: params["logger"] = log_wrapper - return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) + if no_archive and "download_archive" in params: + del params["download_archive"] + + data = yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) + + if follow_redirect and "_type" in data and "url" == data["_type"]: + return extract_info(config, data["url"], debug=debug, no_archive=no_archive, follow_redirect=follow_redirect) + + return data def merge_dict(source: dict, destination: dict) -> dict: From c1939144c9b562bf647c41ebb850dbcb51c5b584 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 11 Mar 2025 00:08:24 +0300 Subject: [PATCH 3/6] parse yt-dlp header cookie for yt-dlp/url/info --- app/library/HttpAPI.py | 10 ++++++++++ app/library/Utils.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index f3a53d23..7287ef1b 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -44,6 +44,7 @@ from .Utils import ( get_file, get_mime_type, get_sidecar_subtitles, + parse_cookies, validate_url, validate_uuid, ) @@ -510,6 +511,15 @@ class HttpAPI(Common): follow_redirect=True, ) + if "formats" in data: + for index, item in enumerate(data["formats"]): + if "cookies" in item: + cookies = parse_cookies(item["cookies"]) + if len(cookies) > 0: + data["formats"][index]["h_cookies"] = "; ".join( + f"{key}={value}" for key, value in cookies.items() + ) + self.cache.set(key=key, value=data, ttl=300) data["_cached"] = { diff --git a/app/library/Utils.py b/app/library/Utils.py index 4bc4515f..a154a7ac 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -606,3 +606,29 @@ def decrypt_data(data: str, key: bytes) -> str: return plaintext.decode() except Exception: return None + + +def parse_cookies(cookie_str: str) -> dict: + """ + Parse a cookie string into a dictionary." + + Args: + cookie_str (str): The cookie string. + + Returns: + dict: The parsed cookies. + + """ + cookie_attributes = {"domain", "path", "expires", "secure", "httponly", "samesite"} + + tokens = cookie_str.split("; ") + + cookies = {} + + for token in tokens: + if "=" in token: + key, value = token.split("=", 1) + if str(key).lower() not in cookie_attributes: + cookies[key] = value + + return cookies From cd977ea28da65b85cc3fc5478307a5f64d562e11 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 11 Mar 2025 02:07:56 +0300 Subject: [PATCH 4/6] minor updates to webui --- ui/layouts/default.vue | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/ui/layouts/default.vue b/ui/layouts/default.vue index d65dff04..88326724 100644 --- a/ui/layouts/default.vue +++ b/ui/layouts/default.vue @@ -17,33 +17,45 @@
- +
- - + + + + Presets +
- - + + + + Terminal +
-
+
- + + + Tasks +
- + + + Notifications +
From 1aeb8efa1ec17eb7ef277015fc1b17e411d5d8e6 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 11 Mar 2025 02:08:13 +0300 Subject: [PATCH 5/6] updated yt-dlp/url/info to give cookies in valid way. --- app/library/HttpAPI.py | 23 ++++++++++++----------- app/library/Utils.py | 28 ++-------------------------- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 7287ef1b..8d4e6521 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -18,6 +18,7 @@ import httpx import magic from aiohttp import web from aiohttp.web import Request, RequestHandler, Response +from yt_dlp.cookies import LenientSimpleCookie from .cache import Cache from .common import Common @@ -44,7 +45,6 @@ from .Utils import ( get_file, get_mime_type, get_sidecar_subtitles, - parse_cookies, validate_url, validate_uuid, ) @@ -496,15 +496,17 @@ class HttpAPI(Common): "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(), "expires": data.get("_cached", {}).get("expires", time.time() + 300), } - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) + return web.Response(body=json.dumps(data, indent=4), status=web.HTTPOk.status_code) opts = {} if self.config.ytdl_options.get("proxy", None): opts["proxy"] = self.config.ytdl_options.get("proxy", None) + ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all() + data = extract_info( - config=YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all(), + config=ytdlp_opts, url=url, debug=False, no_archive=True, @@ -513,14 +515,11 @@ class HttpAPI(Common): if "formats" in data: for index, item in enumerate(data["formats"]): - if "cookies" in item: - cookies = parse_cookies(item["cookies"]) + if "cookies" in item and len(item["cookies"]) > 0: + cookies = [f"{c.key}={c.value}" for c in LenientSimpleCookie(item["cookies"]).values()] if len(cookies) > 0: - data["formats"][index]["h_cookies"] = "; ".join( - f"{key}={value}" for key, value in cookies.items() - ) - - self.cache.set(key=key, value=data, ttl=300) + data["formats"][index]["h_cookies"] = "; ".join(cookies) + data["formats"][index]["h_cookies"] = data["formats"][index]["h_cookies"].strip() data["_cached"] = { "status": "miss", @@ -530,7 +529,9 @@ class HttpAPI(Common): "expires": time.time() + 300, } - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) + self.cache.set(key=key, value=data, ttl=300) + + return web.Response(body=json.dumps(data, indent=4), status=web.HTTPOk.status_code) except Exception as e: LOG.error(f"Error encountered while grabbing video info '{url}'. '{e}'.") LOG.exception(e) diff --git a/app/library/Utils.py b/app/library/Utils.py index a154a7ac..a192ead6 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -76,6 +76,8 @@ def extract_info( config (dict): Configuration options. url (str): URL to extract information from. debug (bool): Enable debug logging. + no_archive (bool): Disable download archive. + follow_redirect (bool): Follow URL redirects. Returns: dict: Video information. @@ -606,29 +608,3 @@ def decrypt_data(data: str, key: bytes) -> str: return plaintext.decode() except Exception: return None - - -def parse_cookies(cookie_str: str) -> dict: - """ - Parse a cookie string into a dictionary." - - Args: - cookie_str (str): The cookie string. - - Returns: - dict: The parsed cookies. - - """ - cookie_attributes = {"domain", "path", "expires", "secure", "httponly", "samesite"} - - tokens = cookie_str.split("; ") - - cookies = {} - - for token in tokens: - if "=" in token: - key, value = token.split("=", 1) - if str(key).lower() not in cookie_attributes: - cookies[key] = value - - return cookies From 6b38c5fcd6198c3df657063b87711a6984df1f49 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 11 Mar 2025 02:19:57 +0300 Subject: [PATCH 6/6] add advanced iOS shortcut --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 57842429..42daa75b 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,13 @@ This shortcut is powerful, as it's allow you to select your preset on the fly pu Combined with the new and powerful presets system, you could add presets for specific websites that need cookies, and use that preset to download directly from your iOS device. +#### Advanced iOS Shortcut + +This shortcut [YTPTube To Media](https://www.icloud.com/shortcuts/6e3db0bd532843e3aec70e6ce211be08) is more advanced, as it's parses +the `yt-dlp` output and attempt to download the media directly to your iOS device. It doesn't always work, but it's a good +starting point for those who want to download media directly to their iOS device. We provide no support for this use case +other than the shortcut itself. this shortcut missing support for parsing the http_headers, it's only parse the cookies. + ## Running behind a reverse proxy It's advisable to run YTPTube behind a reverse proxy, if authentication and/or HTTPS support are required.