diff --git a/.vscode/settings.json b/.vscode/settings.json index ff695099..9315ac71 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,6 +19,9 @@ "arrowless", "attl", "autonumber", + "bgutil", + "bgutilhttp", + "brainicism", "brotlicffi", "consoletitle", "cookiesfrombrowser", @@ -111,7 +114,8 @@ "vconvert", "writedescription", "xerror", - "youtu" + "youtu", + "youtubepot" ], "css.styleSheets": [ "ui/app/assets/css/*.css" diff --git a/Dockerfile b/Dockerfile index c90cc940..6c070c15 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONFAULTHANDLER=1 RUN mkdir /config /downloads && ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone && \ - apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump fribidi && \ + apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump fribidi git && \ useradd -u ${USER_ID:-1000} -U -d /app -s /bin/bash app && \ rm -rf /var/cache/apk/* diff --git a/FAQ.md b/FAQ.md index a605e451..bc8776c1 100644 --- a/FAQ.md +++ b/FAQ.md @@ -39,3 +39,61 @@ whenever the link includes a playlist id. > [!NOTE] > You can also do the same via advanced options `Command options for yt-dlp` field, but presets are more convenient. + +# Install specific yt-dlp version? + +You can force specific version of `yt-dlp` by setting the `YTP_YTDLP_VERSION` environment variable for example + +```env +YTP_YTDLP_VERSION=2025.07.21 or master or nightly +``` + +Then restart the container to apply the changes. + +# How to generate POT tokens? + +You need to start up a pot provider server we already have extractor `bgutil-ytdlp-pot-provider` pre-installed in the container. +You can simply do the following to enable the support for it. + +```yaml +services: + ytptube: + user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000" + image: ghcr.io/arabcoders/ytptube:latest + container_name: ytptube + restart: unless-stopped + ports: + - "8081:8081" + volumes: + - ./config:/config:rw + - ./downloads:/downloads:rw + tmpfs: + - /tmp + depends_on: + - bgutil_provider + bgutil_provider: + init: true + image: brainicism/bgutil-ytdlp-pot-provider:latest + container_name: bgutil_provider + restart: unless-stopped + ports: + - "127.0.0.1:4416:4416" +``` + +Then simply create a new preset, and in the `Command options for yt-dlp` field set the following: + +```bash +--extractor-args "youtubepot-bgutilhttp:base_url=http://bgutil_provider:4416" +--extractor-args "youtube:player-client=default,tv,mweb;formats=incomplete" +``` + +you and also enable the fallback by using the follow extractor args + +```bash +--extractor-args "youtubepot-bgutilhttp:base_url=http://bgutil_provider:4416;disable_innertube=1" +--extractor-args "youtube:player-client=default,tv,mweb;formats=incomplete" +``` + +Use this settings in case the extractor fails to get the pot tokens from the bgutil provider server. + +For more information please visit [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider) project. diff --git a/README.md b/README.md index 695bf527..df6589b9 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ postprocessing, permissions, other `yt-dlp options` configurations which seem no concerns the workings of the underlying yt-dlp library, need not be opened on the YTPTube project. In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, -and once that is working, importing the options that worked for you into a new `preset` or `ytdlp.cli` file. +and once that is working, importing the options that worked for you into a new `preset`. ## Via HTTP @@ -186,20 +186,6 @@ yt-dlp .... Once there, you can use the yt-dlp command freely. -# ytdlp.cli file - -The `config/ytdlp.cli`, is a command line options file for `yt-dlp` it will be globally applied to all downloads. - -We strongly recommend not use this file for options that aren't **truly global**, everything that can be done via the -file can also be done via the presets which is dynamic can be altered per download. Example of good global options -are to be used for all downloads are: - -```bash ---continue --windows-filenames --live-from-start -``` - -Everything else can be done via the presets, and it's more flexible and easier to manage. - # Authentication To enable basic authentication, set the `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD` environment variables. And restart @@ -291,7 +277,6 @@ Certain configuration values can be set via environment variables, using the `-e | YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | | YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | | YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | -| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | | YTP_HOST | Which IP address to bind to | `0.0.0.0` | | YTP_PORT | Which port to bind to | `8081` | | YTP_LOG_LEVEL | Log level | `info` | @@ -312,6 +297,8 @@ Certain configuration values can be set via environment variables, using the `-e | YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | | YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | | YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `empty string` | | YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | | YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | | YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | diff --git a/app/library/Download.py b/app/library/Download.py index 7cdc29d6..88b2f379 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -111,7 +111,7 @@ class Download: self.info = info self.id = info._id self.debug = bool(config.debug) - self.debug_ytdl = bool(config.ytdl_debug) + self.debug_ytdl = bool(config.ytdlp_debug) self.cancelled = False self.tmpfilename = None self.status_queue = None diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 16ec0d74..1ba547c9 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -676,7 +676,7 @@ class DownloadQueue(metaclass=Singleton): extract_info, config=yt_conf, url=item.url, - debug=bool(self.config.ytdl_debug), + debug=bool(self.config.ytdlp_debug), no_archive=False, follow_redirect=True, ), diff --git a/app/library/PackageInstaller.py b/app/library/PackageInstaller.py index 65cceba2..b0ed0463 100644 --- a/app/library/PackageInstaller.py +++ b/app/library/PackageInstaller.py @@ -49,10 +49,18 @@ class PackageInstaller: sys.path.insert(0, str(self.user_site)) def action(self, pkg: str, upgrade: bool = False): - current_version = self._get_installed_version(pkg) + version: str | None = None + if "==" in pkg: + pkg, version = pkg.split("==", 1) - if upgrade and current_version: - latest_version = self._get_latest_version(pkg) + current_version: str | None = self._get_installed_version(pkg) + + if current_version and version and self.compare_versions(current_version, version): + LOG.info(f"'{pkg}' is already installed with the specified version ({version}). Skipping installation.") + return + + if upgrade and current_version and not version: + latest_version: str | None = self._get_latest_version(pkg) if latest_version and parse_version(current_version) >= parse_version(latest_version): LOG.info(f"'{pkg}' is already the latest version ({current_version}). Skipping upgrade.") return @@ -62,7 +70,7 @@ class PackageInstaller: else: LOG.info(f"'{pkg}' is not installed. Installing...") - self._install_pkg(pkg) + self._install_pkg(pkg, version=version) def _get_installed_version(self, pkg: str) -> str | None: try: @@ -82,23 +90,37 @@ class PackageInstaller: LOG.warning(f"Error while querying PyPI for '{pkg}': {e}") return None - def _install_pkg(self, pkg: str): - subprocess.run( - [ - sys.executable, - "-m", - "pip", - "install", - "--upgrade", - "--disable-pip-version-check", - "--target", - str(self.user_site), - "--no-warn-script-location", - pkg, - ], - check=True, - creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, - ) + def _install_pkg(self, pkg: str, version: str | None = None): + cmd: list[str] = [ + sys.executable, + "-m", + "pip", + "install", + "--no-warn-script-location", + "--upgrade", + "--target", + str(self.user_site), + ] + + if version: + if "nightly" == version and pkg == "yt_dlp": + cmd.extend(["--pre", "yt-dlp[default]"]) + elif "master" == version and pkg == "yt_dlp": + cmd.append("git+https://github.com/yt-dlp/yt-dlp.git@master") + else: + cmd.append(version if str(version).startswith("git+") else f"{pkg}=={version}") + else: + cmd.extend(["--disable-pip-version-check", pkg]) + + try: + subprocess.run( + cmd, + check=True, + creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, + ) + except subprocess.CalledProcessError as e: + LOG.error(f"Failed to install package '{pkg}'. Error: {e}") + raise def check(self, pkgs: Packages): """ @@ -118,3 +140,28 @@ class PackageInstaller: except Exception as e: LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}") LOG.exception(e) + + def compare_versions(self, current: str, target: str) -> bool: + """ + Compare versions, handling yt-dlp format where pip uses 2025.7.21 but actual is 2025.07.21 + Returns True if versions match, False otherwise + """ + if current == target: + return True + + # Handle yt-dlp version format differences + current_parts = current.split(".") + target_parts = target.split(".") + + if len(current_parts) == len(target_parts): + # Normalize parts by zero-padding single digits + current_normalized: list[str] = [ + part.zfill(2) if part.isdigit() and len(part) == 1 else part for part in current_parts + ] + target_normalized: list[str] = [ + part.zfill(2) if part.isdigit() and len(part) == 1 else part for part in target_parts + ] + if current_normalized == target_normalized: + return True + + return False diff --git a/app/library/config.py b/app/library/config.py index 1d40a753..5ca6bcad 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -46,9 +46,6 @@ class Config: keep_archive: bool = True """Keep the download archive file.""" - ytdl_debug: bool = False - """Enable yt-dlp debugging.""" - host: str = "0.0.0.0" """The host to bind the server to.""" @@ -178,6 +175,12 @@ class Config: _ytdlp_cli_mutable: str = "" """The command line options to use for yt-dlp.""" + ytdlp_version: str | None = None + """The version of yt-dlp to use, if not set, the latest version will be used.""" + + ytdlp_debug: bool = False + """Enable yt-dlp debugging.""" + is_native: bool = False "Is the application running in webview." @@ -228,7 +231,7 @@ class Config: _boolean_vars: tuple = ( "keep_archive", - "ytdl_debug", + "ytdlp_debug", "debug", "temp_keep", "access_log", @@ -531,11 +534,11 @@ class Config: if not data.get("keep_archive", False) and ytdlp_args.get("download_archive", None): data["keep_archive"] = True - data["ytdlp_version"] = Config.ytdlp_version() + data["ytdlp_version"] = Config._ytdlp_version() return data @staticmethod - def ytdlp_version(): + def _ytdlp_version(): try: from yt_dlp.version import __version__ as YTDLP_VERSION diff --git a/app/routes/api/history.py b/app/routes/api/history.py index 07f69f65..568ed480 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -1,10 +1,12 @@ import asyncio import logging +from pathlib import Path from typing import TYPE_CHECKING from aiohttp import web from aiohttp.web import Request, Response +from app.library.config import Config from app.library.DownloadQueue import DownloadQueue from app.library.encoder import Encoder from app.library.Events import EventBus, Events @@ -70,7 +72,7 @@ async def item_delete(request: Request, queue: DownloadQueue, encoder: Encoder) @route("GET", "api/history/{id}", "item_view") -async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: +async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder, config: Config) -> Response: """ Update an item in the history. @@ -79,6 +81,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) -> queue (DownloadQueue): The download queue instance. encoder (Encoder): The encoder instance. notify (EventBus): The event bus instance. + config (Config): The configuration instance. Returns: Response: The response object. @@ -92,7 +95,35 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) -> if not item: return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) - return web.json_response(data=item.info, status=web.HTTPOk.status_code, dumps=encoder.encode) + if not item.info: + return web.json_response(data={"error": "item has no info."}, status=web.HTTPNotFound.status_code) + + info = { + **item.info.serialize(), + "ffprobe": {}, + } + + if item.info.filename: + try: + from app.library.ffprobe import ffprobe + from app.library.Utils import get_file + + filename = Path(config.download_path) + if item.info.folder: + filename: Path = filename / item.info.folder + + filename: Path = filename / item.info.filename + + if filename.exists(): + realFile, status = get_file( + download_path=config.download_path, file=str(filename.relative_to(config.download_path)) + ) + if status in (web.HTTPOk.status_code, web.HTTPFound.status_code): + info["ffprobe"] = await ffprobe(str(realFile)) + except Exception: + pass + + return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode) @route("POST", "api/history/{id}", "item_update") diff --git a/app/upgrader.py b/app/upgrader.py index cb353122..ec86102f 100644 --- a/app/upgrader.py +++ b/app/upgrader.py @@ -21,6 +21,8 @@ class Upgrader: config_path: Path = Path(__file__).parent.parent / "var" / "config" if env_path := os.environ.get("YTP_CONFIG_PATH", None): config_path = Path(env_path) + else: + os.environ.update({"YTP_CONFIG_PATH": str(config_path)}) if config_path.exists(): envFile: Path = config_path / ".env" @@ -33,11 +35,17 @@ class Upgrader: pkg_installer = PackageInstaller() ytdlp_auto_update: bool = os.environ.get("YTP_YTDLP_AUTO_UPDATE", "true").strip().lower() == "true" + ytdlp_version: str | None = os.environ.get("YTP_YTDLP_VERSION", "").strip() or None if ytdlp_auto_update: try: LOG.info("Checking for newer versions of 'yt-dlp' package.") - pkg_installer.action(pkg="yt_dlp", upgrade=True) + pkg_name = "yt_dlp" + if ytdlp_version: + LOG.info(f"Using specified version '{ytdlp_version}' for '{pkg_name}'.") + pkg_name += f"=={ytdlp_version}" + + pkg_installer.action(pkg=pkg_name, upgrade=True) from yt_dlp.version import __version__ as YTDLP_VERSION diff --git a/pyproject.toml b/pyproject.toml index 48a2205c..27e9433d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "defusedxml>=0.7.1", "zipstream-ng>=1.8.0", "apprise>=1.9.3", + "bgutil-ytdlp-pot-provider>=1.2.1", ] [tool.ruff] diff --git a/ui/app/components/History.vue b/ui/app/components/History.vue index 9489f9c3..73e056fb 100644 --- a/ui/app/components/History.vue +++ b/ui/app/components/History.vue @@ -508,9 +508,7 @@ const dialog_confirm = ref<{ title: 'Confirm Action', confirm: () => { }, message: '', - options: [ - { key: 'remove_history', label: 'Also, Remove from history.' }, - ], + options: [], }) const showThumbnails = computed(() => (props.thumbnails || true) && !hideThumbnail.value) @@ -735,6 +733,7 @@ const addArchiveDialog = (item: StoreItem) => { dialog_confirm.value.visible = true dialog_confirm.value.title = 'Archive Item' dialog_confirm.value.message = `Archive '${item.title || item.id || item.url || '??'}'?` + dialog_confirm.value.options = [{ key: 'remove_history', label: 'Also, Remove from history.' }] dialog_confirm.value.confirm = (opts: any) => archiveItem(item, opts) } @@ -864,10 +863,15 @@ const removeFromArchiveDialog = (item: StoreItem) => { dialog_confirm.value.visible = true dialog_confirm.value.title = 'Remove from Archive' dialog_confirm.value.message = `Remove '${item.title || item.id || item.url || '??'}' from archive?` - dialog_confirm.value.confirm = () => removeFromArchive(item) + dialog_confirm.value.options = [ + { key: 'remove_history', label: 'Also, Remove from history.' }, + { key: 're_add', label: 'Re-add to download form.' }, + ] + dialog_confirm.value.confirm = (opts: any) => removeFromArchive(item, opts) } -const removeFromArchive = async (item: StoreItem, opts?: { remove_history?: boolean }) => { +const removeFromArchive = async (item: StoreItem, opts?: { re_add?: boolean, remove_history?: boolean }) => { + console.log('Removing from archive:', item, opts) try { const req = await request(`/api/archive/${item._id}`, { credentials: 'include', @@ -876,15 +880,21 @@ const removeFromArchive = async (item: StoreItem, opts?: { remove_history?: bool const data = await req.json() if (!req.ok) { toast.error(data.error) - return + } else { + toast.success(data.message || `Removed '${item.title || item.id || item.url || '??'}' from archive.`) } - toast.success(data.message || `Removed '${item.title || item.id || item.url || '??'}' from archive.`) } catch (e: any) { console.error(e) toast.error(`Error: ${e.message}`) } finally { dialog_confirm.value.visible = false } + + if (opts?.re_add) { + retryItem(item, true) + return + } + if (opts?.remove_history) { socket.emit('item_delete', { id: item._id, remove_file: false }) } diff --git a/uv.lock b/uv.lock index 78052cb1..2244d049 100644 --- a/uv.lock +++ b/uv.lock @@ -164,6 +164,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815 }, ] +[[package]] +name = "bgutil-ytdlp-pot-provider" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/3f/62fc3c3668a0518cc6a0dab159362bf2c213c0baf7631d15899a98b83176/bgutil_ytdlp_pot_provider-1.2.1.tar.gz", hash = "sha256:771173df631451046c982dcac27482cc89b300ce0aedc487043f80e45774b9f6", size = 8889 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/37/c66e32bdb34b84581b8a794bdc86775b1161208f915f103e39bd1ad665d3/bgutil_ytdlp_pot_provider-1.2.1-py3-none-any.whl", hash = "sha256:ddf9fee9857fb64c7fdfb4db96db81ca5864a6004fff5549a2043693776184ef", size = 9473 }, +] + [[package]] name = "bidict" version = "0.23.1" @@ -1575,6 +1584,7 @@ dependencies = [ { name = "anyio" }, { name = "apprise" }, { name = "async-timeout" }, + { name = "bgutil-ytdlp-pot-provider" }, { name = "brotli" }, { name = "brotlicffi" }, { name = "caribou" }, @@ -1615,6 +1625,7 @@ requires-dist = [ { name = "anyio" }, { name = "apprise", specifier = ">=1.9.3" }, { name = "async-timeout" }, + { name = "bgutil-ytdlp-pot-provider", specifier = ">=1.2.1" }, { name = "brotli" }, { name = "brotlicffi" }, { name = "caribou", specifier = ">=0.3.0" },