Merge pull request #181 from arabcoders/dev

Handle yt-dlp CLI options

.dockerignore

@@ -1,3 +1,13 @@
-.git
+**/.idea
-.venv
+**/.git
-cd 
+**/.venv
+.github/
+.vscode/
+.devcontainer/
+ui/.nuxt
+ui/.output
+ui/node_modules
+ui/dist
+**/.env
+./var/*
+!./var/.gitignore

.github/workflows/main.yml

@@ -12,9 +12,26 @@ on:
           - info
           - warning
           - debug
+      build:
+        description: "Build"
+        required: false
+        default: false
+        type: boolean
+      update_readme:
+        description: "Update Readme"
+        required: false
+        default: false
+        type: boolean
+      create_release:
+        description: "Create Release (requires build)"
+        required: false
+        default: false
+        type: boolean
+
   push:
     branches:
-      - "*"
+      - dev
+      - master
     paths-ignore:
       - "**.md"
       - ".github/**"
@@ -38,6 +55,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install frontend dependencies
+        uses: bahmutov/npm-install@v1
+        with:
+          working-directory: ui
+          install-command: yarn --production --prefer-offline --frozen-lockfile
+
+      - name: Build frontend
+        uses: bahmutov/npm-install@v1
+        with:
+          working-directory: ui
+          install-command: yarn run generate
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -48,12 +77,14 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: ${{ env.PLATFORMS }}
+          platforms: linux/amd64
           push: false
+          cache-from: type=gha, scope=pr_${{ github.workflow }}
+          cache-to: type=gha, scope=pr_${{ github.workflow }}
 
   push-build:
     runs-on: ubuntu-latest
-    if: github.event_name == 'push'
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.build == 'true')
     permissions:
       packages: write
       contents: write
@@ -61,6 +92,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install frontend dependencies
+        uses: bahmutov/npm-install@v1
+        with:
+          working-directory: ui
+          install-command: yarn --production --prefer-offline --frozen-lockfile
+
+      - name: Build frontend
+        uses: bahmutov/npm-install@v1
+        with:
+          working-directory: ui
+          install-command: yarn run generate
+
       - name: Update Version File
         uses: ArabCoders/write-version-to-file@master
         with:
@@ -75,6 +118,15 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Set Platforms Based on Branch
+        id: set_platforms
+        run: |
+          if [ "${GITHUB_REF}" == "refs/heads/${{ github.repository.default_branch }}" ]; then
+            echo "PLATFORMS=linux/amd64,linux/arm64" >> $GITHUB_ENV
+          else
+            echo "PLATFORMS=linux/amd64" >> $GITHUB_ENV
+          fi
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -123,9 +175,8 @@ jobs:
           regex: 'APP_VERSION\s=\s\"(.+)\"'
 
   dockerhub-sync-readme:
-    needs: push-build
-    if: endsWith(github.ref, github.event.repository.default_branch)
     runs-on: ubuntu-latest
+    if: (github.event_name == 'push' && endsWith(github.ref, github.event.repository.default_branch)) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_readme == 'true')
     steps:
       - name: Sync README
         uses: docker://lsiodev/readme-sync:latest
@@ -142,55 +193,78 @@ jobs:
   create_release:
     needs: push-build
     runs-on: ubuntu-latest
-    if: endsWith(github.ref, github.event.repository.default_branch) && success()
+    if: (endsWith(github.ref, github.event.repository.default_branch) && success()) || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_release == 'true')
     steps:
       - name: Check out code
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0 # so we can do git log for the full history
+          fetch-depth: 0 # so we can see all tags + full history
 
-      - name: Get commits between old and new
+      - name: Determine current branch
-        id: commits
+        id: branch
         run: |
-          PREVIOUS_SHA="${{ github.event.before }}"
+          # github.ref_name should be "master", "main", or your branch name
-          CURRENT_SHA="${{ github.event.after }}"
+          echo "BRANCH_NAME=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
 
-          echo "Previous SHA: $PREVIOUS_SHA"
+      - name: Fetch the two latest tags for this branch
-          echo "Current  SHA: $CURRENT_SHA"
+        id: last_two_tags
+        run: |
+          git fetch --tags
 
-          # If "before" is empty or all zeros, log all commits
+          BRANCH_NAME="${{ steps.branch.outputs.BRANCH_NAME }}"
-          if [ -z "$PREVIOUS_SHA" ] || [ "$PREVIOUS_SHA" = "0000000000000000000000000000000000000000" ]; then
+          echo "Current branch: $BRANCH_NAME"
-            LOG=$(git log --pretty=format:"- %h %s by %an")
+
-          else
+          # List tags matching "branchname-*" and sort by *creation date* descending
-            LOG=$(git log "$PREVIOUS_SHA".."$CURRENT_SHA" --pretty=format:"- %h %s by %an")
+          # Then pick the top 2
+          LATEST_TAGS=$(git tag --list "${BRANCH_NAME}-*" --sort=-creatordate | head -n 2)
+          TAG_COUNT=$(echo "$LATEST_TAGS" | wc -l)
+
+          echo "Found tags:"
+          echo "$LATEST_TAGS"
+
+          if [ "$TAG_COUNT" -lt 2 ]; then
+            echo "Not enough tags found (need at least 2) to compare commits."
+            echo "NOT_ENOUGH_TAGS=true" >> "$GITHUB_OUTPUT"
+            exit 0
           fi
 
+          # The first line is the newest tag
+          TAG_NEWEST=$(echo "$LATEST_TAGS" | sed -n '1p')
+          # The second line is the previous newest
+          TAG_PREVIOUS=$(echo "$LATEST_TAGS" | sed -n '2p')
+
+          echo "Newest tag:   $TAG_NEWEST"
+          echo "Previous tag: $TAG_PREVIOUS"
+
+          # Expose them as outputs for next step
+          echo "NOT_ENOUGH_TAGS=false" >> "$GITHUB_OUTPUT"
+          echo "TAG_NEWEST=$TAG_NEWEST" >> "$GITHUB_OUTPUT"
+          echo "TAG_PREVIOUS=$TAG_PREVIOUS" >> "$GITHUB_OUTPUT"
+
+      - name: Generate commit log for newest tag
+        id: commits
+        if: steps.last_two_tags.outputs.NOT_ENOUGH_TAGS != 'true'
+        run: |
+          TAG_NEWEST="${{ steps.last_two_tags.outputs.TAG_NEWEST }}"
+          TAG_PREVIOUS="${{ steps.last_two_tags.outputs.TAG_PREVIOUS }}"
+
+          echo "Comparing commits between: $TAG_PREVIOUS..$TAG_NEWEST"
+          LOG=$(git log "$TAG_PREVIOUS".."$TAG_NEWEST" --pretty=format:"- %h %s by %an")
+
           echo "LOG<<EOF" >> "$GITHUB_ENV"
           echo "$LOG" >> "$GITHUB_ENV"
           echo "EOF" >> "$GITHUB_ENV"
 
-      - name: Fetch the most recent tag
+      - name: Create / Update GitHub Release for the newest tag
-        id: last_tag
+        if: steps.last_two_tags.outputs.NOT_ENOUGH_TAGS != 'true'
-        run: |
-          # Make sure we have all tags
-          git fetch --tags
-
-          # Get the latest tag by commit date (most recent)
-          # If there are no tags at all, set a fallback
-          LAST_TAG=$(git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) 2>/dev/null || echo "no-tags-found")
-
-          echo "Latest tag found: $LAST_TAG"
-          echo "LAST_TAG=$LAST_TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Create or update GitHub Release using last tag
         uses: softprops/action-gh-release@master
         with:
-          tag_name: ${{ steps.last_tag.outputs.LAST_TAG }}
+          tag_name: ${{ steps.last_two_tags.outputs.TAG_NEWEST }}
-          name: "${{ steps.last_tag.outputs.LAST_TAG }}"
+          name: "${{ steps.last_two_tags.outputs.TAG_NEWEST }}"
           body: ${{ env.LOG }}
+          append_body: true
+          generate_release_notes: true
+          make_latest: true
           draft: false
           prerelease: false
-          generate_release_notes: true
-          append_body: true
-          make_latest: true
           token: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -2,7 +2,7 @@ FROM node:lts-alpine AS node_builder
 
 WORKDIR /app
 COPY ui ./
-RUN yarn install --production --prefer-offline --frozen-lockfile && yarn run generate
+RUN if [ ! -d /app/exported ]; then yarn install --production --prefer-offline --frozen-lockfile && yarn run generate; else echo "Skipping UI build, already built."; fi
 
 FROM python:3.11-alpine AS python_builder
 

Pipfile.lock

@@ -446,36 +446,36 @@
         },
         "debugpy": {
             "hashes": [
-                "sha256:0e22f846f4211383e6a416d04b4c13ed174d24cc5d43f5fd52e7821d0ebc8920",
+                "sha256:086b32e233e89a2740c1615c2f775c34ae951508b28b308681dbbb87bba97d06",
-                "sha256:116bf8342062246ca749013df4f6ea106f23bc159305843491f64672a55af2e5",
+                "sha256:22a11c493c70413a01ed03f01c3c3a2fc4478fc6ee186e340487b2edcd6f4180",
-                "sha256:189058d03a40103a57144752652b3ab08ff02b7595d0ce1f651b9acc3a3a35a0",
+                "sha256:274b6a2040349b5c9864e475284bce5bb062e63dce368a394b8cc865ae3b00c6",
-                "sha256:23dc34c5e03b0212fa3c49a874df2b8b1b8fda95160bd79c01eb3ab51ea8d851",
+                "sha256:2ae5df899732a6051b49ea2632a9ea67f929604fd2b036613a9f12bc3163b92d",
-                "sha256:28e45b3f827d3bf2592f3cf7ae63282e859f3259db44ed2b129093ca0ac7940b",
+                "sha256:36f4829839ef0afdfdd208bb54f4c3d0eea86106d719811681a8627ae2e53dd5",
-                "sha256:2b26fefc4e31ff85593d68b9022e35e8925714a10ab4858fb1b577a8a48cb8cd",
+                "sha256:39dfbb6fa09f12fae32639e3286112fc35ae976114f1f3d37375f3130a820969",
-                "sha256:32db46ba45849daed7ccf3f2e26f7a386867b077f39b2a974bb5c4c2c3b0a280",
+                "sha256:4703575b78dd697b294f8c65588dc86874ed787b7348c65da70cfc885efdf1e1",
-                "sha256:40499a9979c55f72f4eb2fc38695419546b62594f8af194b879d2a18439c97a9",
+                "sha256:4ad9a94d8f5c9b954e0e3b137cc64ef3f579d0df3c3698fe9c3734ee397e4abb",
-                "sha256:44b1b8e6253bceada11f714acf4309ffb98bfa9ac55e4fce14f9e5d4484287a1",
+                "sha256:557cc55b51ab2f3371e238804ffc8510b6ef087673303890f57a24195d096e61",
-                "sha256:52c3cf9ecda273a19cc092961ee34eb9ba8687d67ba34cc7b79a521c1c64c4c0",
+                "sha256:5cc45235fefac57f52680902b7d197fb2f3650112379a6fa9aa1b1c1d3ed3f02",
-                "sha256:52d8a3166c9f2815bfae05f386114b0b2d274456980d41f320299a8d9a5615a7",
+                "sha256:646530b04f45c830ceae8e491ca1c9320a2d2f0efea3141487c82130aba70dce",
-                "sha256:61bc8b3b265e6949855300e84dc93d02d7a3a637f2aec6d382afd4ceb9120c9f",
+                "sha256:696d8ae4dff4cbd06bf6b10d671e088b66669f110c7c4e18a44c43cf75ce966f",
-                "sha256:654130ca6ad5de73d978057eaf9e582244ff72d4574b3e106fb8d3d2a0d32458",
+                "sha256:7e94b643b19e8feb5215fa508aee531387494bf668b2eca27fa769ea11d9f498",
-                "sha256:6ad2688b69235c43b020e04fecccdf6a96c8943ca9c2fb340b8adc103c655e57",
+                "sha256:88a77f422f31f170c4b7e9ca58eae2a6c8e04da54121900651dfa8e66c29901a",
-                "sha256:6c1f6a173d1140e557347419767d2b14ac1c9cd847e0b4c5444c7f3144697e4e",
+                "sha256:898fba72b81a654e74412a67c7e0a81e89723cfe2a3ea6fcd3feaa3395138ca9",
-                "sha256:84e511a7545d11683d32cdb8f809ef63fc17ea2a00455cc62d0a4dbb4ed1c308",
+                "sha256:9649eced17a98ce816756ce50433b2dd85dfa7bc92ceb60579d68c053f98dff9",
-                "sha256:85de8474ad53ad546ff1c7c7c89230db215b9b8a02754d41cb5a76f70d0be296",
+                "sha256:9af40506a59450f1315168d47a970db1a65aaab5df3833ac389d2899a5d63b3f",
-                "sha256:8988f7163e4381b0da7696f37eec7aca19deb02e500245df68a7159739bbd0d3",
+                "sha256:a28ed481d530e3138553be60991d2d61103ce6da254e51547b79549675f539b7",
-                "sha256:8da1db4ca4f22583e834dcabdc7832e56fe16275253ee53ba66627b86e304da1",
+                "sha256:a2ba7ffe58efeae5b8fad1165357edfe01464f9aef25e814e891ec690e7dd82a",
-                "sha256:8ffc382e4afa4aee367bf413f55ed17bd91b191dcaf979890af239dda435f2a1",
+                "sha256:a4042edef80364239f5b7b5764e55fd3ffd40c32cf6753da9bda4ff0ac466018",
-                "sha256:987bce16e86efa86f747d5151c54e91b3c1e36acc03ce1ddb50f9d09d16ded0e",
+                "sha256:b0232cd42506d0c94f9328aaf0d1d0785f90f87ae72d9759df7e5051be039738",
-                "sha256:ad7efe588c8f5cf940f40c3de0cd683cc5b76819446abaa50dc0829a30c094db",
+                "sha256:b202f591204023b3ce62ff9a47baa555dc00bb092219abf5caf0e3718ac20e7c",
-                "sha256:bb3b15e25891f38da3ca0740271e63ab9db61f41d4d8541745cfc1824252cb28",
+                "sha256:b5c6c967d02fee30e157ab5227706f965d5c37679c687b1e7bbc5d9e7128bd41",
-                "sha256:c928bbf47f65288574b78518449edaa46c82572d340e2750889bbf8cd92f3737",
+                "sha256:cbbd4149c4fc5e7d508ece083e78c17442ee13b0e69bfa6bd63003e486770f45",
-                "sha256:ce291a5aca4985d82875d6779f61375e959208cdf09fcec40001e65fb0a54768",
+                "sha256:f30b03b0f27608a0b26c75f0bb8a880c752c0e0b01090551b9d87c7d783e2069",
-                "sha256:d8768edcbeb34da9e11bcb8b5c2e0958d25218df7a6e56adf415ef262cd7b6d1"
+                "sha256:fdb3c6d342825ea10b90e43d7f20f01535a72b3a1997850c0c3cefa5c27a4a2c"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==1.8.11"
+            "version": "==1.8.12"
         },
         "frozenlist": {
             "hashes": [

README.md

@@ -12,7 +12,7 @@ YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since
 * Completely redesigned the frontend UI.
 * Switched out of binary file storage in favor of SQLite.
 * Handle live streams.
-* Support per link, `yt-dlp config` and `cookies`. and `output format`.
+* Support per link `yt-dlp JSON config or cli options`, `cookies` & `output format`.
 * Tasks Runner. It allow you to queue channels for downloading using simple `json` file.
 * Webhook sender. It allow you to add webhook endpoints that receive events related to downloads using simple `json` file.
 * Multi-downloads support.

app/library/HttpAPI.py

@@ -24,7 +24,7 @@ from .M3u8 import M3u8
 from .Playlist import Playlist
 from .Segments import Segments
 from .Subtitle import Subtitle
-from .Utils import StreamingError, calcDownloadPath, getVideoInfo, validate_url
+from .Utils import StreamingError, arg_converter, calcDownloadPath, getVideoInfo, validate_url
 
 LOG = logging.getLogger("http_api")
 MIME = magic.Magic(mime=True)
@@ -217,6 +217,24 @@ class HttpAPI(common):
         await self.queue.test()
         return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code)
 
+    @route("POST", "api/yt-dlp/convert")
+    async def yt_dlp_convert(self, request: Request) -> Response:
+        post = await request.json()
+        args: str | None = post.get("args")
+
+        if not args:
+            return web.json_response(data={"error": "args is required."}, status=web.HTTPBadRequest.status_code)
+
+        try:
+            return web.json_response(data=arg_converter(args), status=web.HTTPOk.status_code)
+        except Exception as e:
+            err = str(e).strip()
+            err = err.split("\n")[-1] if "\n" in err else err
+            LOG.error(f"Failed to convert args. '{err}'.")
+            return web.json_response(
+                data={"error": f"Failed to convert args. '{err}'."}, status=web.HTTPBadRequest.status_code
+            )
+
     @route("POST", "api/add")
     async def add_url(self, request: Request) -> Response:
         post = await request.json()
@@ -637,6 +655,10 @@ class HttpAPI(common):
     async def add_cors(self, _: Request) -> Response:
         return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
 
+    @route("OPTIONS", "api/yt-dlp/convert")
+    async def cors_ytdlp_convert(self, _: Request) -> Response:
+        return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
+
     @route("OPTIONS", "api/delete")
     async def delete_cors(self, _: Request) -> Response:
         return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)

app/library/HttpSocket.py

@@ -17,7 +17,7 @@ from .config import Config
 from .DownloadQueue import DownloadQueue
 from .Emitter import Emitter
 from .encoder import Encoder
-from .Utils import isDownloaded
+from .Utils import isDownloaded, arg_converter
 
 LOG = logging.getLogger("socket_api")
 
@@ -276,3 +276,24 @@ class HttpSocket(common):
     async def resume(self, sid: str, _=None):
         self.queue.resume()
         await self.emitter.emit("paused", {"paused": False, "at": time.time()})
+
+    @ws_event
+    async def ytdlp_convert(self, sid: str, data: dict):
+        if not isinstance(data, dict) or "args" not in data:
+            await self.emitter.warning("Invalid request or no options were given.", to=sid)
+            return
+
+        args: str | None = data.get("args")
+
+        if not args:
+            await self.emitter.warning("no options were given.", to=sid)
+            return
+
+        try:
+            await self.emitter.emit("ytdlp_convert", arg_converter(args), to=sid)
+            return
+        except Exception as e:
+            err = str(e).strip()
+            err = err.split("\n")[-1] if "\n" in err else err
+            LOG.error(f"Failed to convert args. '{err}'.")
+            await self.emitter.error(f"Failed to convert options. '{e}'.", to=sid)

app/library/Utils.py

@@ -5,6 +5,7 @@ import logging
 import os
 import pathlib
 import re
+import shlex
 import socket
 import uuid
 from datetime import datetime, timezone
@@ -109,7 +110,7 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) ->
     return yt_dlp.YoutubeDL().extract_info(url, download=False)
 
 
-def calcDownloadPath(basePath: str, folder: str|None = None, createPath: bool = True) -> str:
+def calcDownloadPath(basePath: str, folder: str | None = None, createPath: bool = True) -> str:
     """Calculates download path and prevents folder traversal.
 
     Returns:
@@ -501,3 +502,36 @@ def validate_url(url: str) -> bool:
         raise ValueError("Access to internal urls or private networks is not allowed.")
 
     return True
+
+
+def arg_converter(args: str) -> dict:
+    """
+    Convert yt-dlp options to a dictionary.
+
+    Args:
+        opts (str): yt-dlp options string.
+
+    Returns:
+        dict: yt-dlp options dictionary.
+    """
+    import yt_dlp.options
+
+    create_parser = yt_dlp.options.create_parser
+
+    def defaultOpts(args: str):
+        patched_parser = create_parser()
+        try:
+            yt_dlp.options.create_parser = lambda: patched_parser
+            return yt_dlp.parse_options(args)
+        finally:
+            yt_dlp.options.create_parser = create_parser
+
+    default_opts = defaultOpts([]).ydl_opts
+
+    opts = yt_dlp.parse_options(shlex.split(args)).ydl_opts
+
+    diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+    if "postprocessors" in diff:
+        diff["postprocessors"] = [pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]]
+
+    return json.loads(json.dumps(diff))

ui/components/NewDownload.vue

@@ -76,11 +76,11 @@
             <div class="field">
               <label class="label is-inline" for="ytdlpConfig"
                 v-tooltip="'Extends current global yt-dlp config. (JSON)'">
-                JSON yt-dlp config
+                JSON yt-dlp config or CLI options.
               </label>
               <div class="control">
-                <textarea class="textarea" id="ytdlpConfig" v-model="ytdlpConfig"
+                <textarea class="textarea" id="ytdlpConfig" v-model="ytdlpConfig" :disabled="!socket.isConnected"
-                  :disabled="!socket.isConnected"></textarea>
+                  placeholder="--no-embed-metadata --no-embed-thumbnail"></textarea>
               </div>
               <span class="subtitle is-6">
                 Some config fields are ignored like cookiefile, path, and output_format etc.
@@ -152,7 +152,35 @@ const url = useStorage('downloadUrl', null)
 const showAdvanced = useStorage('show_advanced', false)
 const addInProgress = ref(false)
 
-const addDownload = () => {
+const addDownload = async () => {
+  // -- send request to convert cli options to JSON
+  if (ytdlpConfig.value && ytdlpConfig.value.length > 2 && !ytdlpConfig.value.trim().startsWith('{')) {
+    const response = await fetch(config.app.url_host + config.app.url_prefix + 'api/yt-dlp/convert', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ args: ytdlpConfig.value }),
+    });
+
+    const data = await response.json()
+    if (200 !== response.status) {
+      toast.error(`Error: (${response.status}): ${data.error}`)
+      return
+    }
+
+    ytdlpConfig.value = JSON.stringify(data, null, 2)
+  }
+
+  if (ytdlpConfig.value && ytdlpConfig.value.length > 2) {
+    try {
+      JSON.parse(ytdlpConfig.value)
+    } catch (e) {
+      toast.error(`Invalid JSON yt-dlp config. ${e.message}`)
+      return
+    }
+  }
+
   addInProgress.value = true
   url.value.split(',').forEach(url => {
     if (!url.trim()) {