diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2276e50d..0669ec66 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,7 +92,7 @@ jobs: cache-to: type=gha, scope=${{ github.workflow }} - name: Version tag - if: github.event_name != 'pull_request' + if: github.event_name == 'disabled_now' uses: arabcoders/action-python-autotagger@master with: token: ${{ secrets.GITHUB_TOKEN }} @@ -102,7 +102,7 @@ jobs: dockerhub-sync-readme: needs: push-build - if: github.event_name != 'pull_request' + if: github.event_name == 'push' runs-on: ubuntu-latest steps: - name: Sync README @@ -116,3 +116,72 @@ jobs: with: entrypoint: node args: /opt/docker-readme-sync/sync + + create_release: + needs: push-build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' && github.event_name == 'push' && success() + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # so we can git log the full history + + - name: Get commits between old and new + id: commits + run: | + PREVIOUS_SHA="${{ github.event.before }}" + CURRENT_SHA="${{ github.event.after }}" + + echo "Previous SHA: $PREVIOUS_SHA" + echo "Current SHA: $CURRENT_SHA" + + # If "before" is empty or all zeros, log all commits + if [ -z "$PREVIOUS_SHA" ] || [ "$PREVIOUS_SHA" = "0000000000000000000000000000000000000000" ]; then + LOG=$(git log --pretty=format:"- %h %s by %an") + else + LOG=$(git log "$PREVIOUS_SHA".."$CURRENT_SHA" --pretty=format:"- %h %s by %an") + fi + + echo "LOG<> "$GITHUB_ENV" + echo "$LOG" >> "$GITHUB_ENV" + echo "EOF" >> "$GITHUB_ENV" + + - name: Create local tag matching Docker's raw pattern + id: createtag + run: | + # We'll replicate raw format: branch + base_ref + date + short sha + # If base_ref is empty (normal push to master, no PR?), it won't appear. + BRANCH_NAME="${{ github.ref_name }}" + BASE_REF="${{ github.base_ref }}" + CURRENT_SHA="${{ github.event.after }}" + DATE=$(date +%Y%m%d) + SHORT_SHA=$(echo "$CURRENT_SHA" | cut -c1-7) + + TAG="${BRANCH_NAME}${BASE_REF}-${DATE}-${SHORT_SHA}" + + echo "Using tag: $TAG" + + git config user.name "github-actions" + git config user.email "github-actions@github.com" + + # Create a lightweight local tag + git tag "$TAG" "$CURRENT_SHA" + + # Push the tag back to GitHub + git push origin "$TAG" + + # Expose as an output + echo "TAG=$TAG" >> "$GITHUB_OUTPUT" + - name: Create GitHub Release + uses: softprops/action-gh-release@master + with: + tag_name: ${{ steps.createtag.outputs.TAG }} + name: "${{ steps.createtag.outputs.TAG }}" + body: ${{ env.LOG }} + draft: false + prerelease: false + generate_release_notes: true + append_body: true + make_latest: true + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/app/library/Download.py b/app/library/Download.py index 9d77a58a..f5eb42d9 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -95,13 +95,13 @@ class Download: def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} - if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", False): + if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", None): dataDict["filename"] = data["info_dict"]["filename"] self.status_queue.put({"id": self.id, **dataDict}) def _postprocessor_hook(self, data: dict): - if data.get("postprocessor") != "MoveFiles" or data.get("status") != "finished": + if "MoveFiles" != data.get("postprocessor") or "finished" != data.get("status"): return if self.debug: @@ -116,24 +116,22 @@ class Download: def _download(self): try: - params: dict = { - "color": "no_color", - "paths": { - "home": self.download_dir, - "temp": self.tempPath, - }, - "outtmpl": {"default": self.output_template, "chapter": self.output_template_chapter}, - "noprogress": True, - "break_on_existing": True, - "progress_hooks": [self._progress_hook], - "postprocessor_hooks": [self._postprocessor_hook], - **get_opts(self.preset, mergeConfig(self.default_ytdl_opts, self.ytdl_opts)), - } + params: dict = get_opts(self.preset, mergeConfig(self.default_ytdl_opts, self.ytdl_opts)) + params.update( + { + "color": "no_color", + "paths": {"home": self.download_dir, "temp": self.tempPath}, + "outtmpl": {"default": self.output_template, "chapter": self.output_template_chapter}, + "noprogress": True, + "break_on_existing": True, + "progress_hooks": [self._progress_hook], + "postprocessor_hooks": [self._postprocessor_hook], + "ignoreerrors": False, + } + ) if "format" not in params and self.default_ytdl_opts.get("format", None): - params["format"] = self.default_ytdl_opts["format"] - - params["ignoreerrors"] = False + params["format"] = "best" if self.debug: params["verbose"] = True @@ -151,7 +149,7 @@ class Download: params["cookiefile"] = f.name except ValueError as e: - LOG.error(f"Invalid cookies: was provided for {self.info.title} - {str(e)}") + LOG.error(f"Invalid cookies: was provided for '{self.info.title}'. '{str(e)}'.") if self.is_live or self.is_manifestless: hasDeletedOptions = False @@ -176,7 +174,7 @@ class Download: cls = yt_dlp.YoutubeDL(params=params) if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: - LOG.debug("Downloading using pre-info.") + LOG.debug(f"Downloading '{self.info.url}' using pre-info.") cls.process_ie_result(self.info_dict, download=True) ret = cls._download_retcode else: @@ -297,7 +295,7 @@ class Download: if self.info.status != "finished" and self.is_live: LOG.warning( - f"Keeping live temp folder [{self.tempPath}], as the reported status is not finished [{self.info.status}]." + f"Keeping live temp folder '{self.tempPath}', as the reported status is not finished '{self.info.status}'." ) return @@ -310,7 +308,7 @@ class Download: ) return - LOG.info(f"Deleting Temp folder: {self.tempPath}") + LOG.info(f"Deleting Temp folder '{self.tempPath}'.") shutil.rmtree(self.tempPath, ignore_errors=True) async def progress_update(self): diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 27db6d8d..c70badde 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -80,8 +80,8 @@ class DownloadQueue: live_in: str | None = None eventType = entry.get("_type") or "video" - if eventType == "playlist": - entries = entry["entries"] + if "playlist" == eventType: + entries = entry.get("entries", []) playlist_index_digits = len(str(len(entries))) results = [] for i, etr in enumerate(entries, start=1): @@ -106,16 +106,16 @@ class DownloadQueue: ) ) - if any(res["status"] == "error" for res in results): + if any("error" == res["status"] for res in results): return { "status": "error", "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res), } return {"status": "ok"} - elif (eventType == "video" or eventType.startswith("url")) and "id" in entry and "title" in entry: + elif ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry: # check if the video is live stream. - if "live_status" in entry and entry.get("live_status") == "is_upcoming": + if "live_status" in entry and "is_upcoming" == entry.get("live_status"): if "release_timestamp" in entry and entry.get("release_timestamp"): live_in = formatdate(entry.get("release_timestamp")) else: @@ -230,10 +230,12 @@ class DownloadQueue: already=None, ): ytdlp_config = ytdlp_config if ytdlp_config else {} - folder = str(folder) + folder = str(folder) if folder else "" + + filePath = calcDownloadPath(basePath=self.config.download_path, folder=folder) LOG.info( - f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'." + f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'." ) if isinstance(ytdlp_config, str): @@ -279,13 +281,13 @@ class DownloadQueue: f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'." ) except yt_dlp.utils.ExistingVideoReached as exc: - LOG.error(f"Video has been downloaded already and recorded in archive.log file. {str(exc)}") + LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{str(exc)}'.") return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} except yt_dlp.utils.YoutubeDLError as exc: - LOG.error(f"YoutubeDLError: Unable to extract info. {str(exc)}") + LOG.error(f"YoutubeDLError: Unable to extract info. '{str(exc)}'.") return {"status": "error", "msg": str(exc)} except asyncio.exceptions.TimeoutError as exc: - LOG.error(f"TimeoutError: Unable to extract info. {str(exc)}") + LOG.error(f"TimeoutError: Unable to extract info. '{str(exc)}'.") return { "status": "error", "msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.", @@ -366,7 +368,7 @@ class DownloadQueue: os.remove(realFile) fileDeleted = True else: - LOG.warning(f"File '{filename}' does not exist.") + LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.") except Exception as e: LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {str(e)}") @@ -448,8 +450,9 @@ class DownloadQueue: await asyncio.sleep(1) async def __downloadFile(self, id: str, entry: Download): + filePath = calcDownloadPath(basePath=self.config.download_path, folder=entry.info.folder) LOG.info( - f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {entry.info.folder}'." + f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {filePath}'." ) try: diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index d4117031..9e299677 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -222,24 +222,32 @@ class HttpAPI(common): post = await request.json() url: str = post.get("url") - preset: str = post.get("preset", "default") if not url: return web.json_response(data={"error": "url is required."}, status=web.HTTPBadRequest.status_code) - folder: str = post.get("folder") - ytdlp_cookies: str = post.get("ytdlp_cookies") - ytdlp_config: dict | None = post.get("ytdlp_config") - output_template: str = post.get("output_template") - if ytdlp_config is None: - ytdlp_config = {} + preset: str = str(post.get("preset", self.config.default_preset)) + folder: str = str(post.get("folder")) if post.get("folder") else "" + ytdlp_cookies: str = str(post.get("ytdlp_cookies")) if post.get("ytdlp_cookies") else "" + output_template: str = str(post.get("output_template")) if post.get("output_template") else "" + + ytdlp_config = post.get("ytdlp_config") + if isinstance(ytdlp_config, str) and ytdlp_config: + try: + ytdlp_config = json.loads(ytdlp_config) + except Exception as e: + LOG.error(f"Failed to parse json yt-dlp config for '{url}'. {str(e)}") + return web.json_response( + data={"error": f"Failed to parse json yt-dlp config. {str(e)}"}, + status=web.HTTPBadRequest.status_code, + ) status = await self.add( url=url, preset=preset, folder=folder, ytdlp_cookies=ytdlp_cookies, - ytdlp_config=ytdlp_config, + ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {}, output_template=output_template, ) diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index e401d6f5..9cfd47fd 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -1,12 +1,13 @@ import asyncio import errno import functools +import json import logging import os import pty import shlex -from datetime import datetime import time +from datetime import datetime import socketio from aiohttp import web @@ -161,20 +162,25 @@ class HttpSocket(common): await self.emitter.warning("No URL provided.", to=sid) return - preset: str = str(data.get("preset", "default")) - folder: str = str(data.get("folder")) - ytdlp_cookies: str = str(data.get("ytdlp_cookies")) - ytdlp_config: dict | None = data.get("ytdlp_config") - output_template: str = data.get("output_template") - if ytdlp_config is None: - ytdlp_config = {} + preset: str = str(data.get("preset", self.config.default_preset)) + folder: str = str(data.get("folder")) if data.get("folder") else "" + ytdlp_cookies: str = str(data.get("ytdlp_cookies")) if data.get("ytdlp_cookies") else "" + output_template: str = str(data.get("output_template")) if data.get("output_template") else "" + + ytdlp_config = data.get("ytdlp_config") + if isinstance(ytdlp_config, str) and ytdlp_config: + try: + ytdlp_config = json.loads(ytdlp_config) + except Exception as e: + await self.emitter.warning(f"Failed to parse json yt-dlp config. {str(e)}", to=sid) + return status = await self.add( url=url, preset=preset, folder=folder, ytdlp_cookies=ytdlp_cookies, - ytdlp_config=ytdlp_config, + ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {}, output_template=output_template, ) diff --git a/app/library/Utils.py b/app/library/Utils.py index bf33cbcf..dbd7c737 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -109,7 +109,7 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> return yt_dlp.YoutubeDL().extract_info(url, download=False) -def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) -> str: +def calcDownloadPath(basePath: str, folder: str|None = None, createPath: bool = True) -> str: """Calculates download path and prevents folder traversal. Returns: diff --git a/app/library/common.py b/app/library/common.py index 80396f3b..97324298 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -22,9 +22,6 @@ class common: async def add( self, url: str, preset: str, folder: str, ytdlp_cookies: str, ytdlp_config: dict, output_template: str ) -> dict[str, str]: - if ytdlp_config is None: - ytdlp_config = {} - if ytdlp_cookies and isinstance(ytdlp_cookies, dict): ytdlp_cookies = self.encoder.encode(ytdlp_cookies) @@ -33,7 +30,7 @@ class common: preset=preset if preset else "default", folder=folder, ytdlp_cookies=ytdlp_cookies, - ytdlp_config=ytdlp_config, + ytdlp_config=ytdlp_config if isinstance(ytdlp_config, dict) else {}, output_template=output_template, ) diff --git a/ui/stores/SocketStore.js b/ui/stores/SocketStore.js index c48cb9a2..f0c156cd 100644 --- a/ui/stores/SocketStore.js +++ b/ui/stores/SocketStore.js @@ -79,8 +79,8 @@ export const useSocketStore = defineStore('socket', () => { socket.value.on("updated", stream => { const data = JSON.parse(stream); - if (true === stateStore.has('history', item._id)) { - stateStore.update('history', item._id, item); + if (true === stateStore.has('history', data._id)) { + stateStore.update('history', data._id, data); return; }