diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b6ef0843..a9b34bad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -234,7 +234,7 @@ jobs: dockerhub-sync-readme: runs-on: ubuntu-latest - if: (github.event_name == 'push' && endsWith(github.ref, github.event.repository.default_branch)) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_readme == 'true') + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_readme == 'true') steps: - name: Sync README uses: docker://lsiodev/readme-sync:latest diff --git a/README.md b/README.md index 11670832..f43e96aa 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,6 @@ Certain configuration values can be set via environment variables, using the `-e | YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | | YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | | YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | -| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer. | `*15 */1 * * *` | +| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer. | `15 */1 * * *` | | YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time. | `1` | diff --git a/app/library/DataStore.py b/app/library/DataStore.py index cce7766a..96c64a3d 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -1,3 +1,4 @@ +import asyncio import copy import json import logging @@ -54,7 +55,7 @@ class DataStore: if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url): return self.dict[i] - msg = f"{key=} or {url=} not found." + msg: str = f"{key=} or {url=} not found." raise KeyError(msg) def get_by_id(self, id: str) -> Download | None: @@ -82,6 +83,11 @@ class DataStore: return items def put(self, value: Download) -> Download: + if "error" == value.info.status: + from app.library.Events import EventBus, Events + + asyncio.create_task(EventBus.get_instance().emit(Events.ITEM_ERROR, value.info), name="emit_item_error") + self.dict.update({value.info._id: value}) self._update_store_item(self.type, value.info) @@ -121,7 +127,7 @@ class DataStore: ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ? """ - stored = copy.deepcopy(item) + stored: ItemDTO = copy.deepcopy(item) if hasattr(stored, "datetime"): try: diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index f885e46b..60e357c6 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedelta from email.utils import formatdate from pathlib import Path from sqlite3 import Connection +from typing import TYPE_CHECKING import yt_dlp from aiohttp import web @@ -38,6 +39,9 @@ from .Utils import ( ) from .YTDLPOpts import YTDLPOpts +if TYPE_CHECKING: + from app.library.Presets import Preset + LOG = logging.getLogger("DownloadQueue") @@ -490,7 +494,7 @@ class DownloadQueue(metaclass=Singleton): { "status": "text" } """ - _preset = Presets.get_instance().get(item.preset) + _preset: Preset | None = Presets.get_instance().get(item.preset) if item.has_cli(): try: @@ -520,9 +524,9 @@ class DownloadQueue(metaclass=Singleton): already.add(item.url) try: - logs = [] + logs: list = [] - yt_conf = { + yt_conf: dict = { "callback": { "func": lambda _, msg: logs.append(msg), "level": logging.WARNING, @@ -542,7 +546,7 @@ class DownloadQueue(metaclass=Singleton): await self._notify.emit(Events.LOG_WARNING, data=event_warning(message)) return {"status": "error", "msg": message} - started = time.perf_counter() + started: float = time.perf_counter() if item.cookies: try: diff --git a/app/library/Events.py b/app/library/Events.py index b07ab345..937b743e 100644 --- a/app/library/Events.py +++ b/app/library/Events.py @@ -109,6 +109,7 @@ class Events: INITIAL_DATA = "initial_data" ITEM_DELETE = "item_delete" ITEM_CANCEL = "item_cancel" + ITEM_ERROR = "item_error" STATUS = "status" CLI_CLOSE = "cli_close" CLI_OUTPUT = "cli_output" diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index 68c0e804..b66c54fc 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -235,6 +235,9 @@ class ItemDTO: def get_id(self) -> str: return self._id + def name(self) -> str: + return f'id="{self.id}", title="{self.title}"' + @staticmethod def removed_fields() -> tuple: """Fields that once existed but are no longer used.""" @@ -247,5 +250,5 @@ class ItemDTO: "output_template", "output_template_chapter", "config", - "temp_path" + "temp_path", ) diff --git a/app/library/task_handlers/youtube.py b/app/library/task_handlers/youtube.py index a97dfd08..40dc31ad 100644 --- a/app/library/task_handlers/youtube.py +++ b/app/library/task_handlers/youtube.py @@ -2,19 +2,31 @@ import asyncio import logging import re from pathlib import Path +from typing import TYPE_CHECKING from app.library.config import Config from app.library.DownloadQueue import DownloadQueue from app.library.Events import EventBus, Events +from app.library.ItemDTO import ItemDTO from app.library.Tasks import Task from app.library.Utils import is_downloaded from app.library.YTDLPOpts import YTDLPOpts +if TYPE_CHECKING: + from app.library.Download import Download + LOG: logging.Logger = logging.getLogger(__name__) +EventBus.get_instance().subscribe( + Events.ITEM_ERROR, + lambda data, _, **__: YoutubeHandler.on_error(data.data), + f"{__name__}.on_error", +) + class YoutubeHandler: queued_ids: set[str] = set() + failure_count: dict[str, int] = {} FEED = "https://www.youtube.com/feeds/videos.xml?{type}={id}" FEED_PLAYLIST = "https://www.youtube.com/feeds/videos.xml?playlist_id={id}" @@ -72,7 +84,7 @@ class YoutubeHandler: }, } - items = [] + items: list = [] async with httpx.AsyncClient(**opts) as client: response = await client.request(method="GET", url=feed_url, timeout=120) @@ -101,17 +113,26 @@ class YoutubeHandler: LOG.warning(f"No entries found in '{task.id}: {task.name}' feed. URL: {feed_url}") return - filtered = [] + filtered: list = [] for item in items: status, _ = is_downloaded(archive_file, url=item["url"]) if status is True or item["id"] in YoutubeHandler.queued_ids: continue - if queue.done.exists(url=item["url"]) or queue.queue.exists(url=item["url"]): - LOG.debug(f"Item '{item['id']}' already exists in the queue or download history.") + if queue.queue.exists(url=item["url"]): + LOG.debug(f"Item '{item['id']}' exists in the queue.") YoutubeHandler.queued_ids.add(item["id"]) continue + try: + done: Download = queue.done.get(url=item["url"]) + if "error" != done.info.status: + LOG.debug(f"Item '{item['id']}' exists in the download history.") + YoutubeHandler.queued_ids.add(item["id"]) + continue + except KeyError: + pass + YoutubeHandler.queued_ids.add(item["id"]) filtered.append(item) @@ -188,6 +209,31 @@ class YoutubeHandler: return None + @staticmethod + async def on_error(item: ItemDTO) -> None: + """ + Handle errors by logging them and removing the queued ID if it exists. + + Args: + item (ItemDTO): The error data containing the URL and other information. + + """ + cls = YoutubeHandler + if not item or not isinstance(item, ItemDTO): + return + + cls.queued_ids.add(item.id) + + if item.id not in cls.queued_ids: + return + + currentFailureCount: int = cls.failure_count.get(item.id, 0) + + LOG.info(f"Removing '{item.name()}' from queued IDs due to error. Failure count: '{currentFailureCount + 1}'.") + cls.queued_ids.remove(item.id) + + cls.failure_count[item.id] = cls.failure_count.get(item.id, 0) + 1 + @staticmethod def tests() -> list[str]: """ diff --git a/ui/components/ConditionForm.vue b/ui/components/ConditionForm.vue index a39dbc3c..39619f5d 100644 --- a/ui/components/ConditionForm.vue +++ b/ui/components/ConditionForm.vue @@ -209,6 +209,7 @@ diff --git a/ui/pages/notifications.vue b/ui/pages/notifications.vue index 9a3457a4..846f27ed 100644 --- a/ui/pages/notifications.vue +++ b/ui/pages/notifications.vue @@ -139,6 +139,7 @@ div.is-centered { diff --git a/ui/pages/presets.vue b/ui/pages/presets.vue index d5fd477e..8da337fd 100644 --- a/ui/pages/presets.vue +++ b/ui/pages/presets.vue @@ -213,6 +213,7 @@ div.is-centered { diff --git a/ui/utils/importer.ts b/ui/utils/importer.ts new file mode 100644 index 00000000..f3d4484b --- /dev/null +++ b/ui/utils/importer.ts @@ -0,0 +1,21 @@ +const encode = (obj: Record): string => { + const jsonStr = JSON.stringify(obj); + const utf8Bytes = new TextEncoder().encode(jsonStr); + const binary = String.fromCharCode(...utf8Bytes); + const base64 = btoa(binary); + return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +const decode = (str: string): object => { + const base64 = str + .replace(/-/g, '+') + .replace(/_/g, '/') + .padEnd(str.length + (4 - str.length % 4) % 4, '='); + + const binary = atob(base64); + const bytes = Uint8Array.from(binary, c => c.charCodeAt(0)); + const jsonStr = new TextDecoder().decode(bytes); + return JSON.parse(jsonStr); +} + +export { encode, decode } diff --git a/ui/utils/index.js b/ui/utils/index.js index 7ec8308d..20a27c32 100644 --- a/ui/utils/index.js +++ b/ui/utils/index.js @@ -406,33 +406,6 @@ const convertCliOptions = async opts => { return data } -/** - * URL Safe Base64 Encode. - * - * @param {String} data The data to encode - * - * @returns {String} The encoded data - */ -const base64UrlEncode = data => btoa(data).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); - -/** - * URL Safe Base64 Decode. - * - * @param {String} input The input string to decode - * - * @returns {String} The decoded data - */ -const base64UrlDecode = input => { - let base64 = input.replace(/-/g, '+').replace(/_/g, '/'); - const pad = base64.length % 4; - if (pad) { - base64 += '='.repeat(4 - pad); - } - - return atob(base64); -} - - /** * Check if array or object has data. * @@ -539,8 +512,6 @@ export { makeDownload, formatBytes, convertCliOptions, - base64UrlEncode, - base64UrlDecode, has_data, toggleClass, cleanObject,