Merge pull request #305 from arabcoders/dev

When item download fails, trigger item_error event

.github/workflows/main.yml

@@ -234,7 +234,7 @@ jobs:
 
   dockerhub-sync-readme:
     runs-on: ubuntu-latest
-    if: (github.event_name == 'push' && endsWith(github.ref, github.event.repository.default_branch)) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_readme == 'true')
+    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_readme == 'true')
     steps:
       - name: Sync README
         uses: docker://lsiodev/readme-sync:latest

README.md

@@ -312,6 +312,6 @@ Certain configuration values can be set via environment variables, using the `-e
 | YTP_YTDLP_AUTO_UPDATE          | Whether to enable the auto update for yt-dlp                       | `true`                             |
 | YTP_BASE_PATH                  | Set this if you are serving YTPTube from sub-folder                | `/`                                |
 | YTP_PREVENT_LIVE_PREMIERE      | Prevents the initial youtube premiere stream from being downloaded | `false`                            |
-| YTP_TASKS_HANDLER_TIMER        | The cron expression for the tasks handler timer.                   | `*15 */1 * * *`                    |
+| YTP_TASKS_HANDLER_TIMER        | The cron expression for the tasks handler timer.                   | `15 */1 * * *`                     |
 | YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time.         | `1`                                |
 

app/library/DataStore.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 import json
 import logging
@@ -54,7 +55,7 @@ class DataStore:
             if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url):
                 return self.dict[i]
 
-        msg = f"{key=} or {url=} not found."
+        msg: str = f"{key=} or {url=} not found."
         raise KeyError(msg)
 
     def get_by_id(self, id: str) -> Download | None:
@@ -82,6 +83,11 @@ class DataStore:
         return items
 
     def put(self, value: Download) -> Download:
+        if "error" == value.info.status:
+            from app.library.Events import EventBus, Events
+
+            asyncio.create_task(EventBus.get_instance().emit(Events.ITEM_ERROR, value.info), name="emit_item_error")
+
         self.dict.update({value.info._id: value})
         self._update_store_item(self.type, value.info)
 
@@ -121,7 +127,7 @@ class DataStore:
         ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ?
         """
 
-        stored = copy.deepcopy(item)
+        stored: ItemDTO = copy.deepcopy(item)
 
         if hasattr(stored, "datetime"):
             try:

app/library/DownloadQueue.py

@@ -8,6 +8,7 @@ from datetime import UTC, datetime, timedelta
 from email.utils import formatdate
 from pathlib import Path
 from sqlite3 import Connection
+from typing import TYPE_CHECKING
 
 import yt_dlp
 from aiohttp import web
@@ -38,6 +39,9 @@ from .Utils import (
 )
 from .YTDLPOpts import YTDLPOpts
 
+if TYPE_CHECKING:
+    from app.library.Presets import Preset
+
 LOG = logging.getLogger("DownloadQueue")
 
 
@@ -490,7 +494,7 @@ class DownloadQueue(metaclass=Singleton):
             { "status": "text" }
 
         """
-        _preset = Presets.get_instance().get(item.preset)
+        _preset: Preset | None = Presets.get_instance().get(item.preset)
 
         if item.has_cli():
             try:
@@ -520,9 +524,9 @@ class DownloadQueue(metaclass=Singleton):
         already.add(item.url)
 
         try:
-            logs = []
+            logs: list = []
 
-            yt_conf = {
+            yt_conf: dict = {
                 "callback": {
                     "func": lambda _, msg: logs.append(msg),
                     "level": logging.WARNING,
@@ -542,7 +546,7 @@ class DownloadQueue(metaclass=Singleton):
                 await self._notify.emit(Events.LOG_WARNING, data=event_warning(message))
                 return {"status": "error", "msg": message}
 
-            started = time.perf_counter()
+            started: float = time.perf_counter()
 
             if item.cookies:
                 try:

app/library/Events.py

@@ -109,6 +109,7 @@ class Events:
     INITIAL_DATA = "initial_data"
     ITEM_DELETE = "item_delete"
     ITEM_CANCEL = "item_cancel"
+    ITEM_ERROR = "item_error"
     STATUS = "status"
     CLI_CLOSE = "cli_close"
     CLI_OUTPUT = "cli_output"

app/library/ItemDTO.py

@@ -235,6 +235,9 @@ class ItemDTO:
     def get_id(self) -> str:
         return self._id
 
+    def name(self) -> str:
+        return f'id="{self.id}",  title="{self.title}"'
+
     @staticmethod
     def removed_fields() -> tuple:
         """Fields that once existed but are no longer used."""
@@ -247,5 +250,5 @@ class ItemDTO:
             "output_template",
             "output_template_chapter",
             "config",
-            "temp_path"
+            "temp_path",
         )

app/library/task_handlers/youtube.py

@@ -2,19 +2,31 @@ import asyncio
 import logging
 import re
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from app.library.config import Config
 from app.library.DownloadQueue import DownloadQueue
 from app.library.Events import EventBus, Events
+from app.library.ItemDTO import ItemDTO
 from app.library.Tasks import Task
 from app.library.Utils import is_downloaded
 from app.library.YTDLPOpts import YTDLPOpts
 
+if TYPE_CHECKING:
+    from app.library.Download import Download
+
 LOG: logging.Logger = logging.getLogger(__name__)
 
+EventBus.get_instance().subscribe(
+    Events.ITEM_ERROR,
+    lambda data, _, **__: YoutubeHandler.on_error(data.data),
+    f"{__name__}.on_error",
+)
+
 
 class YoutubeHandler:
     queued_ids: set[str] = set()
+    failure_count: dict[str, int] = {}
 
     FEED = "https://www.youtube.com/feeds/videos.xml?{type}={id}"
     FEED_PLAYLIST = "https://www.youtube.com/feeds/videos.xml?playlist_id={id}"
@@ -72,7 +84,7 @@ class YoutubeHandler:
             },
         }
 
-        items = []
+        items: list = []
 
         async with httpx.AsyncClient(**opts) as client:
             response = await client.request(method="GET", url=feed_url, timeout=120)
@@ -101,17 +113,26 @@ class YoutubeHandler:
             LOG.warning(f"No entries found in '{task.id}: {task.name}' feed. URL: {feed_url}")
             return
 
-        filtered = []
+        filtered: list = []
         for item in items:
             status, _ = is_downloaded(archive_file, url=item["url"])
             if status is True or item["id"] in YoutubeHandler.queued_ids:
                 continue
 
-            if queue.done.exists(url=item["url"]) or queue.queue.exists(url=item["url"]):
+            if queue.queue.exists(url=item["url"]):
-                LOG.debug(f"Item '{item['id']}' already exists in the queue or download history.")
+                LOG.debug(f"Item '{item['id']}' exists in the queue.")
                 YoutubeHandler.queued_ids.add(item["id"])
                 continue
 
+            try:
+                done: Download = queue.done.get(url=item["url"])
+                if "error" != done.info.status:
+                    LOG.debug(f"Item '{item['id']}' exists in the download history.")
+                    YoutubeHandler.queued_ids.add(item["id"])
+                    continue
+            except KeyError:
+                pass
+
             YoutubeHandler.queued_ids.add(item["id"])
             filtered.append(item)
 
@@ -188,6 +209,31 @@ class YoutubeHandler:
 
         return None
 
+    @staticmethod
+    async def on_error(item: ItemDTO) -> None:
+        """
+        Handle errors by logging them and removing the queued ID if it exists.
+
+        Args:
+            item (ItemDTO): The error data containing the URL and other information.
+
+        """
+        cls = YoutubeHandler
+        if not item or not isinstance(item, ItemDTO):
+            return
+
+        cls.queued_ids.add(item.id)
+
+        if item.id not in cls.queued_ids:
+            return
+
+        currentFailureCount: int = cls.failure_count.get(item.id, 0)
+
+        LOG.info(f"Removing '{item.name()}' from queued IDs due to error. Failure count: '{currentFailureCount + 1}'.")
+        cls.queued_ids.remove(item.id)
+
+        cls.failure_count[item.id] = cls.failure_count.get(item.id, 0) + 1
+
     @staticmethod
     def tests() -> list[str]:
         """

ui/components/ConditionForm.vue

@@ -209,6 +209,7 @@
 <script setup>
 import { useStorage } from '@vueuse/core'
 const emitter = defineEmits(['cancel', 'submit'])
+import { decode } from '~/utils/importer'
 
 const props = defineProps({
   reference: {
@@ -321,18 +322,8 @@ const importItem = async () => {
     return
   }
 
-  if (false === val.startsWith('{')) {
-    try {
-      val = base64UrlDecode(val)
-    } catch (e) {
-      console.error(e)
-      toast.error(`Failed to decode string. ${e.message}`)
-      return
-    }
-  }
-
   try {
-    const item = JSON.parse(val)
+    const item = decode(val)
 
     if (!item?._type || 'condition' !== item._type) {
       toast.error(`Invalid import string. Expected type 'condition', got '${item._type ?? 'unknown'}'.`)

ui/components/NotificationForm.vue

@@ -267,6 +267,8 @@
 
 <script setup>
 import { useStorage } from '@vueuse/core'
+import { decode } from '~/utils/importer'
+
 const emitter = defineEmits(['cancel', 'submit']);
 const toast = useNotification();
 const props = defineProps({
@@ -339,18 +341,8 @@ const importItem = async () => {
     return
   }
 
-  if (false === val.startsWith('{')) {
-    try {
-      val = base64UrlDecode(val)
-    } catch (e) {
-      console.error(e)
-      toast.error(`Failed to decode string. ${e.message}`)
-      return
-    }
-  }
-
   try {
-    const item = JSON.parse(val)
+    const item = decode(val)
 
     if ('notification' !== item._type) {
       toast.error(`Invalid import string. Expected type 'notification', got '${item._type}'.`)

ui/components/PresetForm.vue

@@ -232,6 +232,8 @@
 
 <script setup>
 import { useStorage } from '@vueuse/core'
+import { decode } from '~/utils/importer'
+
 const emitter = defineEmits(['cancel', 'submit']);
 
 const props = defineProps({
@@ -351,18 +353,8 @@ const importItem = async () => {
     return
   }
 
-  if (false === val.startsWith('{')) {
-    try {
-      val = base64UrlDecode(val)
-    } catch (e) {
-      console.error(e)
-      toast.error(`Failed to decode string. ${e.message}`)
-      return
-    }
-  }
-
   try {
-    const item = JSON.parse(val)
+    const item = decode(val)
 
     if (!item?._type || 'preset' !== item._type) {
       toast.error(`Invalid import string. Expected type 'preset', got '${item._type ?? 'unknown'}'.`)

ui/components/TaskForm.vue

@@ -271,6 +271,7 @@
 <script setup>
 import { useStorage } from '@vueuse/core'
 import { CronExpressionParser } from 'cron-parser'
+import { decode } from '~/utils/importer'
 
 const props = defineProps({
   reference: {
@@ -352,18 +353,8 @@ const importItem = async () => {
     return
   }
 
-  if (false === val.startsWith('{')) {
-    try {
-      val = base64UrlDecode(val)
-    } catch (e) {
-      console.error(e)
-      toast.error(`Failed to decode string. ${e.message}`)
-      return
-    }
-  }
-
   try {
-    const item = JSON.parse(val)
+    const item = decode(val)
 
     if ('task' !== item._type) {
       toast.error(`Invalid import string. Expected type 'task', got '${item._type}'.`)

ui/pages/conditions.vue

@@ -116,6 +116,7 @@
 
 <script setup>
 import { request } from '~/utils/index'
+import { encode } from '~/utils/importer'
 
 const toast = useNotification()
 const config = useConfigStore()
@@ -286,6 +287,6 @@ const exportItem = item => {
   userData['_type'] = 'condition'
   userData['_version'] = '1.0'
 
-  return copyText(base64UrlEncode(JSON.stringify(userData)))
+  return copyText(encode(userData))
 }
 </script>

ui/pages/notifications.vue

@@ -139,6 +139,7 @@ div.is-centered {
 
 <script setup>
 import { request } from '~/utils/index'
+import { encode } from '~/utils/importer'
 
 const toast = useNotification()
 const config = useConfigStore()
@@ -337,6 +338,6 @@ const exportItem = async item => {
   data['_type'] = 'notification'
   data['_version'] = '1.0'
 
-  return copyText(base64UrlEncode(JSON.stringify(data)))
+  return copyText(encode(data))
 }
 </script>

ui/pages/presets.vue

@@ -213,6 +213,7 @@ div.is-centered {
 <script setup>
 import { useStorage } from '@vueuse/core'
 import { request } from '~/utils/index'
+import { encode } from '~/utils/importer'
 
 const toast = useNotification()
 const config = useConfigStore()
@@ -400,7 +401,7 @@ const exportItem = item => {
   userData['_type'] = 'preset'
   userData['_version'] = '2.5'
 
-  return copyText(base64UrlEncode(JSON.stringify(userData)))
+  return copyText(encode(userData))
 }
 
 const calcPath = (path) => {

ui/pages/tasks.vue

@@ -20,6 +20,19 @@ div.is-centered {
         </span>
         <div class="is-pulled-right">
           <div class="field is-grouped">
+            <p class="control has-icons-left" v-if="toggleFilter && tasks && tasks.length > 0">
+              <input type="search" v-model.lazy="query" class="input" id="filter"
+                placeholder="Filter displayed content">
+              <span class="icon is-left"><i class="fas fa-filter" /></span>
+            </p>
+
+            <p class="control" v-if="tasks && tasks.length > 0">
+              <button class="button is-danger is-light" v-tooltip.bottom="'Filter'"
+                @click="toggleFilter = !toggleFilter">
+                <span class="icon"><i class="fas fa-filter" /></span>
+              </button>
+            </p>
+
             <p class="control">
               <button class="button is-primary" @click="resetForm(false); toggleForm = !toggleForm"
                 v-tooltip.bottom="'Toggle Add form'">
@@ -58,7 +71,7 @@ div.is-centered {
       </div>
     </div>
 
-    <div class="columns is-multiline" v-if="!isLoading && !toggleForm && tasks && tasks.length > 0">
+    <div class="columns is-multiline" v-if="!isLoading && !toggleForm && filteredTasks && filteredTasks.length > 0">
       <template v-if="'list' === display_style">
         <div class="column is-12">
           <div class="table-container">
@@ -81,7 +94,7 @@ div.is-centered {
                 </tr>
               </thead>
               <tbody>
-                <tr v-for="item in tasks" :key="item.id">
+                <tr v-for="item in filteredTasks" :key="item.id">
                   <td class="is-vcentered">
                     <div class="is-text-overflow">
                       <NuxtLink target="_blank" :href="item.url" class="is-bold">
@@ -139,7 +152,7 @@ div.is-centered {
       </template>
 
       <template v-else>
-        <div class="column is-6" v-for="item in tasks" :key="item.id">
+        <div class="column is-6" v-for="item in filteredTasks" :key="item.id">
           <div class="card is-flex is-full-height is-flex-direction-column">
             <header class="card-header">
               <div class="card-header-title is-text-overflow is-block">
@@ -209,10 +222,15 @@ div.is-centered {
       </template>
     </div>
 
-    <div class="columns is-multiline" v-if="!tasks || tasks.length < 1">
+    <div class="columns is-multiline" v-if="!filteredTasks || filteredTasks.length < 1">
       <div class="column is-12">
         <Message message_class="has-background-info-90 has-text-dark" title="Loading" icon="fas fa-spinner fa-spin"
           message="Loading data. Please wait..." v-if="isLoading" />
+        <Message title="No Results" class="is-background-warning-80 has-text-dark" icon="fas fa-search"
+          v-else-if="query" :useClose="true" @close="query = ''">
+          <p>No results found for the query: <strong>{{ query }}</strong>.</p>
+          <p>Please try a different search term.</p>
+        </Message>
         <Message title="No tasks" message="No tasks are defined." class="is-background-warning-80 has-text-dark"
           icon="fas fa-exclamation-circle" v-else />
       </div>
@@ -225,6 +243,7 @@ import moment from 'moment'
 import { useStorage } from '@vueuse/core'
 import { CronExpressionParser } from 'cron-parser'
 import { request } from '~/utils/index'
+import { encode } from '~/utils/importer'
 import type { task_item, exported_task, error_response } from '~/@types/tasks'
 
 const box = useConfirm()
@@ -241,6 +260,15 @@ const initialLoad = ref<boolean>(true)
 const addInProgress = ref<boolean>(false)
 const display_style = useStorage<string>("tasks_display_style", "cards")
 const remove_keys = ['in_progress']
+const query = ref()
+const toggleFilter = ref(false)
+
+watch(toggleFilter, () => {
+  if (!toggleFilter.value) {
+    query.value = ''
+  }
+});
+
 
 watch(() => config.app.basic_mode, async () => {
   if (!config.app.basic_mode) {
@@ -257,6 +285,15 @@ watch(() => socket.isConnected, async () => {
   }
 })
 
+const filteredTasks = computed<task_item[]>(() => {
+  const q = query.value?.toLowerCase();
+  if (!q) return tasks.value;
+
+  return tasks.value.filter(
+    task => Object.values(task).some(value => typeof value === 'string' && value.toLowerCase().includes(q))
+  );
+});
+
 const reloadContent = async (fromMounted: boolean = false) => {
   try {
     isLoading.value = true
@@ -460,8 +497,7 @@ const exportItem = async (item: task_item) => {
   data._type = 'task'
   data._version = '2.0'
 
-  return copyText(base64UrlEncode(JSON.stringify(data)));
+  return copyText(encode(data));
 }
 
-
 </script>

ui/utils/importer.ts

@@ -0,0 +1,21 @@
+const encode = (obj: Record<string, any>): string => {
+  const jsonStr = JSON.stringify(obj);
+  const utf8Bytes = new TextEncoder().encode(jsonStr);
+  const binary = String.fromCharCode(...utf8Bytes);
+  const base64 = btoa(binary);
+  return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+
+const decode = (str: string): object => {
+  const base64 = str
+    .replace(/-/g, '+')
+    .replace(/_/g, '/')
+    .padEnd(str.length + (4 - str.length % 4) % 4, '=');
+
+  const binary = atob(base64);
+  const bytes = Uint8Array.from(binary, c => c.charCodeAt(0));
+  const jsonStr = new TextDecoder().decode(bytes);
+  return JSON.parse(jsonStr);
+}
+
+export { encode, decode }

ui/utils/index.js

@@ -406,33 +406,6 @@ const convertCliOptions = async opts => {
   return data
 }
 
-/**
- * URL Safe Base64 Encode.
- *
- * @param {String} data The data to encode
- *
- * @returns {String} The encoded data
- */
-const base64UrlEncode = data => btoa(data).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
-
-/**
- * URL Safe Base64 Decode.
- *
- * @param {String} input The input string to decode
- *
- * @returns {String} The decoded data
- */
-const base64UrlDecode = input => {
-  let base64 = input.replace(/-/g, '+').replace(/_/g, '/');
-  const pad = base64.length % 4;
-  if (pad) {
-    base64 += '='.repeat(4 - pad);
-  }
-
-  return atob(base64);
-}
-
-
 /**
  * Check if array or object has data.
  *
@@ -539,8 +512,6 @@ export {
   makeDownload,
   formatBytes,
   convertCliOptions,
-  base64UrlEncode,
-  base64UrlDecode,
   has_data,
   toggleClass,
   cleanObject,