Merge pull request #409 from arabcoders/dev

FEAT: new TwitchHandler

app/library/ItemDTO.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import re
 import time
 import uuid
 from dataclasses import dataclass, field
@@ -115,6 +116,12 @@ class Item:
             msg = "url param is required."
             raise ValueError(msg)
 
+        url = url.strip()
+
+        # If it's only a YouTube video ID, convert to a full URL.
+        if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
+            url = f"https://www.youtube.com/watch?v={url}"
+
         data: dict[str, str] = {"url": url}
 
         preset: str | None = item.get("preset")

app/library/Presets.py

@@ -19,33 +19,34 @@ DEFAULT_PRESETS: list[dict[int, dict[str, str | bool]]] = [
         "id": "3e163c6c-64eb-4448-924f-814b629b3810",
         "name": "default",
         "default": True,
+        "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log",
         "description": "Default preset for yt-dlp. It will download whatever yt-dlp decides is the best quality for the video and audio.",
     },
     {
         "id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48d",
         "name": "Mobile",
-        "cli": '-t mp4 --merge-output-format mp4 --add-chapters --remux-video mp4 \n--embed-metadata --embed-thumbnail \n--postprocessor-args "-movflags +faststart"',
+        "cli": '--socket-timeout 30 --download-archive %(config_path)s/archive.log\n-t mp4 --merge-output-format mp4 --add-chapters --remux-video mp4 \n--embed-metadata --embed-thumbnail \n--postprocessor-args "-movflags +faststart"',
         "default": True,
         "description": "This preset is designed for mobile devices. It will download the best quality video and audio in mp4 format, merge them, and add chapters, metadata, and thumbnail.",
     },
     {
         "id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48b",
         "name": "1080p",
-        "cli": "-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
+        "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log\n-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
         "default": True,
         "description": "Download the best quality video and audio in mp4 format for 1080p resolution.",
     },
     {
         "id": "9719fcc3-4cf2-4d88-b1e4-74dff3dba00e",
         "name": "720p",
-        "cli": "-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
+        "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log\n-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
         "default": True,
         "description": "Download the best quality video and audio in mp4 format for 720p resolution.",
     },
     {
         "id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330",
         "name": "Audio Only",
-        "cli": "--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'",
+        "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log\n--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'",
         "default": True,
         "description": "This preset is designed to download only the audio of the video. It will extract the audio, add chapters, metadata, and thumbnail.",
     },
@@ -55,7 +56,7 @@ DEFAULT_PRESETS: list[dict[int, dict[str, str | bool]]] = [
         "description": 'This preset generate specific filename format and metadata to work with yt-dlp info reader plugins for jellyfin/emby and plex and to make play state sync work for WatchState.\n\nThere is one more step you need to do via Other > Terminal if you have it enabled or directly from container shell\n\nyt-dlp -I0 --write-info-json --write-thumbnail --convert-thumbnails jpg --paths /downloads/youtube -o "%(channel|Unknown_title)s [%(channel_id|Unknown_id)s]/%(title).180B [%(channel_id|Unknown_id)s].%(ext)s" -- https://www.youtube.com/channel/UCClfFsWcT3N2I7VTXXyt84A\n\nChange the url to the channel you want to download.\n\nFor more information please visit \nhttps://github.com/arabcoders/watchstate/blob/master/FAQ.md#how-to-get-watchstate-working-with-youtube-contentlibrary',
         "folder": "youtube",
         "template": "%(channel)s %(channel_id|Unknown_id)s/Season %(release_date>%Y,upload_date>%Y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s",
-        "cli": "--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata",
+        "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log\n--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata",
         "default": True,
     },
 ]
@@ -92,6 +93,7 @@ class Preset:
 
     def json(self) -> str:
         from .encoder import Encoder
+
         return Encoder().encode(self.serialize())
 
     def get(self, key: str, default: Any = None) -> Any:

app/library/Tasks.py

@@ -504,6 +504,10 @@ class HandleTask:
             if not task.handler_enabled:
                 continue
 
+            if not task.get_ytdlp_opts().get_all().get("download_archive"):
+                LOG.debug(f"Task '{task.name}' does not have an archive file configured.")
+                continue
+
             try:
                 handler = self._find_handler(task)
                 if handler is None:
@@ -565,6 +569,9 @@ class HandleTask:
         handlers: list[type] = []
 
         for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
+            if module_name.startswith("_"):
+                continue
+
             module = importlib.import_module(f"{handlers_pkg.__name__}.{module_name}")
             for _, cls in inspect.getmembers(module, inspect.isclass):
                 if cls.__module__ != module.__name__:

app/library/YTDLPOpts.py

@@ -22,7 +22,7 @@ class YTDLPOpts:
     """The command options for yt-dlp from preset."""
 
     def __init__(self):
-        self._config = Config.get_instance()
+        self._config: Config = Config.get_instance()
 
     @staticmethod
     def get_instance() -> "YTDLPOpts":
@@ -96,7 +96,7 @@ class YTDLPOpts:
 
         """
         preset: Preset | None = Presets.get_instance().get(name)
-        if not preset or "default" == name:
+        if not preset:
             return self
 
         if preset.cli:
@@ -169,7 +169,11 @@ class YTDLPOpts:
 
         if len(self._item_cli) > 0:
             try:
-                user_cli: dict = arg_converter(args="\n".join(self._item_cli), level=True)
+                user_cli = "\n".join(self._item_cli)
+                for k, v in self._config.get_replacers().items():
+                    user_cli = user_cli.replace(f"%({k})s", v if isinstance(v, str) else str(v))
+
+                user_cli: dict = arg_converter(args=user_cli, level=True)
             except Exception as e:
                 msg = f"Invalid command options for yt-dlp were given. '{e!s}'."
                 raise ValueError(msg) from e

app/library/config.py

@@ -152,9 +152,6 @@ class Config:
     file_logging: bool = True
     "Enable file logging."
 
-    sentry_dsn: str | None = None
-    "The Sentry DSN to use for error reporting."
-
     secret_key: str
     "The secret key to use for the application."
 
@@ -266,7 +263,6 @@ class Config:
         "basic_mode",
         "default_preset",
         "instance_title",
-        "sentry_dsn",
         "console_enabled",
         "browser_enabled",
         "browser_control_enabled",
@@ -406,6 +402,9 @@ class Config:
         self._ytdlp_cli_mutable += f"\n--socket-timeout {self.socket_timeout}"
 
         if self.keep_archive:
+            LOG.warning(
+                "The global 'keep_archive' option is deprecated and will be removed in future releases. please use presets instead."
+            )
             archive_file: Path = Path(self.archive_file)
             if not archive_file.exists():
                 LOG.info(f"Creating archive file '{archive_file}'.")
@@ -541,6 +540,17 @@ class Config:
         data["ytdlp_version"] = Config._ytdlp_version()
         return data
 
+    def get_replacers(self) -> dict:
+        """
+        Get the variables that can be used in Command options for yt-dlp.
+
+        Returns:
+            dict: The replacer variables.
+
+        """
+        keys: tuple[str] = ("download_path", "temp_path", "config_path")
+        return {k: getattr(self, k) for k in keys}
+
     @staticmethod
     def _ytdlp_version() -> str:
         try:

app/library/task_handlers/_base_handler.py

@@ -0,0 +1,105 @@
+# flake8: noqa: ARG004
+import logging
+from typing import Any
+
+import httpx
+from yt_dlp.utils.networking import random_user_agent
+
+from app.library.config import Config
+from app.library.DownloadQueue import DownloadQueue
+from app.library.Events import EventBus, Events
+from app.library.ItemDTO import ItemDTO
+from app.library.Tasks import Task
+
+LOG: logging.Logger = logging.getLogger(__name__)
+
+
+class BaseHandler:
+    queued: set[str] = set()
+    failure_count: dict[str, int] = {}
+
+    def __init_subclass__(cls, **kwargs):
+        """Ensure each subclass has its own state containers."""
+        super().__init_subclass__(**kwargs)
+        if "queued" not in cls.__dict__:
+            cls.queued = set()
+        if "failure_count" not in cls.__dict__:
+            cls.failure_count = {}
+
+        EventBus.get_instance().subscribe(
+            Events.ITEM_ERROR,
+            lambda data, _, **__: cls.on_error(data.data),
+            f"{cls.__name__}.on_error",
+        )
+
+    @staticmethod
+    def can_handle(task: Task) -> bool:
+        return False
+
+    @staticmethod
+    async def handle(task: Task, notify: EventBus, config: Config, queue: DownloadQueue):
+        pass
+
+    @staticmethod
+    def parse(url: str) -> Any | None:
+        return None
+
+    @classmethod
+    async def on_error(cls, item: ItemDTO) -> None:
+        """
+        Handle errors by logging them and removing the queued ID if it exists.
+
+        Args:
+            item (ItemDTO): The error data containing the URL and other information.
+
+        """
+        if not item or not isinstance(item, ItemDTO):
+            return
+
+        if not item.archive_id or not cls.failure_count.get(item.archive_id, None):
+            LOG.debug(f"Item '{item.name()}' not queued by the handler.")
+            return
+
+        failCount: int = int(cls.failure_count.get(item.archive_id, 0))
+
+        LOG.info(f"Removing '{item.name()}' from queued IDs due to error. Failure count: '{failCount + 1}'.")
+        if item.archive_id in cls.queued:
+            cls.queued.remove(item.archive_id)
+
+        cls.failure_count[item.archive_id] = 1 + failCount
+
+    @staticmethod
+    def tests() -> list[tuple[str, bool]]:
+        return []
+
+    @staticmethod
+    async def request(url: str, headers: dict | None = None, ytdlp_opts: dict | None = None) -> httpx.Response:
+        headers = {} if not isinstance(headers, dict) else headers
+        ytdlp_opts = {} if not isinstance(ytdlp_opts, dict) else ytdlp_opts
+
+        opts: dict[str, Any] = {
+            "headers": {
+                "User-Agent": random_user_agent(),
+            },
+        }
+
+        try:
+            from httpx_curl_cffi import AsyncCurlTransport, CurlOpt
+
+            opts["transport"] = AsyncCurlTransport(
+                impersonate="chrome",
+                default_headers=True,
+                curl_options={CurlOpt.FRESH_CONNECT: True},
+            )
+            opts["headers"].pop("User-Agent", None)
+        except Exception:
+            pass
+
+        for k, v in headers.items():
+            opts["headers"][k] = v
+
+        if proxy := ytdlp_opts.get("proxy", None):
+            opts["proxy"] = proxy
+
+        async with httpx.AsyncClient(**opts) as client:
+            return await client.request(method="GET", url=url, timeout=ytdlp_opts.get("socket_timeout", 120))

app/library/task_handlers/twitch.py

@@ -0,0 +1,189 @@
+import asyncio
+import logging
+import re
+from typing import TYPE_CHECKING
+from xml.etree.ElementTree import Element
+
+from app.library.DownloadQueue import DownloadQueue
+from app.library.Events import EventBus, Events
+from app.library.ItemDTO import Item
+from app.library.Tasks import Task
+from app.library.Utils import archive_read, get_archive_id
+
+from ._base_handler import BaseHandler
+
+if TYPE_CHECKING:
+    from xml.etree.ElementTree import Element
+
+    from app.library.Download import Download
+
+LOG: logging.Logger = logging.getLogger(__name__)
+
+
+class TwitchHandler(BaseHandler):
+    FEED = "https://twitchrss.appspot.com/vodonly/{handle}"
+
+    RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?twitch\.tv\/(?P<id>[a-z0-9_]{3,25})(?:\/.*)?$")
+
+    @staticmethod
+    def can_handle(task: Task) -> bool:
+        LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}")
+        return TwitchHandler.parse(task.url) is not None
+
+    @staticmethod
+    async def handle(task: Task, notify: EventBus, queue: DownloadQueue):
+        """
+        Fetch the RSS feed for a Twitch channel VODs, parse entries,
+        and enqueue new items that are not in the archive/queue already.
+
+        Args:
+            task (Task): The task containing the Twitch channel URL.
+            notify (EventBus): The event bus for notifications.
+            queue (DownloadQueue): The download queue instance.
+
+        """
+        from defusedxml.ElementTree import fromstring
+
+        handleName: str | None = TwitchHandler.parse(task.url)
+        if not handleName:
+            LOG.error(f"Cannot parse '{task.name}' URL: {task.url}")
+            return
+
+        params: dict = task.get_ytdlp_opts().get_all()
+        archive_file: str | None = params.get("download_archive")
+        if not archive_file:
+            LOG.error(f"Task '{task.name}' does not have an archive file.")
+            return
+
+        feed_url: str = TwitchHandler.FEED.format(handle=handleName)
+
+        LOG.debug(f"Fetching '{task.name}' feed.")
+        response = await TwitchHandler.request(url=feed_url, ytdlp_opts=params)
+        response.raise_for_status()
+
+        items: list = []
+        has_items = False
+
+        root: Element[str] = fromstring(response.text)
+        for entry in root.findall("channel/item"):
+            link_elem: Element[str] | None = entry.find("link")
+            url: str = link_elem.text.strip() if link_elem is not None and link_elem.text else ""
+            if not url:
+                LOG.warning(f"Entry in '{task.name}' feed is missing URL. Skipping entry.")
+                continue
+
+            m: re.Match[str] | None = re.search(r"^https?://(?:www\.)?twitch\.tv/videos/(?P<id>\d+)(?:[/?].*)?$", url)
+            if not m:
+                LOG.warning(f"URL in '{task.name}' feed does not look like a VOD link: {url}")
+                continue
+
+            vid: str = m.group("id")
+
+            title_elem: Element[str] | None = entry.find("title")
+            title: str = title_elem.text.strip() if title_elem is not None and title_elem.text else ""
+
+            has_items = True
+
+            id_dict = get_archive_id(url)
+            archive_id: str | None = id_dict.get("archive_id")
+            if not archive_id:
+                LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
+                continue
+
+            if archive_id in TwitchHandler.queued:
+                continue
+
+            items.append({"id": vid, "url": url, "title": title, "archive_id": archive_id})
+
+        if len(items) < 1:
+            if not has_items:
+                LOG.warning(f"No entries found in '{task.name}' feed. URL: {feed_url}")
+            else:
+                LOG.debug(f"No new items found in '{task.name}' feed.")
+            return
+
+        filtered: list = []
+
+        downloaded: list[str] = archive_read(archive_file, [item["archive_id"] for item in items])
+
+        for item in items:
+            TwitchHandler.queued.add(item["archive_id"])
+
+            if item["archive_id"] in downloaded:
+                continue
+
+            if queue.queue.exists(url=item["url"]):
+                continue
+
+            try:
+                done: Download = queue.done.get(url=item["url"])
+                if "error" != done.info.status:
+                    continue
+            except KeyError:
+                pass
+
+            if item["archive_id"] not in TwitchHandler.failure_count:
+                TwitchHandler.failure_count[item["archive_id"]] = 0
+
+            filtered.append(item)
+
+        if len(filtered) < 1:
+            LOG.debug(f"No new items found in '{task.name}' feed.")
+            return
+
+        LOG.info(f"Found '{len(filtered)}' new items from '{task.name}' feed.")
+
+        rItem: Item = Item.format(
+            {
+                "url": feed_url,
+                "preset": task.preset,
+                "folder": task.folder if task.folder else "",
+                "template": task.template if task.template else "",
+                "cli": task.cli if task.cli else "",
+                "auto_start": task.auto_start,
+                "extras": {"source_task": task.id},
+            }
+        )
+
+        try:
+            await asyncio.gather(
+                *[notify.emit(Events.ADD_URL, data=rItem.new_with(url=item["url"]).serialize()) for item in filtered]
+            )
+        except Exception as e:
+            LOG.exception(e)
+            LOG.error(f"Error while adding items from '{task.name}'. {e!s}")
+            return
+
+    @staticmethod
+    def parse(url: str) -> str | None:
+        """
+        Parse twitch URL to extract the channel.
+
+        Args:
+            url (str): The url to check.
+
+        Returns:
+            str | None: The parsed ID if successful, None otherwise.
+
+        """
+        match: re.Match[str] | None = TwitchHandler.RX.match(url)
+        return match.group("id") if match else None
+
+    @staticmethod
+    def tests() -> list[tuple[str, bool]]:
+        """
+        Test cases for the URL parser.
+
+        Returns:
+            list[tuple[str, bool]]: A list of tuples containing the URL and expected result.
+
+        """
+        return [
+            ("https://www.twitch.tv/test_username", True),
+            ("https://twitch.tv/test_username", True),
+            ("http://m.twitch.tv/test_username,", True),
+            ("https://www.twitch.tv/test_username/", True),
+            ("https://twitch.tv/test_username/", True),
+            ("http://m.twitch.tv/test_username/,", True),
+            ("twitch.tv/test_username/", False),
+        ]

app/library/task_handlers/youtube.py

@@ -1,15 +1,16 @@
 import asyncio
 import logging
 import re
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
 from xml.etree.ElementTree import Element
 
-from app.library.config import Config
 from app.library.DownloadQueue import DownloadQueue
 from app.library.Events import EventBus, Events
-from app.library.ItemDTO import Item, ItemDTO
+from app.library.ItemDTO import Item
 from app.library.Tasks import Task
-from app.library.Utils import archive_read
+from app.library.Utils import archive_read, get_archive_id
+
+from ._base_handler import BaseHandler
 
 if TYPE_CHECKING:
     from xml.etree.ElementTree import Element
@@ -18,22 +19,17 @@ if TYPE_CHECKING:
 
 LOG: logging.Logger = logging.getLogger(__name__)
 
-EventBus.get_instance().subscribe(
-    Events.ITEM_ERROR,
-    lambda data, _, **__: YoutubeHandler.on_error(data.data),
-    f"{__name__}.on_error",
-)
-
-
-class YoutubeHandler:
-    queued: set[str] = set()
-    failure_count: dict[str, int] = {}
 
+class YoutubeHandler(BaseHandler):
     FEED = "https://www.youtube.com/feeds/videos.xml?{type}={id}"
 
-    CHANNEL_REGEX = re.compile(r"^https?://(?:www\.)?youtube\.com/(?:channel/(?P<id>UC[0-9A-Za-z_-]{22})|)/?$")
+    CHANNEL_REGEX: re.Pattern[str] = re.compile(
+        r"^https?://(?:www\.)?youtube\.com/(?:channel/(?P<id>UC[0-9A-Za-z_-]{22})|)/?$"
+    )
 
-    PLAYLIST_REGEX = re.compile(r"^https?://(?:www\.)?youtube\.com/(?:playlist\?list=(?P<id>[A-Za-z0-9_-]+)|).*$")
+    PLAYLIST_REGEX: re.Pattern[str] = re.compile(
+        r"^https?://(?:www\.)?youtube\.com/(?:playlist\?list=(?P<id>[A-Za-z0-9_-]+)|).*$"
+    )
 
     @staticmethod
     def can_handle(task: Task) -> bool:
@@ -45,7 +41,7 @@ class YoutubeHandler:
         return YoutubeHandler.parse(task.url) is not None
 
     @staticmethod
-    async def handle(task: Task, notify: EventBus, config: Config, queue: DownloadQueue):
+    async def handle(task: Task, notify: EventBus, queue: DownloadQueue):
         """
         Fetch the Atom feed for a YouTube channel or playlist, parse entries,
         and return a list of videos with metadata.
@@ -53,16 +49,9 @@ class YoutubeHandler:
         Args:
             task (Task): The task containing the YouTube URL.
             notify (EventBus): The event bus for notifications.
-            config (Config): The configuration instance.
             queue (DownloadQueue): The download queue instance.
 
         """
-        params: dict = task.get_ytdlp_opts().get_all()
-        if not (archive_file := params.get("download_archive")):
-            LOG.error(f"Task '{task.name}' does not have an archive file.")
-            return
-
-        import httpx
         from defusedxml.ElementTree import fromstring
 
         parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
@@ -70,65 +59,49 @@ class YoutubeHandler:
             LOG.error(f"Cannot parse '{task.name}' URL: {task.url}")
             return
 
+        params: dict = task.get_ytdlp_opts().get_all()
+
         feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"])
-
         LOG.debug(f"Fetching '{task.name}' feed.")
-        opts: dict[str, Any] = {
-            "proxy": params.get("proxy"),
-            "headers": {
-                "User-Agent": params.get(
-                    "user_agent",
-                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
-                )
-            },
-        }
-
-        try:
-            from httpx_curl_cffi import AsyncCurlTransport, CurlOpt
-
-            opts["transport"] = AsyncCurlTransport(
-                impersonate="chrome",
-                default_headers=True,
-                curl_options={CurlOpt.FRESH_CONNECT: True},
-            )
-            opts.pop("headers", None)
-        except Exception:
-            pass
 
         items: list = []
         has_items = False
 
-        async with httpx.AsyncClient(**opts) as client:
-            response: httpx.Response = await client.request(method="GET", url=feed_url, timeout=120)
-            response.raise_for_status()
+        response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params)
+        response.raise_for_status()
 
-            root: Element[str] = fromstring(response.text)
-            ns: dict[str, str] = {
-                "atom": "http://www.w3.org/2005/Atom",
-                "yt": "http://www.youtube.com/xml/schemas/2015",
-            }
+        root: Element[str] = fromstring(response.text)
+        ns: dict[str, str] = {
+            "atom": "http://www.w3.org/2005/Atom",
+            "yt": "http://www.youtube.com/xml/schemas/2015",
+        }
 
-            for entry in root.findall("atom:entry", ns):
-                vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
-                vid: str | None = vid_elem.text if vid_elem is not None else ""
-                if not vid:
-                    LOG.warning(f"Entry in '{task.name}' feed is missing a video ID. Skipping entry.")
-                    continue
+        for entry in root.findall("atom:entry", ns):
+            vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
+            vid: str | None = vid_elem.text if vid_elem is not None else ""
+            if not vid:
+                LOG.warning(f"Entry in '{task.name}' feed is missing a video ID. Skipping entry.")
+                continue
 
-                archive_id: str = f"youtube {vid}"
-                url: str = f"https://www.youtube.com/watch?v={vid}"
+            url: str = f"https://www.youtube.com/watch?v={vid}"
 
-                title_elem: Element[str] | None = entry.find("atom:title", ns)
-                title: str | None = title_elem.text if title_elem is not None else ""
+            id_dict: dict[str, str | None] = get_archive_id(url)
+            archive_id: str | None = id_dict.get("archive_id")
+            if not archive_id:
+                LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
+                continue
 
-                pub_elem: Element[str] | None = entry.find("atom:published", ns)
-                published: str | None = pub_elem.text if pub_elem is not None else ""
-                has_items = True
+            title_elem: Element[str] | None = entry.find("atom:title", ns)
+            title: str | None = title_elem.text if title_elem is not None else ""
 
-                if archive_id in YoutubeHandler.queued:
-                    continue
+            pub_elem: Element[str] | None = entry.find("atom:published", ns)
+            published: str | None = pub_elem.text if pub_elem is not None else ""
+            has_items = True
 
-                items.append({"id": vid, "url": url, "title": title, "published": published, "archive_id": archive_id})
+            if archive_id in YoutubeHandler.queued:
+                continue
+
+            items.append({"id": vid, "url": url, "title": title, "published": published, "archive_id": archive_id})
 
         if len(items) < 1:
             if not has_items:
@@ -139,7 +112,7 @@ class YoutubeHandler:
 
         filtered: list = []
 
-        downloaded: list[str] = archive_read(archive_file, [item["archive_id"] for item in items])
+        downloaded: list[str] = archive_read(params.get("download_archive"), [item["archive_id"] for item in items])
 
         for item in items:
             YoutubeHandler.queued.add(item["archive_id"])
@@ -170,7 +143,7 @@ class YoutubeHandler:
         rItem: Item = Item.format(
             {
                 "url": feed_url,
-                "preset": str(task.preset or config.default_preset),
+                "preset": task.preset,
                 "folder": task.folder if task.folder else "",
                 "template": task.template if task.template else "",
                 "cli": task.cli if task.cli else "",
@@ -210,31 +183,6 @@ class YoutubeHandler:
 
         return None
 
-    @staticmethod
-    async def on_error(item: ItemDTO) -> None:
-        """
-        Handle errors by logging them and removing the queued ID if it exists.
-
-        Args:
-            item (ItemDTO): The error data containing the URL and other information.
-
-        """
-        cls = YoutubeHandler
-        if not item or not isinstance(item, ItemDTO):
-            return
-
-        if not item.archive_id or not cls.failure_count.get(item.archive_id, None):
-            LOG.debug(f"Item '{item.name()}' not queued by the handler.")
-            return
-
-        failCount: int = int(cls.failure_count.get(item.archive_id, 0))
-
-        LOG.info(f"Removing '{item.name()}' from queued IDs due to error. Failure count: '{failCount + 1}'.")
-        if item.archive_id in cls.queued:
-            cls.queued.remove(item.archive_id)
-
-        cls.failure_count[item.archive_id] = 1 + failCount
-
     @staticmethod
     def tests() -> list[tuple[str, bool]]:
         """

ui/app/components/History.vue

@@ -13,7 +13,7 @@
 
   <div v-if="showCompleted">
     <div class="columns is-multiline is-mobile has-text-centered" v-if="hasItems">
-      <div class="column is-half-mobile" v-if="display_style === 'cards'">
+      <div class="column is-half-mobile" v-if="display_style === 'grid'">
         <button type="button" class="button is-fullwidth is-ghost is-inverted"
           @click="masterSelectAll = !masterSelectAll">
           <span class="icon-text is-block">
@@ -176,7 +176,7 @@
                       </button>
                     </div>
                     <div class="control is-expanded" v-if="item.url && !config.app.basic_mode">
-                      <Dropdown icons="fa-solid fa-cogs" @open_state="s => table_container = !s"
+                      <Dropdown icons="fa-solid fa-cogs" @open_state="(s: boolean) => table_container = !s"
                         :button_classes="'is-small'" label="Actions">
                         <template v-if="'finished' === item.status && item.filename">
                           <NuxtLink @click="playVideo(item)" class="dropdown-item">
@@ -481,7 +481,7 @@ const box = useConfirm()
 const showCompleted = useStorage<boolean>('showCompleted', true)
 const hideThumbnail = useStorage<boolean>('hideThumbnailHistory', false)
 const direction = useStorage<'asc' | 'desc'>('sortCompleted', 'desc')
-const display_style = useStorage<'cards' | 'list'>('display_style', 'cards')
+const display_style = useStorage<'grid' | 'list'>('display_style', 'grid')
 const bg_enable = useStorage<boolean>('random_bg', true)
 const bg_opacity = useStorage<number>('random_bg_opacity', 0.95)
 
@@ -622,6 +622,9 @@ const clearCompleted = async () => {
   for (const key in stateStore.history) {
     if ('finished' === ag(stateStore.get('history', key, {} as StoreItem), 'status')) {
       socket.emit('item_delete', { id: stateStore.history[key]?._id, remove_file: false, })
+      if (selectedElms.value.includes(stateStore.history[key]?._id || '')) {
+        selectedElms.value = selectedElms.value.filter(i => i !== stateStore.history[key]?._id)
+      }
     }
   }
 }
@@ -636,6 +639,11 @@ const clearIncomplete = async () => {
         id: stateStore.history[key]?._id,
         remove_file: false,
       })
+
+      if (selectedElms.value.includes(stateStore.history[key]?._id || '')) {
+        selectedElms.value = selectedElms.value.filter(i => i !== stateStore.history[key]?._id)
+      }
+
     }
   }
 }
@@ -698,7 +706,7 @@ const setStatus = (item: StoreItem) => {
     if (item.extras?.is_premiere) {
       return 'Premiere'
     }
-    return display_style.value === 'cards' ? 'Stream' : 'Live'
+    return display_style.value === 'grid' ? 'Stream' : 'Live'
   }
   if ('skip' === item.status) {
     return 'Skipped'
@@ -754,10 +762,15 @@ const removeItem = async (item: StoreItem) => {
   if (false === (await box.confirm(msg, Boolean(item.filename && config.app.remove_files)))) {
     return false
   }
+
   socket.emit('item_delete', {
     id: item._id,
     remove_file: config.app.remove_files
   })
+
+  if (selectedElms.value.includes(item._id || '')) {
+    selectedElms.value = selectedElms.value.filter(i => i !== item._id)
+  }
 }
 
 const retryItem = (item: StoreItem, re_add = false) => {
@@ -773,6 +786,11 @@ const retryItem = (item: StoreItem, re_add = false) => {
   }
 
   socket.emit('item_delete', { id: item._id, remove_file: false })
+
+  if (selectedElms.value.includes(item._id || '')) {
+    selectedElms.value = selectedElms.value.filter(i => i !== item._id)
+  }
+
   if (true === re_add) {
     toast.info('Cleared the item from history, and added it to the new download form.')
     emitter('add_new', item_req)

ui/app/components/Queue.vue

@@ -12,7 +12,7 @@
 
   <div v-if="showQueue">
     <div class="columns is-multiline is-mobile has-text-centered" v-if="filteredItems.length > 0">
-      <div class="column is-half-mobile" v-if="'cards' === display_style">
+      <div class="column is-half-mobile" v-if="'grid' === display_style">
         <button type="button" class="button is-fullwidth is-ghost" @click="masterSelectAll = !masterSelectAll">
           <span class="icon-text is-block">
             <span class="icon">
@@ -36,7 +36,7 @@
           <span>Pause</span>
         </button>
       </div>
-      <div class="column is-half-mobile" v-if="('cards' === display_style || hasSelected)">
+      <div class="column is-half-mobile" v-if="('grid' === display_style || hasSelected)">
         <button type="button" class="button is-fullwidth is-warning" :disabled="!hasSelected" @click="cancelSelected">
           <span class="icon"><i class="fa-solid fa-eject" /></span>
           <span>Cancel</span>
@@ -115,7 +115,7 @@
                     :data-datetime="item.datetime" v-rtime="item.datetime" />
                 </td>
                 <td class="is-vcentered is-items-center">
-                  <Dropdown icons="fa-solid fa-cogs" @open_state="s => table_container = !s"
+                  <Dropdown icons="fa-solid fa-cogs" @open_state="(s: boolean) => table_container = !s"
                     :button_classes="'is-small'" label="Actions">
                     <template v-if="isEmbedable(item.url)">
                       <NuxtLink class="dropdown-item has-text-danger"
@@ -265,7 +265,7 @@
                 </button>
               </div>
               <div class="column is-half-mobile">
-                <Dropdown icons="fa-solid fa-cogs" @open_state="s => table_container = !s" label="Actions">
+                <Dropdown icons="fa-solid fa-cogs" @open_state="(s: boolean) => table_container = !s" label="Actions">
                   <template v-if="isEmbedable(item.url)">
                     <NuxtLink class="dropdown-item has-text-danger"
                       @click="embed_url = getEmbedable(item.url) as string">
@@ -337,7 +337,7 @@ const toast = useNotification()
 
 const showQueue = useStorage('showQueue', true)
 const hideThumbnail = useStorage('hideThumbnailQueue', false)
-const display_style = useStorage('display_style', 'cards')
+const display_style = useStorage('display_style', 'grid')
 const bg_enable = useStorage('random_bg', true)
 const bg_opacity = useStorage('random_bg_opacity', 0.95)
 

ui/app/layouts/default.vue

@@ -158,7 +158,6 @@ import 'assets/css/style.css'
 import 'assets/css/all.css'
 import { useStorage } from '@vueuse/core'
 import moment from 'moment'
-import * as Sentry from '@sentry/nuxt'
 import type { YTDLPOption } from '~/types/ytdlp'
 import { useDialog } from '~/composables/useDialog'
 import Dialog from '~/components/Dialog.vue'
@@ -249,14 +248,6 @@ const applyPreferredColorScheme = (scheme: string) => {
   }
 }
 
-watch(() => config.app.sentry_dsn, dsn => {
-  if (!dsn) {
-    return
-  }
-  console.warn('Loading sentry module.')
-  Sentry.init({ dsn: dsn })
-})
-
 onMounted(async () => {
   try {
     await handleImage(bg_enable.value)

ui/app/pages/index.vue

@@ -72,25 +72,16 @@
           </p>
           <ul>
             <li>
-              The environment variables <code>YTP_KEEP_ARCHIVE</code> and <code>YTP_SOCKET_TIMEOUT</code> will no
-              longer be user-configurable. Their behavior will be part of the <strong>default presets</strong>. To keep
-              your current behavior <strong>and avoid re-downloading</strong>, please add the following <strong>Command
-                options for yt-dlp</strong> to your presets:
-              <code>--socket-timeout 30 --download-archive /config/archive.log</code>
+              The following ENVs <strong>YTP_KEEP_ARCHIVE</strong> and <strong>YTP_SOCKET_TIMEOUT</strong> will be
+              removed.
+              Their behavior will be part of the <strong>default presets</strong>. To keep your current behavior
+              <strong>and avoid re-downloading</strong>, please add the following <strong>Command options for
+                yt-dlp</strong> to your presets:
+              <code>--socket-timeout 30 --download-archive %(config_path)s/archive.log</code>
             </li>
             <li>
-              The global yt-dlp config file <code>/config/ytdlp.cli</code> is deprecated and will be removed. Please
-              migrate any global options into your presets.
-            </li>
-            <li>
-              The <strong>Basic mode</strong> (which limited the interface to the new download form) is being removed.
-              Everything except what is available behind configurable flag will become part of the standard interface.
-            </li>
-            <li>
-              The file browser feature will be enabled by <strong>default</strong>, and the associated environment
-              variable <code>YTP_BROWSER_ENABLED</code> will be removed, We will keep the
-              <code>YTP_BROWSER_CONTROL_ENABLED</code> to control whether you want to enable the file management
-              features or not. it will default to <code>false</code>.
+              The global yt-dlp config file <strong>/config/ytdlp.cli</strong> will be removed. Please migrate to
+              presets.
             </li>
             <li>The <strong>archive.manual.log</strong> feature has been removed.</li>
           </ul>
@@ -98,6 +89,22 @@
             These changes help reduce confusion from multiple sources of truth. Going forward, <strong>presets</strong>
             and the <strong>Command options for yt-dlp</strong> will be the single source of truth.
           </p>
+          <p>
+            Notable changes in <strong>v0.10.x</strong>:
+          </p>
+          <ul>
+            <li>
+              The file browser feature is going to be enabled by default. and the associated ENV
+              <strong>YTP_BROWSER_ENABLED</strong> will be removed, <strong>YTP_BROWSER_CONTROL_ENABLED</strong> will
+              remain and
+              will default to <strong>false</strong>.
+            </li>
+            <li>
+              The <strong>Basic mode</strong> (which limited the interface to just the new download form) along it's
+              associated ENV <strong>YTP_BASIC_MODE</strong> is being removed. Everything except what is available
+              behind configurable flag will become part of the standard interface.
+            </li>
+          </ul>
         </DeprecatedNotice>
       </div>
     </div>

ui/app/stores/ConfigStore.ts

@@ -6,7 +6,6 @@ export const useConfigStore = defineStore('config', () => {
     showForm: useStorage('showForm', false),
     app: {
       download_path: '/downloads',
-      keep_archive: false,
       remove_files: false,
       ui_update_title: true,
       output_template: '',
@@ -15,7 +14,6 @@ export const useConfigStore = defineStore('config', () => {
       basic_mode: true,
       default_preset: 'default',
       instance_title: null,
-      sentry_dsn: null,
       console_enabled: false,
       browser_enabled: false,
       browser_control_enabled: false,
@@ -27,6 +25,7 @@ export const useConfigStore = defineStore('config', () => {
       app_build_date: '',
       app_branch: '',
       started: 0,
+      app_env: 'production',
     },
     presets: [
       {

ui/app/types/config.d.ts

@@ -4,8 +4,6 @@ import type { DLField } from "./dl_fields"
 type AppConfig = {
   /** Path where downloaded files will be saved */
   download_path: string
-  /** Indicates if the app should keep an archive of downloaded files */
-  keep_archive: boolean
   /** Indicates if files should be removed after download */
   remove_files: boolean
   /** Indicates if the UI should update the title with the current download status */
@@ -22,8 +20,6 @@ type AppConfig = {
   default_preset: string
   /** Instance title for the app, null if not set */
   instance_title: string | null
-  /** Sentry DSN for error tracking, null if not configured */
-  sentry_dsn: string | null
   /** Indicates if the console is enabled */
   console_enabled: boolean
   /** Indicates if the file browser is enabled */
@@ -47,7 +43,7 @@ type AppConfig = {
   /** When the app started */
   started: number,
   /** Application environment, e.g. "production", "development" */
-  app_env: string
+  app_env: "production" | "development"
 }
 
 type Preset = {

ui/nuxt.config.ts

@@ -30,8 +30,7 @@ export default defineNuxtConfig({
   runtimeConfig: {
     public: {
       APP_ENV: process.env.NODE_ENV,
-      wss: process.env.NUXT_PUBLIC_WSS ?? '',
-      sentry: process.env.NUXT_PUBLIC_SENTRY_DSN ?? '',
+      wss: process.env.NUXT_PUBLIC_WSS ?? ''
     }
   },
   build: {
@@ -61,7 +60,6 @@ export default defineNuxtConfig({
     '@pinia/nuxt',
     '@vueuse/nuxt',
     'floating-vue/nuxt',
-    '@sentry/nuxt/module',
   ],
 
   nitro: {

ui/package.json

@@ -12,7 +12,6 @@
   "web-types": "./web-types.json",
   "dependencies": {
     "@pinia/nuxt": "^0.11.2",
-    "@sentry/nuxt": "^10.8.0",
     "@vueuse/core": "^13.9.0",
     "@vueuse/nuxt": "^13.9.0",
     "@xterm/addon-fit": "^0.10.0",
@@ -22,10 +21,10 @@
     "floating-vue": "^5.2.2",
     "hls.js": "^1.6.11",
     "moment": "^2.30.1",
-    "nuxt": "^4.0.3",
+    "nuxt": "4.0.3",
     "pinia": "^3.0.3",
     "socket.io-client": "^4.8.1",
-    "vue": "^3.5.20",
+    "vue": "^3.5.21",
     "vue-router": "^4.5.1",
     "vue-toastification": "2.0.0-rc.5"
   },