Refactored HandleTask to support a generic extractor and made it more testable

2025-09-21 16:33:40 +03:00 · 2025-09-21 16:33:40 +03:00 · 7dc5e55eb8
commit 7dc5e55eb8
parent 4986aea300
16 changed files with 3011 additions and 250 deletions
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -191,5 +191,13 @@
    "app/tests"
  ],
  "python.testing.unittestEnabled": true,
-  "python.testing.pytestEnabled": true
+  "python.testing.pytestEnabled": true,
+  "json.schemas": [
+    {
+      "fileMatch": [
+        "**/var/config/tasks/*.json"
+      ],
+      "url": "./app/library/task_handlers/task_definition.schema.json"
+    }
+  ]
 }
--- a/FAQ.md
+++ b/FAQ.md
@ -145,6 +145,125 @@ YTP_YTDLP_VERSION=2025.07.21 or master or nightly

 Then restart the container to apply the changes.

+# How can I monitor sites without RSS feeds?
+
+YTPTube includes a **generic task handler** that turns JSON definition files into site-specific scrapers. You can use it
+to watch pages that do not expose RSS or public APIs and automatically enqueue new links into the download queue.
+
+1. Create definition files under `/config/tasks/*.json` (for Docker this is the mounted `config/tasks/` folder).
+2. Keep your scheduled task in `tasks.json` pointing at the page you want to monitor and make sure it uses a preset that
+   enables a download archive (`--download-archive`).
+3. When the task runs, the handler scans the JSON files, picks the first definition whose `match` rule covers the task
+   URL, fetches the page, extracts items, and queues the unseen ones.
+
+### Definition schema
+
+Each file must contain a single JSON object with the following keys:
+
+```json5
+{
+  "name": "example",                  // Friendly identifier shown in logs
+  "match": [
+    "https://example.com/articles/*", // Glob strings, or objects with {"regex": "..."} or {"glob": "..."}
+    { "regex": "https://example.com/post/[0-9]+" }
+  ],
+  "engine": {                         // Optional, defaults to HTTPX
+    "type": "httpx",                  // "httpx" (default) or "selenium"
+    "options": {
+      "url": "http://selenium:4444/wd/hub", // Selenium-only: remote hub URL
+      "arguments": ["--headless", "--disable-gpu"],
+      "wait_for": { "type": "css", "expression": ".article" },
+      "wait_timeout": 15,
+      "page_load_timeout": 60
+    }
+  },
+  "request": {                      // Optional HTTP settings
+    "method": "GET",                // GET or POST
+    "url": "https://example.com/articles/latest", // Override the task URL if needed
+    "headers": { "User-Agent": "MyAgent/1.0" },
+    "params": { "page": 1 },
+    "data": null,
+    "json": null,
+    "timeout": 30
+  },
+  "response": {                      // Optional: how to interpret the body
+    "type": "html"                   // "html" (default) or "json"
+  },
+  "parse": {
+    "items": {                        // Optional container for per-item extraction
+      "selector": ".columns .card",   // Defaults to CSS; set "type": "xpath" to use XPath
+      "fields": {
+        "link": {                     // Required inside fields: the per-item URL
+          "type": "css",
+          "expression": ".card-header a",
+          "attribute": "href"
+        },
+        "title": {
+          "type": "css",
+          "expression": ".card-header a",
+          "attribute": "text"
+        },
+        "poet": {
+          "type": "css",
+          "expression": "footer .card-footer-item:first-child a",
+          "attribute": "text"
+        }
+      }
+    },
+    "page_title": {                    // Optional global field outside the container
+      "type": "css",
+      "expression": "title",
+      "attribute": "text"
+    }
+  }
+}
+```
+
+For JSON endpoints, switch the response format and use `jsonpath` selectors:
+
+```json5
+{
+  "response": { "type": "json" },
+  "parse": {
+    "items": {
+      "type": "jsonpath",
+      "selector": "items",
+      "fields": {
+        "link": { "type": "jsonpath", "expression": "url" },
+        "title": { "type": "jsonpath", "expression": "title" }
+      }
+    }
+  }
+}
+```
+
+### Parsing rules
+
+- Every definition must provide a `link` field either at the top level or inside `parse.items.fields`. Other fields are optional metadata attached to the queued item.
+- CSS and XPath rules may specify `attribute`:
+  - `text` / `inner_text` applies `normalize-space()`.
+  - `html` / `outer_html` returns the raw HTML fragment.
+  - Any other value reads that attribute from the element. When omitted, the handler uses text and, for `link`, falls
+    back to `href` automatically.
+- Regex rules scan the HTML fragment associated with the current scope (page-level or container). Set `attribute` to a named/numbered capture group or rely on the first group.
+- `post_filter` lets you run a final regex on the extracted value and pick a named (`value`) group.
+- When you declare `parse.items`, each matching container is processed independently so missing fields in one card do not shift values for the rest.
+- For JSON responses (`response.type = "json"`), set the container `type` and field `type` to `jsonpath` and supply [JMESPath](https://jmespath.org/) expressions. Relative values are resolved against each container object and converted to strings automatically.
+
+### Fetch engines
+
+- **httpx (default)**: supports custom headers, query params, JSON/body payloads, proxy inherited from the task preset,
+  and optional timeout.
+- **selenium**: uses a remote Chrome session. Provide the hub URL under `engine.options.url`; only Chrome is supported at
+  the moment. Optional keys: `arguments` (list or string), `wait_for` (type `css`/`xpath` + `expression`), `wait_timeout`,
+  and `page_load_timeout`.
+
+Definitions are reloaded automatically when files change, so you can tweak them without restarting YTPTube. Check
+`var/config/tasks/01-*.json` for sample files.
+
+> [!NOTE]
+> A machine-readable schema is available at `app/library/task_handlers/task_definition.schema.json` if you want to validate your JSON with editors or CI tools.
+
 # How to generate POT tokens?

 You need a pot provider server we already have the extractor `bgutil-ytdlp-pot-provider` pre-installed in the container.
--- a/README.md
+++ b/README.md
@ -16,6 +16,7 @@ includes features like scheduling downloads, sending notifications, and built-in
 * Random beautiful background.
 * Handles live and upcoming streams.
 * Schedule channels or playlists to be downloaded automatically.
+* Create your own custom task handler feeds for downloads, See [Feeds documentation](FAQ.md#how-can-i-monitor-sites-without-rss-feeds).
 * Send notification to targets based on selected events. includes [Apprise](https://github.com/caronc/apprise?tab=readme-ov-file#readme) support.
 * Support per link options.
 * Support for limits per extractor and overall global limit.
--- a/app/library/Tasks.py
+++ b/app/library/Tasks.py
@ -4,7 +4,8 @@ import json
 import logging
 import pkgutil
 import time
-from dataclasses import dataclass
+import uuid
+from dataclasses import asdict, dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@ -15,11 +16,11 @@ from .config import Config
 from .DownloadQueue import DownloadQueue
 from .encoder import Encoder
 from .Events import EventBus, Events
-from .ItemDTO import Item
+from .ItemDTO import Item, ItemDTO
 from .Scheduler import Scheduler
 from .Services import Services
 from .Singleton import Singleton
-from .Utils import archive_add, archive_delete, extract_info, init_class, validate_url
+from .Utils import archive_add, archive_delete, archive_read, extract_info, init_class, validate_url
 from .YTDLPOpts import YTDLPOpts

 LOG: logging.Logger = logging.getLogger("tasks")
@ -127,6 +128,95 @@ class Task:
        return {"file": archive_file, "items": items}


+def _split_inspect_metadata(metadata: dict[str, Any] | None) -> tuple[dict[str, Any], dict[str, Any]]:
+    """
+    Split commonly consumed metadata keys from the rest.
+
+    Args:
+        metadata (dict[str, Any]|None): The metadata to split.
+
+    Returns:
+        tuple[dict[str, Any], dict[str, Any]]: The primary and extra metadata.
+
+    """
+    metadata = dict(metadata or {})
+    primary: dict[str, Any] = {}
+
+    for key in ("matched", "handler", "supported"):
+        if key in metadata:
+            primary[key] = metadata.pop(key)
+
+    return primary, metadata
+
+
+@dataclass(slots=True)
+class TaskItem:
+    url: str
+    "The URL of the item."
+    title: str | None = None
+    "The title of the item."
+    archive_id: str | None = None
+    "The archive ID of the item."
+    metadata: dict[str, Any] = field(default_factory=dict)
+    "Additional metadata related to the item."
+
+
+@dataclass(slots=True)
+class TaskResult:
+    items: list[TaskItem] = field(default_factory=list)
+    "The list of items."
+    metadata: dict[str, Any] = field(default_factory=dict)
+    "Additional metadata related to the result."
+
+    def serialize(self) -> dict[str, Any]:
+        """
+        Serialize the task result.
+
+        Returns:
+            dict[str, Any]: The serialized task result.
+
+        """
+        primary, extra = _split_inspect_metadata(self.metadata)
+        payload: dict[str, Any] = {**primary, "items": [asdict(item) for item in self.items]}
+
+        if extra:
+            payload["metadata"] = extra
+
+        return payload
+
+
+@dataclass(slots=True)
+class TaskFailure:
+    message: str
+    "A human-readable message describing the failure."
+    error: str | None = None
+    "An optional error code or string."
+    metadata: dict[str, Any] = field(default_factory=dict)
+    "Additional metadata related to the failure."
+
+    def serialize(self) -> dict[str, Any]:
+        """
+        Serialize the task failure.
+
+        Returns:
+            dict[str, Any]: The serialized task failure.
+
+        """
+        primary, extra = _split_inspect_metadata(self.metadata)
+        payload: dict[str, Any] = dict(primary)
+
+        if self.error:
+            payload["error"] = self.error
+
+        if self.message and (not self.error or self.message != self.error):
+            payload["message"] = self.message
+
+        if extra:
+            payload["metadata"] = extra
+
+        return payload
+
+
 class Tasks(metaclass=Singleton):
    """
    This class is used to manage the tasks.
@ -199,6 +289,7 @@ class Tasks(metaclass=Singleton):

        """
        self.load()
+        Services.get_instance().add("tasks", self)

        async def event_handler(data, _):
            if data and data.data:
@ -211,6 +302,10 @@ class Tasks(metaclass=Singleton):
        """Return the tasks."""
        return self._tasks

+    def get_handler(self) -> "HandleTask":
+        """Expose the handle task helper."""
+        return self._task_handler
+
    def get(self, task_id: str) -> Task | None:
        """
        Get a task by its ID.
@ -471,8 +566,21 @@ class HandleTask:
        "The configuration."
        self._task_name: str = f"{__class__.__name__}._dispatcher"
        "The task name for the scheduler."
+        self._queued: dict[str, set[str]] = {}
+        "Queued archive IDs per handler."
+        self._failure_count: dict[str, dict[str, int]] = {}
+        "Failure counts per handler and archive ID."
+
+        EventBus.get_instance().subscribe(
+            Events.ITEM_ERROR,
+            self._handle_item_error,
+            f"{__class__.__name__}.item_error",
+        )

    def load(self) -> None:
+        """
+        Load the available handlers and schedule the dispatcher.
+        """
        self._handlers: list[type] = self._discover()

        timer: str = self._config.tasks_handler_timer
@ -551,7 +659,7 @@ class HandleTask:

        return None

-    async def dispatch(self, task: Task, handler: type | None = None, **kwargs) -> Any | None:
+    async def dispatch(self, task: Task, handler: type | None = None, **kwargs) -> TaskResult | TaskFailure | None:  # noqa: ARG002
        """
        Dispatch a task to the appropriate handler.

@ -561,7 +669,7 @@ class HandleTask:
            **kwargs: Additional context to pass to the handler.

        Returns:
-            Any|None: The result of the handler's execution, or None if no handler found.
+            TaskResult|TaskFailure|None: The extraction outcome, or None if no handler matched.

        """
        if not handler:
@ -569,12 +677,225 @@ class HandleTask:
            if handler is None:
                return None

+        services: Services = Services.get_instance()
+
        try:
-            return await Services.get_instance().handle_async(handler=handler.handle, task=task, **kwargs)
-        except Exception as e:
-            LOG.exception(e)
+            extraction: TaskResult | TaskFailure = await services.handle_async(
+                handler=handler.extract, task=task, config=self._config
+            )
+        except NotImplementedError:
+            LOG.error(f"Handler '{handler.__name__}' does not implement extract().")
+            return TaskFailure(message="Handler does not support extraction.")
+        except Exception as exc:
+            LOG.exception(exc)
            raise

+        if isinstance(extraction, TaskFailure):
+            LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}")
+            return extraction
+
+        if not isinstance(extraction, TaskResult):
+            LOG.error(
+                f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.",
+            )
+            return TaskFailure(
+                message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__}
+            )
+
+        raw_items: list[TaskItem] = extraction.items or []
+        metadata: dict[str, Any] = extraction.metadata or {}
+
+        handler_name: str = handler.__name__
+        queued: set[str] = self._queued.setdefault(handler_name, set())
+        failures: dict[str, int] = self._failure_count.setdefault(handler_name, {})
+
+        params: dict = task.get_ytdlp_opts().get_all()
+        archive_file: str | None = params.get("download_archive")
+
+        download_queue: DownloadQueue = services.get("queue") or DownloadQueue.get_instance()
+        notify: EventBus = services.get("notify") or EventBus.get_instance()
+
+        archive_ids: list[str] = [
+            item.archive_id for item in raw_items if isinstance(item, TaskItem) and item.archive_id
+        ]
+        downloaded: list[str] = archive_read(archive_file, archive_ids) if archive_file else []
+
+        filtered: list[TaskItem] = []
+
+        for item in raw_items:
+            if not isinstance(item, TaskItem):
+                LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}")
+                continue
+
+            url: str = item.url
+            if not url:
+                continue
+
+            archive_id: str | None = item.archive_id
+            if not archive_id:
+                LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.")
+                continue
+
+            if archive_id in queued:
+                continue
+
+            queued.add(archive_id)
+
+            if archive_file and archive_id in downloaded:
+                continue
+
+            if download_queue.queue.exists(url=url):
+                continue
+
+            try:
+                done = download_queue.done.get(url=url)
+                if "error" != done.info.status:
+                    continue
+            except KeyError:
+                pass
+
+            if archive_id not in failures:
+                failures[archive_id] = 0
+
+            filtered.append(item)
+
+        if not filtered:
+            if raw_items:
+                LOG.debug(
+                    f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering."
+                )
+            return TaskResult(items=[], metadata=metadata)
+
+        LOG.info(
+            f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})."
+        )
+
+        base_item = Item.format(
+            {
+                "url": task.url,
+                "preset": task.preset or self._config.default_preset,
+                "folder": task.folder or "",
+                "template": task.template or "",
+                "cli": task.cli or "",
+                "auto_start": task.auto_start,
+                "extras": {"source_task": task.id, "source_handler": handler.__name__},
+            }
+        )
+
+        for item in filtered:
+            metadata_entry: dict[str, Any] = item.metadata if isinstance(item.metadata, dict) else {}
+            extras: dict[str, Any] = base_item.extras.copy()
+            if metadata_entry:
+                extras["metadata"] = metadata_entry
+
+            notify.emit(
+                Events.ADD_URL,
+                data=base_item.new_with(url=item.url, extras=extras).serialize(),
+            )
+
+        return TaskResult(items=filtered, metadata=metadata)
+
+    async def inspect(
+        self,
+        url: str,
+        preset: str | None = None,
+        handler_name: str | None = None,
+    ) -> TaskResult | TaskFailure:
+        if not self._handlers:
+            self._handlers = self._discover()
+
+        task = Task(
+            id=str(uuid.uuid4()),
+            name="Inspector",
+            url=url,
+            preset=preset or self._config.default_preset,
+            auto_start=False,
+        )
+
+        services = Services.get_instance()
+
+        handler_cls: type | None
+        if handler_name:
+            handler_cls = next((cls for cls in self._handlers if cls.__name__.lower() == handler_name.lower()), None)
+            if handler_cls is None:
+                message: str = f"Handler '{handler_name}' not found."
+                return TaskFailure(
+                    message=message,
+                    error=message,
+                    metadata={"matched": False, "handler": handler_name},
+                )
+
+            try:
+                matched = services.handle_sync(handler=handler_cls.can_handle, task=task)
+            except Exception as exc:  # pragma: no cover - defensive
+                LOG.exception(exc)
+                message = str(exc)
+                return TaskFailure(
+                    message=message,
+                    error=message,
+                    metadata={"matched": False, "handler": handler_cls.__name__},
+                )
+
+            if not matched:
+                return TaskFailure(
+                    message="Handler cannot process the supplied URL.",
+                    metadata={"matched": False, "handler": handler_cls.__name__},
+                )
+        else:
+            handler_cls = self._find_handler(task)
+            if handler_cls is None:
+                message = "No handler matched the supplied URL."
+                return TaskFailure(
+                    message=message,
+                    error=message,
+                    metadata={"matched": False, "handler": None},
+                )
+
+        base_metadata: dict[str, Any] = {"matched": True, "handler": handler_cls.__name__}
+
+        try:
+            extraction: TaskResult | TaskFailure = await services.handle_async(
+                handler=handler_cls.extract, task=task, config=self._config
+            )
+        except NotImplementedError:
+            return TaskFailure(
+                message="Handler does not support manual inspection.",
+                metadata={**base_metadata, "supported": False},
+            )
+        except Exception as exc:
+            LOG.exception(exc)
+            message = str(exc)
+            return TaskFailure(
+                message=message,
+                error=message,
+                metadata={**base_metadata, "supported": True},
+            )
+
+        if isinstance(extraction, TaskFailure):
+            combined_failure_metadata: dict[str, Any] = {**base_metadata, "supported": True}
+            if extraction.metadata:
+                combined_failure_metadata.update(extraction.metadata)
+
+            failure_error = extraction.error if extraction.error else extraction.message
+
+            return TaskFailure(
+                message=extraction.message,
+                error=failure_error,
+                metadata=combined_failure_metadata,
+            )
+
+        if not isinstance(extraction, TaskResult):
+            LOG.error(
+                f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.",
+            )
+            extraction = TaskResult()
+
+        combined_metadata: dict[str, Any] = {**base_metadata, "supported": True}
+        if extraction.metadata:
+            combined_metadata.update(extraction.metadata)
+
+        return TaskResult(items=list(extraction.items), metadata=combined_metadata)
+
    def _discover(self) -> list[type]:
        import importlib

@ -591,7 +912,28 @@ class HandleTask:
                if cls.__module__ != module.__name__:
                    continue

-                if callable(getattr(cls, "can_handle", None)) and callable(getattr(cls, "handle", None)):
+                if callable(getattr(cls, "can_handle", None)) and callable(getattr(cls, "extract", None)):
                    handlers.append(cls)

        return handlers
+
+    async def _handle_item_error(self, event, _name, **_kwargs):
+        item = getattr(event, "data", None)
+        if not isinstance(item, ItemDTO):
+            return
+
+        extras = getattr(item, "extras", {}) or {}
+        handler_name = extras.get("source_handler")
+        if not handler_name:
+            return
+
+        archive_id = item.archive_id
+        if not archive_id:
+            return
+
+        queued = self._queued.get(handler_name)
+        if queued:
+            queued.discard(archive_id)
+
+        failures = self._failure_count.setdefault(handler_name, {})
+        failures[archive_id] = failures.get(archive_id, 0) + 1
--- a/app/library/task_handlers/_base_handler.py
+++ b/app/library/task_handlers/_base_handler.py
@ -1,73 +1,30 @@
 # flake8: noqa: ARG004
-import logging
 from typing import Any

 import httpx
 from yt_dlp.utils.networking import random_user_agent

 from app.library.config import Config
-from app.library.DownloadQueue import DownloadQueue
-from app.library.Events import EventBus, Events
-from app.library.ItemDTO import ItemDTO
-from app.library.Tasks import Task
-
-LOG: logging.Logger = logging.getLogger(__name__)
+from app.library.Tasks import Task, TaskFailure, TaskResult


 class BaseHandler:
-    queued: set[str] = set()
-    failure_count: dict[str, int] = {}
-
-    def __init_subclass__(cls, **kwargs):
-        """Ensure each subclass has its own state containers."""
-        super().__init_subclass__(**kwargs)
-        if "queued" not in cls.__dict__:
-            cls.queued = set()
-        if "failure_count" not in cls.__dict__:
-            cls.failure_count = {}
-
-        async def event_handler(data, _):
-            if data and data.data:
-                await cls.on_error(data.data)
-
-        EventBus.get_instance().subscribe(Events.ITEM_ERROR, event_handler, f"{cls.__name__}.on_error")
-
    @staticmethod
    def can_handle(task: Task) -> bool:
        return False

    @staticmethod
-    async def handle(task: Task, notify: EventBus, config: Config, queue: DownloadQueue):
-        pass
+    async def extract(task: Task, config: Config | None = None) -> TaskResult | TaskFailure:
+        raise NotImplementedError
+
+    @classmethod
+    async def inspect(cls, task: Task, config: Config | None = None) -> TaskResult | TaskFailure:
+        return await cls.extract(task=task, config=config)

    @staticmethod
    def parse(url: str) -> Any | None:
        return None

-    @classmethod
-    async def on_error(cls, item: ItemDTO) -> None:
-        """
-        Handle errors by logging them and removing the queued ID if it exists.
-
-        Args:
-            item (ItemDTO): The error data containing the URL and other information.
-
-        """
-        if not item or not isinstance(item, ItemDTO):
-            return
-
-        if not item.archive_id or not cls.failure_count.get(item.archive_id, None):
-            LOG.debug(f"Item '{item.name()}' not queued by the handler.")
-            return
-
-        failCount: int = int(cls.failure_count.get(item.archive_id, 0))
-
-        LOG.info(f"Removing '{item.name()}' from queued IDs due to error. Failure count: '{failCount + 1}'.")
-        if item.archive_id in cls.queued:
-            cls.queued.remove(item.archive_id)
-
-        cls.failure_count[item.archive_id] = 1 + failCount
-
    @staticmethod
    def tests() -> list[tuple[str, bool]]:
        return []
--- a/app/library/task_handlers/generic.py
+++ b/app/library/task_handlers/generic.py
--- a/app/library/task_handlers/task_definition.schema.json
+++ b/app/library/task_handlers/task_definition.schema.json
@ -0,0 +1,456 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://ytptube.app/schemas/task-definition.json",
+  "title": "YTPTube Generic Task Definition",
+  "type": "object",
+  "required": [
+    "name",
+    "match",
+    "parse"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Human-readable identifier for this definition."
+    },
+    "match": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "oneOf": [
+          {
+            "type": "string",
+            "minLength": 1,
+            "description": "Glob pattern matched against task URLs."
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "regex": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Regular expression applied to task URLs."
+              },
+              "glob": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Glob pattern applied to task URLs."
+              }
+            },
+            "anyOf": [
+              {
+                "required": [
+                  "regex"
+                ]
+              },
+              {
+                "required": [
+                  "glob"
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      "description": "Patterns that determine which tasks use this definition."
+    },
+    "engine": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "httpx",
+            "selenium"
+          ],
+          "default": "httpx",
+          "description": "Fetch engine to use (HTTPX or Selenium)."
+        },
+        "options": {
+          "type": "object",
+          "description": "Engine-specific configuration.",
+          "additionalProperties": true,
+          "properties": {
+            "url": {
+              "type": "string",
+              "format": "uri",
+              "description": "Remote Selenium hub URL."
+            },
+            "browser": {
+              "type": "string",
+              "enum": [
+                "chrome"
+              ],
+              "description": "Selenium browser name (currently only chrome)."
+            },
+            "arguments": {
+              "oneOf": [
+                {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                {
+                  "type": "string"
+                }
+              ],
+              "description": "Browser launch arguments for Selenium."
+            },
+            "wait_for": {
+              "$ref": "#/definitions/waitFor"
+            },
+            "wait_timeout": {
+              "type": "number",
+              "minimum": 0,
+              "description": "Seconds to wait for the wait_for selector."
+            },
+            "page_load_timeout": {
+              "type": "number",
+              "minimum": 0,
+              "description": "Seconds to allow page load to complete."
+            }
+          }
+        }
+      },
+      "description": "Optional engine configuration (defaults to HTTPX)."
+    },
+    "request": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "method": {
+          "type": "string",
+          "enum": [
+            "GET",
+            "POST"
+          ],
+          "default": "GET",
+          "description": "HTTP method to use when fetching the page."
+        },
+        "url": {
+          "type": "string",
+          "format": "uri",
+          "description": "Override URL to fetch instead of the task URL."
+        },
+        "headers": {
+          "$ref": "#/definitions/stringMap",
+          "description": "Additional request headers."
+        },
+        "params": {
+          "$ref": "#/definitions/stringMap",
+          "description": "Query string parameters."
+        },
+        "data": {
+          "description": "Form payload for POST requests.",
+          "oneOf": [
+            {
+              "$ref": "#/definitions/stringMap"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "json": {
+          "description": "JSON payload for POST requests.",
+          "type": [
+            "object",
+            "array",
+            "string",
+            "number",
+            "boolean",
+            "null"
+          ]
+        },
+        "timeout": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Timeout in seconds for the request."
+        }
+      },
+      "description": "Optional HTTP request overrides."
+    },
+    "response": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "html",
+            "json"
+          ],
+          "default": "html",
+          "description": "Body format returned by the target URL."
+        }
+      }
+    },
+    "parse": {
+      "type": "object",
+      "minProperties": 1,
+      "description": "Field extraction rules and optional container definition.",
+      "additionalProperties": false,
+      "properties": {
+        "items": {
+          "$ref": "#/definitions/container"
+        }
+      },
+      "patternProperties": {
+        "^(?!items$).+$": {
+          "$ref": "#/definitions/extractionRule"
+        }
+      },
+      "allOf": [
+        {
+          "if": {
+            "not": {
+              "required": [
+                "items"
+              ]
+            }
+          },
+          "then": {
+            "required": [
+              "link"
+            ]
+          }
+        }
+      ]
+    }
+  },
+  "definitions": {
+    "stringMap": {
+      "type": "object",
+      "additionalProperties": {
+        "type": [
+          "string",
+          "number",
+          "boolean",
+          "null"
+        ]
+      }
+    },
+    "waitFor": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "css",
+            "xpath"
+          ],
+          "description": "Selector type to wait for."
+        },
+        "expression": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Selector expression."
+        },
+        "value": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Alias for expression for backwards compatibility."
+        }
+      },
+      "anyOf": [
+        {
+          "required": [
+            "expression"
+          ]
+        },
+        {
+          "required": [
+            "value"
+          ]
+        }
+      ]
+    },
+    "postFilter": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "filter"
+      ],
+      "properties": {
+        "filter": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Regular expression applied after extraction."
+        },
+        "value": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Named or numbered capture group to return."
+        }
+      }
+    },
+    "containerFields": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": false,
+      "patternProperties": {
+        "^.+$": {
+          "$ref": "#/definitions/extractionRule"
+        }
+      },
+      "required": [
+        "link"
+      ]
+    },
+    "container": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "css",
+            "xpath",
+            "jsonpath"
+          ],
+          "default": "css",
+          "description": "Selector type to use for locating repeatable elements."
+        },
+        "selector": {
+          "type": "string",
+          "minLength": 1,
+          "description": "CSS/XPath selector identifying each item container."
+        },
+        "expression": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Alias for selector."
+        },
+        "fields": {
+          "$ref": "#/definitions/containerFields"
+        }
+      },
+      "required": [
+        "fields"
+      ],
+      "anyOf": [
+        {
+          "required": [
+            "selector"
+          ]
+        },
+        {
+          "required": [
+            "expression"
+          ]
+        }
+      ]
+    },
+    "extractionRule": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "type",
+        "expression"
+      ],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "css",
+            "xpath",
+            "regex",
+            "jsonpath"
+          ],
+          "description": "Extraction strategy."
+        },
+        "expression": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Selector or pattern used to extract values."
+        },
+        "attribute": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Optional attribute or group to return."
+        },
+        "post_filter": {
+          "$ref": "#/definitions/postFilter"
+        }
+      }
+    }
+  },
+  "examples": [
+    {
+      "name": "example",
+      "match": [
+        "https://example.com/articles/*",
+        {
+          "regex": "https://example.com/post/[0-9]+"
+        }
+      ],
+      "engine": {
+        "type": "httpx"
+      },
+      "request": {
+        "method": "GET",
+        "headers": {
+          "User-Agent": "MyCustomAgent/1.0"
+        }
+      },
+      "parse": {
+        "link": {
+          "type": "css",
+          "expression": ".article a.link",
+          "attribute": "href"
+        },
+        "title": {
+          "type": "css",
+          "expression": ".article .title",
+          "attribute": "text"
+        },
+        "id": {
+          "type": "regex",
+          "expression": "id=(?P<id>[0-9]+)",
+          "post_filter": {
+            "filter": "(?P<id>[0-9]+)",
+            "value": "id"
+          }
+        }
+      }
+    },
+    {
+      "name": "container-example",
+      "match": [
+        "https://example.com/list"
+      ],
+      "parse": {
+        "items": {
+          "type": "css",
+          "selector": ".cards .card",
+          "fields": {
+            "link": {
+              "type": "css",
+              "expression": ".card-header a",
+              "attribute": "href"
+            },
+            "title": {
+              "type": "css",
+              "expression": ".card-header a",
+              "attribute": "text"
+            },
+            "poet": {
+              "type": "css",
+              "expression": "footer .card-footer-item:first-child a",
+              "attribute": "text"
+            }
+          }
+        }
+      }
+    }
+  ]
+}
--- a/app/library/task_handlers/twitch.py
+++ b/app/library/task_handlers/twitch.py
@ -3,19 +3,14 @@ import re
 from typing import TYPE_CHECKING
 from xml.etree.ElementTree import Element

-from app.library.DownloadQueue import DownloadQueue
-from app.library.Events import EventBus, Events
-from app.library.ItemDTO import Item
-from app.library.Tasks import Task
-from app.library.Utils import archive_read, get_archive_id
+from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
+from app.library.Utils import get_archive_id

 from ._base_handler import BaseHandler

 if TYPE_CHECKING:
    from xml.etree.ElementTree import Element

-    from app.library.Download import Download
-
 LOG: logging.Logger = logging.getLogger(__name__)


@ -30,40 +25,23 @@ class TwitchHandler(BaseHandler):
        return TwitchHandler.parse(task.url) is not None

    @staticmethod
-    async def handle(task: Task, notify: EventBus, queue: DownloadQueue):
-        """
-        Fetch the RSS feed for a Twitch channel VODs, parse entries,
-        and enqueue new items that are not in the archive/queue already.
-
-        Args:
-            task (Task): The task containing the Twitch channel URL.
-            notify (EventBus): The event bus for notifications.
-            queue (DownloadQueue): The download queue instance.
-
-        """
+    async def _collect_feed(
+        task: Task,
+        params: dict,
+        handle_name: str,
+    ) -> tuple[str, list[dict[str, str]], bool]:
        from defusedxml.ElementTree import fromstring

-        handleName: str | None = TwitchHandler.parse(task.url)
-        if not handleName:
-            LOG.error(f"Cannot parse '{task.name}' URL: {task.url}")
-            return
-
-        params: dict = task.get_ytdlp_opts().get_all()
-        archive_file: str | None = params.get("download_archive")
-        if not archive_file:
-            LOG.error(f"Task '{task.name}' does not have an archive file.")
-            return
-
-        feed_url: str = TwitchHandler.FEED.format(handle=handleName)
+        feed_url: str = TwitchHandler.FEED.format(handle=handle_name)

        LOG.debug(f"Fetching '{task.name}' feed.")
        response = await TwitchHandler.request(url=feed_url, ytdlp_opts=params)
        response.raise_for_status()

-        items: list = []
+        root: Element[str] = fromstring(response.text)
+        items: list[dict[str, str]] = []
        has_items = False

-        root: Element[str] = fromstring(response.text)
        for entry in root.findall("channel/item"):
            link_elem: Element[str] | None = entry.find("link")
            url: str = link_elem.text.strip() if link_elem is not None and link_elem.text else ""
@ -71,12 +49,14 @@ class TwitchHandler(BaseHandler):
                LOG.warning(f"Entry in '{task.name}' feed is missing URL. Skipping entry.")
                continue

-            m: re.Match[str] | None = re.search(r"^https?://(?:www\.)?twitch\.tv/videos/(?P<id>\d+)(?:[/?].*)?$", url)
-            if not m:
+            match: re.Match[str] | None = re.search(
+                r"^https?://(?:www\.)?twitch\.tv/videos/(?P<id>\d+)(?:[/?].*)?$", url
+            )
+            if not match:
                LOG.warning(f"URL in '{task.name}' feed does not look like a VOD link: {url}")
                continue

-            vid: str = m.group("id")
+            vid: str = match.group("id")

            title_elem: Element[str] | None = entry.find("title")
            title: str = title_elem.text.strip() if title_elem is not None and title_elem.text else ""
@ -89,68 +69,34 @@ class TwitchHandler(BaseHandler):
                LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
                continue

-            if archive_id in TwitchHandler.queued:
-                continue
-
            items.append({"id": vid, "url": url, "title": title, "archive_id": archive_id})

-        if len(items) < 1:
-            if not has_items:
-                LOG.warning(f"No entries found in '{task.name}' feed. URL: {feed_url}")
-            else:
-                LOG.debug(f"No new items found in '{task.name}' feed.")
-            return
+        return feed_url, items, has_items

-        filtered: list = []
+    @staticmethod
+    async def extract(task: Task) -> TaskResult | TaskFailure:
+        handle_name: str | None = TwitchHandler.parse(task.url)
+        if not handle_name:
+            return TaskFailure(message="Unrecognized Twitch channel URL.")

-        downloaded: list[str] = archive_read(archive_file, [item["archive_id"] for item in items])
-
-        for item in items:
-            TwitchHandler.queued.add(item["archive_id"])
-
-            if item["archive_id"] in downloaded:
-                continue
-
-            if queue.queue.exists(url=item["url"]):
-                continue
-
-            try:
-                done: Download = queue.done.get(url=item["url"])
-                if "error" != done.info.status:
-                    continue
-            except KeyError:
-                pass
-
-            if item["archive_id"] not in TwitchHandler.failure_count:
-                TwitchHandler.failure_count[item["archive_id"]] = 0
-
-            filtered.append(item)
-
-        if len(filtered) < 1:
-            LOG.debug(f"No new items found in '{task.name}' feed.")
-            return
-
-        LOG.info(f"Found '{len(filtered)}' new items from '{task.name}' feed.")
-
-        rItem: Item = Item.format(
-            {
-                "url": feed_url,
-                "preset": task.preset,
-                "folder": task.folder if task.folder else "",
-                "template": task.template if task.template else "",
-                "cli": task.cli if task.cli else "",
-                "auto_start": task.auto_start,
-                "extras": {"source_task": task.id},
-            }
-        )
+        params: dict = task.get_ytdlp_opts().get_all()

        try:
-            for item in filtered:
-                notify.emit(Events.ADD_URL, data=rItem.new_with(url=item["url"]).serialize())
-        except Exception as e:
-            LOG.exception(e)
-            LOG.error(f"Error while adding items from '{task.name}'. {e!s}")
-            return
+            feed_url, items, has_items = await TwitchHandler._collect_feed(task, params, handle_name)
+        except Exception as exc:
+            LOG.exception(exc)
+            return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
+
+        task_items: list[TaskItem] = []
+
+        for entry in items:
+            if not (url := entry.get("url")):
+                continue
+
+            archive_id: str = entry.get("archive_id")
+            task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
+
+        return TaskResult( items=task_items, metadata={ "feed_url": feed_url, "has_entries": has_items } )

    @staticmethod
    def parse(url: str) -> str | None:
--- a/app/library/task_handlers/youtube.py
+++ b/app/library/task_handlers/youtube.py
@ -1,21 +1,16 @@
 import logging
 import re
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 from xml.etree.ElementTree import Element

-from app.library.DownloadQueue import DownloadQueue
-from app.library.Events import EventBus, Events
-from app.library.ItemDTO import Item
-from app.library.Tasks import Task
-from app.library.Utils import archive_read, get_archive_id
+from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
+from app.library.Utils import get_archive_id

 from ._base_handler import BaseHandler

 if TYPE_CHECKING:
    from xml.etree.ElementTree import Element

-    from app.library.Download import Download
-
 LOG: logging.Logger = logging.getLogger(__name__)


@ -32,39 +27,28 @@ class YoutubeHandler(BaseHandler):

    @staticmethod
    def can_handle(task: Task) -> bool:
-        if not task.get_ytdlp_opts().get_all().get("download_archive"):
-            LOG.debug(f"'{task.name}': Task does not have an archive file configured.")
-            return False
-
        LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}")
        return YoutubeHandler.parse(task.url) is not None

    @staticmethod
-    async def handle(task: Task, notify: EventBus, queue: DownloadQueue):
+    async def _get(task: Task, params: dict, parsed: dict[str, str]) -> tuple[str, list[dict[str, str]], int]:
        """
-        Fetch the Atom feed for a YouTube channel or playlist, parse entries,
-        and return a list of videos with metadata.
+        Fetch the feed and return raw entries.

        Args:
            task (Task): The task containing the YouTube URL.
-            notify (EventBus): The event bus for notifications.
-            queue (DownloadQueue): The download queue instance.
+            params (dict): The ytdlp options.
+            parsed (dict): The parsed URL components.
+
+        Returns:
+            tuple[str, list[dict[str, str]], int]: The feed URL, list of

        """
        from defusedxml.ElementTree import fromstring

-        parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
-        if not parsed:
-            LOG.error(f"'{task.name}': Cannot parse task URL: {task.url}")
-            return
-
-        params: dict = task.get_ytdlp_opts().get_all()
-
        feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"])
        LOG.debug(f"'{task.name}': Fetching feed.")

-        items: list = []
-
        response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params)
        response.raise_for_status()

@ -74,10 +58,12 @@ class YoutubeHandler(BaseHandler):
            "yt": "http://www.youtube.com/xml/schemas/2015",
        }

-        real_count: int = 0
+        items: list[dict[str, str]] = []
+        real_count = 0
+
        for entry in root.findall("atom:entry", ns):
            vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
-            vid: str | None = vid_elem.text if vid_elem is not None else ""
+            vid: str = vid_elem.text if vid_elem is not None and vid_elem.text else ""
            if not vid:
                LOG.warning(f"'{task.name}': Entry in the feed is missing a video ID. Skipping.")
                continue
@ -91,73 +77,43 @@ class YoutubeHandler(BaseHandler):
                continue

            title_elem: Element[str] | None = entry.find("atom:title", ns)
-            title: str | None = title_elem.text if title_elem is not None else ""
+            title: str = title_elem.text if title_elem is not None and title_elem.text else ""

            pub_elem: Element[str] | None = entry.find("atom:published", ns)
-            published: str | None = pub_elem.text if pub_elem is not None else ""
-            real_count += 1
+            published: str = pub_elem.text if pub_elem is not None and pub_elem.text else ""

-            if archive_id in YoutubeHandler.queued:
-                continue
+            real_count += 1

            items.append({"id": vid, "url": url, "title": title, "published": published, "archive_id": archive_id})

-        if len(items) < 1:
-            if real_count < 1:
-                LOG.warning(f"'{task.name}': No entries found the RSS feed. URL: {feed_url}")
-            else:
-                LOG.debug(f"'{task.name}': Feed has '{real_count}' entries, all already downloaded/queued.")
-            return
+        return feed_url, items, real_count

-        filtered: list = []
+    @staticmethod
+    async def extract(task: Task) -> TaskResult | TaskFailure:
+        parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
+        if not parsed:
+            return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")

-        downloaded: list[str] = archive_read(params.get("download_archive"), [item["archive_id"] for item in items])
-
-        for item in items:
-            YoutubeHandler.queued.add(item["archive_id"])
-            if item["archive_id"] in downloaded:
-                continue
-
-            if queue.queue.exists(url=item["url"]):
-                continue
-
-            try:
-                done: Download = queue.done.get(url=item["url"])
-                if "error" != done.info.status:
-                    continue
-            except KeyError:
-                pass
-
-            if item["archive_id"] not in YoutubeHandler.failure_count:
-                YoutubeHandler.failure_count[item["archive_id"]] = 0
-
-            filtered.append(item)
-
-        if len(filtered) < 1:
-            LOG.debug(f"'{task.name}': Feed has '{real_count}' entries, all already downloaded/queued.")
-            return
-
-        LOG.info(f"'{task.name}': Found '{len(filtered)}/{real_count}' new items from feed.")
-
-        rItem: Item = Item.format(
-            {
-                "url": feed_url,
-                "preset": task.preset,
-                "folder": task.folder if task.folder else "",
-                "template": task.template if task.template else "",
-                "cli": task.cli if task.cli else "",
-                "auto_start": task.auto_start,
-                "extras": {"source_task": task.id},
-            }
-        )
+        params: dict = task.get_ytdlp_opts().get_all()

        try:
-            for item in filtered:
-                notify.emit(Events.ADD_URL, data=rItem.new_with(url=item["url"]).serialize())
-        except Exception as e:
-            LOG.exception(e)
-            LOG.error(f"'{task.name}': Error while adding items from task feed. {e!s}")
-            return
+            feed_url, items, real_count = await YoutubeHandler._get(task, params, parsed)
+        except Exception as exc:
+            LOG.exception(exc)
+            return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
+
+        task_items: list[TaskItem] = []
+
+        for entry in items:
+            if not (url := entry.get("url")):
+                continue
+
+            archive_id: str = entry.get("archive_id")
+            metadata: dict[str, Any] = {"published": entry.get("published")}
+
+            task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
+
+        return TaskResult(items=task_items, metadata={"feed_url": feed_url, "entry_count": real_count})

    @staticmethod
    def parse(url: str) -> dict[str, str] | None:
--- a/app/routes/api/tasks.py
+++ b/app/routes/api/tasks.py
@ -6,12 +6,45 @@ from aiohttp.web import Request, Response

 from app.library.encoder import Encoder
 from app.library.router import route
-from app.library.Tasks import Task, Tasks
-from app.library.Utils import init_class, validate_uuid
+from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks
+from app.library.Utils import init_class, validate_url, validate_uuid

 LOG: logging.Logger = logging.getLogger(__name__)


+@route("POST", "api/tasks/inspect", "task_handler_inspect")
+async def task_handler_inspect(request: Request, tasks: Tasks, encoder: Encoder) -> Response:
+    data = await request.json()
+
+    url: str | None = data.get("url") if isinstance(data, dict) else None
+    if not url:
+        return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code)
+    try:
+        validate_url(url)
+    except ValueError as e:
+        return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
+
+    preset: str = data.get("preset", "") if isinstance(data, dict) else ""
+    handler_name: str | None = data.get("handler") if isinstance(data, dict) else None
+
+    try:
+        result: TaskResult | TaskFailure = await tasks.get_handler().inspect(
+            url=url, preset=preset, handler_name=handler_name
+        )
+    except Exception as e:
+        LOG.exception(e)
+        return web.json_response(
+            {"error": "Failed to inspect handler.", "message": str(e)},
+            status=web.HTTPInternalServerError.status_code,
+        )
+
+    return web.json_response(
+        data=result,
+        status=web.HTTPBadRequest.status_code if isinstance(result, TaskFailure) else web.HTTPOk.status_code,
+        dumps=encoder.encode,
+    )
+
+
@route("GET", "api/tasks/", "tasks")
 async def tasks(encoder: Encoder) -> Response:
    """
--- a/app/tests/test_generic_task_handler.py
+++ b/app/tests/test_generic_task_handler.py
@ -0,0 +1,356 @@
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from app.library.task_handlers.generic import (
+    ContainerDefinition,
+    EngineConfig,
+    ExtractionRule,
+    GenericTaskHandler,
+    RequestConfig,
+    ResponseConfig,
+    TaskDefinition,
+    load_task_definitions,
+)
+from app.library.Tasks import Task, TaskFailure, TaskResult
+
+
+@pytest.fixture(autouse=True)
+def reset_generic_handler(monkeypatch):
+    monkeypatch.setattr(GenericTaskHandler, "_definitions", [])
+    monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {})
+
+
+def test_load_task_definitions_parses_valid_file(tmp_path: Path):
+    tasks_dir = tmp_path / "tasks"
+    tasks_dir.mkdir()
+
+    definition_content = {
+        "name": "example",
+        "match": ["https://example.com/articles/*"],
+        "parse": {
+            "link": {"type": "css", "expression": ".article a.link::attr(href)"},
+            "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
+        },
+    }
+
+    (tasks_dir / "01-example.json").write_text(json.dumps(definition_content), encoding="utf-8")
+
+    config = SimpleNamespace(config_path=str(tmp_path))
+    definitions = load_task_definitions(config=config)
+
+    assert len(definitions) == 1
+    definition = definitions[0]
+    assert definition.name == "example"
+    assert definition.matchers[0].matches("https://example.com/articles/123")
+    assert definition.parsers["link"].expression == ".article a.link::attr(href)"
+
+
+def test_load_task_definitions_handles_container(tmp_path: Path):
+    tasks_dir = tmp_path / "tasks"
+    tasks_dir.mkdir()
+
+    definition_content = {
+        "name": "container",
+        "match": ["https://example.com/cards"],
+        "parse": {
+            "items": {
+                "selector": ".cards .card",
+                "fields": {
+                    "link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
+                    "title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
+                },
+            }
+        },
+    }
+
+    (tasks_dir / "02-container.json").write_text(json.dumps(definition_content), encoding="utf-8")
+
+    config = SimpleNamespace(config_path=str(tmp_path))
+    definitions = load_task_definitions(config=config)
+
+    assert len(definitions) == 1
+    definition = definitions[0]
+    assert definition.container is not None
+    assert definition.container.selector == ".cards .card"
+    assert "link" in definition.container.fields
+    assert definition.parsers == {}
+
+
+def test_load_task_definitions_handles_json(tmp_path: Path):
+    tasks_dir = tmp_path / "tasks"
+    tasks_dir.mkdir()
+
+    definition_content = {
+        "name": "json-def",
+        "match": ["https://example.com/api"],
+        "response": {"type": "json"},
+        "parse": {
+            "items": {
+                "type": "jsonpath",
+                "selector": "items",
+                "fields": {
+                    "link": {"type": "jsonpath", "expression": "url"},
+                    "title": {"type": "jsonpath", "expression": "title"},
+                },
+            }
+        },
+    }
+
+    (tasks_dir / "03-json.json").write_text(json.dumps(definition_content), encoding="utf-8")
+
+    config = SimpleNamespace(config_path=str(tmp_path))
+    definitions = load_task_definitions(config=config)
+
+    assert len(definitions) == 1
+    definition = definitions[0]
+    assert definition.response.format == "json"
+    assert definition.container is not None
+    assert definition.container.selector_type == "jsonpath"
+    assert definition.container.fields["link"].type == "jsonpath"
+
+
+def test_parse_items_extracts_values():
+    definition = TaskDefinition(
+        name="example",
+        source=Path("example.json"),
+        matchers=[],
+        engine=EngineConfig(),
+        request=RequestConfig(),
+        parsers={
+            "link": ExtractionRule(type="css", expression=".article a.link::attr(href)", attribute=None),
+            "title": ExtractionRule(type="css", expression=".article .title", attribute="text"),
+            "id": ExtractionRule(type="css", expression=".article", attribute="data-id"),
+        },
+    )
+
+    html = """
+    <div class="article" data-id="101">
+      <a class="link" href="/article-101">First</a>
+      <span class="title">First Title</span>
+    </div>
+    <div class="article" data-id="102">
+      <a class="link" href="https://example.com/article-102">Second</a>
+      <span class="title">Second Title</span>
+    </div>
+    """
+
+    items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/")
+
+    assert len(items) == 2
+    assert items[0]["link"] == "https://example.com/article-101"
+    assert items[0]["title"] == "First Title"
+    assert items[0]["id"] == "101"
+    assert items[1]["link"] == "https://example.com/article-102"
+
+
+def test_parse_items_handles_nested_card_layout():
+    definition = TaskDefinition(
+        name="nested",
+        source=Path("nested.json"),
+        matchers=[],
+        engine=EngineConfig(),
+        request=RequestConfig(),
+        parsers={},
+        container=ContainerDefinition(
+            selector_type="css",
+            selector=".columns .card",
+            fields={
+                "link": ExtractionRule(
+                    type="css",
+                    expression=".card-header a[href]",
+                    attribute="href",
+                ),
+                "title": ExtractionRule(
+                    type="css",
+                    expression=".card-header a[href]",
+                    attribute="text",
+                ),
+                "poet": ExtractionRule(
+                    type="css",
+                    expression="footer .card-footer-item:first-child a",
+                    attribute="text",
+                ),
+                "category": ExtractionRule(
+                    type="css",
+                    expression="footer .card-footer-item:nth-child(2) a",
+                    attribute="text",
+                ),
+            },
+        ),
+    )
+
+    html = """
+    <div class="columns is-multiline">
+      <div class="column is-6">
+        <div class="card">
+          <div class="card-header">
+            <p class="card-header-title is-4 has-text-centered is-block text-truncate">
+              <a href="/poems/view/111" title="First Poem">First Poem</a>
+            </p>
+          </div>
+          <footer class="card-footer has-text-centered">
+            <p class="card-footer-item text-truncate">
+              <span class="text-truncate"> By <a href="/poet/alpha">Poet Alpha</a></span>
+            </p>
+            <p class="card-footer-item text-truncate">
+              <span class="text-truncate"> In <a href="/category/one">Category One</a></span>
+            </p>
+          </footer>
+        </div>
+      </div>
+      <div class="column is-6">
+        <div class="card">
+          <div class="card-header">
+            <p class="card-header-title is-4 has-text-centered is-block text-truncate">
+              <a href="/poems/view/222" title="Second Poem">Second Poem</a>
+            </p>
+          </div>
+          <footer class="card-footer has-text-centered">
+            <p class="card-footer-item text-truncate">
+              <span class="text-truncate"> By <a href="/poet/beta">Poet Beta</a></span>
+            </p>
+          </footer>
+        </div>
+      </div>
+    </div>
+    """
+
+    items = GenericTaskHandler._parse_items(definition, html, "https://example.com")
+
+    assert len(items) == 2
+    assert items[0]["link"] == "https://example.com/poems/view/111"
+    assert items[0]["title"] == "First Poem"
+    assert items[0]["poet"] == "Poet Alpha"
+    assert items[0]["category"] == "Category One"
+
+    assert items[1]["link"] == "https://example.com/poems/view/222"
+    assert items[1]["title"] == "Second Poem"
+    assert items[1]["poet"] == "Poet Beta"
+    assert "category" not in items[1]
+
+
+def test_parse_items_handles_json_container():
+    definition = TaskDefinition(
+        name="json",
+        source=Path("json.json"),
+        matchers=[],
+        engine=EngineConfig(),
+        request=RequestConfig(),
+        parsers={},
+        container=ContainerDefinition(
+            selector_type="jsonpath",
+            selector="entries",
+            fields={
+                "link": ExtractionRule(type="jsonpath", expression="url"),
+                "title": ExtractionRule(type="jsonpath", expression="title"),
+                "id": ExtractionRule(type="jsonpath", expression="id"),
+            },
+        ),
+        response=ResponseConfig(format="json"),
+    )
+
+    payload = {
+        "entries": [
+            {"url": "/video/1", "title": "First", "id": 1},
+            {"url": "https://example.com/video/2", "title": "Second", "id": 2},
+            {"title": "Missing Link", "id": 3},
+        ]
+    }
+
+    items = GenericTaskHandler._parse_items(
+        definition=definition,
+        html="",
+        base_url="https://example.com",
+        json_data=payload,
+    )
+
+    assert len(items) == 2
+    assert items[0]["link"] == "https://example.com/video/1"
+    assert items[0]["title"] == "First"
+    assert items[0]["id"] == "1"
+
+    assert items[1]["link"] == "https://example.com/video/2"
+    assert items[1]["title"] == "Second"
+    assert items[1]["id"] == "2"
+
+
+@pytest.mark.asyncio
+async def test_generic_task_handler_inspect(monkeypatch):
+    definition = TaskDefinition(
+        name="json-inspect",
+        source=Path("json-inspect.json"),
+        matchers=[],
+        engine=EngineConfig(),
+        request=RequestConfig(),
+        parsers={},
+        container=ContainerDefinition(
+            selector_type="jsonpath",
+            selector="items",
+            fields={
+                "link": ExtractionRule(type="jsonpath", expression="url"),
+                "title": ExtractionRule(type="jsonpath", expression="title"),
+            },
+        ),
+        response=ResponseConfig(format="json"),
+    )
+
+    monkeypatch.setattr(
+        GenericTaskHandler,
+        "_find_definition",
+        classmethod(lambda cls, url: definition),  # noqa: ARG005
+    )
+
+    async def fake_fetch_content(url, definition, ytdlp_opts):  # noqa: ARG001
+        return "", {"items": [{"url": "/video/1", "title": "First"}]}
+
+    monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content))
+
+    task = Task(id="inspect", name="Inspect", url="https://example.com/api")
+    result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
+
+    assert isinstance(result, TaskResult)
+    assert len(result.items) == 1
+    item = result.items[0]
+    assert item.url == "https://example.com/video/1"
+    assert item.title == "First"
+
+
+def test_parse_items_handles_json_top_level_list():
+    definition = TaskDefinition(
+        name="json-list",
+        source=Path("json-list.json"),
+        matchers=[],
+        engine=EngineConfig(),
+        request=RequestConfig(),
+        parsers={},
+        container=ContainerDefinition(
+            selector_type="jsonpath",
+            selector="[]",
+            fields={
+                "link": ExtractionRule(type="jsonpath", expression="url"),
+                "title": ExtractionRule(type="jsonpath", expression="title"),
+            },
+        ),
+        response=ResponseConfig(format="json"),
+    )
+
+    payload = [
+        {"url": "/video/1", "title": "First"},
+        {"url": "/video/2", "title": "Second"},
+    ]
+
+    items = GenericTaskHandler._parse_items(
+        definition=definition,
+        html="",
+        base_url="https://example.com",
+        json_data=payload,
+    )
+
+    assert len(items) == 2
+    assert items[0]["link"] == "https://example.com/video/1"
+    assert items[0]["title"] == "First"
+    assert items[1]["link"] == "https://example.com/video/2"
+    assert items[1]["title"] == "Second"
--- a/app/tests/test_twitch_handler.py
+++ b/app/tests/test_twitch_handler.py
@ -0,0 +1,57 @@
+import pytest
+
+from app.library.task_handlers.twitch import TwitchHandler
+from app.library.Tasks import Task, TaskResult
+
+
+class DummyResponse:
+    def __init__(self, text: str):
+        self.text = text
+
+    def raise_for_status(self) -> None:
+        return None
+
+
+class DummyOpts:
+    def __init__(self, data):
+        self._data = data
+
+    def get_all(self):
+        return self._data
+
+
+@pytest.mark.asyncio
+async def test_twitch_handler_inspect(monkeypatch):
+    feed = """
+    <rss><channel>
+      <item>
+        <link>https://www.twitch.tv/videos/111</link>
+        <title>First VOD</title>
+      </item>
+      <item>
+        <link>https://www.twitch.tv/videos/222</link>
+        <title>Second VOD</title>
+      </item>
+    </channel></rss>
+    """.strip()
+
+    async def fake_request(**kwargs):  # noqa: ARG001
+        return DummyResponse(feed)
+
+    monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request))
+    monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+
+    task = Task(
+        id="inspect",
+        name="Inspect",
+        url="https://www.twitch.tv/testchannel",
+        preset="default",
+    )
+
+    result = await TwitchHandler.extract(task)
+
+    assert isinstance(result, TaskResult)
+    assert len(result.items) == 2
+    assert result.items[0].url == "https://www.twitch.tv/videos/111"
+    assert result.items[0].title == "First VOD"
+    assert result.metadata.get("has_entries") is True
--- a/app/tests/test_youtube_handler.py
+++ b/app/tests/test_youtube_handler.py
@ -0,0 +1,61 @@
+import pytest
+
+from app.library.task_handlers.youtube import YoutubeHandler
+from app.library.Tasks import Task, TaskResult
+
+
+class DummyResponse:
+    def __init__(self, text: str):
+        self.text = text
+
+    def raise_for_status(self) -> None:
+        return None
+
+
+class DummyOpts:
+    def __init__(self, data):
+        self._data = data
+
+    def get_all(self):
+        return self._data
+
+
+@pytest.mark.asyncio
+async def test_youtube_handler_inspect(monkeypatch):
+    feed = """
+    <feed xmlns='http://www.w3.org/2005/Atom' xmlns:yt='http://www.youtube.com/xml/schemas/2015'>
+      <entry>
+        <yt:videoId>abc123</yt:videoId>
+        <title>First Video</title>
+        <published>2024-01-01T00:00:00Z</published>
+      </entry>
+      <entry>
+        <yt:videoId>def456</yt:videoId>
+        <title>Second Video</title>
+        <published>2024-01-02T00:00:00Z</published>
+      </entry>
+    </feed>
+    """.strip()
+
+    async def fake_request(**kwargs):  # noqa: ARG001
+        return DummyResponse(feed)
+
+    monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request))
+    monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+
+    task = Task(
+        id="inspect",
+        name="Inspect",
+        url="https://www.youtube.com/channel/UCabcdefghijklmnopqrstuv",
+        preset="default",
+    )
+
+    result = await YoutubeHandler.extract(task)
+
+    assert isinstance(result, TaskResult)
+    assert len(result.items) == 2
+    first = result.items[0]
+    assert first.url == "https://www.youtube.com/watch?v=abc123"
+    assert first.title == "First Video"
+    assert first.metadata.get("published") == "2024-01-01T00:00:00Z"
+    assert result.metadata.get("entry_count") == 2
--- a/pyproject.toml
+++ b/pyproject.toml
@ -40,6 +40,9 @@ dependencies = [
  "bgutil-ytdlp-pot-provider>=1.2.1",
  "pycryptodome>=3.23.0",
  "httpx-curl-cffi>=0.1.4",
+  "selenium>=4.35.0",
+  "parsel>=1.10.0",
+  "jmespath>=1.0.1",
 ]

 [tool.ruff]
--- a/ui/app/components/Dialog.vue
+++ b/ui/app/components/Dialog.vue
@ -44,7 +44,7 @@
            </p>

            <!-- prompt input -->
-            <div v-if="state.current?.type === 'prompt'" class="field">
+            <div v-if="'prompt' === state.current?.type" class="field">
              <div class="control">
                <input ref="inputEl" class="input" type="text" v-model="localInput"
                  :placeholder="(state.current?.opts as any)?.placeholder ?? ''" @keyup.stop />
@ -61,8 +61,8 @@
          </section>

          <footer class="modal-card-foot p-4 is-justify-content-flex-end">
-            <template v-if="state.current?.type === 'alert'">
-              <button class="button is-danger" @click="onEnter">
+            <template v-if="'alert' === state.current?.type">
+              <button id="primaryButton" class="button is-danger" @click="onEnter">
                <span class="icon-text">
                  <span class="icon"><i class="fas fa-check" /></span>
                  <span>{{ (state.current?.opts as any)?.confirmText ?? 'OK' }}</span>
@ -70,10 +70,11 @@
              </button>
            </template>

-            <template v-else-if="state.current?.type === 'confirm' || state.current?.type === 'prompt'">
+            <template v-else-if="'confirm' === state.current?.type || 'prompt' === state.current?.type">
              <div class="field is-grouped">
                <div class="control">
-                  <button class="button" @click="onEnter" :class="state.current?.opts.confirmColor ?? 'is-primary'"
+                  <button id="primaryButton" class="button" @click="onEnter"
+                    :class="state.current?.opts.confirmColor ?? 'is-primary'"
                    :disabled="localInput === (state.current?.opts as PromptOptions)?.initial">
                    <span class="icon-text">
                      <span class="icon"><i class="fas fa-check" /></span>
@ -114,7 +115,7 @@ watch(() => state.current, (cur) => {
    enableOpacity()
  }

-  localInput.value = cur?.type === 'prompt' ? (cur.opts as any).initial ?? '' : ''
+  localInput.value = 'prompt' === cur?.type ? (cur.opts as any).initial ?? '' : ''
 }, { immediate: true })

 const inputEl = ref<HTMLInputElement>()
@ -123,12 +124,12 @@ const focusPrimary = () => {
  if (!root) {
    return
  }
-  const btn = root.querySelector<HTMLButtonElement>('.modal-card-foot .button.is-primary')
+  const btn = root.querySelector<HTMLButtonElement>('#primaryButton')
  btn?.focus()
 }
 const focusInput = async () => {
  await nextTick()
-  if (state.current?.type === 'prompt') {
+  if ('prompt' === state.current?.type) {
    requestAnimationFrame(() => inputEl.value?.focus({ preventScroll: true }))
    return
  }
--- a/uv.lock
+++ b/uv.lock
@ -319,6 +319,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/8c/dd/9c40c4e0f4d3cb6cf52eb335e9cc1fa140c1f3a87146fb6987f465b069da/cronsim-2.6-py3-none-any.whl", hash = "sha256:5e153ff8ed64da7ee8d5caac470dbeda8024ab052c3010b1be149772b4801835", size = 13500 },
 ]

+[[package]]
+name = "cssselect"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/0a/c3ea9573b1dc2e151abfe88c7fe0c26d1892fe6ed02d0cdb30f0d57029d5/cssselect-1.3.0.tar.gz", hash = "sha256:57f8a99424cfab289a1b6a816a43075a4b00948c86b4dcf3ef4ee7e15f7ab0c7", size = 42870 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ee/58/257350f7db99b4ae12b614a36256d9cc870d71d9e451e79c2dc3b23d7c3c/cssselect-1.3.0-py3-none-any.whl", hash = "sha256:56d1bf3e198080cc1667e137bc51de9cadfca259f03c2d4e09037b3e01e30f0d", size = 18786 },
+]
+
 [[package]]
 name = "curl-cffi"
 version = "0.13.0"
@ -501,6 +510,59 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 },
 ]

+[[package]]
+name = "jmespath"
+version = "1.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256 },
+]
+
+[[package]]
+name = "lxml"
+version = "6.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8f/bd/f9d01fd4132d81c6f43ab01983caea69ec9614b913c290a26738431a015d/lxml-6.0.1.tar.gz", hash = "sha256:2b3a882ebf27dd026df3801a87cf49ff791336e0f94b0fad195db77e01240690", size = 4070214 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/c4/cd757eeec4548e6652eff50b944079d18ce5f8182d2b2cf514e125e8fbcb/lxml-6.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:485eda5d81bb7358db96a83546949c5fe7474bec6c68ef3fa1fb61a584b00eea", size = 8405139 },
+    { url = "https://files.pythonhosted.org/packages/ff/99/0290bb86a7403893f5e9658490c705fcea103b9191f2039752b071b4ef07/lxml-6.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d12160adea318ce3d118f0b4fbdff7d1225c75fb7749429541b4d217b85c3f76", size = 4585954 },
+    { url = "https://files.pythonhosted.org/packages/88/a7/4bb54dd1e626342a0f7df6ec6ca44fdd5d0e100ace53acc00e9a689ead04/lxml-6.0.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48c8d335d8ab72f9265e7ba598ae5105a8272437403f4032107dbcb96d3f0b29", size = 4944052 },
+    { url = "https://files.pythonhosted.org/packages/71/8d/20f51cd07a7cbef6214675a8a5c62b2559a36d9303fe511645108887c458/lxml-6.0.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:405e7cf9dbdbb52722c231e0f1257214202dfa192327fab3de45fd62e0554082", size = 5098885 },
+    { url = "https://files.pythonhosted.org/packages/5a/63/efceeee7245d45f97d548e48132258a36244d3c13c6e3ddbd04db95ff496/lxml-6.0.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:299a790d403335a6a057ade46f92612ebab87b223e4e8c5308059f2dc36f45ed", size = 5017542 },
+    { url = "https://files.pythonhosted.org/packages/57/5d/92cb3d3499f5caba17f7933e6be3b6c7de767b715081863337ced42eb5f2/lxml-6.0.1-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:48da704672f6f9c461e9a73250440c647638cc6ff9567ead4c3b1f189a604ee8", size = 5347303 },
+    { url = "https://files.pythonhosted.org/packages/69/f8/606fa16a05d7ef5e916c6481c634f40870db605caffed9d08b1a4fb6b989/lxml-6.0.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21e364e1bb731489e3f4d51db416f991a5d5da5d88184728d80ecfb0904b1d68", size = 5641055 },
+    { url = "https://files.pythonhosted.org/packages/b3/01/15d5fc74ebb49eac4e5df031fbc50713dcc081f4e0068ed963a510b7d457/lxml-6.0.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bce45a2c32032afddbd84ed8ab092130649acb935536ef7a9559636ce7ffd4a", size = 5242719 },
+    { url = "https://files.pythonhosted.org/packages/42/a5/1b85e2aaaf8deaa67e04c33bddb41f8e73d07a077bf9db677cec7128bfb4/lxml-6.0.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:fa164387ff20ab0e575fa909b11b92ff1481e6876835014e70280769920c4433", size = 4717310 },
+    { url = "https://files.pythonhosted.org/packages/42/23/f3bb1292f55a725814317172eeb296615db3becac8f1a059b53c51fc1da8/lxml-6.0.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7587ac5e000e1594e62278422c5783b34a82b22f27688b1074d71376424b73e8", size = 5254024 },
+    { url = "https://files.pythonhosted.org/packages/b4/be/4d768f581ccd0386d424bac615d9002d805df7cc8482ae07d529f60a3c1e/lxml-6.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:57478424ac4c9170eabf540237125e8d30fad1940648924c058e7bc9fb9cf6dd", size = 5055335 },
+    { url = "https://files.pythonhosted.org/packages/40/07/ed61d1a3e77d1a9f856c4fab15ee5c09a2853fb7af13b866bb469a3a6d42/lxml-6.0.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:09c74afc7786c10dd6afaa0be2e4805866beadc18f1d843cf517a7851151b499", size = 4784864 },
+    { url = "https://files.pythonhosted.org/packages/01/37/77e7971212e5c38a55431744f79dff27fd751771775165caea096d055ca4/lxml-6.0.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7fd70681aeed83b196482d42a9b0dc5b13bab55668d09ad75ed26dff3be5a2f5", size = 5657173 },
+    { url = "https://files.pythonhosted.org/packages/32/a3/e98806d483941cd9061cc838b1169626acef7b2807261fbe5e382fcef881/lxml-6.0.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:10a72e456319b030b3dd900df6b1f19d89adf06ebb688821636dc406788cf6ac", size = 5245896 },
+    { url = "https://files.pythonhosted.org/packages/07/de/9bb5a05e42e8623bf06b4638931ea8c8f5eb5a020fe31703abdbd2e83547/lxml-6.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0fa45fb5f55111ce75b56c703843b36baaf65908f8b8d2fbbc0e249dbc127ed", size = 5267417 },
+    { url = "https://files.pythonhosted.org/packages/f2/43/c1cb2a7c67226266c463ef8a53b82d42607228beb763b5fbf4867e88a21f/lxml-6.0.1-cp313-cp313-win32.whl", hash = "sha256:01dab65641201e00c69338c9c2b8a0f2f484b6b3a22d10779bb417599fae32b5", size = 3610051 },
+    { url = "https://files.pythonhosted.org/packages/34/96/6a6c3b8aa480639c1a0b9b6faf2a63fb73ab79ffcd2a91cf28745faa22de/lxml-6.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:bdf8f7c8502552d7bff9e4c98971910a0a59f60f88b5048f608d0a1a75e94d1c", size = 4009325 },
+    { url = "https://files.pythonhosted.org/packages/8c/66/622e8515121e1fd773e3738dae71b8df14b12006d9fb554ce90886689fd0/lxml-6.0.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6aeca75959426b9fd8d4782c28723ba224fe07cfa9f26a141004210528dcbe2", size = 3670443 },
+    { url = "https://files.pythonhosted.org/packages/38/e3/b7eb612ce07abe766918a7e581ec6a0e5212352194001fd287c3ace945f0/lxml-6.0.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:29b0e849ec7030e3ecb6112564c9f7ad6881e3b2375dd4a0c486c5c1f3a33859", size = 8426160 },
+    { url = "https://files.pythonhosted.org/packages/35/8f/ab3639a33595cf284fe733c6526da2ca3afbc5fd7f244ae67f3303cec654/lxml-6.0.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:02a0f7e629f73cc0be598c8b0611bf28ec3b948c549578a26111b01307fd4051", size = 4589288 },
+    { url = "https://files.pythonhosted.org/packages/2c/65/819d54f2e94d5c4458c1db8c1ccac9d05230b27c1038937d3d788eb406f9/lxml-6.0.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:beab5e54de016e730875f612ba51e54c331e2fa6dc78ecf9a5415fc90d619348", size = 4964523 },
+    { url = "https://files.pythonhosted.org/packages/5b/4a/d4a74ce942e60025cdaa883c5a4478921a99ce8607fc3130f1e349a83b28/lxml-6.0.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:92a08aefecd19ecc4ebf053c27789dd92c87821df2583a4337131cf181a1dffa", size = 5101108 },
+    { url = "https://files.pythonhosted.org/packages/cb/48/67f15461884074edd58af17b1827b983644d1fae83b3d909e9045a08b61e/lxml-6.0.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36c8fa7e177649470bc3dcf7eae6bee1e4984aaee496b9ccbf30e97ac4127fa2", size = 5053498 },
+    { url = "https://files.pythonhosted.org/packages/b6/d4/ec1bf1614828a5492f4af0b6a9ee2eb3e92440aea3ac4fa158e5228b772b/lxml-6.0.1-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5d08e0f1af6916267bb7eff21c09fa105620f07712424aaae09e8cb5dd4164d1", size = 5351057 },
+    { url = "https://files.pythonhosted.org/packages/65/2b/c85929dacac08821f2100cea3eb258ce5c8804a4e32b774f50ebd7592850/lxml-6.0.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9705cdfc05142f8c38c97a61bd3a29581ceceb973a014e302ee4a73cc6632476", size = 5671579 },
+    { url = "https://files.pythonhosted.org/packages/d0/36/cf544d75c269b9aad16752fd9f02d8e171c5a493ca225cb46bb7ba72868c/lxml-6.0.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74555e2da7c1636e30bff4e6e38d862a634cf020ffa591f1f63da96bf8b34772", size = 5250403 },
+    { url = "https://files.pythonhosted.org/packages/c2/e8/83dbc946ee598fd75fdeae6151a725ddeaab39bb321354a9468d4c9f44f3/lxml-6.0.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e38b5f94c5a2a5dadaddd50084098dfd005e5a2a56cd200aaf5e0a20e8941782", size = 4696712 },
+    { url = "https://files.pythonhosted.org/packages/f4/72/889c633b47c06205743ba935f4d1f5aa4eb7f0325d701ed2b0540df1b004/lxml-6.0.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a5ec101a92ddacb4791977acfc86c1afd624c032974bfb6a21269d1083c9bc49", size = 5268177 },
+    { url = "https://files.pythonhosted.org/packages/b0/b6/f42a21a1428479b66ea0da7bd13e370436aecaff0cfe93270c7e165bd2a4/lxml-6.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5c17e70c82fd777df586c12114bbe56e4e6f823a971814fd40dec9c0de518772", size = 5094648 },
+    { url = "https://files.pythonhosted.org/packages/51/b0/5f8c1e8890e2ee1c2053c2eadd1cb0e4b79e2304e2912385f6ca666f48b1/lxml-6.0.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:45fdd0415a0c3d91640b5d7a650a8f37410966a2e9afebb35979d06166fd010e", size = 4745220 },
+    { url = "https://files.pythonhosted.org/packages/eb/f9/820b5125660dae489ca3a21a36d9da2e75dd6b5ffe922088f94bbff3b8a0/lxml-6.0.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:d417eba28981e720a14fcb98f95e44e7a772fe25982e584db38e5d3b6ee02e79", size = 5692913 },
+    { url = "https://files.pythonhosted.org/packages/23/8e/a557fae9eec236618aecf9ff35fec18df41b6556d825f3ad6017d9f6e878/lxml-6.0.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:8e5d116b9e59be7934febb12c41cce2038491ec8fdb743aeacaaf36d6e7597e4", size = 5259816 },
+    { url = "https://files.pythonhosted.org/packages/fa/fd/b266cfaab81d93a539040be699b5854dd24c84e523a1711ee5f615aa7000/lxml-6.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c238f0d0d40fdcb695c439fe5787fa69d40f45789326b3bb6ef0d61c4b588d6e", size = 5276162 },
+    { url = "https://files.pythonhosted.org/packages/25/6c/6f9610fbf1de002048e80585ea4719591921a0316a8565968737d9f125ca/lxml-6.0.1-cp314-cp314-win32.whl", hash = "sha256:537b6cf1c5ab88cfd159195d412edb3e434fee880f206cbe68dff9c40e17a68a", size = 3669595 },
+    { url = "https://files.pythonhosted.org/packages/72/a5/506775e3988677db24dc75a7b03e04038e0b3d114ccd4bccea4ce0116c15/lxml-6.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:911d0a2bb3ef3df55b3d97ab325a9ca7e438d5112c102b8495321105d25a441b", size = 4079818 },
+    { url = "https://files.pythonhosted.org/packages/0a/44/9613f300201b8700215856e5edd056d4e58dd23368699196b58877d4408b/lxml-6.0.1-cp314-cp314-win_arm64.whl", hash = "sha256:2834377b0145a471a654d699bdb3a2155312de492142ef5a1d426af2c60a0a31", size = 3753901 },
+]
+
 [[package]]
 name = "macholib"
 version = "1.16.3"
@ -581,6 +643,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065 },
 ]

+[[package]]
+name = "outcome"
+version = "1.3.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "attrs" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692 },
+]
+
 [[package]]
 name = "packaging"
 version = "25.0"
@ -590,6 +664,22 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 },
 ]

+[[package]]
+name = "parsel"
+version = "1.10.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cssselect" },
+    { name = "jmespath" },
+    { name = "lxml" },
+    { name = "packaging" },
+    { name = "w3lib" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f6/df/acd504c154c0b9028b0d8491a77fdd5f86e9c06ee04f986abf85e36d9a5f/parsel-1.10.0.tar.gz", hash = "sha256:14f17db9559f51b43357b9dfe43cec870a8efb5ea4857abb624ec6ff80d8a080", size = 51421 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/12/18/35d1d947553d24909dca37e2ff11720eecb601360d1bac8d7a9a1bc7eb08/parsel-1.10.0-py2.py3-none-any.whl", hash = "sha256:6a0c28bd81f9df34ba665884c88efa0b18b8d2c44c81f64e27f2f0cb37d46169", size = 17266 },
+]
+
 [[package]]
 name = "pefile"
 version = "2023.2.7"
@ -782,6 +872,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178 },
 ]

+[[package]]
+name = "pysocks"
+version = "1.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725 },
+]
+
 [[package]]
 name = "pysubs2"
 version = "1.8.0"
@ -1008,6 +1107,23 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/e1/a3/03216a6a86c706df54422612981fb0f9041dbb452c3401501d4a22b942c9/ruff-0.13.0-py3-none-win_arm64.whl", hash = "sha256:ab80525317b1e1d38614addec8ac954f1b3e662de9d59114ecbf771d00cf613e", size = 12312357 },
 ]

+[[package]]
+name = "selenium"
+version = "4.35.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "trio" },
+    { name = "trio-websocket" },
+    { name = "typing-extensions" },
+    { name = "urllib3", extra = ["socks"] },
+    { name = "websocket-client" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/75/67/9016942b5781843cfea6f5bc1383cea852d9fa08f85f55a0547874525b5c/selenium-4.35.0.tar.gz", hash = "sha256:83937a538afb40ef01e384c1405c0863fa184c26c759d34a1ebbe7b925d3481c", size = 907991 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/17/ef/d0e033e1b3f19a0325ce03863b68d709780908381135fc0f9436dea76a7b/selenium-4.35.0-py3-none-any.whl", hash = "sha256:90bb6c6091fa55805785cf1660fa1e2176220475ccdb466190f654ef8eef6114", size = 9602106 },
+]
+
 [[package]]
 name = "setuptools"
 version = "80.9.0"
@ -1048,12 +1164,52 @@ wheels = [
 ]

 [[package]]
-name = "typing-extensions"
-version = "4.15.0"
+name = "sortedcontainers"
+version = "2.4.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391 }
+sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 },
+    { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 },
+]
+
+[[package]]
+name = "trio"
+version = "0.30.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "attrs" },
+    { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" },
+    { name = "idna" },
+    { name = "outcome" },
+    { name = "sniffio" },
+    { name = "sortedcontainers" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/01/c1/68d582b4d3a1c1f8118e18042464bb12a7c1b75d64d75111b297687041e3/trio-0.30.0.tar.gz", hash = "sha256:0781c857c0c81f8f51e0089929a26b5bb63d57f927728a5586f7e36171f064df", size = 593776 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/69/8e/3f6dfda475ecd940e786defe6df6c500734e686c9cd0a0f8ef6821e9b2f2/trio-0.30.0-py3-none-any.whl", hash = "sha256:3bf4f06b8decf8d3cf00af85f40a89824669e2d033bb32469d34840edcfc22a5", size = 499194 },
+]
+
+[[package]]
+name = "trio-websocket"
+version = "0.12.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "outcome" },
+    { name = "trio" },
+    { name = "wsproto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d1/3c/8b4358e81f2f2cfe71b66a267f023a91db20a817b9425dd964873796980a/trio_websocket-0.12.2.tar.gz", hash = "sha256:22c72c436f3d1e264d0910a3951934798dcc5b00ae56fc4ee079d46c7cf20fae", size = 33549 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c7/19/eb640a397bba49ba49ef9dbe2e7e5c04202ba045b6ce2ec36e9cadc51e04/trio_websocket-0.12.2-py3-none-any.whl", hash = "sha256:df605665f1db533f4a386c94525870851096a223adcb97f72a07e8b4beba45b6", size = 21221 },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.14.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906 },
 ]

 [[package]]
@ -1086,6 +1242,29 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795 },
 ]

+[package.optional-dependencies]
+socks = [
+    { name = "pysocks" },
+]
+
+[[package]]
+name = "w3lib"
+version = "2.3.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bf/7d/1172cfaa1e29beb9bf938e484c122b3bdc82e8e37b17a4f753ba6d6e009f/w3lib-2.3.1.tar.gz", hash = "sha256:5c8ac02a3027576174c2b61eb9a2170ba1b197cae767080771b6f1febda249a4", size = 49531 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/58/dd/56f0d8af71e475ed194d702f8b4cf9cea812c95e82ad823d239023c6558c/w3lib-2.3.1-py3-none-any.whl", hash = "sha256:9ccd2ae10c8c41c7279cd8ad4fe65f834be894fe7bfdd7304b991fd69325847b", size = 21751 },
+]
+
+[[package]]
+name = "websocket-client"
+version = "1.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e6/30/fba0d96b4b5fbf5948ed3f4681f7da2f9f64512e1d303f94b4cc174c24a5/websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da", size = 54648 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826 },
+]
+
 [[package]]
 name = "wsproto"
 version = "1.2.0"
@ -1176,8 +1355,10 @@ dependencies = [
    { name = "defusedxml" },
    { name = "httpx" },
    { name = "httpx-curl-cffi" },
+    { name = "jmespath" },
    { name = "multidict" },
    { name = "mutagen" },
+    { name = "parsel" },
    { name = "platformdirs" },
    { name = "pycryptodome" },
    { name = "pyjson5" },
@ -1187,6 +1368,7 @@ dependencies = [
    { name = "python-magic-bin", marker = "sys_platform == 'win32'" },
    { name = "python-socketio" },
    { name = "regex" },
+    { name = "selenium" },
    { name = "tzdata", marker = "sys_platform == 'win32'" },
    { name = "yt-dlp" },
    { name = "zipstream-ng" },
@ -1223,8 +1405,10 @@ requires-dist = [
    { name = "defusedxml", specifier = ">=0.7.1" },
    { name = "httpx" },
    { name = "httpx-curl-cffi", specifier = ">=0.1.4" },
+    { name = "jmespath", specifier = ">=1.0.1" },
    { name = "multidict", specifier = "==6.5.1" },
    { name = "mutagen" },
+    { name = "parsel", specifier = ">=1.10.0" },
    { name = "platformdirs" },
    { name = "pycryptodome", specifier = ">=3.23.0" },
    { name = "pyinstaller", marker = "extra == 'installer'" },
@ -1235,6 +1419,7 @@ requires-dist = [
    { name = "python-magic-bin", marker = "sys_platform == 'win32'" },
    { name = "python-socketio", specifier = ">=5.11.1" },
    { name = "regex" },
+    { name = "selenium", specifier = ">=4.35.0" },
    { name = "tzdata", marker = "sys_platform == 'win32'" },
    { name = "yt-dlp" },
    { name = "zipstream-ng", specifier = ">=1.8.0" },