Merge pull request #502 from arabcoders/dev

Feat: Add task handler for tver.jp

app/library/task_handlers/_base_handler.py

@@ -30,7 +30,22 @@ class BaseHandler:
         return []
 
     @staticmethod
-    async def request(url: str, headers: dict | None = None, ytdlp_opts: dict | None = None) -> httpx.Response:
+    async def request(
+        url: str, headers: dict | None = None, ytdlp_opts: dict | None = None, **kwargs
+    ) -> httpx.Response:
+        """
+        Make an HTTP request.
+
+        Args:
+            url (str): The URL to request.
+            headers (dict | None): Additional headers to include in the request.
+            ytdlp_opts (dict | None): yt-dlp options that may affect the request.
+            **kwargs: Additional arguments to pass to httpx request.
+
+        Returns:
+            httpx.Response: The HTTP response.
+
+        """
         headers = {} if not isinstance(headers, dict) else headers
         ytdlp_opts = {} if not isinstance(ytdlp_opts, dict) else ytdlp_opts
 
@@ -59,4 +74,6 @@ class BaseHandler:
             opts["proxy"] = proxy
 
         async with httpx.AsyncClient(**opts) as client:
-            return await client.request(method="GET", url=url, timeout=ytdlp_opts.get("socket_timeout", 120))
+            method = kwargs.pop("method", "GET").upper()
+            timeout = ytdlp_opts.get("timeout", ytdlp_opts.get("socket_timeout", 120))
+            return await client.request(method=method, url=url, timeout=timeout, **kwargs)

app/library/task_handlers/tver.py

@@ -0,0 +1,202 @@
+import logging
+import re
+
+from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
+from app.library.Utils import get_archive_id
+
+from ._base_handler import BaseHandler
+
+LOG: logging.Logger = logging.getLogger(__name__)
+
+
+class TverHandler(BaseHandler):
+    SERIES_API = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}"
+    SESSION_API = "https://platform-api.tver.jp/v2/api/platform_users/browser/create"
+    HEADERS = {
+        "x-tver-platform-type": "web",
+        "Origin": "https://tver.jp",
+        "Referer": "https://tver.jp/",
+    }
+
+    RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?P<id>sr[a-z0-9_]+)$")
+
+    @staticmethod
+    def can_handle(task: Task) -> bool:
+        LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}")
+        return TverHandler.parse(task.url) is not None
+
+    @staticmethod
+    async def _get_session_tokens() -> dict[str, str] | None:
+        """
+        Create a session and retrieve platform_uid and platform_token.
+
+        Returns:
+            dict[str, str] | None: Dictionary with platform_uid and platform_token, or None on failure.
+
+        """
+        try:
+            response = await TverHandler.request(
+                url=TverHandler.SESSION_API,
+                headers=TverHandler.HEADERS,
+                method="POST",
+                data={"device_type": "pc"},
+            )
+            response.raise_for_status()
+
+            data = response.json()
+            platform_uid = data.get("result", {}).get("platform_uid")
+            platform_token = data.get("result", {}).get("platform_token")
+
+            if not platform_uid or not platform_token:
+                LOG.warning("Failed to extract platform_uid and platform_token from session response.")
+                return None
+
+            return {"platform_uid": platform_uid, "platform_token": platform_token}
+        except Exception as exc:
+            LOG.warning(f"Failed to create tver session: {exc}")
+            return None
+
+    @staticmethod
+    async def _collect_feed(
+        task: Task,
+        params: dict,
+        series_id: str,
+    ) -> tuple[str, list[dict[str, str]], bool]:
+        """
+        Fetch episodes from tver series API.
+
+        Args:
+            task (Task): The task containing the tver series URL.
+            params (dict): The ytdlp options.
+            series_id (str): The tver series ID.
+
+        Returns:
+            tuple[str, list[dict[str, str]], bool]: The feed URL, list of episodes, and whether items were found.
+
+        """
+        tokens = await TverHandler._get_session_tokens()
+        if not tokens:
+            msg = "Could not obtain tver session tokens"
+            raise RuntimeError(msg)
+
+        feed_url = TverHandler.SERIES_API.format(id=series_id)
+
+        LOG.debug(f"Fetching '{task.name}' episodes from tver series {series_id}.")
+
+        response = await TverHandler.request(
+            url=feed_url,
+            headers=TverHandler.HEADERS,
+            ytdlp_opts=params,
+            params={
+                "platform_uid": tokens["platform_uid"],
+                "platform_token": tokens["platform_token"],
+            },
+        )
+        response.raise_for_status()
+
+        data = response.json()
+
+        items: list[dict[str, str]] = []
+        has_items = False
+
+        # Parse episodes from result.contents[0].contents
+        try:
+            contents = data.get("result", {}).get("contents", [])
+            if not contents:
+                LOG.warning(f"No contents found in tver series response for '{task.name}'.")
+                return feed_url, items, has_items
+
+            season_block = contents[0] if contents else {}
+            episodes = season_block.get("contents", [])
+
+            for episode_data in episodes:
+                if "episode" != episode_data.get("type"):
+                    continue
+
+                content = episode_data.get("content", {})
+                episode_id = content.pop("id")
+                if not episode_id:
+                    LOG.warning(f"Episode missing ID in '{task.name}' feed. Skipping.")
+                    continue
+
+                url = f"https://tver.jp/episodes/{episode_id}"
+
+                title = content.pop("title", "")
+
+                id_dict = get_archive_id(url)
+                archive_id = id_dict.get("archive_id")
+                if not archive_id:
+                    LOG.warning(
+                        f"Could not compute archive ID for episode '{episode_id}' in '{task.name}' feed. Skipping."
+                    )
+                    continue
+
+                has_items = True
+                items.append(
+                    {"id": episode_id, "url": url, "title": title, "archive_id": archive_id, "metadata": content}
+                )
+
+        except Exception as exc:
+            LOG.warning(f"Error parsing tver episodes for '{task.name}': {exc}")
+
+        return feed_url, items, has_items
+
+    @staticmethod
+    async def extract(task: Task) -> TaskResult | TaskFailure:
+        series_id: str | None = TverHandler.parse(task.url)
+        if not series_id:
+            return TaskFailure(message="Unrecognized Tver series URL.")
+
+        params: dict = task.get_ytdlp_opts().get_all()
+
+        try:
+            feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id)
+        except Exception as exc:
+            LOG.exception(exc)
+            return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
+
+        task_items: list[TaskItem] = []
+
+        for entry in items:
+            if not (url := entry.get("url")):
+                continue
+
+            archive_id: str = entry.get("archive_id")
+            task_items.append(
+                TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
+            )
+
+        return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
+
+    @staticmethod
+    def parse(url: str) -> str | None:
+        """
+        Parse URL to extract series ID.
+
+        Args:
+            url (str): The url to check.
+
+        Returns:
+            str | None: The parsed ID if successful, None otherwise.
+
+        """
+        match: re.Match[str] | None = TverHandler.RX.match(url)
+        return match.group("id") if match else None
+
+    @staticmethod
+    def tests() -> list[tuple[str, bool]]:
+        """
+        Test cases for the URL parser.
+
+        Returns:
+            list[tuple[str, bool]]: A list of tuples containing the URL and expected result.
+
+        """
+        return [
+            ("https://tver.jp/series/sr1jg44pbb", True),
+            ("https://www.tver.jp/series/sr1jg44pbb", True),
+            ("https://m.tver.jp/series/sr_test_id", True),
+            ("http://tver.jp/series/sr123abc456", True),
+            ("https://tver.jp/videos/sr123abc456", False),
+            ("https://youtube.com/watch?v=123", False),
+        ]

app/tests/test_tver_handler.py

@@ -0,0 +1,156 @@
+import pytest
+
+from app.library.task_handlers.tver import TverHandler
+from app.library.Tasks import Task, TaskResult
+
+
+class DummyResponse:
+    def __init__(self, data: dict):
+        self.data = data
+
+    def json(self):
+        return self.data
+
+    def raise_for_status(self) -> None:
+        return None
+
+
+class DummyOpts:
+    def __init__(self, data):
+        self._data = data
+
+    def get_all(self):
+        return self._data
+
+
+@pytest.mark.asyncio
+async def test_tver_handler_extract(monkeypatch):
+    """Test tver handler extraction of episodes from series."""
+    session_response = {
+        "result": {
+            "platform_uid": "test_uid_123",
+            "platform_token": "test_token_456",
+        }
+    }
+
+    episodes_response = {
+        "result": {
+            "contents": [
+                {
+                    "seasonTitle": "本編",
+                    "hasNext": False,
+                    "contents": [
+                        {
+                            "type": "episode",
+                            "content": {
+                                "id": "eppcxhym1e",
+                                "version": 12,
+                                "title": "木村拓哉がタクシー 運転手！目黒蓮が木村と二人旅",
+                                "seriesID": "sr1jg44pbb",
+                                "endAt": 1766404799,
+                                "broadcastDateLabel": "11月24日(月)放送 分",
+                                "isNHKContent": False,
+                                "isSubtitle": False,
+                                "ribbonID": 0,
+                                "seriesTitle": "ウルトラタクシー",
+                                "isAvailable": True,
+                                "broadcasterName": "TBS",
+                                "productionProviderName": "TBS",
+                                "isEndingSoon": False,
+                                "thumbnailPath": "/images/content/thumbnail/episode/xlarge/eppcxhym1e.jpg?v=84b0ee80",
+                            },
+                            "resume": {"lastViewedDuration": 0, "contentDuration": 3796, "completed": False},
+                            "isFavorite": False,
+                            "isGood": False,
+                            "isLater": False,
+                            "goodCount": 5553,
+                        },
+                        {
+                            "type": "episode",
+                            "content": {
+                                "id": "epejwb9mvx",
+                                "version": 9,
+                                "title": "木村拓哉がタクシー運転手！蒼井優＆上戸彩高校の同級生コンビがディズニーリゾートを大満喫！",
+                                "seriesID": "sr1jg44pbb",
+                                "endAt": 1764590399,
+                                "broadcastDateLabel": "11月24日(月)放送分",
+                                "isNHKContent": False,
+                                "isSubtitle": False,
+                                "ribbonID": 0,
+                                "seriesTitle": "ウルトラタクシー",
+                                "isAvailable": True,
+                                "broadcasterName": "TBS",
+                                "productionProviderName": "TBS",
+                                "isEndingSoon": False,
+                                "thumbnailPath": "/images/content/thumbnail/episode/xlarge/epejwb9mvx.jpg?v=ab478281",
+                            },
+                            "resume": {"lastViewedDuration": 0, "contentDuration": 1896, "completed": False},
+                            "isFavorite": False,
+                            "isGood": False,
+                            "isLater": False,
+                            "goodCount": 1298,
+                        },
+                    ],
+                }
+            ],
+            "seasons": [
+                {
+                    "type": "season",
+                    "content": {"id": "sshbu1ycq8", "version": 2, "title": "本編", "season_extended": None},
+                }
+            ],
+        }
+    }
+
+    call_count = 0
+
+    async def fake_request(url: str, **kwargs):  # noqa: ARG001
+        nonlocal call_count
+        call_count += 1
+
+        if "browser/create" in url:
+            return DummyResponse(session_response)
+
+        if "callSeriesEpisodes" in url:
+            return DummyResponse(episodes_response)
+
+        msg = f"Unexpected URL: {url}"
+        raise RuntimeError(msg)
+
+    monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request))
+    monkeypatch.setattr(Task, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"}))
+
+    task = Task(id="test_tver", name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
+
+    result = await TverHandler.extract(task)
+
+    assert isinstance(result, TaskResult)
+    assert len(result.items) == 2
+    assert result.items[0].url == "https://tver.jp/episodes/eppcxhym1e"
+    assert "木村拓哉がタクシー 運転手！目黒蓮が木村と二人旅" == result.items[0].title
+
+    assert result.items[1].url == "https://tver.jp/episodes/epejwb9mvx"
+    assert "木村拓哉がタクシー運転手！蒼井優＆上戸彩高校の同級生コンビがディズニーリゾートを大満喫！" == result.items[1].title
+
+    assert result.metadata.get("has_entries") is True
+    assert "callSeriesEpisodes" in result.metadata.get("feed_url", "")
+    assert call_count == 2
+
+
+@pytest.mark.parametrize(("url", "should_match"), TverHandler.tests())
+def test_tver_handler_parse(url: str, should_match: bool):
+    """Test tver URL parsing."""
+    result = TverHandler.parse(url)
+    if should_match:
+        assert result is not None
+        assert result.startswith("sr")
+    else:
+        assert result is None
+
+
+def test_tver_handler_can_handle():
+    """Test tver handler can_handle method."""
+    task_valid = Task(id="test1", name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
+    task_invalid = Task(id="test2", name="Test", url="https://youtube.com/watch?v=123", preset="default")
+    assert TverHandler.can_handle(task_valid) is True
+    assert TverHandler.can_handle(task_invalid) is False

app/yt_dlp_plugins/postprocessor/nfo_maker.py

@@ -30,7 +30,7 @@ class NFOMakerPP(PostProcessor):
         r"(?i)\b(?:https?://|ftp://|www\.)\S+|\b(?!@)[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?){1,}\b(?:/[^\s<>()]*)?"
     )
     _MD_LINK = re.compile(r"\[([^\]]+)\]\((?:[^)]+)\)")
-    _TIME_LINE_PAT = re.compile(r"^\s*(?:\d+:)?\d{1,2}:\d{2}(?::\d{2})?(?:\s*[-–—•:]\s*.*)?$", re.IGNORECASE)  # noqa: RUF001
+    _TIME_LINE_PAT = re.compile(r"^\s*(?:\d+:)?\d{1,2}:\d{2}(?::\d{2})?(?:\s*[-–—•:]\s*.*)?$", re.IGNORECASE)
     _HASHTAGS_LINE = re.compile(r"^\s*(?:#[\w\-]+(?:\s+|$))+$")
     _MENTION_LINE = re.compile(r"^\s*@[\w.\-]{2,}\s*$")
     _PROMO_LINE_PAT = re.compile(
@@ -402,7 +402,7 @@ class NFOMakerPP(PostProcessor):
 
             # collapse leftover multiple spaces and stray separators
             ln = re.sub(r"\s{2,}", " ", ln)
-            ln = re.sub(r"\s*[-–—•·]+\s*$", "", ln)  # noqa: RUF001
+            ln = re.sub(r"\s*[-–—•·]+\s*$", "", ln)
 
             if ln:
                 cleaned.append(ln)

pyproject.toml

@@ -167,6 +167,7 @@ ignore = [
   "D203",
   "TRY004", # Like it's our choice to use ValuesError :|
   "PT011",
+  "RUF001", # We like unicode chars.
 ]
 
 # Allow fix for all enabled rules (when `--fix`) is provided.