Merge pull request #502 from arabcoders/dev
Feat: Add task handler for tver.jp
This commit is contained in:
Abdulmohsen
GitHub
commit
7e6b6f0544
5 changed files with 380 additions and 4 deletions
|
|
@ -30,7 +30,22 @@ class BaseHandler:
|
|||
return []
|
||||
|
||||
@staticmethod
|
||||
async def request(url: str, headers: dict | None = None, ytdlp_opts: dict | None = None) -> httpx.Response:
|
||||
async def request(
|
||||
url: str, headers: dict | None = None, ytdlp_opts: dict | None = None, **kwargs
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Make an HTTP request.
|
||||
|
||||
Args:
|
||||
url (str): The URL to request.
|
||||
headers (dict | None): Additional headers to include in the request.
|
||||
ytdlp_opts (dict | None): yt-dlp options that may affect the request.
|
||||
**kwargs: Additional arguments to pass to httpx request.
|
||||
|
||||
Returns:
|
||||
httpx.Response: The HTTP response.
|
||||
|
||||
"""
|
||||
headers = {} if not isinstance(headers, dict) else headers
|
||||
ytdlp_opts = {} if not isinstance(ytdlp_opts, dict) else ytdlp_opts
|
||||
|
||||
|
|
@ -59,4 +74,6 @@ class BaseHandler:
|
|||
opts["proxy"] = proxy
|
||||
|
||||
async with httpx.AsyncClient(**opts) as client:
|
||||
return await client.request(method="GET", url=url, timeout=ytdlp_opts.get("socket_timeout", 120))
|
||||
method = kwargs.pop("method", "GET").upper()
|
||||
timeout = ytdlp_opts.get("timeout", ytdlp_opts.get("socket_timeout", 120))
|
||||
return await client.request(method=method, url=url, timeout=timeout, **kwargs)
|
||||
|
|
|
|||
202
app/library/task_handlers/tver.py
Normal file
202
app/library/task_handlers/tver.py
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
import logging
|
||||
import re
|
||||
|
||||
from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult
|
||||
from app.library.Utils import get_archive_id
|
||||
|
||||
from ._base_handler import BaseHandler
|
||||
|
||||
LOG: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TverHandler(BaseHandler):
|
||||
SERIES_API = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}"
|
||||
SESSION_API = "https://platform-api.tver.jp/v2/api/platform_users/browser/create"
|
||||
HEADERS = {
|
||||
"x-tver-platform-type": "web",
|
||||
"Origin": "https://tver.jp",
|
||||
"Referer": "https://tver.jp/",
|
||||
}
|
||||
|
||||
RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?P<id>sr[a-z0-9_]+)$")
|
||||
|
||||
@staticmethod
|
||||
def can_handle(task: Task) -> bool:
|
||||
LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}")
|
||||
return TverHandler.parse(task.url) is not None
|
||||
|
||||
@staticmethod
|
||||
async def _get_session_tokens() -> dict[str, str] | None:
|
||||
"""
|
||||
Create a session and retrieve platform_uid and platform_token.
|
||||
|
||||
Returns:
|
||||
dict[str, str] | None: Dictionary with platform_uid and platform_token, or None on failure.
|
||||
|
||||
"""
|
||||
try:
|
||||
response = await TverHandler.request(
|
||||
url=TverHandler.SESSION_API,
|
||||
headers=TverHandler.HEADERS,
|
||||
method="POST",
|
||||
data={"device_type": "pc"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
platform_uid = data.get("result", {}).get("platform_uid")
|
||||
platform_token = data.get("result", {}).get("platform_token")
|
||||
|
||||
if not platform_uid or not platform_token:
|
||||
LOG.warning("Failed to extract platform_uid and platform_token from session response.")
|
||||
return None
|
||||
|
||||
return {"platform_uid": platform_uid, "platform_token": platform_token}
|
||||
except Exception as exc:
|
||||
LOG.warning(f"Failed to create tver session: {exc}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _collect_feed(
|
||||
task: Task,
|
||||
params: dict,
|
||||
series_id: str,
|
||||
) -> tuple[str, list[dict[str, str]], bool]:
|
||||
"""
|
||||
Fetch episodes from tver series API.
|
||||
|
||||
Args:
|
||||
task (Task): The task containing the tver series URL.
|
||||
params (dict): The ytdlp options.
|
||||
series_id (str): The tver series ID.
|
||||
|
||||
Returns:
|
||||
tuple[str, list[dict[str, str]], bool]: The feed URL, list of episodes, and whether items were found.
|
||||
|
||||
"""
|
||||
tokens = await TverHandler._get_session_tokens()
|
||||
if not tokens:
|
||||
msg = "Could not obtain tver session tokens"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
feed_url = TverHandler.SERIES_API.format(id=series_id)
|
||||
|
||||
LOG.debug(f"Fetching '{task.name}' episodes from tver series {series_id}.")
|
||||
|
||||
response = await TverHandler.request(
|
||||
url=feed_url,
|
||||
headers=TverHandler.HEADERS,
|
||||
ytdlp_opts=params,
|
||||
params={
|
||||
"platform_uid": tokens["platform_uid"],
|
||||
"platform_token": tokens["platform_token"],
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
items: list[dict[str, str]] = []
|
||||
has_items = False
|
||||
|
||||
# Parse episodes from result.contents[0].contents
|
||||
try:
|
||||
contents = data.get("result", {}).get("contents", [])
|
||||
if not contents:
|
||||
LOG.warning(f"No contents found in tver series response for '{task.name}'.")
|
||||
return feed_url, items, has_items
|
||||
|
||||
season_block = contents[0] if contents else {}
|
||||
episodes = season_block.get("contents", [])
|
||||
|
||||
for episode_data in episodes:
|
||||
if "episode" != episode_data.get("type"):
|
||||
continue
|
||||
|
||||
content = episode_data.get("content", {})
|
||||
episode_id = content.pop("id")
|
||||
if not episode_id:
|
||||
LOG.warning(f"Episode missing ID in '{task.name}' feed. Skipping.")
|
||||
continue
|
||||
|
||||
url = f"https://tver.jp/episodes/{episode_id}"
|
||||
|
||||
title = content.pop("title", "")
|
||||
|
||||
id_dict = get_archive_id(url)
|
||||
archive_id = id_dict.get("archive_id")
|
||||
if not archive_id:
|
||||
LOG.warning(
|
||||
f"Could not compute archive ID for episode '{episode_id}' in '{task.name}' feed. Skipping."
|
||||
)
|
||||
continue
|
||||
|
||||
has_items = True
|
||||
items.append(
|
||||
{"id": episode_id, "url": url, "title": title, "archive_id": archive_id, "metadata": content}
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
LOG.warning(f"Error parsing tver episodes for '{task.name}': {exc}")
|
||||
|
||||
return feed_url, items, has_items
|
||||
|
||||
@staticmethod
|
||||
async def extract(task: Task) -> TaskResult | TaskFailure:
|
||||
series_id: str | None = TverHandler.parse(task.url)
|
||||
if not series_id:
|
||||
return TaskFailure(message="Unrecognized Tver series URL.")
|
||||
|
||||
params: dict = task.get_ytdlp_opts().get_all()
|
||||
|
||||
try:
|
||||
feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id)
|
||||
except Exception as exc:
|
||||
LOG.exception(exc)
|
||||
return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
|
||||
|
||||
task_items: list[TaskItem] = []
|
||||
|
||||
for entry in items:
|
||||
if not (url := entry.get("url")):
|
||||
continue
|
||||
|
||||
archive_id: str = entry.get("archive_id")
|
||||
task_items.append(
|
||||
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
|
||||
)
|
||||
|
||||
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
||||
|
||||
@staticmethod
|
||||
def parse(url: str) -> str | None:
|
||||
"""
|
||||
Parse URL to extract series ID.
|
||||
|
||||
Args:
|
||||
url (str): The url to check.
|
||||
|
||||
Returns:
|
||||
str | None: The parsed ID if successful, None otherwise.
|
||||
|
||||
"""
|
||||
match: re.Match[str] | None = TverHandler.RX.match(url)
|
||||
return match.group("id") if match else None
|
||||
|
||||
@staticmethod
|
||||
def tests() -> list[tuple[str, bool]]:
|
||||
"""
|
||||
Test cases for the URL parser.
|
||||
|
||||
Returns:
|
||||
list[tuple[str, bool]]: A list of tuples containing the URL and expected result.
|
||||
|
||||
"""
|
||||
return [
|
||||
("https://tver.jp/series/sr1jg44pbb", True),
|
||||
("https://www.tver.jp/series/sr1jg44pbb", True),
|
||||
("https://m.tver.jp/series/sr_test_id", True),
|
||||
("http://tver.jp/series/sr123abc456", True),
|
||||
("https://tver.jp/videos/sr123abc456", False),
|
||||
("https://youtube.com/watch?v=123", False),
|
||||
]
|
||||
156
app/tests/test_tver_handler.py
Normal file
156
app/tests/test_tver_handler.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import pytest
|
||||
|
||||
from app.library.task_handlers.tver import TverHandler
|
||||
from app.library.Tasks import Task, TaskResult
|
||||
|
||||
|
||||
class DummyResponse:
|
||||
def __init__(self, data: dict):
|
||||
self.data = data
|
||||
|
||||
def json(self):
|
||||
return self.data
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class DummyOpts:
|
||||
def __init__(self, data):
|
||||
self._data = data
|
||||
|
||||
def get_all(self):
|
||||
return self._data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tver_handler_extract(monkeypatch):
|
||||
"""Test tver handler extraction of episodes from series."""
|
||||
session_response = {
|
||||
"result": {
|
||||
"platform_uid": "test_uid_123",
|
||||
"platform_token": "test_token_456",
|
||||
}
|
||||
}
|
||||
|
||||
episodes_response = {
|
||||
"result": {
|
||||
"contents": [
|
||||
{
|
||||
"seasonTitle": "本編",
|
||||
"hasNext": False,
|
||||
"contents": [
|
||||
{
|
||||
"type": "episode",
|
||||
"content": {
|
||||
"id": "eppcxhym1e",
|
||||
"version": 12,
|
||||
"title": "木村拓哉がタクシー 運転手！目黒蓮が木村と二人旅",
|
||||
"seriesID": "sr1jg44pbb",
|
||||
"endAt": 1766404799,
|
||||
"broadcastDateLabel": "11月24日(月)放送 分",
|
||||
"isNHKContent": False,
|
||||
"isSubtitle": False,
|
||||
"ribbonID": 0,
|
||||
"seriesTitle": "ウルトラタクシー",
|
||||
"isAvailable": True,
|
||||
"broadcasterName": "TBS",
|
||||
"productionProviderName": "TBS",
|
||||
"isEndingSoon": False,
|
||||
"thumbnailPath": "/images/content/thumbnail/episode/xlarge/eppcxhym1e.jpg?v=84b0ee80",
|
||||
},
|
||||
"resume": {"lastViewedDuration": 0, "contentDuration": 3796, "completed": False},
|
||||
"isFavorite": False,
|
||||
"isGood": False,
|
||||
"isLater": False,
|
||||
"goodCount": 5553,
|
||||
},
|
||||
{
|
||||
"type": "episode",
|
||||
"content": {
|
||||
"id": "epejwb9mvx",
|
||||
"version": 9,
|
||||
"title": "木村拓哉がタクシー運転手！蒼井優＆上戸彩高校の同級生コンビがディズニーリゾートを大満喫！",
|
||||
"seriesID": "sr1jg44pbb",
|
||||
"endAt": 1764590399,
|
||||
"broadcastDateLabel": "11月24日(月)放送分",
|
||||
"isNHKContent": False,
|
||||
"isSubtitle": False,
|
||||
"ribbonID": 0,
|
||||
"seriesTitle": "ウルトラタクシー",
|
||||
"isAvailable": True,
|
||||
"broadcasterName": "TBS",
|
||||
"productionProviderName": "TBS",
|
||||
"isEndingSoon": False,
|
||||
"thumbnailPath": "/images/content/thumbnail/episode/xlarge/epejwb9mvx.jpg?v=ab478281",
|
||||
},
|
||||
"resume": {"lastViewedDuration": 0, "contentDuration": 1896, "completed": False},
|
||||
"isFavorite": False,
|
||||
"isGood": False,
|
||||
"isLater": False,
|
||||
"goodCount": 1298,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
"seasons": [
|
||||
{
|
||||
"type": "season",
|
||||
"content": {"id": "sshbu1ycq8", "version": 2, "title": "本編", "season_extended": None},
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def fake_request(url: str, **kwargs): # noqa: ARG001
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
|
||||
if "browser/create" in url:
|
||||
return DummyResponse(session_response)
|
||||
|
||||
if "callSeriesEpisodes" in url:
|
||||
return DummyResponse(episodes_response)
|
||||
|
||||
msg = f"Unexpected URL: {url}"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request))
|
||||
monkeypatch.setattr(Task, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"}))
|
||||
|
||||
task = Task(id="test_tver", name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
|
||||
|
||||
result = await TverHandler.extract(task)
|
||||
|
||||
assert isinstance(result, TaskResult)
|
||||
assert len(result.items) == 2
|
||||
assert result.items[0].url == "https://tver.jp/episodes/eppcxhym1e"
|
||||
assert "木村拓哉がタクシー 運転手！目黒蓮が木村と二人旅" == result.items[0].title
|
||||
|
||||
assert result.items[1].url == "https://tver.jp/episodes/epejwb9mvx"
|
||||
assert "木村拓哉がタクシー運転手！蒼井優＆上戸彩高校の同級生コンビがディズニーリゾートを大満喫！" == result.items[1].title
|
||||
|
||||
assert result.metadata.get("has_entries") is True
|
||||
assert "callSeriesEpisodes" in result.metadata.get("feed_url", "")
|
||||
assert call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("url", "should_match"), TverHandler.tests())
|
||||
def test_tver_handler_parse(url: str, should_match: bool):
|
||||
"""Test tver URL parsing."""
|
||||
result = TverHandler.parse(url)
|
||||
if should_match:
|
||||
assert result is not None
|
||||
assert result.startswith("sr")
|
||||
else:
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_tver_handler_can_handle():
|
||||
"""Test tver handler can_handle method."""
|
||||
task_valid = Task(id="test1", name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
|
||||
task_invalid = Task(id="test2", name="Test", url="https://youtube.com/watch?v=123", preset="default")
|
||||
assert TverHandler.can_handle(task_valid) is True
|
||||
assert TverHandler.can_handle(task_invalid) is False
|
||||
|
|
@ -30,7 +30,7 @@ class NFOMakerPP(PostProcessor):
|
|||
r"(?i)\b(?:https?://|ftp://|www\.)\S+|\b(?!@)[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?){1,}\b(?:/[^\s<>()]*)?"
|
||||
)
|
||||
_MD_LINK = re.compile(r"\[([^\]]+)\]\((?:[^)]+)\)")
|
||||
_TIME_LINE_PAT = re.compile(r"^\s*(?:\d+:)?\d{1,2}:\d{2}(?::\d{2})?(?:\s*[-–—•:]\s*.*)?$", re.IGNORECASE) # noqa: RUF001
|
||||
_TIME_LINE_PAT = re.compile(r"^\s*(?:\d+:)?\d{1,2}:\d{2}(?::\d{2})?(?:\s*[-–—•:]\s*.*)?$", re.IGNORECASE)
|
||||
_HASHTAGS_LINE = re.compile(r"^\s*(?:#[\w\-]+(?:\s+|$))+$")
|
||||
_MENTION_LINE = re.compile(r"^\s*@[\w.\-]{2,}\s*$")
|
||||
_PROMO_LINE_PAT = re.compile(
|
||||
|
|
@ -402,7 +402,7 @@ class NFOMakerPP(PostProcessor):
|
|||
|
||||
# collapse leftover multiple spaces and stray separators
|
||||
ln = re.sub(r"\s{2,}", " ", ln)
|
||||
ln = re.sub(r"\s*[-–—•·]+\s*$", "", ln) # noqa: RUF001
|
||||
ln = re.sub(r"\s*[-–—•·]+\s*$", "", ln)
|
||||
|
||||
if ln:
|
||||
cleaned.append(ln)
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ ignore = [
|
|||
"D203",
|
||||
"TRY004", # Like it's our choice to use ValuesError :|
|
||||
"PT011",
|
||||
"RUF001", # We like unicode chars.
|
||||
]
|
||||
|
||||
# Allow fix for all enabled rules (when `--fix`) is provided.
|
||||
|
|
|
|||
Loading…
Reference in a new issue