chore: use ty as type checker.
This commit is contained in:
parent
18f54a28c6
commit
3631947234
107 changed files with 1563 additions and 1354 deletions
|
|
@ -6,13 +6,16 @@ from app.features.conditions.migration import Migration
|
|||
from app.library.Singleton import Singleton
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, Iterable
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
|
||||
from sqlalchemy.engine.result import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.selectable import Select
|
||||
|
||||
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
|
||||
from app.features.conditions.models import ConditionModel
|
||||
|
|
@ -22,10 +25,26 @@ from app.library.log import get_logger
|
|||
LOG = get_logger()
|
||||
|
||||
|
||||
def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
|
||||
model = ConditionModel()
|
||||
for key, value in payload.items():
|
||||
if not hasattr(model, key):
|
||||
msg = f"'{key}' is an invalid keyword argument for ConditionModel"
|
||||
raise TypeError(msg)
|
||||
setattr(model, key, value)
|
||||
return model
|
||||
|
||||
|
||||
def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
|
||||
if isinstance(payload, ConditionModel):
|
||||
return payload
|
||||
return _model_from_payload(payload)
|
||||
|
||||
|
||||
class ConditionsRepository(metaclass=Singleton):
|
||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
||||
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||
self._migrated = False
|
||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
||||
self.session: SessionFactory = session or get_session
|
||||
|
||||
async def run_migrations(self) -> None:
|
||||
if self._migrated:
|
||||
|
|
@ -38,7 +57,7 @@ class ConditionsRepository(metaclass=Singleton):
|
|||
def get_instance() -> ConditionsRepository:
|
||||
return ConditionsRepository()
|
||||
|
||||
async def list(self) -> list[ConditionModel]:
|
||||
async def all(self) -> list[ConditionModel]:
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[ConditionModel]] = await session.execute(
|
||||
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
|
||||
|
|
@ -93,11 +112,11 @@ class ConditionsRepository(metaclass=Singleton):
|
|||
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def create(self, payload: ConditionModel | dict) -> ConditionModel:
|
||||
async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
|
||||
async with self.session() as session:
|
||||
model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload
|
||||
model = _coerce_model(payload)
|
||||
if model.id is not None:
|
||||
model.id = None
|
||||
model.id = None # ty: ignore
|
||||
|
||||
if await self.get_by_name(name=model.name) is not None:
|
||||
msg: str = f"Condition with name '{model.name}' already exists."
|
||||
|
|
@ -160,13 +179,11 @@ class ConditionsRepository(metaclass=Singleton):
|
|||
await session.commit()
|
||||
return model
|
||||
|
||||
async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]:
|
||||
async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
|
||||
async with self.session() as session:
|
||||
try:
|
||||
await session.execute(delete(ConditionModel))
|
||||
models: list[ConditionModel] = [
|
||||
ConditionModel(**item) if isinstance(item, dict) else item for item in items
|
||||
]
|
||||
models: list[ConditionModel] = [_coerce_model(item) for item in items]
|
||||
session.add_all(models)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -131,6 +131,12 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
|
|||
status=web.HTTPInternalServerError.status_code,
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return web.json_response(
|
||||
data={"error": "Failed to extract video info."},
|
||||
status=web.HTTPInternalServerError.status_code,
|
||||
)
|
||||
|
||||
try:
|
||||
from app.features.ytdlp.mini_filter import match_str
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from collections.abc import Iterable
|
||||
from numbers import Number
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from app.features.conditions.models import ConditionModel
|
||||
from app.features.conditions.repository import ConditionsRepository
|
||||
from app.features.conditions.schemas import Condition
|
||||
from app.features.ytdlp.mini_filter import match_str
|
||||
from app.library.Events import EventBus, Events
|
||||
from app.library.log import get_logger
|
||||
|
|
@ -13,7 +13,7 @@ from app.library.Singleton import Singleton
|
|||
LOG = get_logger()
|
||||
|
||||
|
||||
def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tuple[set[str], bool]:
|
||||
def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
|
||||
ignored: set[str] = set()
|
||||
ignore_all = False
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tu
|
|||
return ignored, ignore_all
|
||||
|
||||
for value in ignore_conditions:
|
||||
if isinstance(value, bool) or not isinstance(value, (str, Number)):
|
||||
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
|
||||
continue
|
||||
|
||||
identifier = str(value).strip()
|
||||
|
|
@ -55,7 +55,7 @@ class Conditions(metaclass=Singleton):
|
|||
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
|
||||
|
||||
async def get_all(self) -> list[ConditionModel]:
|
||||
return await self._repo.list()
|
||||
return await self._repo.all()
|
||||
|
||||
async def save(self, item: ConditionModel | dict) -> ConditionModel:
|
||||
"""
|
||||
|
|
@ -80,7 +80,7 @@ class Conditions(metaclass=Singleton):
|
|||
if item.id is None or 0 == item.id:
|
||||
model = await repo.create(item)
|
||||
else:
|
||||
model = await repo.update(item.id, item.serialize())
|
||||
model = await repo.update(item.id, Condition.model_validate(item).model_dump())
|
||||
except KeyError as exc:
|
||||
raise ValueError(str(exc)) from exc
|
||||
|
||||
|
|
@ -113,13 +113,15 @@ class Conditions(metaclass=Singleton):
|
|||
repo = self._repo
|
||||
return await repo.get(identifier)
|
||||
|
||||
async def match(self, info: dict, ignore_conditions: Iterable[str | Number] | None = None) -> ConditionModel | None:
|
||||
async def match(
|
||||
self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
|
||||
) -> ConditionModel | None:
|
||||
"""
|
||||
Check if any condition matches the info dict.
|
||||
|
||||
Args:
|
||||
info (dict): The info dict to check.
|
||||
ignore_conditions (Iterable[str | Number] | None): Condition ids or names to skip for this match.
|
||||
ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
|
||||
|
||||
Returns:
|
||||
Condition|None: The condition if found, None otherwise.
|
||||
|
|
@ -133,7 +135,7 @@ class Conditions(metaclass=Singleton):
|
|||
return None
|
||||
|
||||
repo = self._repo
|
||||
items: list[ConditionModel] = await repo.list()
|
||||
items: list[ConditionModel] = await repo.all()
|
||||
if len(items) < 1:
|
||||
return None
|
||||
|
||||
|
|
@ -187,7 +189,7 @@ class Conditions(metaclass=Singleton):
|
|||
if not info or not isinstance(info, dict) or len(info) < 1:
|
||||
return None
|
||||
|
||||
if not (item := await self.get(identifier)) or not item.enabled or not item.filter:
|
||||
if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
|
||||
return None
|
||||
|
||||
return item if match_str(item.filter, info) else None
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import pytest
|
|||
import pytest_asyncio
|
||||
from types import SimpleNamespace
|
||||
import pytest
|
||||
from typing import Any
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import make_mocked_request
|
||||
|
||||
|
|
@ -39,13 +40,13 @@ async def repo():
|
|||
|
||||
|
||||
def _json_request(path: str, payload: object) -> web.Request:
|
||||
request = make_mocked_request("POST", path)
|
||||
mock_request: Any = make_mocked_request("POST", path)
|
||||
|
||||
async def _json() -> object:
|
||||
return payload
|
||||
|
||||
request.json = _json # type: ignore[attr-defined]
|
||||
return request
|
||||
mock_request.json = _json
|
||||
return mock_request
|
||||
|
||||
|
||||
class TestAllowInternalUrlsScope:
|
||||
|
|
@ -208,7 +209,7 @@ class TestConditionsRepository:
|
|||
await repo.create({"name": "A", "filter": "test", "priority": 1})
|
||||
await repo.create({"name": "C", "filter": "test", "priority": 2})
|
||||
|
||||
items = await repo.list()
|
||||
items = await repo.all()
|
||||
|
||||
assert items[0].name == "C", "Highest priority should be first"
|
||||
assert items[1].name == "A", "Same priority sorted alphabetically"
|
||||
|
|
@ -240,6 +241,6 @@ class TestConditionsRepository:
|
|||
|
||||
assert len(result) == 2, "Should create 2 new conditions"
|
||||
|
||||
all_items = await repo.list()
|
||||
all_items = await repo.all()
|
||||
assert len(all_items) == 2, "Should only have new conditions"
|
||||
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime as SQLADateTime
|
||||
from sqlalchemy import TypeDecorator
|
||||
from sqlalchemy import Dialect, TypeDecorator
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
|
|
@ -20,16 +20,18 @@ class UTCDateTime(TypeDecorator):
|
|||
impl = SQLADateTime
|
||||
cache_ok = True
|
||||
|
||||
def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None:
|
||||
def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
|
||||
"""Convert datetime to UTC before storing."""
|
||||
_ = dialect
|
||||
if value is not None:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value.astimezone(UTC).replace(tzinfo=None)
|
||||
return value
|
||||
|
||||
def process_result_value(self, value: datetime | None, _dialect) -> datetime | None:
|
||||
def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
|
||||
"""Ensure datetime is timezone-aware (UTC) when loading."""
|
||||
_ = dialect
|
||||
if value is not None and value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
|
||||
|
|
@ -11,20 +11,39 @@ from app.library.log import get_logger
|
|||
from app.library.Singleton import Singleton
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, Iterable
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
|
||||
from sqlalchemy.engine.result import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.selectable import Select
|
||||
|
||||
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||
|
||||
LOG = get_logger()
|
||||
|
||||
|
||||
def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
|
||||
model = DLFieldModel()
|
||||
for key, value in payload.items():
|
||||
if not hasattr(model, key):
|
||||
msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
|
||||
raise TypeError(msg)
|
||||
setattr(model, key, value)
|
||||
return model
|
||||
|
||||
|
||||
def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
|
||||
if isinstance(payload, DLFieldModel):
|
||||
return payload
|
||||
return _model_from_payload(payload)
|
||||
|
||||
|
||||
class DLFieldsRepository(metaclass=Singleton):
|
||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
||||
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||
self._migrated = False
|
||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
||||
self.session: SessionFactory = session or get_session
|
||||
|
||||
async def run_migrations(self) -> None:
|
||||
if self._migrated:
|
||||
|
|
@ -37,7 +56,7 @@ class DLFieldsRepository(metaclass=Singleton):
|
|||
def get_instance() -> DLFieldsRepository:
|
||||
return DLFieldsRepository()
|
||||
|
||||
async def list(self) -> list[DLFieldModel]:
|
||||
async def all(self) -> list[DLFieldModel]:
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[DLFieldModel]] = await session.execute(
|
||||
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
|
||||
|
|
@ -90,9 +109,9 @@ class DLFieldsRepository(metaclass=Singleton):
|
|||
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def create(self, payload: DLFieldModel | dict) -> DLFieldModel:
|
||||
async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
|
||||
async with self.session() as session:
|
||||
model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload
|
||||
model = _coerce_model(payload)
|
||||
|
||||
if await self.get_by_name(name=model.name) is not None:
|
||||
msg: str = f"DL field with name '{model.name}' already exists."
|
||||
|
|
@ -153,13 +172,16 @@ class DLFieldsRepository(metaclass=Singleton):
|
|||
await session.commit()
|
||||
return model
|
||||
|
||||
async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]:
|
||||
async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
|
||||
async with self.session() as session:
|
||||
try:
|
||||
await session.execute(delete(DLFieldModel))
|
||||
models: list[DLFieldModel] = [
|
||||
DLFieldModel(**item) if isinstance(item, dict) else item for item in items
|
||||
]
|
||||
models: list[DLFieldModel] = []
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
models.append(DLFieldModel(**cast("dict[str, Any]", item)))
|
||||
else:
|
||||
models.append(item)
|
||||
session.add_all(models)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
|
|||
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
|
||||
|
||||
async def get_all(self) -> list[DLFieldModel]:
|
||||
return await self._repo.list()
|
||||
return await self._repo.all()
|
||||
|
||||
async def get_all_serialized(self) -> list[dict[str, Any]]:
|
||||
items = await self._repo.list()
|
||||
items = await self._repo.all()
|
||||
return [DLField.model_validate(item).model_dump() for item in items]
|
||||
|
||||
async def save(self, item: DLField | dict) -> DLFieldModel:
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ class TestDLFieldsRepository:
|
|||
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
|
||||
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
|
||||
|
||||
items = await repo.list()
|
||||
items = await repo.all()
|
||||
|
||||
assert items[0].name == "c", "Lowest order should be first"
|
||||
assert items[1].name == "a", "Same order sorted alphabetically"
|
||||
|
|
@ -243,6 +243,6 @@ class TestDLFieldsRepository:
|
|||
|
||||
assert len(result) == 2, "Should create 2 new fields"
|
||||
|
||||
all_items = await repo.list()
|
||||
all_items = await repo.all()
|
||||
assert len(all_items) == 2, "Should only have new fields"
|
||||
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"
|
||||
|
|
|
|||
|
|
@ -11,20 +11,39 @@ from app.library.log import get_logger
|
|||
from app.library.Singleton import Singleton
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
|
||||
from sqlalchemy.engine.result import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.selectable import Select
|
||||
|
||||
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||
|
||||
LOG = get_logger()
|
||||
|
||||
|
||||
def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
|
||||
model = NotificationModel()
|
||||
for key, value in payload.items():
|
||||
if not hasattr(model, key):
|
||||
msg = f"'{key}' is an invalid keyword argument for NotificationModel"
|
||||
raise TypeError(msg)
|
||||
setattr(model, key, value)
|
||||
return model
|
||||
|
||||
|
||||
def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
|
||||
if isinstance(payload, NotificationModel):
|
||||
return payload
|
||||
return _model_from_payload(payload)
|
||||
|
||||
|
||||
class NotificationsRepository(metaclass=Singleton):
|
||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
||||
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||
self._migrated = False
|
||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
||||
self.session: SessionFactory = session or get_session
|
||||
|
||||
async def run_migrations(self) -> None:
|
||||
if self._migrated:
|
||||
|
|
@ -37,7 +56,7 @@ class NotificationsRepository(metaclass=Singleton):
|
|||
def get_instance() -> NotificationsRepository:
|
||||
return NotificationsRepository()
|
||||
|
||||
async def list(self) -> list[NotificationModel]:
|
||||
async def all(self) -> list[NotificationModel]:
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[NotificationModel]] = await session.execute(
|
||||
select(NotificationModel).order_by(NotificationModel.name.asc())
|
||||
|
|
@ -92,11 +111,11 @@ class NotificationsRepository(metaclass=Singleton):
|
|||
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def create(self, payload: NotificationModel | dict) -> NotificationModel:
|
||||
async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
|
||||
async with self.session() as session:
|
||||
model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload
|
||||
model = _coerce_model(payload)
|
||||
if model.id is not None:
|
||||
model.id = None
|
||||
model.id = None # ty: ignore
|
||||
|
||||
if await self.get_by_name(name=model.name) is not None:
|
||||
msg: str = f"Notification target with name '{model.name}' already exists."
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from app.features.notifications.models import NotificationModel
|
||||
from app.features.notifications.repository import NotificationsRepository
|
||||
|
|
@ -64,10 +64,10 @@ class Notifications(metaclass=Singleton):
|
|||
await self._repo.run_migrations()
|
||||
|
||||
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
|
||||
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit")
|
||||
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
|
||||
|
||||
async def list(self) -> list[NotificationModel]:
|
||||
return await self._repo.list()
|
||||
async def all(self) -> list[NotificationModel]:
|
||||
return await self._repo.all()
|
||||
|
||||
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
|
||||
return await self._repo.list_paginated(page, per_page)
|
||||
|
|
@ -92,8 +92,8 @@ class Notifications(metaclass=Singleton):
|
|||
async def delete(self, identifier: int | str) -> NotificationModel:
|
||||
return await self._repo.delete(identifier)
|
||||
|
||||
async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]:
|
||||
targets = await self._repo.list()
|
||||
async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
|
||||
targets = await self._repo.all()
|
||||
if len(targets) < 1:
|
||||
return []
|
||||
|
||||
|
|
@ -126,7 +126,7 @@ class Notifications(metaclass=Singleton):
|
|||
if wait:
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
return tasks
|
||||
return cast("list[dict | Awaitable[dict]]", tasks)
|
||||
|
||||
def emit(self, e: Event, _, **__) -> None:
|
||||
if not NotificationEvents.is_valid(e.event):
|
||||
|
|
@ -271,7 +271,7 @@ class Notifications(metaclass=Singleton):
|
|||
|
||||
if not status:
|
||||
msg = "Apprise failed to send notification."
|
||||
raise RuntimeError(msg) # noqa: TRY301
|
||||
self._raise_apprise_error(msg)
|
||||
except Exception as exc:
|
||||
LOG.exception(
|
||||
"Failed to send Apprise notification for event '%s'.",
|
||||
|
|
@ -288,6 +288,10 @@ class Notifications(metaclass=Singleton):
|
|||
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _raise_apprise_error(msg: str) -> None:
|
||||
raise RuntimeError(msg)
|
||||
|
||||
async def _send(self, target: Notification, ev: Event) -> dict:
|
||||
try:
|
||||
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ class TestNotificationsRepository:
|
|||
@pytest.mark.asyncio
|
||||
async def test_list_empty(self, repo):
|
||||
"""List returns empty when no notifications exist."""
|
||||
notifications = await repo.list()
|
||||
notifications = await repo.all()
|
||||
assert notifications == [], "Should return empty list when no notifications exist"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class Migration(FeatureMigration):
|
|||
return
|
||||
|
||||
inserted = 0
|
||||
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()}
|
||||
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
|
||||
|
||||
for index, item in enumerate(items):
|
||||
if not (normalized := self._normalize(item, index, seen_names)):
|
||||
|
|
|
|||
|
|
@ -30,6 +30,30 @@ if TYPE_CHECKING:
|
|||
LOG = get_logger()
|
||||
|
||||
|
||||
def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(payload, dict):
|
||||
data: dict[str, Any] = {}
|
||||
for key, value in payload.items():
|
||||
if not isinstance(key, str):
|
||||
msg = "Preset payload keys must be strings."
|
||||
raise TypeError(msg)
|
||||
data[key] = value
|
||||
return data
|
||||
|
||||
return {
|
||||
"name": payload.name,
|
||||
"description": payload.description,
|
||||
"folder": payload.folder,
|
||||
"template": payload.template,
|
||||
"cookies": payload.cookies,
|
||||
"cli": payload.cli,
|
||||
"default": payload.default,
|
||||
"priority": payload.priority,
|
||||
"created_at": payload.created_at,
|
||||
"updated_at": payload.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class PresetsRepository(metaclass=Singleton):
|
||||
SORT_FIELDS: dict[str, Any] = {
|
||||
"id": PresetModel.id,
|
||||
|
|
@ -73,15 +97,15 @@ class PresetsRepository(metaclass=Singleton):
|
|||
LOG.debug("Refreshing presets cache due to configuration update.")
|
||||
await self._update_cache()
|
||||
|
||||
Services.get_instance().add(__class__.__name__, self)
|
||||
Services.get_instance().add(PresetsRepository.__name__, self)
|
||||
EventBus.get_instance().subscribe(
|
||||
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
|
||||
Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
|
||||
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
|
||||
|
||||
async def _update_cache(self) -> None:
|
||||
from app.features.presets.service import Presets
|
||||
|
||||
await Presets.get_instance().refresh_cache(await self.list())
|
||||
await Presets.get_instance().refresh_cache(await self.all())
|
||||
|
||||
@staticmethod
|
||||
def get_instance() -> PresetsRepository:
|
||||
|
|
@ -97,7 +121,7 @@ class PresetsRepository(metaclass=Singleton):
|
|||
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
|
||||
config.default_preset = "default"
|
||||
|
||||
async def list(self) -> list[PresetModel]:
|
||||
async def all(self) -> list[PresetModel]:
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[PresetModel]] = await session.execute(
|
||||
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
|
||||
|
|
@ -238,24 +262,9 @@ class PresetsRepository(metaclass=Singleton):
|
|||
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def create(self, payload: PresetModel | dict) -> PresetModel:
|
||||
async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
|
||||
async with self.session() as session:
|
||||
data: dict[str, Any]
|
||||
if isinstance(payload, dict):
|
||||
data = dict(payload)
|
||||
else:
|
||||
data = {
|
||||
"name": payload.name,
|
||||
"description": payload.description,
|
||||
"folder": payload.folder,
|
||||
"template": payload.template,
|
||||
"cookies": payload.cookies,
|
||||
"cli": payload.cli,
|
||||
"default": payload.default,
|
||||
"priority": payload.priority,
|
||||
"created_at": payload.created_at,
|
||||
"updated_at": payload.updated_at,
|
||||
}
|
||||
data = _payload_data(payload)
|
||||
|
||||
data.pop("id", None)
|
||||
model = PresetModel(**data)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
|
|||
def __init__(self, repo: PresetsRepository | None = None) -> None:
|
||||
self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
|
||||
self._cache: list[tuple[int, str, Preset]] = []
|
||||
Services.get_instance().add(__class__.__name__, self)
|
||||
Services.get_instance().add(type(self).__name__, self)
|
||||
|
||||
@staticmethod
|
||||
def get_instance() -> Presets:
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class TestPresetsRepository:
|
|||
await repo.create({"name": "A", "priority": 1})
|
||||
await repo.create({"name": "C", "priority": 2})
|
||||
|
||||
items = await repo.list()
|
||||
items = await repo.all()
|
||||
|
||||
assert items[0].name == "c", "Highest priority should be first"
|
||||
assert items[1].name == "a", "Same priority should sort by name"
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@ import os
|
|||
import subprocess # qa: ignore
|
||||
from pathlib import Path
|
||||
|
||||
import anyio
|
||||
|
||||
from app.features.streaming.types import FFProbeError
|
||||
from app.library.log import get_logger
|
||||
from app.library.Utils import timed_lru_cache
|
||||
|
|
@ -39,21 +37,26 @@ class FFStream:
|
|||
|
||||
def __repr__(self):
|
||||
if "codec_long_name" not in self.__dict__:
|
||||
self.codec_long_name = self.__dict__.get("codec_name", "")
|
||||
self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
|
||||
|
||||
index = self.__dict__.get("index", "?")
|
||||
codec_type = self.__dict__.get("codec_type", "unknown")
|
||||
codec_long_name = self.__dict__.get("codec_long_name", "")
|
||||
|
||||
if self.is_video():
|
||||
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>"
|
||||
return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
|
||||
|
||||
if self.is_audio():
|
||||
return (
|
||||
f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), "
|
||||
"{sample_rate}Hz> "
|
||||
f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
|
||||
f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
|
||||
f"{self.__dict__.get('sample_rate')}Hz> "
|
||||
)
|
||||
|
||||
if self.is_subtitle() or self.is_attachment():
|
||||
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>"
|
||||
return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
|
||||
|
||||
return f"<Stream: #{self.index} [{self.codec_type}]>"
|
||||
return f"<Stream: #{index} [{codec_type}]>"
|
||||
|
||||
def is_audio(self):
|
||||
"""
|
||||
|
|
@ -252,14 +255,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
|
|||
raise OSError(msg)
|
||||
|
||||
try:
|
||||
async with await anyio.open_file(os.devnull, "w") as tempf:
|
||||
await asyncio.create_subprocess_exec(
|
||||
"ffprobe",
|
||||
"-h",
|
||||
stdout=tempf,
|
||||
stderr=tempf,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
||||
)
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ffprobe",
|
||||
"-h",
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
||||
)
|
||||
await proc.wait()
|
||||
except FileNotFoundError as e:
|
||||
msg = "ffprobe not found."
|
||||
raise OSError(msg) from e
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class M3u8:
|
|||
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
|
||||
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
|
||||
|
||||
segmentSize: float = f"{self.duration:.6f}"
|
||||
segmentSize: str = f"{self.duration:.6f}"
|
||||
splits: int = math.ceil(duration / self.duration)
|
||||
|
||||
segmentParams: dict = {}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class Playlist:
|
|||
"The path where files are downloaded."
|
||||
|
||||
async def make(self, file: Path) -> str:
|
||||
ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
|
||||
ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
|
||||
|
||||
try:
|
||||
ff = await ffprobe(file)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# flake8: noqa: ARG002
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from functools import lru_cache
|
||||
|
|
@ -16,9 +16,13 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
|
|||
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
|
||||
values are dicts with keys "full" and "lp" booleans.
|
||||
"""
|
||||
vainfo = shutil.which("vainfo")
|
||||
if not vainfo:
|
||||
return {}
|
||||
|
||||
try:
|
||||
result: subprocess.CompletedProcess[str] = subprocess.run(
|
||||
["vainfo"], # noqa: S607
|
||||
[vainfo],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
|
|
@ -87,9 +91,13 @@ def ffmpeg_encoders() -> set[str]:
|
|||
"""
|
||||
from app.library.config import SUPPORTED_CODECS
|
||||
|
||||
ffmpeg = shutil.which("ffmpeg")
|
||||
if not ffmpeg:
|
||||
return set()
|
||||
|
||||
try:
|
||||
result: subprocess.CompletedProcess[str] = subprocess.run(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607
|
||||
[ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
|
|
@ -149,9 +157,11 @@ class _BaseBuilder:
|
|||
codec_name: str
|
||||
|
||||
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
|
||||
_ = ctx
|
||||
return []
|
||||
|
||||
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
|
||||
_ = ctx
|
||||
return [*args, "-codec:v", self.codec_name]
|
||||
|
||||
|
||||
|
|
@ -312,4 +322,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
|
|||
"libx264": [],
|
||||
}
|
||||
|
||||
return chains.get(codec, chains["libx264"])
|
||||
return tuple(chains.get(codec, chains["libx264"]))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import asyncio
|
||||
import os
|
||||
import subprocess # type: ignore
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
|
@ -178,6 +178,7 @@ class Segments:
|
|||
)
|
||||
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
while True:
|
||||
chunk: bytes = await proc.stdout.read(1024 * 64)
|
||||
if not chunk:
|
||||
|
|
@ -240,7 +241,9 @@ class Segments:
|
|||
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
|
||||
|
||||
async def stream(self, file: Path, resp: web.StreamResponse) -> None:
|
||||
codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec
|
||||
codec: str = self.vcodec
|
||||
if Segments._cache_initialized and Segments._cached_vcodec:
|
||||
codec = Segments._cached_vcodec
|
||||
if not codec or codec not in SUPPORTED_CODECS:
|
||||
codec: str = select_encoder(self.vcodec or "")
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import anyio
|
||||
import pysubs2
|
||||
|
|
@ -48,7 +49,8 @@ def ms_to_timestamp(ms: int) -> str:
|
|||
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
|
||||
|
||||
|
||||
SubstationFormat.ms_to_timestamp = ms_to_timestamp
|
||||
_substation_format: Any = SubstationFormat
|
||||
_substation_format.ms_to_timestamp = ms_to_timestamp
|
||||
|
||||
|
||||
class Subtitle:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ THUMBNAIL_MISS_TTL = 3600.0
|
|||
|
||||
_LOCK = asyncio.Lock()
|
||||
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
|
||||
|
||||
|
||||
_SEM: asyncio.Semaphore | None = None
|
||||
_SEM_LIMIT: int | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from pathlib import Path
|
|||
|
||||
from aiohttp import web
|
||||
from aiohttp.web import Request, Response
|
||||
from aiohttp.web_response import StreamResponse
|
||||
|
||||
from app.features.streaming.library.m3u8 import M3u8
|
||||
from app.features.streaming.library.playlist import Playlist
|
||||
|
|
@ -32,7 +33,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
file: str | None = request.match_info.get("file")
|
||||
|
||||
if not file:
|
||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||
|
|
@ -83,8 +84,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
mode: str = request.match_info.get("mode")
|
||||
file: str | None = request.match_info.get("file")
|
||||
mode: str | None = request.match_info.get("mode")
|
||||
|
||||
if mode not in ["video", "subtitle"]:
|
||||
return web.json_response(
|
||||
|
|
@ -94,13 +95,14 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
|||
if not file:
|
||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
duration = request.query.get("duration", None)
|
||||
duration: float | None = None
|
||||
duration_arg = request.query.get("duration", None)
|
||||
|
||||
if "subtitle" in mode:
|
||||
if not duration:
|
||||
if not duration_arg:
|
||||
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
duration = float(duration)
|
||||
duration = float(duration_arg)
|
||||
|
||||
base_path: str = config.base_path.rstrip("/")
|
||||
|
||||
|
|
@ -124,6 +126,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
|||
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
|
||||
|
||||
if "subtitle" in mode:
|
||||
if duration is None:
|
||||
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
|
||||
text = await cls.make_subtitle(file=realFile, duration=duration)
|
||||
else:
|
||||
text = await cls.make_stream(file=realFile)
|
||||
|
|
@ -149,7 +153,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
|||
|
||||
|
||||
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
|
||||
async def segments_stream(request: Request, config: Config, app: web.Application) -> Response:
|
||||
async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
|
||||
"""
|
||||
Get the segments.
|
||||
|
||||
|
|
@ -162,9 +166,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
segment: int = request.match_info.get("segment")
|
||||
sd: int = request.query.get("sd")
|
||||
file: str | None = request.match_info.get("file")
|
||||
segment: str | None = request.match_info.get("segment")
|
||||
sd: str | None = request.query.get("sd")
|
||||
vc: int = int(request.query.get("vc", 0))
|
||||
ac: int = int(request.query.get("ac", 0))
|
||||
|
||||
|
|
@ -241,7 +245,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
file: str | None = request.match_info.get("file")
|
||||
|
||||
if not file:
|
||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||
|
|
@ -299,7 +303,7 @@ async def subtitles_manifest_get(request: Request, config: Config, app: web.Appl
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
file: str | None = request.match_info.get("file")
|
||||
|
||||
if not file:
|
||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||
|
|
@ -354,7 +358,7 @@ async def subtitles_track_get(request: Request, config: Config, app: web.Applica
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
file: str | None = request.match_info.get("file")
|
||||
source_format: str | None = request.match_info.get("source_format")
|
||||
|
||||
if not file:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from pathlib import Path
|
|||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from app.features.streaming.library.segments import Segments
|
||||
from app.tests.helpers import get_test_system_temp_root
|
||||
|
|
@ -148,18 +149,18 @@ class _FakeProc:
|
|||
self.killed = True
|
||||
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, fail_with: Exception | None = None) -> None:
|
||||
class _FakeResp(web.StreamResponse):
|
||||
def __init__(self, fail_with: BaseException | None = None) -> None:
|
||||
self.data: bytearray = bytearray()
|
||||
self.eof = False
|
||||
self._exc = fail_with
|
||||
|
||||
async def write(self, data: bytes) -> None:
|
||||
async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
|
||||
if self._exc:
|
||||
raise self._exc
|
||||
self.data.extend(data)
|
||||
|
||||
async def write_eof(self) -> None:
|
||||
async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
|
||||
self.eof = True
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -260,7 +261,7 @@ async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
output = temp_dir / ".jpg"
|
||||
media.write_text("video")
|
||||
|
||||
ff_info = FFProbeResult()
|
||||
ff_info: Any = FFProbeResult()
|
||||
ff_info.metadata = {"duration": "120.0"}
|
||||
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
|
||||
|
||||
|
|
@ -321,7 +322,7 @@ async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
media1.write_text("video")
|
||||
media2.write_text("video")
|
||||
|
||||
ff_info = FFProbeResult()
|
||||
ff_info: Any = FFProbeResult()
|
||||
ff_info.metadata = {"duration": "60.0"}
|
||||
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
|
||||
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# flake8: noqa: ARG004
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
|
||||
|
|
@ -12,6 +11,7 @@ if TYPE_CHECKING:
|
|||
class BaseHandler:
|
||||
@staticmethod
|
||||
async def can_handle(task: HandleTask) -> bool:
|
||||
_ = task
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -24,6 +24,7 @@ class BaseHandler:
|
|||
|
||||
@staticmethod
|
||||
def parse(url: str) -> Any | None:
|
||||
_ = url
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class GenericTaskHandler(BaseHandler):
|
|||
from app.features.tasks.definitions.utils import model_to_schema
|
||||
|
||||
repo = TaskDefinitionsRepository.get_instance()
|
||||
models = await repo.list()
|
||||
models = await repo.all()
|
||||
|
||||
cls._definitions = [model_to_schema(model) for model in models]
|
||||
return cls._definitions
|
||||
|
|
@ -141,7 +141,8 @@ class GenericTaskHandler(BaseHandler):
|
|||
return False
|
||||
|
||||
@staticmethod
|
||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004
|
||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||
_ = config
|
||||
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
|
||||
if not definition:
|
||||
return TaskFailure(message="No generic task definition matched the provided URL.")
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from app.features.tasks.definitions.results import HandleTask, TaskFailure, Task
|
|||
from app.features.ytdlp.extractor import fetch_info
|
||||
from app.features.ytdlp.utils import get_archive_id
|
||||
from app.library.cache import Cache
|
||||
from app.library.config import Config
|
||||
from app.library.log import get_logger
|
||||
|
||||
from ._base_handler import BaseHandler
|
||||
|
|
@ -160,12 +161,14 @@ class RssGenericHandler(BaseHandler):
|
|||
return feed_url, items, real_count
|
||||
|
||||
@staticmethod
|
||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||
_ = config
|
||||
"""
|
||||
Extract items from an RSS/Atom feed.
|
||||
|
||||
Args:
|
||||
task (Task): The task containing the feed URL.
|
||||
config (Config | None): Optional handler configuration.
|
||||
|
||||
Returns:
|
||||
TaskResult | TaskFailure: Extraction result with parsed items or failure information.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import httpx
|
|||
|
||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||
from app.features.ytdlp.utils import get_archive_id
|
||||
from app.library.config import Config
|
||||
from app.library.log import get_logger
|
||||
|
||||
from ._base_handler import BaseHandler
|
||||
|
|
@ -177,7 +178,8 @@ class TverHandler(BaseHandler):
|
|||
return feed_url, items, has_items
|
||||
|
||||
@staticmethod
|
||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||
_ = config
|
||||
series_id: str | None = TverHandler.parse(task.url)
|
||||
if not series_id:
|
||||
return TaskFailure(message="Unrecognized Tver series URL.")
|
||||
|
|
@ -208,10 +210,11 @@ class TverHandler(BaseHandler):
|
|||
if not (url := entry.get("url")):
|
||||
continue
|
||||
|
||||
archive_id: str = entry.get("archive_id")
|
||||
task_items.append(
|
||||
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
|
||||
)
|
||||
archive_id: str | None = entry.get("archive_id")
|
||||
metadata = entry.get("metadata", {})
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
|
||||
|
||||
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
|
||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||
from app.features.ytdlp.utils import get_archive_id
|
||||
from app.library.config import Config
|
||||
from app.library.log import get_logger
|
||||
|
||||
from ._base_handler import BaseHandler
|
||||
|
|
@ -92,7 +93,8 @@ class TwitchHandler(BaseHandler):
|
|||
return feed_url, items, has_items
|
||||
|
||||
@staticmethod
|
||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||
_ = config
|
||||
handle_name: str | None = TwitchHandler.parse(task.url)
|
||||
if not handle_name:
|
||||
return TaskFailure(message="Unrecognized Twitch channel URL.")
|
||||
|
|
@ -122,7 +124,7 @@ class TwitchHandler(BaseHandler):
|
|||
if not (url := entry.get("url")):
|
||||
continue
|
||||
|
||||
archive_id: str = entry.get("archive_id")
|
||||
archive_id: str | None = entry.get("archive_id")
|
||||
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
|
||||
|
||||
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
|
||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||
from app.features.ytdlp.utils import get_archive_id
|
||||
from app.library.config import Config
|
||||
from app.library.log import get_logger
|
||||
|
||||
from ._base_handler import BaseHandler
|
||||
|
|
@ -103,7 +104,8 @@ class YoutubeHandler(BaseHandler):
|
|||
return feed_url, items, real_count
|
||||
|
||||
@staticmethod
|
||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||
_ = config
|
||||
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
|
||||
if not parsed:
|
||||
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
|
||||
|
|
@ -133,7 +135,7 @@ class YoutubeHandler(BaseHandler):
|
|||
if not (url := entry.get("url")):
|
||||
continue
|
||||
|
||||
archive_id: str = entry.get("archive_id")
|
||||
archive_id: str | None = entry.get("archive_id")
|
||||
metadata: dict[str, Any] = {"published": entry.get("published")}
|
||||
|
||||
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
|
||||
|
|
|
|||
|
|
@ -14,20 +14,23 @@ from app.library.Services import Services
|
|||
from app.library.Singleton import Singleton
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
|
||||
from sqlalchemy.engine.result import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.selectable import Select
|
||||
|
||||
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||
|
||||
LOG = get_logger()
|
||||
|
||||
|
||||
class TaskDefinitionsRepository(metaclass=Singleton):
|
||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
||||
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||
self._migrated = False
|
||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
||||
self.session: SessionFactory = session or get_session
|
||||
|
||||
async def run_migrations(self) -> None:
|
||||
if self._migrated:
|
||||
|
|
@ -51,12 +54,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
|
|||
LOG.debug("Refreshing task definitions due to configuration update.")
|
||||
await GenericTaskHandler.refresh_definitions(force=True)
|
||||
|
||||
Services.get_instance().add(__class__.__name__, self)
|
||||
Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
|
||||
EventBus.get_instance().subscribe(
|
||||
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
|
||||
Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
|
||||
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
|
||||
|
||||
async def list(self) -> list[TaskDefinitionModel]:
|
||||
async def all(self) -> list[TaskDefinitionModel]:
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
|
||||
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
|
||||
|
|
@ -115,9 +118,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
|
|||
|
||||
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
|
||||
async with self.session() as session:
|
||||
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
|
||||
model: TaskDefinitionModel = TaskDefinitionModel(**payload)
|
||||
if model.id is not None:
|
||||
model.id = None
|
||||
model.id = None # ty: ignore
|
||||
|
||||
if await self.get_by_name(name=model.name) is not None:
|
||||
msg: str = f"Task definition with name '{model.name}' already exists."
|
||||
|
|
|
|||
|
|
@ -48,10 +48,13 @@ class HandleTask(TaskSchema):
|
|||
if isinstance(ret, tuple):
|
||||
return ret
|
||||
|
||||
archive_file: Path = ret.get("file")
|
||||
items: set[str] = ret.get("items", set())
|
||||
archive_file = ret.get("file")
|
||||
items = ret.get("items", set())
|
||||
if not isinstance(archive_file, Path) or not isinstance(items, set):
|
||||
return (False, "Failed to get archive information.")
|
||||
archive_items = [item for item in items if isinstance(item, str)]
|
||||
|
||||
if len(items) < 1 or not archive_add(archive_file, list(items)):
|
||||
if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
|
||||
return (True, "No new items to mark as downloaded.")
|
||||
|
||||
return (True, f"Task '{self.name}' items marked as downloaded.")
|
||||
|
|
@ -70,10 +73,13 @@ class HandleTask(TaskSchema):
|
|||
if isinstance(ret, tuple):
|
||||
return ret
|
||||
|
||||
archive_file: Path = ret.get("file")
|
||||
items: set[str] = ret.get("items", set())
|
||||
archive_file = ret.get("file")
|
||||
items = ret.get("items", set())
|
||||
if not isinstance(archive_file, Path) or not isinstance(items, set):
|
||||
return (False, "Failed to get archive information.")
|
||||
archive_items = [item for item in items if isinstance(item, str)]
|
||||
|
||||
if len(items) < 1 or not archive_delete(archive_file, list(items)):
|
||||
if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
|
||||
return (True, "No items to remove from archive file.")
|
||||
|
||||
return (True, f"Removed '{self.name}' items from archive file.")
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import random
|
|||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.features.tasks.definitions.handlers._base_handler import BaseHandler
|
||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||
from app.features.tasks.models import TaskModel
|
||||
from app.features.ytdlp.utils import archive_read
|
||||
|
|
@ -27,7 +28,7 @@ LOG = get_logger()
|
|||
|
||||
class TaskHandle:
|
||||
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
|
||||
self._handlers: list[type] = []
|
||||
self._handlers: list[type[BaseHandler]] = []
|
||||
"The available handlers."
|
||||
self._repo: TasksRepository = tasks
|
||||
"The tasks manager."
|
||||
|
|
@ -35,7 +36,7 @@ class TaskHandle:
|
|||
"The scheduler."
|
||||
self._config: Config = config
|
||||
"The configuration."
|
||||
self._task_name: str = f"{__class__.__name__}._dispatcher"
|
||||
self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
|
||||
"The task name for the scheduler."
|
||||
self._queued: dict[str, set[str]] = {}
|
||||
"Queued archive IDs per handler."
|
||||
|
|
@ -45,11 +46,11 @@ class TaskHandle:
|
|||
EventBus.get_instance().subscribe(
|
||||
Events.ITEM_ERROR,
|
||||
self._handle_item_error,
|
||||
f"{__class__.__name__}.item_error",
|
||||
f"{TaskHandle.__name__}.item_error",
|
||||
)
|
||||
|
||||
def load(self) -> None:
|
||||
self._handlers: list[type] = self._discover()
|
||||
self._handlers: list[type[BaseHandler]] = self._discover()
|
||||
|
||||
timer: str = self._config.tasks_handler_timer
|
||||
try:
|
||||
|
|
@ -72,16 +73,16 @@ class TaskHandle:
|
|||
self._scheduler.add(
|
||||
timer=timer,
|
||||
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
|
||||
id=f"{__class__.__name__}._dispatcher",
|
||||
id=f"{TaskHandle.__name__}._dispatcher",
|
||||
)
|
||||
|
||||
async def _dispatcher(self):
|
||||
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
|
||||
|
||||
handler_groups: dict[str, list[tuple[HandleTask, type]]] = {}
|
||||
dispatches: list[tuple[HandleTask, type, asyncio.Task[TaskResult | TaskFailure | None]]] = []
|
||||
handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
|
||||
dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
|
||||
|
||||
tasks: list[TaskModel] = await self._repo.list()
|
||||
tasks: list[TaskModel] = await self._repo.all()
|
||||
|
||||
for task_model in tasks:
|
||||
task: HandleTask = HandleTask.model_validate(task_model)
|
||||
|
|
@ -100,7 +101,7 @@ class TaskHandle:
|
|||
continue
|
||||
|
||||
try:
|
||||
handler: type | None = await self._find_handler(task)
|
||||
handler: type[BaseHandler] | None = await self._find_handler(task)
|
||||
if handler is None:
|
||||
s["u"].append(task.name)
|
||||
continue
|
||||
|
|
@ -181,7 +182,9 @@ class TaskHandle:
|
|||
},
|
||||
)
|
||||
|
||||
async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None:
|
||||
async def _dispatch(
|
||||
self, task: HandleTask, handler: type[BaseHandler], delay: float
|
||||
) -> TaskResult | TaskFailure | None:
|
||||
"""
|
||||
Dispatch a task after a random delay to avoid rate limiting.
|
||||
|
||||
|
|
@ -215,7 +218,7 @@ class TaskHandle:
|
|||
exc_info=(type(exc), exc, exc.__traceback__),
|
||||
)
|
||||
|
||||
async def _find_handler(self, task: HandleTask) -> type | None:
|
||||
async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
|
||||
for cls in self._handlers:
|
||||
try:
|
||||
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
|
||||
|
|
@ -239,9 +242,10 @@ class TaskHandle:
|
|||
async def dispatch(
|
||||
self,
|
||||
task: HandleTask,
|
||||
handler: type | None = None,
|
||||
**kwargs, # noqa: ARG002
|
||||
handler: type[BaseHandler] | None = None,
|
||||
**kwargs,
|
||||
) -> TaskResult | TaskFailure | None:
|
||||
_ = kwargs
|
||||
"""
|
||||
Dispatch a task to the appropriate handler.
|
||||
|
||||
|
|
@ -582,11 +586,11 @@ class TaskHandle:
|
|||
|
||||
return TaskResult(items=list(extraction.items), metadata=combined_metadata)
|
||||
|
||||
def _discover(self) -> list[type]:
|
||||
def _discover(self) -> list[type[BaseHandler]]:
|
||||
"""Discover all available task handlers."""
|
||||
import app.features.tasks.definitions.handlers as handlers_pkg
|
||||
|
||||
handlers: list[type] = []
|
||||
handlers: list[type[BaseHandler]] = []
|
||||
|
||||
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
|
||||
if module_name.startswith("_"):
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
|
|||
from app.features.tasks.definitions.schemas import (
|
||||
Definition,
|
||||
EngineConfig,
|
||||
Parse,
|
||||
RequestConfig,
|
||||
ResponseConfig,
|
||||
TaskDefinition,
|
||||
|
|
@ -30,10 +31,12 @@ def test_build_def_payload():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
|
||||
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
|
||||
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
||||
}
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(),
|
||||
|
|
@ -54,15 +57,17 @@ def test_build_def_container():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"items": {
|
||||
"selector": ".cards .card",
|
||||
"fields": {
|
||||
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
|
||||
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"items": {
|
||||
"selector": ".cards .card",
|
||||
"fields": {
|
||||
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
|
||||
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(),
|
||||
|
|
@ -82,16 +87,18 @@ def test_build_def_json():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "items",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "items",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(type="json"),
|
||||
|
|
@ -112,11 +119,13 @@ def test_parse_items_basic():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
|
||||
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
||||
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
|
||||
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
||||
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
|
||||
}
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(),
|
||||
|
|
@ -154,34 +163,36 @@ def test_parse_items_cards():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"items": {
|
||||
"type": "css",
|
||||
"selector": ".columns .card",
|
||||
"fields": {
|
||||
"link": {
|
||||
"type": "css",
|
||||
"expression": ".card-header a[href]",
|
||||
"attribute": "href",
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"items": {
|
||||
"type": "css",
|
||||
"selector": ".columns .card",
|
||||
"fields": {
|
||||
"link": {
|
||||
"type": "css",
|
||||
"expression": ".card-header a[href]",
|
||||
"attribute": "href",
|
||||
},
|
||||
"title": {
|
||||
"type": "css",
|
||||
"expression": ".card-header a[href]",
|
||||
"attribute": "text",
|
||||
},
|
||||
"poet": {
|
||||
"type": "css",
|
||||
"expression": "footer .card-footer-item:first-child a",
|
||||
"attribute": "text",
|
||||
},
|
||||
"category": {
|
||||
"type": "css",
|
||||
"expression": "footer .card-footer-item:nth-child(2) a",
|
||||
"attribute": "text",
|
||||
},
|
||||
},
|
||||
"title": {
|
||||
"type": "css",
|
||||
"expression": ".card-header a[href]",
|
||||
"attribute": "text",
|
||||
},
|
||||
"poet": {
|
||||
"type": "css",
|
||||
"expression": "footer .card-footer-item:first-child a",
|
||||
"attribute": "text",
|
||||
},
|
||||
"category": {
|
||||
"type": "css",
|
||||
"expression": "footer .card-footer-item:nth-child(2) a",
|
||||
"attribute": "text",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(),
|
||||
|
|
@ -249,17 +260,19 @@ def test_parse_items_json():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "entries",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
"id": {"type": "jsonpath", "expression": "id"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "entries",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
"id": {"type": "jsonpath", "expression": "id"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(type="json"),
|
||||
|
|
@ -297,16 +310,18 @@ async def test_generic_task_handler_inspect(monkeypatch):
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "items",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "items",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(type="json"),
|
||||
|
|
@ -351,16 +366,18 @@ def test_parse_items_json_list():
|
|||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
definition=Definition(
|
||||
parse={
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "[]",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
},
|
||||
parse=Parse.model_validate(
|
||||
{
|
||||
"items": {
|
||||
"type": "jsonpath",
|
||||
"selector": "[]",
|
||||
"fields": {
|
||||
"link": {"type": "jsonpath", "expression": "url"},
|
||||
"title": {"type": "jsonpath", "expression": "title"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
engine=EngineConfig(),
|
||||
request=RequestConfig(),
|
||||
response=ResponseConfig(type="json"),
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ class TestTaskDefinitionsRepository:
|
|||
await repo.create(_sample_definition("Alpha", priority=2))
|
||||
await repo.create(_sample_definition("Beta", priority=1))
|
||||
|
||||
items = await repo.list()
|
||||
items = await repo.all()
|
||||
assert len(items) == 2, "Should return two task definitions"
|
||||
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,17 @@
|
|||
from typing import Any
|
||||
from typing import Any, Literal, overload
|
||||
|
||||
from app.features.tasks.definitions.models import TaskDefinitionModel
|
||||
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
|
||||
|
||||
|
||||
@overload
|
||||
def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
|
||||
|
||||
|
||||
@overload
|
||||
def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
|
||||
|
||||
|
||||
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
|
||||
"""
|
||||
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.
|
||||
|
|
|
|||
|
|
@ -10,20 +10,39 @@ from app.library.log import get_logger
|
|||
from app.library.Singleton import Singleton
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
|
||||
from sqlalchemy.engine.result import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.selectable import Select
|
||||
|
||||
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||
|
||||
LOG = get_logger()
|
||||
|
||||
|
||||
def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
|
||||
model = TaskModel()
|
||||
for key, value in payload.items():
|
||||
if not hasattr(model, key):
|
||||
msg = f"'{key}' is an invalid keyword argument for TaskModel"
|
||||
raise TypeError(msg)
|
||||
setattr(model, key, value)
|
||||
return model
|
||||
|
||||
|
||||
def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
|
||||
if isinstance(payload, TaskModel):
|
||||
return payload
|
||||
return _model_from_payload(payload)
|
||||
|
||||
|
||||
class TasksRepository(metaclass=Singleton):
|
||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
||||
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||
self._migrated = False
|
||||
self.session = session or get_session
|
||||
self.session: SessionFactory = session or get_session
|
||||
|
||||
async def run_migrations(self) -> None:
|
||||
if self._migrated:
|
||||
|
|
@ -38,7 +57,7 @@ class TasksRepository(metaclass=Singleton):
|
|||
def get_instance() -> TasksRepository:
|
||||
return TasksRepository()
|
||||
|
||||
async def list(self) -> list[TaskModel]:
|
||||
async def all(self) -> list[TaskModel]:
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
|
||||
return list(result.scalars().all())
|
||||
|
|
@ -90,7 +109,7 @@ class TasksRepository(metaclass=Singleton):
|
|||
"""Get all enabled tasks."""
|
||||
async with self.session() as session:
|
||||
result: Result[tuple[TaskModel]] = await session.execute(
|
||||
select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712
|
||||
select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
|
@ -102,11 +121,11 @@ class TasksRepository(metaclass=Singleton):
|
|||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def create(self, payload: TaskModel | dict) -> TaskModel:
|
||||
async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
|
||||
async with self.session() as session:
|
||||
model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload
|
||||
model = _coerce_model(payload)
|
||||
if model.id is not None:
|
||||
model.id = None
|
||||
model.id = None # ty: ignore
|
||||
|
||||
if await self.get_by_name(name=model.name) is not None:
|
||||
msg: str = f"Task with name '{model.name}' already exists."
|
||||
|
|
|
|||
|
|
@ -705,8 +705,9 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
|
|||
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
|
||||
if info.get("uploader"):
|
||||
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
|
||||
if info.get("tags", []):
|
||||
for tag in info.get("tags", []):
|
||||
tags = info.get("tags", [])
|
||||
if isinstance(tags, list):
|
||||
for tag in tags:
|
||||
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
|
||||
if info.get("year"):
|
||||
xml_content += f" <year>{info.get('year')}</year>\n"
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
|
|||
pass
|
||||
|
||||
async def _load_tasks(self) -> None:
|
||||
tasks = await self._repo.list()
|
||||
tasks = await self._repo.all()
|
||||
|
||||
for task in tasks:
|
||||
if not task.timer or not task.enabled:
|
||||
|
|
@ -134,9 +134,11 @@ class Tasks(metaclass=Singleton):
|
|||
task_id = task.id
|
||||
task_name = task.name
|
||||
try:
|
||||
if not (task := await self._repo.get(task_id)):
|
||||
current_task = await self._repo.get(task_id)
|
||||
if not current_task:
|
||||
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
|
||||
return
|
||||
task = current_task
|
||||
|
||||
if not task.enabled:
|
||||
LOG.debug(
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ class TestTasksRepository:
|
|||
await repo.create({"name": "Alice", "url": "https://example.com"})
|
||||
await repo.create({"name": "Bob", "url": "https://example.com"})
|
||||
|
||||
items = await repo.list()
|
||||
items = await repo.all()
|
||||
|
||||
assert items[0].name == "Alice", "Should be alphabetically first"
|
||||
assert items[1].name == "Bob", "Should be alphabetically second"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
|
@ -41,13 +42,13 @@ def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
|
||||
|
||||
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
|
||||
request = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
|
||||
mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
|
||||
|
||||
async def _json() -> object:
|
||||
return payload
|
||||
|
||||
request.json = _json # type: ignore[attr-defined]
|
||||
return request
|
||||
mock_request.json = _json
|
||||
return mock_request
|
||||
|
||||
|
||||
class _Notify:
|
||||
|
|
@ -85,7 +86,7 @@ async def test_add_requires_timer_without_handler(repo) -> None:
|
|||
|
||||
assert response.status == web.HTTPBadRequest.status_code
|
||||
assert b"requires a timer" in response.body
|
||||
assert await repo.list() == []
|
||||
assert await repo.all() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -109,7 +110,7 @@ async def test_add_all_or_nothing(repo) -> None:
|
|||
|
||||
assert response.status == web.HTTPBadRequest.status_code
|
||||
assert b"requires a timer" in response.body
|
||||
assert await repo.list() == []
|
||||
assert await repo.all() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -123,7 +124,7 @@ async def test_add_allows_handler_only(repo) -> None:
|
|||
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
|
||||
|
||||
assert response.status == web.HTTPOk.status_code
|
||||
items = await repo.list()
|
||||
items = await repo.all()
|
||||
assert len(items) == 1
|
||||
assert items[0].name == "Handler Only"
|
||||
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ class ExtractorPool(metaclass=Singleton):
|
|||
def attach(self, app: web.Application) -> None:
|
||||
"""Attach the extractor pool to the application (no-op for now)."""
|
||||
app.on_shutdown.append(self.shutdown)
|
||||
Services.get_instance().add(__class__.__name__, self)
|
||||
Services.get_instance().add(type(self).__name__, self)
|
||||
|
||||
def _ensure_initialized(self, config: ExtractorConfig) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -286,7 +286,7 @@ async def convert(request: Request) -> Response:
|
|||
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
try:
|
||||
response = {"opts": {}, "output_template": None, "download_path": None}
|
||||
response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
|
||||
|
||||
data = arg_converter(args, dumps=True)
|
||||
|
||||
|
|
@ -384,6 +384,8 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
|
|||
|
||||
if cache.has(key) and not request.query.get("force", False):
|
||||
data: Any | None = cache.get(key)
|
||||
if data is None:
|
||||
data = {}
|
||||
data["_cached"] = {
|
||||
"status": "hit",
|
||||
"preset": preset,
|
||||
|
|
@ -393,7 +395,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
|
|||
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
|
||||
"expires": data.get("_cached", {}).get("expires", time.time() + 300),
|
||||
}
|
||||
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
||||
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
||||
|
||||
ytdlp_opts: dict = opts.get_all()
|
||||
|
||||
|
|
@ -450,7 +452,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
|
|||
|
||||
cache.set(key=key, value=data, ttl=300)
|
||||
|
||||
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
||||
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
||||
except Exception as e:
|
||||
LOG.exception(
|
||||
"Failed to get video info for '%s': %s.",
|
||||
|
|
@ -486,7 +488,7 @@ async def get_options() -> Response:
|
|||
"""
|
||||
from app.features.ytdlp.ytdlp import ytdlp_options
|
||||
|
||||
return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
|
||||
return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
|
||||
|
||||
|
||||
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
|
||||
|
|
@ -508,7 +510,11 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
|
|||
response = []
|
||||
|
||||
for i, url in enumerate(data):
|
||||
dct = {"index": i, "url": url}
|
||||
dct: dict[str, Any] = {"index": i, "url": url}
|
||||
if not isinstance(url, str):
|
||||
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
|
||||
response.append(dct)
|
||||
continue
|
||||
try:
|
||||
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
|
||||
dct.update(get_archive_id(url))
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ class TestExtractInfo:
|
|||
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["id"] == "test123"
|
||||
assert logs == ["[generic_browser] Browser fallback warning"]
|
||||
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
|
||||
|
|
@ -155,6 +156,7 @@ class TestExtractInfo:
|
|||
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["id"] == "test123"
|
||||
assert logs == ["[generic_browser] Browser fallback warning"]
|
||||
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import importlib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -219,7 +220,8 @@ class TestYTDLP:
|
|||
"""Test _delete_downloaded_files skips cleanup when _interrupted is True."""
|
||||
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
|
||||
ytdlp = YTDLP(params={})
|
||||
ytdlp.to_screen = Mock()
|
||||
mock_obj: Any = ytdlp
|
||||
mock_obj.to_screen = Mock()
|
||||
|
||||
# Set interrupted flag
|
||||
ytdlp._interrupted = True
|
||||
|
|
@ -229,7 +231,7 @@ class TestYTDLP:
|
|||
# Should not call super method
|
||||
mock_super_delete.assert_not_called()
|
||||
# Should show message
|
||||
ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
|
||||
mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
|
||||
# Should return None
|
||||
assert result is None
|
||||
|
||||
|
|
|
|||
|
|
@ -776,7 +776,7 @@ class TestYTDLPCli:
|
|||
from app.features.ytdlp.ytdlp_opts import YTDLPCli
|
||||
|
||||
with pytest.raises(ValueError, match="Expected Item instance"):
|
||||
YTDLPCli(item="not an item") # type: ignore
|
||||
YTDLPCli(item="not an item")
|
||||
|
||||
@patch("app.features.presets.service.Presets")
|
||||
@patch("app.features.ytdlp.ytdlp_opts.Config")
|
||||
|
|
|
|||
|
|
@ -46,7 +46,8 @@ class TestLogWrapper:
|
|||
def test_add_target_type_validation(self) -> None:
|
||||
lw = LogWrapper()
|
||||
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
|
||||
lw.add_target(123) # type: ignore[arg-type]
|
||||
bad: Any = 123
|
||||
lw.add_target(bad)
|
||||
|
||||
def test_add_target_names(self) -> None:
|
||||
lw = LogWrapper()
|
||||
|
|
@ -128,7 +129,7 @@ class TestExtractYtdlpLogs:
|
|||
def test_extract_ytdlp_logs_with_filters(self):
|
||||
"""Test log extraction with filters."""
|
||||
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
|
||||
filters = [re.compile(r"ERROR")]
|
||||
filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
|
||||
result = extract_ytdlp_logs(logs, filters)
|
||||
assert result == ["ERROR: Failed"]
|
||||
|
||||
|
|
@ -312,9 +313,12 @@ class TestGetThumbnail:
|
|||
def test_non_list(self):
|
||||
"""Test that None is returned for non-list input."""
|
||||
|
||||
assert get_thumbnail(None) is None
|
||||
assert get_thumbnail("not a list") is None
|
||||
assert get_thumbnail({"not": "list"}) is None
|
||||
bad_none: Any = None
|
||||
bad_str: Any = "not a list"
|
||||
bad_dict: Any = {"not": "list"}
|
||||
assert get_thumbnail(bad_none) is None
|
||||
assert get_thumbnail(bad_str) is None
|
||||
assert get_thumbnail(bad_dict) is None
|
||||
|
||||
def test_thumbnail_preference(self):
|
||||
"""Test that the thumbnail with highest preference is returned."""
|
||||
|
|
@ -349,6 +353,7 @@ class TestGetThumbnail:
|
|||
]
|
||||
|
||||
result = get_thumbnail(thumbnails)
|
||||
assert result is not None
|
||||
assert result["url"] == "with_pref.jpg"
|
||||
|
||||
def test_all_equal(self):
|
||||
|
|
@ -365,12 +370,15 @@ class TestGetThumbnail:
|
|||
class TestGetExtras:
|
||||
def test_none(self):
|
||||
"""Test that empty dict is returned for None input."""
|
||||
assert get_extras(None) == {}
|
||||
bad: Any = None
|
||||
assert get_extras(bad) == {}
|
||||
|
||||
def test_non_dict(self):
|
||||
"""Test that empty dict is returned for non-dict input."""
|
||||
assert get_extras("not a dict") == {}
|
||||
assert get_extras([]) == {}
|
||||
bad_str: Any = "not a dict"
|
||||
bad_list: Any = []
|
||||
assert get_extras(bad_str) == {}
|
||||
assert get_extras(bad_list) == {}
|
||||
|
||||
def test_extracts_video_information(self):
|
||||
"""Test extracting information from a video entry."""
|
||||
|
|
|
|||
|
|
@ -101,12 +101,12 @@ class LogWrapper:
|
|||
name (str|None): The name of the logging target. Defaults to None.
|
||||
|
||||
"""
|
||||
if not isinstance(target, logging.Logger | Callable):
|
||||
if not isinstance(target, logging.Logger) and not callable(target):
|
||||
msg = "Target must be a logging.Logger instance or a callable."
|
||||
raise TypeError(msg)
|
||||
|
||||
if name is None:
|
||||
name = target.name if isinstance(target, logging.Logger) else target.__name__
|
||||
name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
|
||||
|
||||
self.targets.append(
|
||||
LogTarget(
|
||||
|
|
@ -134,8 +134,9 @@ class LogWrapper:
|
|||
if target.logger:
|
||||
log_kwargs = {**kwargs}
|
||||
log_kwargs.setdefault("stacklevel", 3)
|
||||
if isinstance(target.target, logging.Logger):
|
||||
target.target.log(level, msg, *args, **log_kwargs)
|
||||
else:
|
||||
elif callable(target.target):
|
||||
target.target(level, msg, *args, **kwargs)
|
||||
|
||||
def debug(self, msg, *args, **kwargs):
|
||||
|
|
@ -179,13 +180,17 @@ def arg_converter(
|
|||
|
||||
create_parser = yt_dlp.options.create_parser
|
||||
|
||||
def _default_opts(args: str):
|
||||
def _default_opts(args: str | list[str]):
|
||||
patched_parser = create_parser()
|
||||
|
||||
def patched_create_parser():
|
||||
return patched_parser
|
||||
|
||||
try:
|
||||
yt_dlp.options.create_parser = lambda: patched_parser
|
||||
yt_dlp.options.__dict__["create_parser"] = patched_create_parser
|
||||
return yt_dlp.parse_options(args)
|
||||
finally:
|
||||
yt_dlp.options.create_parser = create_parser
|
||||
yt_dlp.options.__dict__["create_parser"] = create_parser
|
||||
|
||||
apply_ytdlp_patches()
|
||||
|
||||
|
|
@ -238,7 +243,7 @@ def arg_converter(
|
|||
return diff
|
||||
|
||||
|
||||
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
|
||||
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
|
||||
"""
|
||||
Extract yt-dlp log lines matching built-in filters plus any extras.
|
||||
|
||||
|
|
@ -332,7 +337,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
|
|||
return _DATA.YTDLP_INFO_CLS
|
||||
|
||||
|
||||
def get_thumbnail(thumbnails: list) -> str | None:
|
||||
def get_thumbnail(thumbnails: list) -> dict | None:
|
||||
"""
|
||||
Extract thumbnail URL from a yt-dlp entry.
|
||||
|
||||
|
|
@ -340,7 +345,7 @@ def get_thumbnail(thumbnails: list) -> str | None:
|
|||
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
|
||||
|
||||
Returns:
|
||||
str | None: The thumbnail URL if available, otherwise None.
|
||||
dict | None: The best thumbnail dict if available, otherwise None.
|
||||
|
||||
"""
|
||||
if not thumbnails or not isinstance(thumbnails, list):
|
||||
|
|
@ -381,7 +386,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
|
|||
extras[f"playlist_{property}"] = val
|
||||
|
||||
if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
|
||||
extras["thumbnail"] = thumbnail.get("url")
|
||||
extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
|
||||
elif thumbnail := entry.get("thumbnail"):
|
||||
extras["thumbnail"] = thumbnail
|
||||
|
||||
|
|
@ -439,7 +444,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
|
|||
}
|
||||
|
||||
"""
|
||||
idDict: dict[str, None] = {
|
||||
idDict: dict[str, str | None] = {
|
||||
"id": None,
|
||||
"ie_key": None,
|
||||
"archive_id": None,
|
||||
|
|
|
|||
|
|
@ -50,12 +50,12 @@ class ARGSMerger:
|
|||
"""
|
||||
return str(self)
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
def as_dict(self) -> list[str]:
|
||||
"""
|
||||
Get all the options as a dict.
|
||||
Get all the options as a shell argument list.
|
||||
|
||||
Returns:
|
||||
dict: The options as a dict
|
||||
list[str]: The options as shell arguments.
|
||||
|
||||
"""
|
||||
return shlex.split(shlex.join(self.args))
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class BackgroundWorker(metaclass=Singleton):
|
|||
"The queue to hold the tasks."
|
||||
self.running = True
|
||||
"Whether the background worker is running or not."
|
||||
self.thread: threading.Thread = None
|
||||
self.thread: threading.Thread | None = None
|
||||
"The thread that runs the background worker."
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -49,7 +49,8 @@ class BackgroundWorker(metaclass=Singleton):
|
|||
try:
|
||||
LOG.debug("Stopping background worker thread.")
|
||||
self.queue.put((CloseThread, (), {}))
|
||||
self.thread.join(timeout=5)
|
||||
if self.thread:
|
||||
self.thread.join(timeout=5)
|
||||
LOG.debug("Background worker thread has been shut down.")
|
||||
except Exception as e:
|
||||
LOG.exception(
|
||||
|
|
|
|||
|
|
@ -1,14 +1,17 @@
|
|||
import base64
|
||||
import hmac
|
||||
import inspect
|
||||
from collections.abc import Awaitable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import anyio
|
||||
from aiohttp import web
|
||||
from aiohttp.web import Request, RequestHandler, Response
|
||||
from aiohttp.typedefs import Handler, Middleware
|
||||
from aiohttp.web import Request, Response
|
||||
from aiohttp.web_log import AccessLogger
|
||||
from aiohttp.web_request import BaseRequest
|
||||
from aiohttp.web_response import StreamResponse
|
||||
|
||||
from app.library.log import get_logger
|
||||
from app.library.Services import Services
|
||||
|
|
@ -24,12 +27,12 @@ LOG = get_logger("http")
|
|||
|
||||
|
||||
class HttpAccessLogger(AccessLogger):
|
||||
def log(self, request: Request, response: Response, time: float) -> None:
|
||||
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
|
||||
try:
|
||||
fmt_info = self._format_line(request, response, time)
|
||||
|
||||
values: list[object] = []
|
||||
extra: dict[str, object] = {"elapsed_ms": round(time * 1000.0, 2)}
|
||||
extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
|
||||
for key, value in fmt_info:
|
||||
values.append(value)
|
||||
|
||||
|
|
@ -37,7 +40,7 @@ class HttpAccessLogger(AccessLogger):
|
|||
extra[key] = value
|
||||
else:
|
||||
parent, child = key
|
||||
group = extra.get(parent, {})
|
||||
group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment]
|
||||
if not isinstance(group, dict):
|
||||
group = {}
|
||||
group[child] = value
|
||||
|
|
@ -177,7 +180,7 @@ class HttpAPI:
|
|||
registered_options.append(route.path)
|
||||
|
||||
@staticmethod
|
||||
def basic_auth(username: str, password: str) -> Awaitable:
|
||||
def basic_auth(username: str, password: str) -> Middleware:
|
||||
"""
|
||||
Middleware to handle basic authentication.
|
||||
|
||||
|
|
@ -191,7 +194,7 @@ class HttpAPI:
|
|||
"""
|
||||
|
||||
@web.middleware
|
||||
async def auth_handler(request: Request, handler: RequestHandler) -> Response:
|
||||
async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
|
||||
# if OPTIONS request, skip auth
|
||||
if "OPTIONS" == request.method:
|
||||
return await handler(request)
|
||||
|
|
@ -204,7 +207,7 @@ class HttpAPI:
|
|||
auth_cookie = request.cookies.get("auth")
|
||||
if auth_cookie is not None:
|
||||
try:
|
||||
data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key)
|
||||
data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
|
||||
if data is not None:
|
||||
data = base64.b64encode(data.encode()).decode()
|
||||
auth_header = f"Basic {data}"
|
||||
|
|
@ -250,7 +253,7 @@ class HttpAPI:
|
|||
},
|
||||
)
|
||||
|
||||
response: Response = await handler(request)
|
||||
response: StreamResponse = await handler(request)
|
||||
|
||||
contentType: str | None = response.headers.get("content-type", None)
|
||||
if contentType and not contentType.startswith("text/html"):
|
||||
|
|
@ -261,7 +264,7 @@ class HttpAPI:
|
|||
"auth",
|
||||
encrypt_data(
|
||||
f"{user_name}:{user_password}",
|
||||
key=Config.get_instance().secret_key,
|
||||
key=cast("bytes", Config.get_instance().secret_key),
|
||||
),
|
||||
max_age=60 * 60 * 24 * 7,
|
||||
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
|
||||
|
|
@ -280,9 +283,9 @@ class HttpAPI:
|
|||
return auth_handler
|
||||
|
||||
@staticmethod
|
||||
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable:
|
||||
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
|
||||
@web.middleware
|
||||
async def middleware_handler(request: Request, handler: RequestHandler) -> Response:
|
||||
async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
|
||||
static_path = str(app.router["download_static"].url_for(filename=""))
|
||||
if request.path.startswith(static_path):
|
||||
realFile, status = get_file(
|
||||
|
|
@ -301,12 +304,13 @@ class HttpAPI:
|
|||
},
|
||||
)
|
||||
|
||||
response: Response = await handler(request)
|
||||
response: StreamResponse = await handler(request)
|
||||
|
||||
contentType: str = str(response.headers.get("content-type", ""))
|
||||
if contentType.startswith("text/html") and getattr(response, "_path", None):
|
||||
rewrite_path: str = base_path.rstrip("/")
|
||||
async with await anyio.open_file(response._path, "rb") as f:
|
||||
response_path = cast("Any", response)._path
|
||||
async with await anyio.open_file(response_path, "rb") as f:
|
||||
content = await f.read()
|
||||
content: str = (
|
||||
content.decode("utf-8")
|
||||
|
|
@ -339,7 +343,7 @@ class HttpAPI:
|
|||
|
||||
return middleware_handler
|
||||
|
||||
async def _handle(self, handler: RequestHandler, request: Request) -> Response:
|
||||
async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
|
||||
"""
|
||||
Call the handler with the request and return the response.
|
||||
|
||||
|
|
|
|||
|
|
@ -104,19 +104,20 @@ class HttpSocket:
|
|||
}
|
||||
)
|
||||
|
||||
self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit")
|
||||
self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
|
||||
|
||||
@staticmethod
|
||||
def ws_event(func): # type: ignore
|
||||
def ws_event(func):
|
||||
"""
|
||||
Decorator to mark a method as a socket event.
|
||||
"""
|
||||
|
||||
@functools.wraps(func) # type: ignore
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
return await func(*args, **kwargs) # type: ignore
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
wrapper._ws_event = func.__name__ # type: ignore
|
||||
wrapper_with_event: Any = wrapper
|
||||
wrapper_with_event._ws_event = func.__name__
|
||||
return wrapper
|
||||
|
||||
async def on_shutdown(self, _: web.Application):
|
||||
|
|
@ -134,9 +135,11 @@ class HttpSocket:
|
|||
if not (data and data.data):
|
||||
return
|
||||
|
||||
await Services.get_instance().get("queue").add(item=Item.format(data.data))
|
||||
queue = Services.get_instance().get("queue")
|
||||
if queue is not None:
|
||||
await queue.add(item=Item.format(data.data))
|
||||
|
||||
self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add")
|
||||
self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
|
||||
|
||||
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
|
||||
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ class Item:
|
|||
bool: True if the item has extras, False otherwise.
|
||||
|
||||
"""
|
||||
return self.extras and len(self.extras) > 0
|
||||
return bool(self.extras and len(self.extras) > 0)
|
||||
|
||||
def add_extras(self, key: str, value: Any) -> None:
|
||||
"""
|
||||
|
|
@ -117,7 +117,7 @@ class Item:
|
|||
bool: True if the item has command options for yt, False otherwise.
|
||||
|
||||
"""
|
||||
return self.cli and len(self.cli) > 2
|
||||
return bool(self.cli and len(self.cli) > 2)
|
||||
|
||||
@staticmethod
|
||||
def _default_preset() -> str:
|
||||
|
|
@ -161,7 +161,7 @@ class Item:
|
|||
Item: The formatted item.
|
||||
|
||||
"""
|
||||
url: str = item.get("url")
|
||||
url = item.get("url")
|
||||
|
||||
if not url or not isinstance(url, str):
|
||||
msg = "url param is required."
|
||||
|
|
@ -173,7 +173,7 @@ class Item:
|
|||
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
|
||||
url = f"https://www.youtube.com/watch?v={url}"
|
||||
|
||||
data: dict[str, str] = {"url": url}
|
||||
data: dict[str, Any] = {"url": url}
|
||||
|
||||
preset: str | None = item.get("preset")
|
||||
if preset and isinstance(preset, str) and preset != Item._default_preset():
|
||||
|
|
@ -185,24 +185,28 @@ class Item:
|
|||
|
||||
data["preset"] = preset
|
||||
|
||||
if item.get("folder") and isinstance(item.get("folder"), str):
|
||||
data["folder"] = item.get("folder")
|
||||
folder = item.get("folder")
|
||||
if isinstance(folder, str) and folder:
|
||||
data["folder"] = folder
|
||||
|
||||
if item.get("cookies") and isinstance(item.get("cookies"), str):
|
||||
data["cookies"] = item.get("cookies")
|
||||
cookies = item.get("cookies")
|
||||
if isinstance(cookies, str) and cookies:
|
||||
data["cookies"] = cookies
|
||||
|
||||
if item.get("template") and isinstance(item.get("template"), str):
|
||||
data["template"] = item.get("template")
|
||||
template = item.get("template")
|
||||
if isinstance(template, str) and template:
|
||||
data["template"] = template
|
||||
|
||||
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
|
||||
data["auto_start"] = bool(item.get("auto_start"))
|
||||
if isinstance(item.get("auto_start"), bool):
|
||||
data["auto_start"] = item["auto_start"]
|
||||
|
||||
extras: dict | None = item.get("extras")
|
||||
if extras and isinstance(extras, dict) and len(extras) > 0:
|
||||
if isinstance(extras, dict) and len(extras) > 0:
|
||||
data["extras"] = extras
|
||||
|
||||
if item.get("requeued") and isinstance(item.get("requeued"), bool):
|
||||
data["requeued"] = item.get("requeued")
|
||||
requeued = item.get("requeued")
|
||||
if isinstance(requeued, bool):
|
||||
data["requeued"] = requeued
|
||||
|
||||
cli: str | None = item.get("cli")
|
||||
if cli and len(cli) > 2:
|
||||
|
|
@ -574,8 +578,10 @@ class ItemDTO:
|
|||
self.is_archivable = bool(self._archive_file)
|
||||
if not self.is_archivable:
|
||||
self.is_archived = False
|
||||
else:
|
||||
elif self.archive_id and self._archive_file:
|
||||
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
|
||||
else:
|
||||
self.is_archived = False
|
||||
|
||||
def get_archive_file(self) -> str | None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
|
|||
|
||||
|
||||
class Packages:
|
||||
def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
|
||||
def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
|
||||
from_env: list[str] = env.split() if env else []
|
||||
from_file = []
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from aiocron import Cron
|
||||
from aiohttp import web
|
||||
|
|
@ -66,7 +68,7 @@ class Scheduler(metaclass=Singleton):
|
|||
if data and data.data:
|
||||
self.add(**data.data)
|
||||
|
||||
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add")
|
||||
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
|
||||
|
||||
def get_all(self) -> dict[str, Cron]:
|
||||
"""Return the jobs."""
|
||||
|
|
@ -99,7 +101,7 @@ class Scheduler(metaclass=Singleton):
|
|||
return id in self._jobs
|
||||
|
||||
def add(
|
||||
self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None
|
||||
self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
|
||||
) -> str:
|
||||
"""
|
||||
Add a job to the schedule.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import inspect
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
|
||||
|
||||
|
|
@ -117,7 +118,7 @@ class Services(metaclass=Singleton):
|
|||
|
||||
return candidates[0]
|
||||
|
||||
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
|
||||
def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
|
||||
sig: inspect.Signature = inspect.signature(handler)
|
||||
|
||||
try:
|
||||
|
|
@ -169,10 +170,10 @@ class Services(metaclass=Singleton):
|
|||
|
||||
return resolved
|
||||
|
||||
async def handle_async(self, handler: callable, **kwargs) -> Any:
|
||||
async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
|
||||
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
|
||||
return await handler(**resolved)
|
||||
|
||||
def handle_sync(self, handler: callable, **kwargs) -> Any:
|
||||
def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
|
||||
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
|
||||
return handler(**resolved)
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ class TerminalSessionManager(metaclass=Singleton):
|
|||
self._cleanup_job_id = Scheduler.get_instance().add(
|
||||
timer=CLEANUP_SCHEDULE,
|
||||
func=self.cleanup,
|
||||
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
|
||||
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
|
||||
)
|
||||
|
||||
async def on_startup(self, _: web.Application) -> None:
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ class UpdateChecker(metaclass=Singleton):
|
|||
self._notify.subscribe(
|
||||
event=Events.STARTED,
|
||||
callback=event_handler,
|
||||
name=f"{__class__.__name__}.{__class__.attach.__name__}",
|
||||
name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
|
||||
)
|
||||
|
||||
self._schedule_check()
|
||||
|
|
@ -108,7 +108,7 @@ class UpdateChecker(metaclass=Singleton):
|
|||
self._job_id = self._scheduler.add(
|
||||
timer=timer,
|
||||
func=lambda: asyncio.create_task(self.check_for_updates()),
|
||||
id=f"{__class__.__name__}.{self.check_for_updates.__name__}",
|
||||
id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
|
||||
)
|
||||
|
||||
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
|
|||
|
||||
|
||||
class FileLogFormatter(logging.Formatter):
|
||||
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
|
||||
def formatTime(self, record, datefmt=None): # noqa: N802
|
||||
_ = datefmt
|
||||
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
|
||||
|
||||
|
||||
|
|
@ -78,7 +79,8 @@ class JsonLogFormatter(logging.Formatter):
|
|||
|
||||
return json.dumps(data, ensure_ascii=False, default=str)
|
||||
|
||||
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
|
||||
def formatTime(self, record, datefmt=None): # noqa: N802
|
||||
_ = datefmt
|
||||
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -210,7 +212,7 @@ class JsonLogFormatter(logging.Formatter):
|
|||
return source
|
||||
|
||||
|
||||
def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
|
||||
def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
|
||||
"""
|
||||
Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
|
||||
Supports both synchronous and asynchronous functions.
|
||||
|
|
@ -280,8 +282,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
|
|||
|
||||
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
|
||||
|
||||
async_wrapper.cache_clear = cache_clear
|
||||
async_wrapper.cache_info = cache_info
|
||||
async_wrapper_cache: Any = async_wrapper
|
||||
async_wrapper_cache.cache_clear = cache_clear
|
||||
async_wrapper_cache.cache_info = cache_info
|
||||
return async_wrapper
|
||||
|
||||
# For sync functions, use the original implementation
|
||||
|
|
@ -309,8 +312,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
|
|||
return result
|
||||
|
||||
# expose cache_clear, cache_info
|
||||
sync_wrapper.cache_clear = cached.cache_clear
|
||||
sync_wrapper.cache_info = cached.cache_info
|
||||
sync_wrapper_cache: Any = sync_wrapper
|
||||
sync_wrapper_cache.cache_clear = cached.cache_clear
|
||||
sync_wrapper_cache.cache_info = cached.cache_info
|
||||
return sync_wrapper
|
||||
|
||||
return decorator
|
||||
|
|
@ -381,7 +385,12 @@ def _is_safe_key(key: Any) -> bool:
|
|||
|
||||
|
||||
def merge_dict(
|
||||
source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None
|
||||
source: dict,
|
||||
destination: dict,
|
||||
max_depth: int = 50,
|
||||
max_list_size: int = 10000,
|
||||
_depth: int = 0,
|
||||
_seen: set | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Merge data from source into destination.
|
||||
|
|
@ -472,7 +481,7 @@ def merge_dict(
|
|||
return destination_copy
|
||||
|
||||
|
||||
def check_id(file: Path) -> bool | str:
|
||||
def check_id(file: Path) -> bool | Path:
|
||||
"""
|
||||
Check if we are able to get an id from the file name.
|
||||
if so check if any video file with the same id exists.
|
||||
|
|
@ -491,6 +500,8 @@ def check_id(file: Path) -> bool | str:
|
|||
return False
|
||||
|
||||
id: str | None = match.groupdict().get("id")
|
||||
if id is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
if not file.parent.exists():
|
||||
|
|
@ -545,9 +556,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
|
|||
from yarl import URL
|
||||
|
||||
parsed_url = URL(url)
|
||||
except ValueError:
|
||||
except ValueError as exc:
|
||||
msg = "Invalid URL."
|
||||
raise ValueError(msg) # noqa: B904
|
||||
raise ValueError(msg) from exc
|
||||
|
||||
# Check allowed schemes
|
||||
if parsed_url.scheme not in ["http", "https"]:
|
||||
|
|
@ -596,7 +607,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
|
|||
|
||||
|
||||
@timed_lru_cache(ttl_seconds=60, max_size=256)
|
||||
def get_file_sidecar(file: Path | None = None) -> dict[dict]:
|
||||
def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
|
||||
"""
|
||||
Get sidecar files for the given file.
|
||||
|
||||
|
|
@ -925,7 +936,7 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
|
|||
download_path = Path(download_path)
|
||||
|
||||
try:
|
||||
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
|
||||
realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
|
||||
if realFile.exists():
|
||||
return (realFile, 200)
|
||||
except Exception as e:
|
||||
|
|
@ -936,11 +947,11 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
|
|||
)
|
||||
return (Path(file), 404)
|
||||
|
||||
possibleFile: bool | str = check_id(file=realFile)
|
||||
if not possibleFile:
|
||||
possibleFile: bool | Path = check_id(file=realFile)
|
||||
if not isinstance(possibleFile, Path):
|
||||
return (realFile, 404)
|
||||
|
||||
return (Path(possibleFile), 302)
|
||||
return (possibleFile, 302)
|
||||
|
||||
|
||||
def encrypt_data(data: str, key: bytes) -> str:
|
||||
|
|
@ -962,7 +973,7 @@ def encrypt_data(data: str, key: bytes) -> str:
|
|||
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
|
||||
|
||||
|
||||
def decrypt_data(data: str, key: bytes) -> str:
|
||||
def decrypt_data(data: str, key: bytes) -> str | None:
|
||||
"""
|
||||
Decrypts AES-GCM encrypted data
|
||||
|
||||
|
|
@ -975,8 +986,8 @@ def decrypt_data(data: str, key: bytes) -> str:
|
|||
|
||||
"""
|
||||
try:
|
||||
data = base64.urlsafe_b64decode(data)
|
||||
iv, ciphertext, tag = data[:12], data[12:-16], data[-16:]
|
||||
raw: bytes = base64.urlsafe_b64decode(data)
|
||||
iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
|
||||
cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
|
||||
plaintext = cipher.decrypt_and_verify(ciphertext, tag)
|
||||
return plaintext.decode()
|
||||
|
|
@ -987,7 +998,7 @@ def decrypt_data(data: str, key: bytes) -> str:
|
|||
def get(
|
||||
data: dict | list,
|
||||
path: str | list | None = None,
|
||||
default: any = None,
|
||||
default: Any = None,
|
||||
separator=".",
|
||||
):
|
||||
"""
|
||||
|
|
@ -1111,7 +1122,7 @@ def get_files(
|
|||
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
|
||||
exc_info=True,
|
||||
)
|
||||
return []
|
||||
return [], 0
|
||||
|
||||
try:
|
||||
dir_path.relative_to(base_path)
|
||||
|
|
@ -1122,7 +1133,7 @@ def get_files(
|
|||
base_path,
|
||||
extra={"path": str(dir_path), "base_path": str(base_path)},
|
||||
)
|
||||
return []
|
||||
return [], 0
|
||||
|
||||
if not str(dir_path).startswith(str(base_path)):
|
||||
LOG.warning(
|
||||
|
|
@ -1131,7 +1142,7 @@ def get_files(
|
|||
base_path,
|
||||
extra={"path": str(dir_path), "base_path": str(base_path)},
|
||||
)
|
||||
return []
|
||||
return [], 0
|
||||
|
||||
if not dir_path.is_dir():
|
||||
LOG.warning(
|
||||
|
|
@ -1139,7 +1150,7 @@ def get_files(
|
|||
dir_path,
|
||||
extra={"path": str(dir_path), "base_path": str(base_path)},
|
||||
)
|
||||
return []
|
||||
return [], 0
|
||||
|
||||
contents: list = []
|
||||
for file in dir_path.iterdir():
|
||||
|
|
@ -1298,7 +1309,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
|
|||
try:
|
||||
from http.cookiejar import MozillaCookieJar
|
||||
|
||||
cookies = MozillaCookieJar(str(file), None, None)
|
||||
cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
|
||||
cookies.load()
|
||||
|
||||
return (True, cookies)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,20 @@
|
|||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
|
||||
if key not in data or data[key] is None:
|
||||
return False, None
|
||||
return True, data[key]
|
||||
|
||||
|
||||
def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
|
||||
if key not in data:
|
||||
return False
|
||||
del data[key]
|
||||
return True
|
||||
|
||||
|
||||
def get_value(value: Any) -> Any:
|
||||
"""
|
||||
Return the result of calling `value` if it is callable, else return `value`.
|
||||
|
|
@ -91,14 +105,19 @@ def ag(
|
|||
return val
|
||||
return get_value(default)
|
||||
key = path
|
||||
if isinstance(array, dict) and key in array and array[key] is not None:
|
||||
return array[key]
|
||||
if isinstance(array, dict):
|
||||
found, value = _dict_value(array, key)
|
||||
if found:
|
||||
return value
|
||||
if not isinstance(key, str) or separator not in key:
|
||||
return get_value(default)
|
||||
current = array
|
||||
for segment in key.split(separator):
|
||||
if isinstance(current, dict) and segment in current and current[segment] is not None:
|
||||
current = current[segment]
|
||||
if isinstance(current, dict):
|
||||
found, value = _dict_value(current, segment)
|
||||
if not found:
|
||||
return get_value(default)
|
||||
current = value
|
||||
elif isinstance(current, list):
|
||||
try:
|
||||
idx = int(segment)
|
||||
|
|
@ -150,7 +169,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
|
|||
except TypeError:
|
||||
return False
|
||||
if isinstance(data, dict):
|
||||
if path in data and data[path] is not None:
|
||||
found, _ = _dict_value(data, path)
|
||||
if found:
|
||||
return True
|
||||
elif isinstance(data, list) and isinstance(path, int):
|
||||
return 0 <= path < len(data) and data[path] is not None
|
||||
|
|
@ -158,8 +178,11 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
|
|||
segments = path_str.split(separator)
|
||||
current = data
|
||||
for seg in segments:
|
||||
if isinstance(current, dict) and seg in current and current[seg] is not None:
|
||||
current = current[seg]
|
||||
if isinstance(current, dict):
|
||||
found, value = _dict_value(current, seg)
|
||||
if not found:
|
||||
return False
|
||||
current = value
|
||||
elif isinstance(current, list):
|
||||
try:
|
||||
idx = int(seg)
|
||||
|
|
@ -189,17 +212,16 @@ def ag_delete(
|
|||
The modified structure.
|
||||
|
||||
"""
|
||||
if isinstance(path, list | tuple):
|
||||
for p in path:
|
||||
ag_delete(data, p, separator)
|
||||
return data
|
||||
if not isinstance(data, dict | list):
|
||||
try:
|
||||
data = vars(data)
|
||||
except TypeError:
|
||||
return data # type: ignore
|
||||
if isinstance(data, dict) and path in data:
|
||||
del data[path]
|
||||
if isinstance(path, list | tuple):
|
||||
for p in path:
|
||||
ag_delete(data, p, separator)
|
||||
return data
|
||||
if isinstance(data, dict) and _dict_delete(data, path):
|
||||
return data
|
||||
if isinstance(data, list) and isinstance(path, int):
|
||||
if 0 <= path < len(data):
|
||||
|
|
@ -210,8 +232,12 @@ def ag_delete(
|
|||
last = segments.pop()
|
||||
current = data
|
||||
for seg in segments:
|
||||
if isinstance(current, dict) and seg in current:
|
||||
current = current[seg]
|
||||
if isinstance(current, dict):
|
||||
found, value = _dict_value(current, seg)
|
||||
if not found:
|
||||
current = None
|
||||
break
|
||||
current = value
|
||||
elif isinstance(current, list):
|
||||
try:
|
||||
idx = int(seg)
|
||||
|
|
@ -228,8 +254,8 @@ def ag_delete(
|
|||
break
|
||||
if current is None:
|
||||
return data
|
||||
if isinstance(current, dict) and last in current:
|
||||
del current[last]
|
||||
if isinstance(current, dict):
|
||||
_dict_delete(current, last)
|
||||
elif isinstance(current, list):
|
||||
try:
|
||||
idx = int(last)
|
||||
|
|
@ -246,10 +272,12 @@ def run_tests():
|
|||
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
|
||||
|
||||
def test_ag_set_overwrite_error():
|
||||
error = ""
|
||||
try:
|
||||
ag_set({"a": 1}, "a.b", 2)
|
||||
except RuntimeError as e:
|
||||
assert "Cannot set value at path" in str(e) # noqa: PT017
|
||||
error = str(e)
|
||||
assert "Cannot set value at path" in error
|
||||
|
||||
def test_ag_get_value_callable_and_value():
|
||||
assert get_value(5) == 5
|
||||
|
|
@ -320,9 +348,9 @@ def run_tests():
|
|||
]:
|
||||
try:
|
||||
test()
|
||||
print(f"PASS: {test.__name__}") # noqa: T201
|
||||
sys.stdout.write(f"PASS: {test.__name__}\n")
|
||||
except AssertionError:
|
||||
print(f"FAIL {test.__name__}") # noqa: T201
|
||||
sys.stdout.write(f"FAIL {test.__name__}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ class Cache(metaclass=ThreadSafe):
|
|||
Scheduler.get_instance().add(
|
||||
timer="* * * * *",
|
||||
func=self.cleanup,
|
||||
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
|
||||
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
|
||||
)
|
||||
|
||||
def set(self, key: str, value: Any, ttl: float | None = None) -> None:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
import http.cookiejar
|
||||
from abc import ABC
|
||||
from collections.abc import Callable
|
||||
from typing import Any, ClassVar
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
from yt_dlp.networking.common import (
|
||||
_REQUEST_HANDLERS,
|
||||
|
|
@ -223,11 +223,13 @@ class CFSolverRH(RequestHandler, ABC):
|
|||
try:
|
||||
response: Response = director.send(request)
|
||||
except HTTPError as error:
|
||||
if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers):
|
||||
if error.response and is_cf_challenge(
|
||||
getattr(error.response, "status", None), cast("Any", error.response.headers)
|
||||
):
|
||||
return self._retry_with_clearance(request, error.response, director)
|
||||
raise
|
||||
|
||||
if is_cf_challenge(getattr(response, "status", None), response.headers):
|
||||
if is_cf_challenge(getattr(response, "status", None), cast("Any", response.headers)):
|
||||
return self._retry_with_clearance(request, response, director)
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
# flake8: noqa: S310
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.library.log import get_logger
|
||||
|
||||
from .cache import Cache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
|
||||
CACHE: Cache = Cache()
|
||||
LOG = get_logger()
|
||||
|
||||
|
|
@ -57,7 +59,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
|
|||
if user_agent:
|
||||
payload.setdefault("headers", {})["User-Agent"] = user_agent
|
||||
|
||||
req = urllib.request.Request(
|
||||
req = urllib.request.Request( # noqa: S310
|
||||
endpoint,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
|
|
@ -68,7 +70,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
|
|||
"Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint}
|
||||
)
|
||||
start_time = time.time()
|
||||
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
|
||||
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: # noqa: S310
|
||||
result = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
if "ok" != result.get("status"):
|
||||
|
|
@ -98,7 +100,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
|
|||
return CACHE.get(host)
|
||||
|
||||
|
||||
def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool:
|
||||
def is_cf_challenge(status: int | None, headers: Mapping[str, Any] | None) -> bool:
|
||||
"""
|
||||
Determine whether a response indicates a Cloudflare challenge.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import logging
|
|||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
|
|
@ -30,7 +31,7 @@ APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = (
|
|||
if TYPE_CHECKING:
|
||||
from subprocess import CompletedProcess
|
||||
|
||||
SUPPORTED_CODECS: tuple[str] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
|
||||
SUPPORTED_CODECS: tuple[str, ...] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
|
||||
"Supported encoder names in order of preference."
|
||||
|
||||
|
||||
|
|
@ -179,7 +180,7 @@ class Config(metaclass=Singleton):
|
|||
file_logging: bool = True
|
||||
"Enable file logging."
|
||||
|
||||
secret_key: str
|
||||
secret_key: str | bytes
|
||||
"The secret key to use for the application."
|
||||
|
||||
tasks_handler_timer: str = "15 */1 * * *"
|
||||
|
|
@ -377,7 +378,7 @@ class Config(metaclass=Singleton):
|
|||
LOG = get_logger()
|
||||
|
||||
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
|
||||
envFile: str = Path(self.config_path) / ".env"
|
||||
envFile: Path = Path(self.config_path) / ".env"
|
||||
|
||||
if envFile.exists():
|
||||
LOG.info("Loading environment variables from '%s'.", envFile)
|
||||
|
|
@ -388,7 +389,7 @@ class Config(metaclass=Singleton):
|
|||
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
|
||||
Path(baseDefaultPath) / "var" / "downloads"
|
||||
)
|
||||
self.app_path = Path(__file__).parent.parent.absolute()
|
||||
self.app_path = str(Path(__file__).parent.parent.absolute())
|
||||
|
||||
for k, v in self._get_attributes().items():
|
||||
if k.startswith("_") or k in self._manual_vars:
|
||||
|
|
@ -501,7 +502,7 @@ class Config(metaclass=Singleton):
|
|||
handler.setFormatter(formatter)
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
key_file: str = Path(self.config_path) / "secret.key"
|
||||
key_file: Path = Path(self.config_path) / "secret.key"
|
||||
|
||||
if key_file.exists() and key_file.stat().st_size > 2:
|
||||
with open(key_file, "rb") as f:
|
||||
|
|
@ -511,7 +512,7 @@ class Config(metaclass=Singleton):
|
|||
with open(key_file, "wb") as f:
|
||||
f.write(self.secret_key)
|
||||
|
||||
self.started = time.time()
|
||||
self.started = int(time.time())
|
||||
|
||||
for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS:
|
||||
logging.getLogger(_tool).setLevel(_level)
|
||||
|
|
@ -545,7 +546,7 @@ class Config(metaclass=Singleton):
|
|||
|
||||
def _get_attributes(self) -> dict:
|
||||
attrs: dict = {}
|
||||
vClass: str = self.__class__
|
||||
vClass: type = self.__class__
|
||||
|
||||
for attribute in vClass.__dict__:
|
||||
if attribute.startswith("_"):
|
||||
|
|
@ -601,7 +602,7 @@ class Config(metaclass=Singleton):
|
|||
dict[str, str]: The replacer variables.
|
||||
|
||||
"""
|
||||
keys: tuple[str] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
|
||||
keys: tuple[str, ...] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
|
||||
return {k: getattr(self, k) for k in keys}
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -619,15 +620,20 @@ class Config(metaclass=Singleton):
|
|||
This is used to set the version to the latest git tag.
|
||||
"""
|
||||
LOG = get_logger()
|
||||
git_path: str = Path(__file__).parent / ".." / ".." / ".git"
|
||||
git_path: Path = Path(__file__).parent / ".." / ".." / ".git"
|
||||
if not git_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
git = shutil.which("git")
|
||||
if not git:
|
||||
LOG.warning("Git executable was not found.")
|
||||
return
|
||||
|
||||
branch_result: CompletedProcess[str] = subprocess.run(
|
||||
["git", "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607
|
||||
[git, "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=os.path.dirname(git_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
|
@ -645,7 +651,7 @@ class Config(metaclass=Singleton):
|
|||
return
|
||||
|
||||
commit_result: CompletedProcess[str] = subprocess.run(
|
||||
["git", "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], # noqa: S607
|
||||
[git, "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"],
|
||||
cwd=os.path.dirname(git_path),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
|
|
|||
|
|
@ -39,6 +39,14 @@ if TYPE_CHECKING:
|
|||
class Download:
|
||||
update_task: asyncio.Task | None = None
|
||||
|
||||
@staticmethod
|
||||
def _raise_no_formats(msg: str) -> None:
|
||||
raise ValueError(msg)
|
||||
|
||||
@staticmethod
|
||||
def _raise_interrupted() -> None:
|
||||
raise SystemExit(130)
|
||||
|
||||
def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None):
|
||||
"""
|
||||
Initialize download task.
|
||||
|
|
@ -51,7 +59,7 @@ class Download:
|
|||
"""
|
||||
config: Config = Config.get_instance()
|
||||
|
||||
self.download_dir: str = info.download_dir
|
||||
self.download_dir: str = info.download_dir or ""
|
||||
self.temp_dir: str | None = info.temp_dir
|
||||
self.template: str | None = info.template
|
||||
self.template_chapter: str | None = info.template_chapter
|
||||
|
|
@ -98,6 +106,16 @@ class Download:
|
|||
params: dict[str, Any] = {}
|
||||
download_skipped = False
|
||||
|
||||
hook_handlers = self._hook_handlers
|
||||
if hook_handlers is None:
|
||||
msg = "_hook_handlers must be initialized before _download(). Call start() first."
|
||||
raise RuntimeError(msg)
|
||||
|
||||
status_queue = self.status_queue
|
||||
if status_queue is None:
|
||||
msg = "status_queue must be initialized before _download(). Call start() first."
|
||||
raise RuntimeError(msg)
|
||||
|
||||
try:
|
||||
params = (
|
||||
self.info.get_ytdlp_opts()
|
||||
|
|
@ -125,9 +143,9 @@ class Download:
|
|||
|
||||
params.update(
|
||||
{
|
||||
"progress_hooks": [self._hook_handlers.progress_hook],
|
||||
"postprocessor_hooks": [self._hook_handlers.postprocessor_hook],
|
||||
"post_hooks": [self._hook_handlers.post_hook],
|
||||
"progress_hooks": [hook_handlers.progress_hook],
|
||||
"postprocessor_hooks": [hook_handlers.postprocessor_hook],
|
||||
"post_hooks": [hook_handlers.post_hook],
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -137,7 +155,10 @@ class Download:
|
|||
|
||||
if self.info.cookies:
|
||||
try:
|
||||
cookie_file = Path(self._temp_manager.temp_path or self.temp_dir) / f"cookie_{self.info._id}.txt"
|
||||
cookie_file = (
|
||||
Path(self._temp_manager.temp_path or self.temp_dir or self.download_dir)
|
||||
/ f"cookie_{self.info._id}.txt"
|
||||
)
|
||||
self.logger.debug(
|
||||
f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.",
|
||||
extra={
|
||||
|
|
@ -173,7 +194,9 @@ class Download:
|
|||
f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info."
|
||||
)
|
||||
self.info_dict = None
|
||||
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default:
|
||||
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and (
|
||||
(preset := self.info.get_preset()) is not None and preset.default
|
||||
):
|
||||
self.logger.debug(
|
||||
"Removing download archive option for generic extractor on '%s'.",
|
||||
self.info.url,
|
||||
|
|
@ -267,7 +290,7 @@ class Download:
|
|||
if filtered_logs := extract_ytdlp_logs(self.logs):
|
||||
msg += " " + ", ".join(filtered_logs)
|
||||
|
||||
raise ValueError(msg) # noqa: TRY301
|
||||
self._raise_no_formats(msg)
|
||||
|
||||
self.logger.info(
|
||||
f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.",
|
||||
|
|
@ -327,11 +350,11 @@ class Download:
|
|||
def mark_cancelled(*_) -> None:
|
||||
cls._interrupted = True
|
||||
cls.to_screen("[info] Interrupt received, exiting cleanly...")
|
||||
raise SystemExit(130) # noqa: TRY301
|
||||
self._raise_interrupted()
|
||||
|
||||
signal.signal(signal.SIGUSR1, mark_cancelled)
|
||||
|
||||
self.status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
|
||||
status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
|
||||
|
||||
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
|
||||
self.logger.debug(
|
||||
|
|
@ -386,7 +409,7 @@ class Download:
|
|||
)
|
||||
ret = cls.download(url_list=[self.info.url])
|
||||
|
||||
self.status_queue.put(
|
||||
status_queue.put(
|
||||
{
|
||||
"id": self.id,
|
||||
"status": "finished" if 0 == ret else "error",
|
||||
|
|
@ -407,7 +430,7 @@ class Download:
|
|||
}
|
||||
},
|
||||
)
|
||||
self.status_queue.put(
|
||||
status_queue.put(
|
||||
{
|
||||
"id": self.id,
|
||||
"status": "skip",
|
||||
|
|
@ -432,7 +455,7 @@ class Download:
|
|||
}
|
||||
},
|
||||
)
|
||||
self.status_queue.put(
|
||||
status_queue.put(
|
||||
{
|
||||
"id": self.id,
|
||||
"status": "error",
|
||||
|
|
@ -442,7 +465,7 @@ class Download:
|
|||
}
|
||||
)
|
||||
finally:
|
||||
self.status_queue.put(Terminator())
|
||||
status_queue.put(Terminator())
|
||||
if cookie_file and cookie_file.exists():
|
||||
try:
|
||||
cookie_file.unlink()
|
||||
|
|
@ -499,24 +522,24 @@ class Download:
|
|||
Process exit code or None
|
||||
|
||||
"""
|
||||
self.status_queue = Config.get_manager().Queue()
|
||||
status_queue: Any = Config.get_manager().Queue()
|
||||
self.status_queue = status_queue
|
||||
|
||||
if temp_path := self._temp_manager.create_temp_path():
|
||||
self.info.temp_path = str(temp_path)
|
||||
temp_path = self._temp_manager.create_temp_path()
|
||||
|
||||
self._status_tracker = StatusTracker(
|
||||
info=self.info,
|
||||
download_id=self.id,
|
||||
download_dir=self.download_dir,
|
||||
temp_path=temp_path,
|
||||
status_queue=self.status_queue,
|
||||
status_queue=status_queue,
|
||||
logger=self.logger,
|
||||
debug=self.debug,
|
||||
)
|
||||
|
||||
self._hook_handlers = HookHandlers(
|
||||
download_id=self.id,
|
||||
status_queue=self.status_queue,
|
||||
status_queue=status_queue,
|
||||
logger=self.logger,
|
||||
debug=self.debug,
|
||||
)
|
||||
|
|
@ -529,7 +552,8 @@ class Download:
|
|||
EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info)
|
||||
asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}")
|
||||
|
||||
ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join)
|
||||
proc = self._process_manager.proc
|
||||
ret = await asyncio.get_running_loop().run_in_executor(None, proc.join) if proc else None
|
||||
|
||||
if self._status_tracker.final_update:
|
||||
return ret
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import time
|
|||
import uuid
|
||||
from numbers import Number
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import yt_dlp.utils
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ async def add_item(
|
|||
already=None,
|
||||
logs: list | None = None,
|
||||
yt_params: dict | None = None,
|
||||
) -> dict[str, str]:
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Route an entry to the appropriate processor based on type.
|
||||
|
||||
|
|
@ -102,7 +102,7 @@ async def add_item(
|
|||
|
||||
async def add(
|
||||
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
|
||||
) -> dict[str, str]:
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Add an item to the download queue.
|
||||
|
||||
|
|
@ -204,7 +204,7 @@ async def add(
|
|||
)
|
||||
|
||||
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
|
||||
dlInfo.info.msg += f" Found in archive '{archive_file}'."
|
||||
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{archive_file}'."
|
||||
|
||||
await queue.done.put(dlInfo)
|
||||
|
||||
|
|
@ -306,9 +306,12 @@ async def add(
|
|||
new_archive_id: str | None = None
|
||||
|
||||
if entry.get("extractor_key") and entry.get("id"):
|
||||
new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}"
|
||||
if new_archive_id != archive_id:
|
||||
extra_ids.append(new_archive_id)
|
||||
_extractor_key = entry.get("extractor_key")
|
||||
_entry_id = entry.get("id")
|
||||
if isinstance(_extractor_key, str) and _entry_id is not None:
|
||||
new_archive_id = f"{_extractor_key.lower()} {_entry_id!s}"
|
||||
if new_archive_id != archive_id:
|
||||
extra_ids.append(new_archive_id)
|
||||
|
||||
if len(extra_ids) > 0:
|
||||
archive_ids: list[str] = archive_read(_archive_file, extra_ids)
|
||||
|
|
@ -336,7 +339,7 @@ async def add(
|
|||
)
|
||||
)
|
||||
|
||||
dlInfo.info.msg += f" Found in archive '{_archive_file}'."
|
||||
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{_archive_file}'."
|
||||
await queue.done.put(dlInfo)
|
||||
|
||||
queue._notify.emit(
|
||||
|
|
@ -374,7 +377,7 @@ async def add(
|
|||
|
||||
if condition.extras.get("ignore_download", False):
|
||||
extra_msg: str = ""
|
||||
if _archive_file and not condition.extras.get("no_archive", False):
|
||||
if _archive_file and not condition.extras.get("no_archive", False) and archive_id:
|
||||
archive_add(_archive_file, [archive_id])
|
||||
extra_msg = f" and added to archive file '{_archive_file}'"
|
||||
|
||||
|
|
@ -385,7 +388,7 @@ async def add(
|
|||
if not store_type:
|
||||
dlInfo = Download(
|
||||
info=ItemDTO(
|
||||
id=entry.get("id"),
|
||||
id=str(entry.get("id") or item.url),
|
||||
title=_name,
|
||||
url=item.url,
|
||||
preset=item.preset,
|
||||
|
|
@ -401,6 +404,9 @@ async def add(
|
|||
|
||||
LOG.info(log_message)
|
||||
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
|
||||
if dlInfo is None:
|
||||
msg = "Download info is not available."
|
||||
raise RuntimeError(msg)
|
||||
queue._notify.emit(
|
||||
Events.ITEM_MOVED,
|
||||
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ async def process_playlist(
|
|||
|
||||
max_downloads: int = -1
|
||||
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
|
||||
if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int):
|
||||
max_downloads: int = ytdlp_opts.get("max_downloads")
|
||||
if isinstance(ytdlp_opts.get("max_downloads"), int):
|
||||
max_downloads = ytdlp_opts["max_downloads"]
|
||||
|
||||
results: list[dict[str, str]] = []
|
||||
for i, etr in enumerate(entries, start=1):
|
||||
|
|
|
|||
|
|
@ -99,15 +99,19 @@ class ProcessManager:
|
|||
if not self.running():
|
||||
return False
|
||||
|
||||
procId: int | None = self.proc.ident
|
||||
proc = self.proc
|
||||
if proc is None:
|
||||
return False
|
||||
|
||||
procId: int | None = proc.ident
|
||||
try:
|
||||
self.logger.info(
|
||||
"Stopping download process PID=%s.",
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"process_ident": procId,
|
||||
"is_live": self.is_live,
|
||||
}
|
||||
|
|
@ -117,25 +121,25 @@ class ProcessManager:
|
|||
if self.is_live:
|
||||
self.logger.debug(
|
||||
"Requesting graceful live cancellation for PID=%s.",
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"is_live": self.is_live,
|
||||
}
|
||||
},
|
||||
)
|
||||
self.cancel_event.set()
|
||||
|
||||
if wait_for_process_with_timeout(self.proc, 10):
|
||||
if wait_for_process_with_timeout(proc, 10):
|
||||
self.logger.debug(
|
||||
"Download process PID=%s stopped gracefully.",
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"is_live": self.is_live,
|
||||
}
|
||||
},
|
||||
|
|
@ -143,43 +147,43 @@ class ProcessManager:
|
|||
return True
|
||||
self.logger.warning(
|
||||
"Download process PID=%s did not respond to live cancellation; forcing termination.",
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"is_live": self.is_live,
|
||||
"force": True,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
elif self.proc.pid and "posix" == os.name:
|
||||
elif proc.pid and "posix" == os.name:
|
||||
signal_name = "SIGUSR1"
|
||||
|
||||
try:
|
||||
self.logger.debug(
|
||||
"Sending %s signal to download process PID=%s.",
|
||||
signal_name,
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"signal": signal_name,
|
||||
}
|
||||
},
|
||||
)
|
||||
os.kill(self.proc.pid, signal.SIGUSR1)
|
||||
os.kill(proc.pid, signal.SIGUSR1)
|
||||
|
||||
if wait_for_process_with_timeout(self.proc, 5):
|
||||
if wait_for_process_with_timeout(proc, 5):
|
||||
self.logger.debug(
|
||||
"Download process PID=%s stopped gracefully.",
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"signal": signal_name,
|
||||
}
|
||||
},
|
||||
|
|
@ -187,12 +191,12 @@ class ProcessManager:
|
|||
return True
|
||||
self.logger.warning(
|
||||
"Download process PID=%s did not respond to %s; forcing termination.",
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
signal_name,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"signal": signal_name,
|
||||
"force": True,
|
||||
}
|
||||
|
|
@ -203,51 +207,49 @@ class ProcessManager:
|
|||
self.logger.debug(
|
||||
"Failed to send %s signal to download process PID=%s. %s",
|
||||
signal_name,
|
||||
self.proc.pid,
|
||||
proc.pid,
|
||||
e,
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid,
|
||||
"signal": signal_name,
|
||||
"exception_type": type(e).__name__,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
if self.proc.is_alive():
|
||||
if proc.is_alive():
|
||||
self.logger.info(
|
||||
"Terminating download process PID=%s.",
|
||||
self.proc.pid,
|
||||
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}},
|
||||
proc.pid,
|
||||
extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
|
||||
)
|
||||
self.proc.terminate()
|
||||
proc.terminate()
|
||||
|
||||
if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2):
|
||||
if not wait_for_process_with_timeout(proc, 1 if self.is_live else 2):
|
||||
self.logger.warning(
|
||||
"Download process PID=%s did not terminate; killing forcefully.",
|
||||
self.proc.pid,
|
||||
extra={
|
||||
"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}
|
||||
},
|
||||
proc.pid,
|
||||
extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
|
||||
)
|
||||
self.proc.kill()
|
||||
wait_for_process_with_timeout(self.proc, 1)
|
||||
proc.kill()
|
||||
wait_for_process_with_timeout(proc, 1)
|
||||
|
||||
self.logger.info(
|
||||
"Download process PID=%s stopped.",
|
||||
self.proc.pid,
|
||||
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid}},
|
||||
proc.pid,
|
||||
extra={"download": {"download_id": self.download_id, "process_id": proc.pid}},
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.exception(
|
||||
f"Failed to stop download process PID={self.proc.pid}, ident={procId}.",
|
||||
f"Failed to stop download process PID={proc.pid if proc else None}, ident={procId}.",
|
||||
extra={
|
||||
"download": {
|
||||
"download_id": self.download_id,
|
||||
"process_id": self.proc.pid,
|
||||
"process_id": proc.pid if proc else None,
|
||||
"process_ident": procId,
|
||||
"is_live": self.is_live,
|
||||
"exception_type": type(e).__name__,
|
||||
|
|
@ -271,11 +273,12 @@ class ProcessManager:
|
|||
return False
|
||||
|
||||
self.cancel_in_progress = True
|
||||
procId: int | None = self.proc.ident
|
||||
proc = self.proc
|
||||
procId: int | None = proc.ident if proc else None
|
||||
|
||||
if not procId:
|
||||
if self.proc:
|
||||
self.proc.close()
|
||||
if proc:
|
||||
proc.close()
|
||||
self.proc = None
|
||||
self.logger.warning("Attempted to close download process, but it is not running.")
|
||||
return False
|
||||
|
|
@ -293,13 +296,14 @@ class ProcessManager:
|
|||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
if self.proc.is_alive():
|
||||
current = self.proc
|
||||
if current is not None and current.is_alive():
|
||||
self.logger.debug(
|
||||
"Waiting for download process PID='%s' to close.",
|
||||
procId,
|
||||
extra={"download": {"download_id": self.download_id, "process_ident": procId}},
|
||||
)
|
||||
await loop.run_in_executor(None, self.proc.join)
|
||||
await loop.run_in_executor(None, current.join)
|
||||
self.logger.debug(
|
||||
"Download process PID='%s' closed.",
|
||||
procId,
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class DownloadQueue(metaclass=Singleton):
|
|||
async def event_handler(_, __):
|
||||
await self.initialize()
|
||||
|
||||
self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}")
|
||||
self._notify.subscribe(Events.STARTED, event_handler, f"{DownloadQueue.__name__}.initialize")
|
||||
|
||||
Scheduler.get_instance().add(
|
||||
timer="* * * * *",
|
||||
|
|
@ -689,7 +689,7 @@ class DownloadQueue(metaclass=Singleton):
|
|||
)
|
||||
return {"deleted": deleted_count}
|
||||
|
||||
async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
|
||||
async def get(self, mode: str = "all") -> dict[str, dict[str, ItemDTO]]:
|
||||
"""
|
||||
Get the download queue and the download history.
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ class StatusTracker:
|
|||
self.final_update = False
|
||||
self._terminator_sent: bool = False
|
||||
self._candidate_filepath: Path | None = None
|
||||
self.update_task: asyncio.Task | None = None
|
||||
self.update_task: asyncio.Future[Any] | None = None
|
||||
self._last_progress_time: float = 0.0
|
||||
self._pending_progress: bool = False
|
||||
self._progress_interval: float = 0.5
|
||||
|
|
@ -256,7 +256,7 @@ class StatusTracker:
|
|||
self.info.percent = status["downloaded_bytes"] / total * 100
|
||||
except ZeroDivisionError:
|
||||
self.info.percent = 0
|
||||
self.info.total_bytes = total
|
||||
self.info.total_bytes = int(total)
|
||||
|
||||
self.info.speed = status.get("speed")
|
||||
self.info.eta = status.get("eta")
|
||||
|
|
@ -280,8 +280,9 @@ class StatusTracker:
|
|||
"""
|
||||
while True:
|
||||
try:
|
||||
self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
|
||||
status = await self.update_task
|
||||
update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
|
||||
self.update_task = asyncio.ensure_future(update_task)
|
||||
status = await update_task
|
||||
if status is None or isinstance(status, Terminator):
|
||||
self._flush_progress()
|
||||
return
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import functools
|
||||
import importlib.util
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, cast, overload
|
||||
|
|
@ -98,12 +99,7 @@ def _normalize_proxy(proxy: str | dict[str, str] | None) -> str | None:
|
|||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _curl_available() -> bool:
|
||||
try:
|
||||
import httpx_curl_cffi # noqa: F401
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
return importlib.util.find_spec("httpx_curl_cffi") is not None
|
||||
|
||||
|
||||
def resolve_curl_transport(use_curl: bool = True) -> bool:
|
||||
|
|
@ -121,7 +117,7 @@ def _build_async_curl_transport(
|
|||
from httpx_curl_cffi import AsyncCurlTransport
|
||||
|
||||
return AsyncCurlTransport(
|
||||
impersonate=curl_impersonate,
|
||||
impersonate=cast("Any", curl_impersonate),
|
||||
default_headers=curl_default_headers,
|
||||
)
|
||||
|
||||
|
|
@ -308,10 +304,7 @@ def get_async_client(
|
|||
curl_default_headers=curl_default_headers,
|
||||
)
|
||||
client = httpx.AsyncClient(
|
||||
transport=cast(
|
||||
"httpx.AsyncBaseTransport",
|
||||
_get_transport(enable_cf, is_async=True, transport=transport),
|
||||
),
|
||||
transport=_get_transport(enable_cf, is_async=True, transport=transport),
|
||||
proxy=cast("Any", proxy),
|
||||
)
|
||||
Globals.SHARED_ASYNC_CLIENTS[key] = client
|
||||
|
|
@ -330,10 +323,7 @@ def get_sync_client(
|
|||
return Globals.SHARED_SYNC_CLIENTS[key]
|
||||
|
||||
client = httpx.Client(
|
||||
transport=cast(
|
||||
"httpx.BaseTransport",
|
||||
_get_transport(enable_cf, is_async=False, transport=None),
|
||||
),
|
||||
transport=_get_transport(enable_cf, is_async=False, transport=None),
|
||||
proxy=cast("Any", proxy),
|
||||
)
|
||||
Globals.SHARED_SYNC_CLIENTS[key] = client
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# flake8: noqa: S608
|
||||
"""
|
||||
This module is a fork of the Caribou library
|
||||
(https://pypi.org/project/caribou/) modified to work for our specific use case.
|
||||
|
|
@ -9,10 +8,11 @@ import datetime
|
|||
import glob
|
||||
import os.path
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator, Iterable, Sequence
|
||||
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
|
||||
from importlib.machinery import ModuleSpec, SourceFileLoader
|
||||
from importlib.util import module_from_spec, spec_from_loader
|
||||
from typing import TYPE_CHECKING
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection
|
||||
|
|
@ -44,7 +44,9 @@ class InvalidNameError(Error):
|
|||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator:
|
||||
async def execute(
|
||||
conn: AsyncConnection, sql: str, params: Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None
|
||||
) -> AsyncIterator:
|
||||
params = {} if params is None else params
|
||||
result = await conn.execute(text(sql), params)
|
||||
try:
|
||||
|
|
@ -126,12 +128,12 @@ class Database:
|
|||
|
||||
self.version_table: str = version_table
|
||||
|
||||
self._owns_connection = bool(isinstance(db_url, str))
|
||||
if self._owns_connection:
|
||||
self._owns_connection = isinstance(db_url, str)
|
||||
if isinstance(db_url, str):
|
||||
self.conn: AsyncConnection | None = None
|
||||
self.db_url: str = db_url
|
||||
else:
|
||||
self.conn: AsyncConnection = db_url
|
||||
self.conn = db_url
|
||||
|
||||
async def __aenter__(self):
|
||||
if self._owns_connection and self.conn is None:
|
||||
|
|
@ -183,7 +185,8 @@ class Database:
|
|||
async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
|
||||
assert self.conn
|
||||
if target_version not in (0, "0"):
|
||||
_assert_migration_exists(migrations, target_version)
|
||||
_assert_migration_exists(migrations, str(target_version))
|
||||
target = str(target_version)
|
||||
|
||||
migrations.sort(key=lambda x: x.get_version(), reverse=True)
|
||||
database_version: str | None = await self.get_version()
|
||||
|
|
@ -192,7 +195,7 @@ class Database:
|
|||
current_version: str = migration.get_version()
|
||||
if database_version is not None and current_version > database_version:
|
||||
continue
|
||||
if current_version <= target_version:
|
||||
if current_version <= target:
|
||||
break
|
||||
await migration.downgrade(self.conn)
|
||||
next_version: str | int = 0
|
||||
|
|
@ -210,14 +213,14 @@ class Database:
|
|||
assert self.conn
|
||||
if not await self.is_version_controlled():
|
||||
return None
|
||||
sql: str = f"SELECT version FROM {self.version_table}"
|
||||
sql: str = f"SELECT version FROM {self.version_table}" # noqa: S608
|
||||
async with execute(self.conn, sql) as result:
|
||||
rows = result.fetchall()
|
||||
return rows[0][0] if rows else "0"
|
||||
|
||||
async def update_version(self, version: str) -> None:
|
||||
assert self.conn
|
||||
sql: str = f"UPDATE {self.version_table} SET version = :version"
|
||||
sql: str = f"UPDATE {self.version_table} SET version = :version" # noqa: S608
|
||||
async with transaction(self.conn):
|
||||
await self.conn.execute(text(sql), {"version": version})
|
||||
|
||||
|
|
@ -226,7 +229,7 @@ class Database:
|
|||
sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
|
||||
async with transaction(self.conn):
|
||||
await self.conn.execute(text(sql))
|
||||
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"})
|
||||
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) # noqa: S608
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
|
||||
|
|
@ -238,7 +241,7 @@ def _assert_migration_exists(migrations: Iterable[Migration], version: str | int
|
|||
raise Error(msg)
|
||||
|
||||
|
||||
def load_migrations(directory: str) -> list[Migration]:
|
||||
def load_migrations(directory: str | Path) -> list[Migration]:
|
||||
"""Return the migrations contained in the given directory."""
|
||||
directory = str(directory)
|
||||
if not os.path.exists(directory) or not os.path.isdir(directory):
|
||||
|
|
@ -247,7 +250,7 @@ def load_migrations(directory: str) -> list[Migration]:
|
|||
return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
|
||||
|
||||
|
||||
async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None:
|
||||
async def upgrade(db_url: AsyncConnection | str, migration_dir: str | Path, version: str | None = None) -> None:
|
||||
"""
|
||||
Upgrade the given database with the migrations contained in the
|
||||
migrations directory. If a version is not specified, upgrade
|
||||
|
|
@ -260,7 +263,7 @@ async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: st
|
|||
await db.upgrade(load_migrations(migration_dir), version)
|
||||
|
||||
|
||||
async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None:
|
||||
async def downgrade(db_url: str | AsyncConnection, migration_dir: str | Path, version: str) -> None:
|
||||
"""
|
||||
Downgrade the database to the given version with the migrations
|
||||
contained in the given migration directory.
|
||||
|
|
|
|||
|
|
@ -2,14 +2,16 @@ import asyncio
|
|||
import contextlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import fields
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import formatdate
|
||||
from typing import Any
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from aiohttp import web
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine.row import RowMapping
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.ext.asyncio.engine import AsyncConnection
|
||||
|
||||
|
|
@ -36,13 +38,13 @@ def _memory_db_url(db_path: str) -> str:
|
|||
return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true"
|
||||
|
||||
|
||||
def _decode_row(row: dict) -> ItemDTO:
|
||||
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
|
||||
def _decode_row(row: Mapping[str, Any] | RowMapping) -> ItemDTO:
|
||||
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC)
|
||||
data = json.loads(row["data"])
|
||||
data.pop("_id", None)
|
||||
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
|
||||
item._id = row["id"]
|
||||
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
|
||||
item.datetime = formatdate(row_date.timestamp())
|
||||
return item
|
||||
|
||||
|
||||
|
|
@ -149,8 +151,8 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
return self._sessionmaker
|
||||
|
||||
async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
|
||||
await self.get_connection()
|
||||
result = await self._conn.execute(
|
||||
conn = await self.get_connection()
|
||||
result = await conn.execute(
|
||||
text(
|
||||
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC'
|
||||
),
|
||||
|
|
@ -163,12 +165,12 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
|
||||
return await self.get(type_value, key=key, url=url) is not None
|
||||
|
||||
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
|
||||
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> ItemDTO | None:
|
||||
if not key and not url:
|
||||
msg = "key or url must be provided."
|
||||
raise KeyError(msg)
|
||||
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
|
||||
clauses: list[str] = []
|
||||
params: dict[str, str] = {"type_value": type_value}
|
||||
|
|
@ -186,7 +188,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608
|
||||
)
|
||||
|
||||
result = await self._conn.execute(text(query), params)
|
||||
result = await conn.execute(text(query), params)
|
||||
row = result.mappings().first()
|
||||
|
||||
if not row:
|
||||
|
|
@ -195,8 +197,8 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
return _decode_row(row)
|
||||
|
||||
async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
|
||||
await self.get_connection()
|
||||
result = await self._conn.execute(
|
||||
conn = await self.get_connection()
|
||||
result = await conn.execute(
|
||||
text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'),
|
||||
{"type_value": type_value, "id": id},
|
||||
)
|
||||
|
|
@ -212,24 +214,24 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
if not ids_list:
|
||||
return []
|
||||
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list)))
|
||||
params: dict[str, str] = {"type_value": type_value}
|
||||
params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)})
|
||||
|
||||
result = await self._conn.execute(
|
||||
result = await conn.execute(
|
||||
text(
|
||||
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608
|
||||
),
|
||||
params,
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
rows = list(result.mappings().all())
|
||||
order = {item_id: index for index, item_id in enumerate(ids_list)}
|
||||
rows.sort(key=lambda row: order.get(row["id"], len(ids_list)))
|
||||
return [(row["id"], _decode_row(row)) for row in rows]
|
||||
|
||||
async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]:
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
params: dict[str, str] = {"type_value": type_value}
|
||||
where_clauses = ['"type" = :type_value']
|
||||
|
||||
|
|
@ -239,7 +241,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
params.update(status_params)
|
||||
|
||||
query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608
|
||||
result = await self._conn.execute(text(query), params)
|
||||
result = await conn.execute(text(query), params)
|
||||
rows = result.mappings().all()
|
||||
return [(row["id"], _decode_row(row)) for row in rows]
|
||||
|
||||
|
|
@ -254,7 +256,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
if not kwargs:
|
||||
return None
|
||||
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
|
||||
clauses: list[str] = []
|
||||
params: dict[str, str | float | int] = {"type_value": type_value}
|
||||
|
|
@ -321,7 +323,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
where_clause: str = " OR ".join(f"({clause})" for clause in clauses)
|
||||
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608
|
||||
|
||||
result = await self._conn.execute(text(query), params)
|
||||
result = await conn.execute(text(query), params)
|
||||
row = result.mappings().first()
|
||||
|
||||
if not row:
|
||||
|
|
@ -332,7 +334,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None
|
||||
|
||||
async def count(self, type_value: str, status_filter: str | None = None) -> int:
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
where_clauses: list[str] = ['"type" = :type_value']
|
||||
params: dict[str, str] = {"type_value": type_value}
|
||||
|
||||
|
|
@ -344,7 +346,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
where_clause: str = " AND ".join(where_clauses)
|
||||
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
|
||||
|
||||
result = await self._conn.execute(text(count_query), params)
|
||||
result = await conn.execute(text(count_query), params)
|
||||
row = result.mappings().first()
|
||||
|
||||
return row["count"] if row else 0
|
||||
|
|
@ -357,7 +359,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
order: str,
|
||||
status_filter: str | None = None,
|
||||
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
where_clauses: list[str] = ['"type" = :type_value']
|
||||
params: dict[str, str | int] = {"type_value": type_value}
|
||||
|
||||
|
|
@ -369,7 +371,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
where_clause: str = " AND ".join(where_clauses)
|
||||
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
|
||||
|
||||
result = await self._conn.execute(text(count_query), params)
|
||||
result = await conn.execute(text(count_query), params)
|
||||
count_row = result.mappings().first()
|
||||
|
||||
total_items = count_row["count"] if count_row else 0
|
||||
|
|
@ -384,7 +386,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
|
||||
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608
|
||||
|
||||
result = await self._conn.execute(text(query), params)
|
||||
result = await conn.execute(text(query), params)
|
||||
rows = result.mappings().all()
|
||||
|
||||
items = [(row["id"], _decode_row(row)) for row in rows]
|
||||
|
|
@ -396,15 +398,15 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
await self._upsert_now(type_value, item)
|
||||
|
||||
async def delete(self, type_value: str, key: str) -> None:
|
||||
await self.get_connection()
|
||||
await self._conn.execute(
|
||||
conn = await self.get_connection()
|
||||
await conn.execute(
|
||||
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
|
||||
{"type_value": type_value, "key": key},
|
||||
)
|
||||
await self._conn.commit()
|
||||
await conn.commit()
|
||||
|
||||
async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
keys_list: list[str] = list(keys)
|
||||
if not keys_list:
|
||||
return 0
|
||||
|
|
@ -412,15 +414,15 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
params: dict[str, str] = {"type_value": type_value}
|
||||
params.update({f"key_{i}": key for i, key in enumerate(keys_list)})
|
||||
|
||||
result = await self._conn.execute(
|
||||
result = await conn.execute(
|
||||
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
|
||||
params,
|
||||
)
|
||||
await self._conn.commit()
|
||||
await conn.commit()
|
||||
return result.rowcount if result else 0
|
||||
|
||||
async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int:
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
params: dict[str, str] = {"type_value": type_value}
|
||||
where_clauses = ['"type" = :type_value']
|
||||
|
||||
|
|
@ -429,11 +431,11 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
where_clauses.append(status_clause)
|
||||
params.update(status_params)
|
||||
|
||||
result = await self._conn.execute(
|
||||
result = await conn.execute(
|
||||
text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608
|
||||
params,
|
||||
)
|
||||
await self._conn.commit()
|
||||
await conn.commit()
|
||||
return result.rowcount if result else 0
|
||||
|
||||
async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
|
||||
|
|
@ -484,6 +486,9 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
|
||||
async def _enqueue(self, op: _Op) -> None:
|
||||
self._ensure_worker()
|
||||
if self._queue is None:
|
||||
msg = "Queue was not initialized by _ensure_worker."
|
||||
raise RuntimeError(msg)
|
||||
await self._queue.put(op)
|
||||
|
||||
def _ensure_worker(self) -> None:
|
||||
|
|
@ -492,10 +497,14 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
|
||||
|
||||
async def _writer(self) -> None:
|
||||
queue = self._queue
|
||||
if queue is None:
|
||||
msg = "Writer started before queue was initialized."
|
||||
raise RuntimeError(msg)
|
||||
while True:
|
||||
op = await self._queue.get()
|
||||
op = await queue.get()
|
||||
if isinstance(op.op, Terminator):
|
||||
self._queue.task_done()
|
||||
queue.task_done()
|
||||
break
|
||||
try:
|
||||
async with self._lock:
|
||||
|
|
@ -507,34 +516,34 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__},
|
||||
)
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
queue.task_done()
|
||||
await asyncio.sleep(self._flush_interval)
|
||||
|
||||
async def _apply(self, op: _Op) -> None:
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
if op.op == "upsert" and op.item:
|
||||
await self._upsert_now_conn(self._conn, op.type_value, op.item)
|
||||
await self._conn.commit()
|
||||
await self._upsert_now_conn(conn, op.type_value, op.item)
|
||||
await conn.commit()
|
||||
elif op.op == "delete" and op.key:
|
||||
await self._conn.execute(
|
||||
await conn.execute(
|
||||
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
|
||||
{"type_value": op.type_value, "key": op.key},
|
||||
)
|
||||
await self._conn.commit()
|
||||
await conn.commit()
|
||||
elif op.op == "bulk_delete" and op.keys:
|
||||
placeholders = ",".join(f":key_{i}" for i in range(len(op.keys)))
|
||||
params: dict[str, str] = {"type_value": op.type_value}
|
||||
params.update({f"key_{i}": key for i, key in enumerate(op.keys)})
|
||||
await self._conn.execute(
|
||||
await conn.execute(
|
||||
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
|
||||
params,
|
||||
)
|
||||
await self._conn.commit()
|
||||
await conn.commit()
|
||||
|
||||
async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
|
||||
await self.get_connection()
|
||||
await self._upsert_now_conn(self._conn, type_value, item)
|
||||
await self._conn.commit()
|
||||
conn = await self.get_connection()
|
||||
await self._upsert_now_conn(conn, type_value, item)
|
||||
await conn.commit()
|
||||
|
||||
async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None:
|
||||
"""Helper to upsert using an existing connection."""
|
||||
|
|
@ -560,7 +569,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
|
||||
async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None:
|
||||
"""Execute raw SQL query (for testing purposes)."""
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
if isinstance(params, tuple):
|
||||
# Convert positional params to dict for SQLAlchemy
|
||||
# Assuming queries use ? placeholders, we need to count them
|
||||
|
|
@ -575,16 +584,16 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
# Replace ? with :p0, :p1, etc.
|
||||
for i in range(len(params)):
|
||||
query = query.replace("?", f":p{i}", 1)
|
||||
await self._conn.execute(text(query), param_dict)
|
||||
await conn.execute(text(query), param_dict)
|
||||
elif isinstance(params, dict):
|
||||
await self._conn.execute(text(query), params)
|
||||
await conn.execute(text(query), params)
|
||||
else:
|
||||
await self._conn.execute(text(query))
|
||||
await self._conn.commit()
|
||||
await conn.execute(text(query))
|
||||
await conn.commit()
|
||||
|
||||
async def fetch_raw(self, query: str, params: dict | tuple | None = None):
|
||||
"""Fetch results from raw SQL query (for testing purposes)."""
|
||||
await self.get_connection()
|
||||
conn = await self.get_connection()
|
||||
if isinstance(params, tuple):
|
||||
# Convert positional params to dict for SQLAlchemy
|
||||
placeholders = query.count("?")
|
||||
|
|
@ -596,11 +605,11 @@ class SqliteStore(metaclass=ThreadSafe):
|
|||
param_dict = {f"p{i}": params[i] for i in range(len(params))}
|
||||
for i in range(len(params)):
|
||||
query = query.replace("?", f":p{i}", 1)
|
||||
result = await self._conn.execute(text(query), param_dict)
|
||||
result = await conn.execute(text(query), param_dict)
|
||||
elif isinstance(params, dict):
|
||||
result = await self._conn.execute(text(query), params)
|
||||
result = await conn.execute(text(query), params)
|
||||
else:
|
||||
result = await self._conn.execute(text(query))
|
||||
result = await conn.execute(text(query))
|
||||
return result.mappings().all()
|
||||
|
||||
async def get_connection(self) -> AsyncConnection:
|
||||
|
|
|
|||
23
app/local.py
23
app/local.py
|
|
@ -1,14 +1,14 @@
|
|||
#!/usr/bin/env python3
|
||||
# flake8: noqa: S310 T201
|
||||
|
||||
import argparse
|
||||
import http.client
|
||||
import os
|
||||
import pathlib
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import webbrowser
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
if __name__ == "__main__":
|
||||
from multiprocessing import freeze_support
|
||||
|
|
@ -26,7 +26,7 @@ APP_ROOT = str((pathlib.Path(__file__).parent / "..").resolve())
|
|||
if APP_ROOT not in sys.path:
|
||||
sys.path.insert(0, APP_ROOT)
|
||||
|
||||
import platformdirs # type: ignore
|
||||
import platformdirs
|
||||
|
||||
|
||||
def set_env():
|
||||
|
|
@ -55,18 +55,27 @@ def open_browser_when_ready(url: str, timeout: float = 5.0) -> None:
|
|||
import threading
|
||||
|
||||
def wait_then_open():
|
||||
parsed = urlsplit(url)
|
||||
path = parsed.path or "/"
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1) as response:
|
||||
if 200 == response.status:
|
||||
conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 80, timeout=1)
|
||||
try:
|
||||
conn.request("GET", path)
|
||||
response = conn.getresponse()
|
||||
if response.status == 200:
|
||||
os.environ.pop("LD_LIBRARY_PATH", None)
|
||||
webbrowser.open_new(url)
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
|
||||
print(f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.")
|
||||
sys.stderr.write(
|
||||
f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.\n"
|
||||
)
|
||||
|
||||
threading.Thread(target=wait_then_open, daemon=True).start()
|
||||
|
||||
|
|
@ -115,7 +124,7 @@ def main():
|
|||
|
||||
set_env()
|
||||
|
||||
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH")) / ".env"
|
||||
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH", "")) / ".env"
|
||||
|
||||
port = None
|
||||
if env_file.exists():
|
||||
|
|
|
|||
22
app/main.py
22
app/main.py
|
|
@ -199,18 +199,16 @@ class Main:
|
|||
)
|
||||
HTTP_LOGGER_CLASS = HttpAccessLogger
|
||||
|
||||
run_args = {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"loop": asyncio.get_event_loop(),
|
||||
"access_log": HTTP_LOGGER,
|
||||
"print": started,
|
||||
"handle_signals": cb is None,
|
||||
}
|
||||
if HTTP_LOGGER_CLASS is not None:
|
||||
run_args["access_log_class"] = HTTP_LOGGER_CLASS
|
||||
|
||||
web.run_app(self._app, **run_args)
|
||||
web.run_app(
|
||||
self._app,
|
||||
host=host,
|
||||
port=port,
|
||||
loop=asyncio.get_event_loop(),
|
||||
access_log=HTTP_LOGGER,
|
||||
print=started,
|
||||
handle_signals=cb is None,
|
||||
access_log_class=HTTP_LOGGER_CLASS if HTTP_LOGGER_CLASS is not None else web.AccessLogger,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ async def get_ffprobe(request: Request, config: Config, encoder: Encoder, app: w
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
file: str | None = request.match_info.get("file")
|
||||
if not file:
|
||||
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ async def get_file_info(request: Request, config: Config, encoder: Encoder, app:
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
file: str = request.match_info.get("file")
|
||||
file: str | None = request.match_info.get("file")
|
||||
if not file:
|
||||
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ async def file_browser(request: Request, config: Config, encoder: Encoder) -> Re
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
req_path: str = request.match_info.get("path")
|
||||
req_path: str | None = request.match_info.get("path")
|
||||
req_path: str = "/" if not req_path else unquote_plus(req_path)
|
||||
|
||||
raw_req: str = (req_path or "").strip()
|
||||
|
|
@ -461,7 +461,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
|
|||
record(path, ok=False, error=str(e), action=action, extra={"item": params})
|
||||
continue
|
||||
else:
|
||||
extra_info: dict[str, Path] = {"new_path": renamed}
|
||||
extra_info: dict[str, Any] = {"new_path": renamed}
|
||||
if sidecar_renamed:
|
||||
extra_info["sidecar_count"] = len(sidecar_renamed)
|
||||
record(path, ok=True, action=action, extra=extra_info)
|
||||
|
|
@ -591,7 +591,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
|
|||
record(path, ok=False, error=str(e), action=action, extra={"item": params})
|
||||
continue
|
||||
else:
|
||||
extra_info: dict[str, Path] = {"new_path": moved}
|
||||
extra_info: dict[str, Any] = {"new_path": moved}
|
||||
if sidecar_moved:
|
||||
extra_info["sidecar_count"] = len(sidecar_moved)
|
||||
|
||||
|
|
@ -617,7 +617,7 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
|
|||
if not json or not isinstance(json, list):
|
||||
return web.json_response({"error": "Invalid parameters."}, status=400)
|
||||
|
||||
files: list[str] = []
|
||||
files: list[Path] = []
|
||||
for f in json:
|
||||
if not isinstance(f, str):
|
||||
continue
|
||||
|
|
@ -627,12 +627,10 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
|
|||
continue
|
||||
files.append(ref)
|
||||
|
||||
sc: list[dict] = get_file_sidecar(ref)
|
||||
sc: dict[str, list[dict[str, Any]]] = get_file_sidecar(ref)
|
||||
if sc:
|
||||
for side in sc:
|
||||
for scf in sc[side]:
|
||||
if isinstance(scf, dict) and "file" in scf:
|
||||
files.append(scf["file"]) # noqa: PERF401
|
||||
for value in sc.values():
|
||||
files.extend(scf["file"] for scf in value if isinstance(scf, dict) and "file" in scf)
|
||||
|
||||
if not files:
|
||||
return web.json_response({"error": "No valid files."}, status=400)
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response:
|
|||
|
||||
cache.set(cache_key, dct, ttl=3600)
|
||||
|
||||
return web.Response(**dct)
|
||||
return web.Response(body=dct["body"], headers=dct["headers"])
|
||||
except Exception:
|
||||
LOG.exception(
|
||||
"Failed to request doc '%s' from '%s'.",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from aiohttp import web
|
||||
from aiohttp.web import Request, Response
|
||||
from aiohttp.web import Request, Response, StreamResponse
|
||||
|
||||
from app.library.config import Config
|
||||
from app.library.log import get_logger
|
||||
|
|
@ -10,7 +10,7 @@ LOG = get_logger()
|
|||
|
||||
|
||||
@route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static")
|
||||
async def download_file(request: Request, config: Config, app: web.Application) -> Response:
|
||||
async def download_file(request: Request, config: Config, app: web.Application) -> StreamResponse:
|
||||
"""
|
||||
Serve download files.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import Any
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.web import Request, Response
|
||||
from aiohttp.web_response import StreamResponse
|
||||
|
||||
from app.features.presets.schemas import Preset
|
||||
from app.features.presets.service import Presets
|
||||
|
|
@ -20,10 +21,6 @@ from app.library.log import get_logger
|
|||
from app.library.router import route
|
||||
from app.library.Utils import calc_download_path, get_file_sidecar, rename_file
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from library.downloads import Download
|
||||
|
||||
|
||||
LOG = get_logger()
|
||||
|
||||
|
||||
|
|
@ -280,7 +277,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
id: str = request.match_info.get("id")
|
||||
id: str | None = request.match_info.get("id")
|
||||
if not id:
|
||||
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
|
|
@ -314,7 +311,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
|
|||
|
||||
|
||||
@route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail")
|
||||
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> Response:
|
||||
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> StreamResponse:
|
||||
if not (id := request.match_info.get("id")):
|
||||
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
|
|
@ -458,7 +455,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
|
|||
Response: The response object.
|
||||
|
||||
"""
|
||||
id: str = request.match_info.get("id")
|
||||
id: str | None = request.match_info.get("id")
|
||||
if not id:
|
||||
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
||||
|
||||
|
|
|
|||
|
|
@ -116,8 +116,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp
|
|||
safe_backend: str = _safe_url(backend)
|
||||
await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24)
|
||||
else:
|
||||
backend = await cache.aget(CACHE_KEY_BING)
|
||||
safe_backend = _safe_url(backend if isinstance(backend, str) else None)
|
||||
cached_backend = await cache.aget(CACHE_KEY_BING)
|
||||
backend = cached_backend if isinstance(cached_backend, str) else ""
|
||||
safe_backend = _safe_url(backend)
|
||||
|
||||
if not isinstance(backend, str) or not backend:
|
||||
return web.json_response(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
|
@ -110,7 +111,7 @@ async def _read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict:
|
|||
return {"logs": [], "next_offset": None, "end_is_reached": True}
|
||||
|
||||
|
||||
async def _tail_log(file: Path, emitter: callable, sleep_time: float = 0.5):
|
||||
async def _tail_log(file: Path, emitter: Callable[[dict], Awaitable[None]], sleep_time: float = 0.5):
|
||||
"""
|
||||
Live tail.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.web import Request, Response
|
||||
|
|
@ -489,7 +490,7 @@ async def system_limits(queue: DownloadQueue, config: Config, encoder: Encoder)
|
|||
|
||||
for download in [*active_non_live, *queued_non_live]:
|
||||
extractor = (download.info.get_extractor() or "unknown").lower()
|
||||
entry = per_extractor.setdefault(
|
||||
entry: dict[str, Any] = per_extractor.setdefault(
|
||||
extractor,
|
||||
{
|
||||
"name": extractor,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from pathlib import Path
|
|||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
|
||||
|
||||
APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve())
|
||||
if APP_ROOT not in sys.path:
|
||||
|
|
@ -69,7 +69,7 @@ def parse_args() -> argparse.Namespace:
|
|||
return parser.parse_args()
|
||||
|
||||
|
||||
async def connect_db(db_file: str) -> AsyncConnection:
|
||||
async def connect_db(db_file: Path) -> tuple[AsyncConnection, AsyncEngine]:
|
||||
"""Create async SQLAlchemy connection."""
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}")
|
||||
conn = await engine.connect()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -91,7 +92,7 @@ class TestAgSet:
|
|||
|
||||
def test_ag_set_error_on_non_dict_final(self):
|
||||
"""Test error when final target is not a dict."""
|
||||
data = "not_a_dict"
|
||||
data: Any = "not_a_dict"
|
||||
with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"):
|
||||
ag_set(data, "key", "value")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import http.cookiejar
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -12,10 +13,10 @@ try:
|
|||
YTDLP_AVAILABLE = True
|
||||
except ImportError:
|
||||
YTDLP_AVAILABLE = False
|
||||
Request = Mock
|
||||
Response = Mock
|
||||
RequestDirector = Mock
|
||||
HTTPError = Exception
|
||||
Request: Any = Mock
|
||||
Response: Any = Mock
|
||||
RequestDirector: Any = Mock
|
||||
HTTPError: Any = Exception
|
||||
|
||||
pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -24,7 +25,7 @@ class TestConditionIgnoreMatching:
|
|||
second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
||||
|
||||
service = Conditions.get_instance()
|
||||
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
|
||||
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
|
||||
|
||||
matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"])
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ class TestConditionIgnoreMatching:
|
|||
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
||||
|
||||
service = Conditions.get_instance()
|
||||
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
|
||||
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
|
||||
|
||||
matched = await service.match(info={"duration": 60}, ignore_conditions=["123"])
|
||||
|
||||
|
|
@ -48,7 +49,7 @@ class TestConditionIgnoreMatching:
|
|||
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
||||
|
||||
service = Conditions.get_instance()
|
||||
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
|
||||
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
|
||||
|
||||
matched = await service.match(info={"duration": 60}, ignore_conditions=[123])
|
||||
|
||||
|
|
@ -59,7 +60,7 @@ class TestConditionIgnoreMatching:
|
|||
first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20)
|
||||
|
||||
service = Conditions.get_instance()
|
||||
service._repo = Mock(list=AsyncMock(return_value=[first]))
|
||||
service._repo = Mock(all=AsyncMock(return_value=[first]))
|
||||
|
||||
matched = await service.match(info={"duration": 60}, ignore_conditions=["*"])
|
||||
|
||||
|
|
@ -78,10 +79,11 @@ class TestConditionIgnorePropagation:
|
|||
preset="default",
|
||||
extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]},
|
||||
)
|
||||
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
||||
item.get_archive_id = Mock(return_value=None)
|
||||
item.get_archive_file = Mock(return_value=None)
|
||||
item.is_archived = Mock(return_value=False)
|
||||
mock_item: Any = item
|
||||
mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
||||
mock_item.get_archive_id = Mock(return_value=None)
|
||||
mock_item.get_archive_file = Mock(return_value=None)
|
||||
mock_item.is_archived = Mock(return_value=False)
|
||||
|
||||
matcher = Mock(match=AsyncMock(return_value=None))
|
||||
entry = {"id": "video-1", "duration": 60}
|
||||
|
|
@ -113,10 +115,11 @@ class TestConditionIgnorePropagation:
|
|||
preset="default",
|
||||
extras={"ignore_conditions": [123, "Primary", True, " * "]},
|
||||
)
|
||||
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
||||
item.get_archive_id = Mock(return_value=None)
|
||||
item.get_archive_file = Mock(return_value=None)
|
||||
item.is_archived = Mock(return_value=False)
|
||||
mock_item: Any = item
|
||||
mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
||||
mock_item.get_archive_id = Mock(return_value=None)
|
||||
mock_item.get_archive_file = Mock(return_value=None)
|
||||
mock_item.is_archived = Mock(return_value=False)
|
||||
|
||||
matcher = Mock(match=AsyncMock(return_value=None))
|
||||
entry = {"id": "video-1", "duration": 60}
|
||||
|
|
@ -135,7 +138,7 @@ class TestConditionIgnorePropagation:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_playlist_keeps_parent_ignore(self) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class FakeItem:
|
||||
def __init__(self) -> None:
|
||||
|
|
@ -158,7 +161,8 @@ class TestConditionIgnorePropagation:
|
|||
}
|
||||
|
||||
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
|
||||
result = await process_playlist(queue=queue, entry=entry, item=FakeItem())
|
||||
fake: Any = FakeItem()
|
||||
result = await process_playlist(queue=queue, entry=entry, item=fake)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
assert captured["url"] == "https://example.com/v/1"
|
||||
|
|
@ -209,7 +213,8 @@ class TestConditionIgnorePropagation:
|
|||
}
|
||||
|
||||
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
|
||||
result = await process_playlist(queue=queue, entry=entry, item=FakeItem())
|
||||
fake: Any = FakeItem()
|
||||
result = await process_playlist(queue=queue, entry=entry, item=fake)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
assert queue.add.await_count == 2
|
||||
|
|
|
|||
|
|
@ -4,10 +4,13 @@ from collections import OrderedDict
|
|||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import formatdate
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.library.DataStore import DataStore, StoreType
|
||||
from app.library.downloads import Download
|
||||
from app.library.ItemDTO import ItemDTO
|
||||
from app.library.operations import Operation
|
||||
from app.library.sqlite_store import SqliteStore
|
||||
|
|
@ -62,17 +65,18 @@ async def make_db(data: int = 0) -> SqliteStore:
|
|||
return ins
|
||||
|
||||
|
||||
class StubDownload:
|
||||
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False):
|
||||
class StubDownload(Download):
|
||||
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False) -> None:
|
||||
self.info = info
|
||||
self._started: bool = started
|
||||
self._cancelled: bool = cancelled
|
||||
self.id = info._id
|
||||
self._stub_started: bool = started
|
||||
self._stub_cancelled: bool = cancelled
|
||||
|
||||
def started(self) -> bool:
|
||||
return self._started
|
||||
return self._stub_started
|
||||
|
||||
def is_cancelled(self) -> bool:
|
||||
return self._cancelled
|
||||
return self._stub_cancelled
|
||||
|
||||
|
||||
async def make_store_async(store_type: StoreType) -> DataStore:
|
||||
|
|
@ -357,7 +361,8 @@ class TestDataStore:
|
|||
def __init__(self):
|
||||
self.info = None
|
||||
|
||||
store._dict["broken"] = BrokenDownload()
|
||||
broken: Any = BrokenDownload()
|
||||
store._dict["broken"] = broken
|
||||
|
||||
# Should still find the valid item
|
||||
result = await store.get_item(title="Video 1")
|
||||
|
|
@ -960,10 +965,10 @@ class TestDataStoreOperations:
|
|||
|
||||
item1 = make_item(id="vid1", title="Video 1")
|
||||
item1._id = "id1"
|
||||
item1.filesize = 1000
|
||||
setattr(item1, "filesize", 1000)
|
||||
item2 = make_item(id="vid2", title="Video 2")
|
||||
item2._id = "id2"
|
||||
item2.filesize = 2000
|
||||
setattr(item2, "filesize", 2000)
|
||||
|
||||
await store.put(StubDownload(info=item1))
|
||||
await store.put(StubDownload(info=item2))
|
||||
|
|
@ -986,10 +991,10 @@ class TestDataStoreOperations:
|
|||
|
||||
item1 = make_item(id="vid1", title="Video 1")
|
||||
item1._id = "id1"
|
||||
item1.filesize = 1000
|
||||
setattr(item1, "filesize", 1000)
|
||||
item2 = make_item(id="vid2", title="Video 2")
|
||||
item2._id = "id2"
|
||||
item2.filesize = 2000
|
||||
setattr(item2, "filesize", 2000)
|
||||
|
||||
await store.put(StubDownload(info=item1))
|
||||
await store.put(StubDownload(info=item2))
|
||||
|
|
@ -1007,7 +1012,7 @@ class TestDataStoreOperations:
|
|||
|
||||
item1 = make_item(id="vid1", title="Video 1")
|
||||
item1._id = "id1"
|
||||
item1.filesize = 1000
|
||||
setattr(item1, "filesize", 1000)
|
||||
|
||||
await store.put(StubDownload(info=item1))
|
||||
|
||||
|
|
@ -1033,7 +1038,7 @@ class TestDataStoreOperations:
|
|||
|
||||
item1 = make_item(id="vid1", title="Video 1")
|
||||
item1._id = "id1"
|
||||
item1.filesize = 1000
|
||||
setattr(item1, "filesize", 1000)
|
||||
|
||||
await store.put(StubDownload(info=item1))
|
||||
|
||||
|
|
@ -1084,7 +1089,7 @@ class TestDataStoreOperations:
|
|||
|
||||
item1 = make_item(id="vid1", title="Video 1")
|
||||
item1._id = "id1"
|
||||
item1.description = None
|
||||
setattr(item1, "description", None)
|
||||
|
||||
await store.put(StubDownload(info=item1))
|
||||
|
||||
|
|
|
|||
|
|
@ -307,7 +307,8 @@ class TestDownloadFlow:
|
|||
postprocessor_hook=Mock(),
|
||||
post_hook=Mock(),
|
||||
)
|
||||
download.info.get_ytdlp_opts = Mock(
|
||||
download_any: Any = download.info
|
||||
download_any.get_ytdlp_opts = Mock(
|
||||
return_value=Mock(
|
||||
add=Mock(
|
||||
return_value=Mock(
|
||||
|
|
@ -379,7 +380,8 @@ class TestDownloadFlow:
|
|||
postprocessor_hook=Mock(),
|
||||
post_hook=Mock(),
|
||||
)
|
||||
download.info.get_ytdlp_opts = Mock(
|
||||
download_any2: Any = download.info
|
||||
download_any2.get_ytdlp_opts = Mock(
|
||||
return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={}))))
|
||||
)
|
||||
|
||||
|
|
@ -481,7 +483,8 @@ class TestDownloadFlow:
|
|||
)
|
||||
queue.put(Terminator())
|
||||
|
||||
download._download = fake_download
|
||||
download_mock: Any = download
|
||||
download_mock._download = fake_download
|
||||
|
||||
class InlineProcess:
|
||||
def __init__(self, target):
|
||||
|
|
@ -578,7 +581,8 @@ class TestDownloadFlow:
|
|||
queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)})
|
||||
queue.put(Terminator())
|
||||
|
||||
download._download = fake_download
|
||||
download_mock: Any = download
|
||||
download_mock._download = fake_download
|
||||
|
||||
class InlineProcess:
|
||||
def __init__(self, target):
|
||||
|
|
@ -1028,7 +1032,7 @@ class TestStatusTracker:
|
|||
"debug": False,
|
||||
}
|
||||
|
||||
def test_init_sets_attributes(self, mock_config: dict) -> None:
|
||||
def test_init_sets_attributes(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
assert st.id == "test-id", "Should set download ID"
|
||||
assert st.info == mock_config["info"], "Should set info reference"
|
||||
|
|
@ -1036,7 +1040,7 @@ class TestStatusTracker:
|
|||
assert st.final_update is False, "Should initialize final_update as False"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_ignores_bad_id(self, mock_config: dict) -> None:
|
||||
async def test_status_ignores_bad_id(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "wrong-id", "status": "downloading"}
|
||||
|
||||
|
|
@ -1044,14 +1048,14 @@ class TestStatusTracker:
|
|||
assert st.info.status != "downloading", "Should not update status for wrong ID"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_ignores_short(self, mock_config: dict) -> None:
|
||||
async def test_status_ignores_short(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "test-id"}
|
||||
|
||||
await st.process_status_update(status)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_status_update_sets_status(self, mock_config: dict) -> None:
|
||||
async def test_process_status_update_sets_status(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000}
|
||||
|
||||
|
|
@ -1059,7 +1063,7 @@ class TestStatusTracker:
|
|||
assert st.info.status == "downloading", "Should update info status"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_sets_skipped(self, mock_config: dict) -> None:
|
||||
async def test_status_sets_skipped(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "test-id", "status": "downloading", "download_skipped": True}
|
||||
|
||||
|
|
@ -1067,7 +1071,7 @@ class TestStatusTracker:
|
|||
assert st.info.download_skipped is True, "Should update download_skipped from status queue"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None:
|
||||
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"}
|
||||
|
||||
|
|
@ -1075,7 +1079,7 @@ class TestStatusTracker:
|
|||
assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_sets_percent(self, mock_config: dict) -> None:
|
||||
async def test_status_sets_percent(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {
|
||||
"id": "test-id",
|
||||
|
|
@ -1090,7 +1094,7 @@ class TestStatusTracker:
|
|||
assert st.info.percent == 50.0, "Should calculate percent correctly"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_uses_estimate(self, mock_config: dict) -> None:
|
||||
async def test_status_uses_estimate(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {
|
||||
"id": "test-id",
|
||||
|
|
@ -1104,7 +1108,7 @@ class TestStatusTracker:
|
|||
assert st.info.percent == 30.0, "Should calculate percent from estimate"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_percent(self, mock_config: dict) -> None:
|
||||
async def test_status_percent(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {
|
||||
"id": "test-id",
|
||||
|
|
@ -1117,7 +1121,7 @@ class TestStatusTracker:
|
|||
assert st.info.percent == 50.0, "Should calculate percent correctly with valid total"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_sets_speed_eta(self, mock_config: dict) -> None:
|
||||
async def test_status_sets_speed_eta(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60}
|
||||
|
||||
|
|
@ -1126,7 +1130,7 @@ class TestStatusTracker:
|
|||
assert st.info.eta == 60, "Should set eta"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_status_update_sets_error(self, mock_config: dict) -> None:
|
||||
async def test_process_status_update_sets_error(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
status = {"id": "test-id", "status": "error", "error": "Download failed"}
|
||||
|
||||
|
|
@ -1135,7 +1139,7 @@ class TestStatusTracker:
|
|||
assert st.info.error == "Download failed", "Should set error message"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None:
|
||||
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
|
||||
test_file = tmp_path / "test.mp4"
|
||||
test_file.write_text("test content")
|
||||
|
||||
|
|
@ -1148,20 +1152,20 @@ class TestStatusTracker:
|
|||
assert st.info.filename == "test.mp4", "Should set relative filename"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None:
|
||||
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict[str, Any]) -> None:
|
||||
queue = DummyQueue()
|
||||
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100})
|
||||
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200})
|
||||
queue.put(Terminator())
|
||||
|
||||
config = {**mock_config, "status_queue": queue}
|
||||
config: dict[str, Any] = {**mock_config, "status_queue": queue}
|
||||
st = StatusTracker(**config)
|
||||
|
||||
await st.drain_queue(max_iterations=10)
|
||||
assert st.info.downloaded_bytes == 200, "Should process all queued updates"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None:
|
||||
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
|
||||
test_file = tmp_path / "test.mp4"
|
||||
test_file.write_text("test content")
|
||||
|
||||
|
|
@ -1169,25 +1173,25 @@ class TestStatusTracker:
|
|||
queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)})
|
||||
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999})
|
||||
|
||||
config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
|
||||
config: dict[str, Any] = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
|
||||
st = StatusTracker(**config)
|
||||
|
||||
await st.drain_queue(max_iterations=10)
|
||||
assert st.final_update is True, "Should stop draining after final update"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_queue_skips_invalid(self, mock_config: dict) -> None:
|
||||
async def test_drain_queue_skips_invalid(self, mock_config: dict[str, Any]) -> None:
|
||||
queue = DummyQueue()
|
||||
queue.put({"id": "test-id", "status": "downloading"})
|
||||
queue.put(None)
|
||||
|
||||
config = {**mock_config, "status_queue": queue}
|
||||
config: dict[str, Any] = {**mock_config, "status_queue": queue}
|
||||
st = StatusTracker(**config)
|
||||
|
||||
await st.drain_queue(max_iterations=5)
|
||||
assert st.info.status == "downloading", "valid updates should still be processed"
|
||||
|
||||
def test_cancel_update_task(self, mock_config: dict) -> None:
|
||||
def test_cancel_update_task(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
st.update_task = Mock()
|
||||
st.update_task.done = Mock(return_value=False)
|
||||
|
|
@ -1196,15 +1200,15 @@ class TestStatusTracker:
|
|||
st.cancel_update_task()
|
||||
st.update_task.cancel.assert_called_once()
|
||||
|
||||
def test_cancel_update_task_noop(self, mock_config: dict) -> None:
|
||||
def test_cancel_update_task_noop(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
|
||||
st.cancel_update_task()
|
||||
assert st.update_task is None, "missing tasks should be ignored"
|
||||
|
||||
def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None:
|
||||
def test_put_terminator_adds_to_queue(self, mock_config: dict[str, Any]) -> None:
|
||||
queue = DummyQueue()
|
||||
config = {**mock_config, "status_queue": queue}
|
||||
config: dict[str, Any] = {**mock_config, "status_queue": queue}
|
||||
st = StatusTracker(**config)
|
||||
|
||||
st.put_terminator()
|
||||
|
|
@ -1212,7 +1216,7 @@ class TestStatusTracker:
|
|||
assert isinstance(queue.items[0], Terminator), "Should add Terminator instance"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_emits_item_progress(self, mock_config: dict) -> None:
|
||||
async def test_progress_emits_item_progress(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
st.info.status = "downloading"
|
||||
calls: list = []
|
||||
|
|
@ -1233,7 +1237,7 @@ class TestStatusTracker:
|
|||
assert "options" not in payload
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_change_emits_item_updated(self, mock_config: dict) -> None:
|
||||
async def test_status_change_emits_item_updated(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
st.info.status = "started"
|
||||
calls: list = []
|
||||
|
|
@ -1249,7 +1253,7 @@ class TestStatusTracker:
|
|||
assert updated_calls[0][1]["data"] is st.info
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_throttled(self, mock_config: dict) -> None:
|
||||
async def test_progress_throttled(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
st.info.status = "downloading"
|
||||
st._progress_interval = 0.5
|
||||
|
|
@ -1272,7 +1276,7 @@ class TestStatusTracker:
|
|||
assert st._pending_progress is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flush_on_status_change(self, mock_config: dict) -> None:
|
||||
async def test_flush_on_status_change(self, mock_config: dict[str, Any]) -> None:
|
||||
st = StatusTracker(**mock_config)
|
||||
st.info.status = "downloading"
|
||||
st._progress_interval = 10.0
|
||||
|
|
@ -1345,23 +1349,29 @@ class TestQueueManager:
|
|||
auto_start=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _any_video_item() -> Any:
|
||||
return TestQueueManager._video_item()
|
||||
|
||||
def test_live_queue_caps_visible_items(self) -> None:
|
||||
queue_manager = object.__new__(DownloadQueue)
|
||||
items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
|
||||
items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
|
||||
queue_manager.queue = self.LiveStore(items)
|
||||
queue_manager.pool = Mock()
|
||||
queue_manager.pool.get_active_downloads.return_value = {}
|
||||
|
||||
snapshot = DownloadQueue.live_queue(queue_manager, limit=2)
|
||||
|
||||
assert list(snapshot["queue"].keys()) == ["id0", "id1"]
|
||||
queue_view = snapshot["queue"]
|
||||
assert isinstance(queue_view, dict)
|
||||
assert list(queue_view.keys()) == ["id0", "id1"]
|
||||
assert snapshot["queue_count"] == 5
|
||||
assert snapshot["queue_loaded"] == 2
|
||||
assert snapshot["queue_limit"] == 2
|
||||
|
||||
def test_live_queue_keeps_active(self) -> None:
|
||||
queue_manager = object.__new__(DownloadQueue)
|
||||
items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
|
||||
items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
|
||||
queue_manager.queue = self.LiveStore(items)
|
||||
queue_manager.pool = Mock()
|
||||
queue_manager.pool.get_active_downloads.return_value = {
|
||||
|
|
@ -1371,7 +1381,9 @@ class TestQueueManager:
|
|||
|
||||
snapshot = DownloadQueue.live_queue(queue_manager, limit=1)
|
||||
|
||||
assert list(snapshot["queue"].keys()) == ["id3", "id4"]
|
||||
queue_view = snapshot["queue"]
|
||||
assert isinstance(queue_view, dict)
|
||||
assert list(queue_view.keys()) == ["id3", "id4"]
|
||||
assert snapshot["queue_count"] == 5
|
||||
assert snapshot["queue_loaded"] == 2
|
||||
assert snapshot["queue_limit"] == 1
|
||||
|
|
@ -1388,7 +1400,7 @@ class TestQueueManager:
|
|||
|
||||
result = await add_video(
|
||||
queue=self._video_queue(),
|
||||
item=self._video_item(),
|
||||
item=self._any_video_item(),
|
||||
entry={
|
||||
"id": "live-id",
|
||||
"title": "Live stream",
|
||||
|
|
@ -1420,7 +1432,7 @@ class TestQueueManager:
|
|||
"formats": [{"format_id": "18"}],
|
||||
}
|
||||
|
||||
result = await add_video(queue=self._video_queue(), item=self._video_item(), entry=entry)
|
||||
result = await add_video(queue=self._video_queue(), item=self._any_video_item(), entry=entry)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
assert seen == [entry]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -132,7 +133,8 @@ class TestEncoder:
|
|||
return True
|
||||
return original_isinstance(obj, cls)
|
||||
|
||||
builtins.isinstance = mock_isinstance
|
||||
builtins_any: Any = builtins
|
||||
builtins_any.isinstance = mock_isinstance
|
||||
|
||||
try:
|
||||
result = self.encoder.default(mock_daterange)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -7,7 +8,7 @@ import pytest
|
|||
from app.yt_dlp_plugins.extractor import generic_browser
|
||||
|
||||
|
||||
def _make_ie(config: dict[str, str | None] | None = None) -> generic_browser.GenericBrowserIE:
|
||||
def _make_ie(config: dict[str, str | None] | None = None) -> Any:
|
||||
ie = object.__new__(generic_browser.GenericBrowserIE)
|
||||
values = config or {}
|
||||
ie._configuration_arg = lambda name, default: [values.get(name)]
|
||||
|
|
@ -250,6 +251,7 @@ def test_extract_network_formats_playlist_entries_keep_own_urls() -> None:
|
|||
{"title": "Title", "webpage_url": "https://example.com/page", "original_url": "https://example.com/page"},
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["_type"] == "playlist"
|
||||
assert result["entries"][0]["url"] == "https://cdn.example/1.mp3"
|
||||
assert result["entries"][0]["webpage_url"] == "https://cdn.example/1.mp3"
|
||||
|
|
|
|||
|
|
@ -36,7 +36,8 @@ def _make_download(
|
|||
) -> SimpleNamespace:
|
||||
base_dir = download_dir or "/downloads"
|
||||
original_post_init = ItemDTO.__post_init__
|
||||
ItemDTO.__post_init__ = lambda self: None
|
||||
cls: Any = ItemDTO
|
||||
cls.__post_init__ = lambda self: None
|
||||
|
||||
try:
|
||||
item = ItemDTO(
|
||||
|
|
|
|||
|
|
@ -73,5 +73,5 @@ async def test_bg_log_redact(caplog: pytest.LogCaptureFixture, monkeypatch: pyte
|
|||
)
|
||||
assert "apitoken=secret" not in record.getMessage()
|
||||
assert "user:pass@" not in record.getMessage()
|
||||
assert record.url == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted"
|
||||
assert record.exception_type == "RuntimeError"
|
||||
assert getattr(record, "url", None) == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted"
|
||||
assert getattr(record, "exception_type", None) == "RuntimeError"
|
||||
|
|
|
|||
|
|
@ -386,7 +386,7 @@ class TestItemAddExtras:
|
|||
def test_add_extras_none(self):
|
||||
"""Test adding extras when extras is None."""
|
||||
item = Item(url="https://example.com")
|
||||
item.extras = None
|
||||
setattr(item, "extras", None)
|
||||
|
||||
item.add_extras("key1", "value1")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -21,7 +22,7 @@ if "aiocron" not in sys.modules:
|
|||
def uuid(self) -> str:
|
||||
return "stub-uuid"
|
||||
|
||||
aiocron_stub.Cron = _CronImportStub
|
||||
setattr(aiocron_stub, "Cron", _CronImportStub)
|
||||
sys.modules["aiocron"] = aiocron_stub
|
||||
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ class DummyCron:
|
|||
self,
|
||||
*,
|
||||
spec: str,
|
||||
func: callable,
|
||||
func: Callable,
|
||||
args: tuple = (),
|
||||
kwargs: dict | None = None,
|
||||
uuid: str | None = None,
|
||||
|
|
@ -231,7 +232,7 @@ class TestScheduler:
|
|||
self,
|
||||
*_,
|
||||
spec: str,
|
||||
func: callable,
|
||||
func: Callable,
|
||||
args: tuple = (),
|
||||
kwargs: dict | None = None,
|
||||
uuid: str | None = None,
|
||||
|
|
@ -266,7 +267,7 @@ class TestScheduler:
|
|||
self,
|
||||
*_,
|
||||
spec: str,
|
||||
func: callable,
|
||||
func: Callable,
|
||||
args: tuple = (),
|
||||
kwargs: dict | None = None,
|
||||
uuid: str | None = None,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -182,12 +183,14 @@ async def test_paginate_order_and_bounds():
|
|||
encoded = itm.json()
|
||||
created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
_list.append(itm)
|
||||
assert store._conn is not None
|
||||
await store._conn.execute(
|
||||
text(
|
||||
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)'
|
||||
),
|
||||
{"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at},
|
||||
)
|
||||
assert store._conn is not None
|
||||
await store._conn.commit()
|
||||
|
||||
items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC")
|
||||
|
|
@ -208,7 +211,9 @@ async def test_get_by_id():
|
|||
item = make_item(99)
|
||||
await store.enqueue_upsert("queue", item)
|
||||
await store.flush()
|
||||
assert item._id == (await store.get_by_id("queue", item._id))._id
|
||||
fetched = await store.get_by_id("queue", item._id)
|
||||
assert fetched is not None
|
||||
assert item._id == fetched._id
|
||||
await store.close()
|
||||
|
||||
|
||||
|
|
@ -358,7 +363,8 @@ async def test_on_shutdown_closes_connection():
|
|||
await store.enqueue_upsert("queue", make_item(1))
|
||||
await store.flush()
|
||||
|
||||
await store.on_shutdown(None)
|
||||
app: Any = None
|
||||
await store.on_shutdown(app)
|
||||
assert store._conn is None
|
||||
# Note: on_shutdown already closes the connection, so no need to call close() again
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
|
@ -10,6 +10,7 @@ import uuid
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -861,16 +862,20 @@ class TestMergeDict:
|
|||
def test_merge_dict_type_validation(self):
|
||||
"""Test that non-dict parameters are properly rejected."""
|
||||
# Test with non-dict source
|
||||
bad_src: Any = "not_a_dict"
|
||||
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
|
||||
merge_dict("not_a_dict", {"key": "value"})
|
||||
merge_dict(bad_src, {"key": "value"})
|
||||
|
||||
# Test with non-dict destination
|
||||
bad_dst: Any = "not_a_dict"
|
||||
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
|
||||
merge_dict({"key": "value"}, "not_a_dict")
|
||||
merge_dict({"key": "value"}, bad_dst)
|
||||
|
||||
# Test with both non-dict
|
||||
bad_src2: Any = "not_a_dict"
|
||||
bad_dst2: Any = ["also_not_dict"]
|
||||
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
|
||||
merge_dict("not_a_dict", ["also_not_dict"])
|
||||
merge_dict(bad_src2, bad_dst2)
|
||||
|
||||
def test_merge_dict_immutability(self):
|
||||
"""Test that original dictionaries are not modified (immutability)."""
|
||||
|
|
@ -1084,7 +1089,8 @@ class TestEncryptDecrypt:
|
|||
assert encrypted != data
|
||||
assert isinstance(encrypted, str)
|
||||
|
||||
decrypted: str = decrypt_data(encrypted, key)
|
||||
decrypted = decrypt_data(encrypted, key)
|
||||
assert decrypted is not None
|
||||
assert decrypted == data
|
||||
|
||||
def test_encrypt_empty_string(self):
|
||||
|
|
@ -1093,7 +1099,8 @@ class TestEncryptDecrypt:
|
|||
data: str = ""
|
||||
|
||||
encrypted: str = encrypt_data(data, key)
|
||||
decrypted: str = decrypt_data(encrypted, key)
|
||||
decrypted = decrypt_data(encrypted, key)
|
||||
assert decrypted is not None
|
||||
assert decrypted == data
|
||||
|
||||
|
||||
|
|
@ -1489,7 +1496,8 @@ class TestGet:
|
|||
def test_get_list_access(self):
|
||||
"""Test getting from list."""
|
||||
data = ["item0", "item1", "item2"]
|
||||
result = get(data, 1)
|
||||
key: Any = 1
|
||||
result = get(data, key)
|
||||
assert result == "item1"
|
||||
|
||||
def test_get_empty_path(self):
|
||||
|
|
|
|||
|
|
@ -10,9 +10,8 @@ import logging
|
|||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from library.PackageInstaller import PackageInstaller, Packages
|
||||
|
||||
from app.library.log import get_logger
|
||||
from app.library.PackageInstaller import PackageInstaller, Packages
|
||||
|
||||
LOG = get_logger()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import base64
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
|
@ -190,12 +191,7 @@ class RemoteBrowserUnavailableError(ExtractorError):
|
|||
class PlaywrightDriver:
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
try:
|
||||
import playwright.sync_api # noqa: F401
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
return importlib.util.find_spec("playwright.sync_api") is not None
|
||||
|
||||
@staticmethod
|
||||
def connect(ws_url: str, timeout: int | None = None):
|
||||
|
|
@ -336,15 +332,11 @@ class PlaywrightDriver:
|
|||
class SeleniumDriver:
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
try:
|
||||
import selenium.webdriver # noqa: F401
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
return importlib.util.find_spec("selenium.webdriver") is not None
|
||||
|
||||
@staticmethod
|
||||
def connect(ws_url: str, timeout: int | None = None): # noqa: ARG004
|
||||
def connect(ws_url: str, timeout: int | None = None):
|
||||
_ = timeout
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
|
|
@ -441,6 +433,10 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
_failed: bool = False
|
||||
_remote_browser_failures: dict[str, str] = {}
|
||||
_url: str = ""
|
||||
__wrapped__: Any
|
||||
|
||||
def _fallback_extract(self, url: str) -> dict[str, Any]:
|
||||
return self.__wrapped__._real_extract(self, url)
|
||||
|
||||
def _get_config(self, name: str, env_name: str) -> str | None:
|
||||
value = self._configuration_arg(name, [None])[0]
|
||||
|
|
@ -473,7 +469,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
self._url = url
|
||||
|
||||
if not (browser_url := self._get_config("url", "YTP_BROWSER_URL")) or self._failed:
|
||||
return self.__wrapped__._real_extract(self, url)
|
||||
return self._fallback_extract(url)
|
||||
|
||||
safe_url = self._safe_url(browser_url)
|
||||
|
||||
|
|
@ -496,7 +492,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
message = str(e)
|
||||
self._failed = True
|
||||
self.report_warning(f"Remote browser unavailable: {message}, marking as failed.", video_id)
|
||||
return self.__wrapped__._real_extract(self, url)
|
||||
return self._fallback_extract(url)
|
||||
|
||||
try:
|
||||
self.report_extraction(url)
|
||||
|
|
@ -513,17 +509,18 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
requests = self._merge_requests(session.get_requests(), session.get_media_requests())
|
||||
self.write_debug(f"Captured {len(requests)} network requests")
|
||||
|
||||
if self._downloader.params.get("dump_intermediate_pages"):
|
||||
downloader = self._downloader
|
||||
if downloader and downloader.params.get("dump_intermediate_pages"):
|
||||
self.to_screen(f"Browser content dump for: {url}")
|
||||
self.to_screen(base64.b64encode(webpage.encode("utf-8")).decode("ascii"))
|
||||
|
||||
if self._downloader.params.get("write_pages"):
|
||||
filename = _request_dump_filename(url, video_id, None, self._downloader.params.get("trim_file_name"))
|
||||
if downloader and downloader.params.get("write_pages"):
|
||||
filename = _request_dump_filename(url, video_id, None, downloader.params.get("trim_file_name"))
|
||||
self.to_screen(f"Saving request to {filename}")
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
f.write(webpage)
|
||||
|
||||
if self._downloader.params.get("debug_printtraffic"):
|
||||
if downloader and downloader.params.get("debug_printtraffic"):
|
||||
self.to_screen(f"[browser] {url}")
|
||||
self.to_screen(f"[browser] Captured '{len(requests)}' network requests")
|
||||
for req in requests:
|
||||
|
|
@ -599,9 +596,9 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
self, requests: list[dict], video_id: str, base_info: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
candidates = self._pick_network_candidates(requests)
|
||||
formats = []
|
||||
direct_formats = []
|
||||
subtitles = {}
|
||||
formats: list[dict[str, Any]] = []
|
||||
direct_formats: list[dict[str, Any]] = []
|
||||
subtitles: dict[str, Any] = {}
|
||||
source_counts = {}
|
||||
has_manifest_formats = False
|
||||
|
||||
|
|
@ -674,7 +671,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
self.report_warning(
|
||||
"Generic browser extractor found no media formats. falling back to generic extractor.", video_id
|
||||
)
|
||||
return self.__wrapped__._real_extract(self, self._url)
|
||||
return self._fallback_extract(self._url)
|
||||
|
||||
if not has_manifest_formats and len(direct_formats) > 1:
|
||||
base_title = (base_info.get("title") or "").strip() or video_id
|
||||
|
|
@ -698,7 +695,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
)
|
||||
return {"_type": "playlist", "entries": entries}
|
||||
|
||||
result = {"formats": formats, "direct": True}
|
||||
result: dict[str, Any] = {"formats": formats, "direct": True}
|
||||
if subtitles:
|
||||
result["subtitles"] = subtitles
|
||||
if formats and formats[0].get("url"):
|
||||
|
|
@ -733,9 +730,13 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
has_media_header_ext = header_ext and header_ext in MEDIA_EXTENSIONS
|
||||
|
||||
resource_type = entry.get("resourceType")
|
||||
if resource_type and resource_type.lower() not in MEDIA_RESOURCE_TYPES: # noqa: SIM102
|
||||
if not has_media_ext and not has_media_header_ext:
|
||||
continue
|
||||
if (
|
||||
resource_type
|
||||
and resource_type.lower() not in MEDIA_RESOURCE_TYPES
|
||||
and not has_media_ext
|
||||
and not has_media_header_ext
|
||||
):
|
||||
continue
|
||||
|
||||
if not ext and not header_ext:
|
||||
rt = (resource_type or "").lower()
|
||||
|
|
@ -779,7 +780,8 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
|
|||
)
|
||||
|
||||
def _get_timeout_ms(self) -> int | None:
|
||||
socket_timeout = self._downloader.params.get("socket_timeout")
|
||||
downloader = self._downloader
|
||||
socket_timeout = downloader.params.get("socket_timeout") if downloader else None
|
||||
if isinstance(socket_timeout, (int, float)) and socket_timeout > 0:
|
||||
return int(socket_timeout * 1000)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -119,7 +119,8 @@ class NFOMakerPP(PostProcessor):
|
|||
def pp_key(cls) -> str:
|
||||
return "NFOMaker"
|
||||
|
||||
def run(self, info: dict | None = None) -> tuple[list, dict]:
|
||||
def run(self, information: Any) -> tuple[list, dict[str, Any]]:
|
||||
info = information
|
||||
if "fail" == self.mode:
|
||||
msg = "Intentionally failing due to mode=fail."
|
||||
raise Exception(msg)
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue