chore: use ty as type checker.

This commit is contained in:
arabcoders 2026-06-14 10:43:58 +03:00
parent 18f54a28c6
commit 3631947234
107 changed files with 1563 additions and 1354 deletions

View file

@ -6,13 +6,16 @@ from app.features.conditions.migration import Migration
from app.library.Singleton import Singleton from app.library.Singleton import Singleton
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Iterable from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
from sqlalchemy import delete, func, or_, select from sqlalchemy import delete, func, or_, select
from app.features.conditions.models import ConditionModel from app.features.conditions.models import ConditionModel
@ -22,10 +25,26 @@ from app.library.log import get_logger
LOG = get_logger() LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
model = ConditionModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for ConditionModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
if isinstance(payload, ConditionModel):
return payload
return _model_from_payload(payload)
class ConditionsRepository(metaclass=Singleton): class ConditionsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None: async def run_migrations(self) -> None:
if self._migrated: if self._migrated:
@ -38,7 +57,7 @@ class ConditionsRepository(metaclass=Singleton):
def get_instance() -> ConditionsRepository: def get_instance() -> ConditionsRepository:
return ConditionsRepository() return ConditionsRepository()
async def list(self) -> list[ConditionModel]: async def all(self) -> list[ConditionModel]:
async with self.session() as session: async with self.session() as session:
result: Result[tuple[ConditionModel]] = await session.execute( result: Result[tuple[ConditionModel]] = await session.execute(
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc()) select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
@ -93,11 +112,11 @@ class ConditionsRepository(metaclass=Singleton):
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1)) result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none() return result.scalar_one_or_none()
async def create(self, payload: ConditionModel | dict) -> ConditionModel: async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
async with self.session() as session: async with self.session() as session:
model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload model = _coerce_model(payload)
if model.id is not None: if model.id is not None:
model.id = None model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None: if await self.get_by_name(name=model.name) is not None:
msg: str = f"Condition with name '{model.name}' already exists." msg: str = f"Condition with name '{model.name}' already exists."
@ -160,13 +179,11 @@ class ConditionsRepository(metaclass=Singleton):
await session.commit() await session.commit()
return model return model
async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]: async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
async with self.session() as session: async with self.session() as session:
try: try:
await session.execute(delete(ConditionModel)) await session.execute(delete(ConditionModel))
models: list[ConditionModel] = [ models: list[ConditionModel] = [_coerce_model(item) for item in items]
ConditionModel(**item) if isinstance(item, dict) else item for item in items
]
session.add_all(models) session.add_all(models)
await session.commit() await session.commit()
except Exception: except Exception:

View file

@ -131,6 +131,12 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
status=web.HTTPInternalServerError.status_code, status=web.HTTPInternalServerError.status_code,
) )
if not isinstance(data, dict):
return web.json_response(
data={"error": "Failed to extract video info."},
status=web.HTTPInternalServerError.status_code,
)
try: try:
from app.features.ytdlp.mini_filter import match_str from app.features.ytdlp.mini_filter import match_str

View file

@ -1,10 +1,10 @@
from collections.abc import Iterable from collections.abc import Iterable
from numbers import Number
from aiohttp import web from aiohttp import web
from app.features.conditions.models import ConditionModel from app.features.conditions.models import ConditionModel
from app.features.conditions.repository import ConditionsRepository from app.features.conditions.repository import ConditionsRepository
from app.features.conditions.schemas import Condition
from app.features.ytdlp.mini_filter import match_str from app.features.ytdlp.mini_filter import match_str
from app.library.Events import EventBus, Events from app.library.Events import EventBus, Events
from app.library.log import get_logger from app.library.log import get_logger
@ -13,7 +13,7 @@ from app.library.Singleton import Singleton
LOG = get_logger() LOG = get_logger()
def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tuple[set[str], bool]: def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
ignored: set[str] = set() ignored: set[str] = set()
ignore_all = False ignore_all = False
@ -21,7 +21,7 @@ def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tu
return ignored, ignore_all return ignored, ignore_all
for value in ignore_conditions: for value in ignore_conditions:
if isinstance(value, bool) or not isinstance(value, (str, Number)): if isinstance(value, bool) or not isinstance(value, (str, int, float)):
continue continue
identifier = str(value).strip() identifier = str(value).strip()
@ -55,7 +55,7 @@ class Conditions(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations") EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
async def get_all(self) -> list[ConditionModel]: async def get_all(self) -> list[ConditionModel]:
return await self._repo.list() return await self._repo.all()
async def save(self, item: ConditionModel | dict) -> ConditionModel: async def save(self, item: ConditionModel | dict) -> ConditionModel:
""" """
@ -80,7 +80,7 @@ class Conditions(metaclass=Singleton):
if item.id is None or 0 == item.id: if item.id is None or 0 == item.id:
model = await repo.create(item) model = await repo.create(item)
else: else:
model = await repo.update(item.id, item.serialize()) model = await repo.update(item.id, Condition.model_validate(item).model_dump())
except KeyError as exc: except KeyError as exc:
raise ValueError(str(exc)) from exc raise ValueError(str(exc)) from exc
@ -113,13 +113,15 @@ class Conditions(metaclass=Singleton):
repo = self._repo repo = self._repo
return await repo.get(identifier) return await repo.get(identifier)
async def match(self, info: dict, ignore_conditions: Iterable[str | Number] | None = None) -> ConditionModel | None: async def match(
self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
) -> ConditionModel | None:
""" """
Check if any condition matches the info dict. Check if any condition matches the info dict.
Args: Args:
info (dict): The info dict to check. info (dict): The info dict to check.
ignore_conditions (Iterable[str | Number] | None): Condition ids or names to skip for this match. ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
Returns: Returns:
Condition|None: The condition if found, None otherwise. Condition|None: The condition if found, None otherwise.
@ -133,7 +135,7 @@ class Conditions(metaclass=Singleton):
return None return None
repo = self._repo repo = self._repo
items: list[ConditionModel] = await repo.list() items: list[ConditionModel] = await repo.all()
if len(items) < 1: if len(items) < 1:
return None return None
@ -187,7 +189,7 @@ class Conditions(metaclass=Singleton):
if not info or not isinstance(info, dict) or len(info) < 1: if not info or not isinstance(info, dict) or len(info) < 1:
return None return None
if not (item := await self.get(identifier)) or not item.enabled or not item.filter: if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
return None return None
return item if match_str(item.filter, info) else None return item if match_str(item.filter, info) else None

View file

@ -6,6 +6,7 @@ import pytest
import pytest_asyncio import pytest_asyncio
from types import SimpleNamespace from types import SimpleNamespace
import pytest import pytest
from typing import Any
from aiohttp import web from aiohttp import web
from aiohttp.test_utils import make_mocked_request from aiohttp.test_utils import make_mocked_request
@ -39,13 +40,13 @@ async def repo():
def _json_request(path: str, payload: object) -> web.Request: def _json_request(path: str, payload: object) -> web.Request:
request = make_mocked_request("POST", path) mock_request: Any = make_mocked_request("POST", path)
async def _json() -> object: async def _json() -> object:
return payload return payload
request.json = _json # type: ignore[attr-defined] mock_request.json = _json
return request return mock_request
class TestAllowInternalUrlsScope: class TestAllowInternalUrlsScope:
@ -208,7 +209,7 @@ class TestConditionsRepository:
await repo.create({"name": "A", "filter": "test", "priority": 1}) await repo.create({"name": "A", "filter": "test", "priority": 1})
await repo.create({"name": "C", "filter": "test", "priority": 2}) await repo.create({"name": "C", "filter": "test", "priority": 2})
items = await repo.list() items = await repo.all()
assert items[0].name == "C", "Highest priority should be first" assert items[0].name == "C", "Highest priority should be first"
assert items[1].name == "A", "Same priority sorted alphabetically" assert items[1].name == "A", "Same priority sorted alphabetically"
@ -240,6 +241,6 @@ class TestConditionsRepository:
assert len(result) == 2, "Should create 2 new conditions" assert len(result) == 2, "Should create 2 new conditions"
all_items = await repo.list() all_items = await repo.all()
assert len(all_items) == 2, "Should only have new conditions" assert len(all_items) == 2, "Should only have new conditions"
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items" assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"

View file

@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import UTC, datetime from datetime import UTC, datetime
from sqlalchemy import DateTime as SQLADateTime from sqlalchemy import DateTime as SQLADateTime
from sqlalchemy import TypeDecorator from sqlalchemy import Dialect, TypeDecorator
from sqlalchemy.orm import DeclarativeBase from sqlalchemy.orm import DeclarativeBase
@ -20,16 +20,18 @@ class UTCDateTime(TypeDecorator):
impl = SQLADateTime impl = SQLADateTime
cache_ok = True cache_ok = True
def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None: def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
"""Convert datetime to UTC before storing.""" """Convert datetime to UTC before storing."""
_ = dialect
if value is not None: if value is not None:
if value.tzinfo is None: if value.tzinfo is None:
return value.replace(tzinfo=UTC) return value.replace(tzinfo=UTC)
return value.astimezone(UTC).replace(tzinfo=None) return value.astimezone(UTC).replace(tzinfo=None)
return value return value
def process_result_value(self, value: datetime | None, _dialect) -> datetime | None: def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
"""Ensure datetime is timezone-aware (UTC) when loading.""" """Ensure datetime is timezone-aware (UTC) when loading."""
_ = dialect
if value is not None and value.tzinfo is None: if value is not None and value.tzinfo is None:
return value.replace(tzinfo=UTC) return value.replace(tzinfo=UTC)
return value return value

View file

@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any, cast
from sqlalchemy import delete, func, or_, select from sqlalchemy import delete, func, or_, select
@ -11,20 +11,39 @@ from app.library.log import get_logger
from app.library.Singleton import Singleton from app.library.Singleton import Singleton
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Iterable from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger() LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
model = DLFieldModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
if isinstance(payload, DLFieldModel):
return payload
return _model_from_payload(payload)
class DLFieldsRepository(metaclass=Singleton): class DLFieldsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None: async def run_migrations(self) -> None:
if self._migrated: if self._migrated:
@ -37,7 +56,7 @@ class DLFieldsRepository(metaclass=Singleton):
def get_instance() -> DLFieldsRepository: def get_instance() -> DLFieldsRepository:
return DLFieldsRepository() return DLFieldsRepository()
async def list(self) -> list[DLFieldModel]: async def all(self) -> list[DLFieldModel]:
async with self.session() as session: async with self.session() as session:
result: Result[tuple[DLFieldModel]] = await session.execute( result: Result[tuple[DLFieldModel]] = await session.execute(
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc()) select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
@ -90,9 +109,9 @@ class DLFieldsRepository(metaclass=Singleton):
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1)) result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none() return result.scalar_one_or_none()
async def create(self, payload: DLFieldModel | dict) -> DLFieldModel: async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
async with self.session() as session: async with self.session() as session:
model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload model = _coerce_model(payload)
if await self.get_by_name(name=model.name) is not None: if await self.get_by_name(name=model.name) is not None:
msg: str = f"DL field with name '{model.name}' already exists." msg: str = f"DL field with name '{model.name}' already exists."
@ -153,13 +172,16 @@ class DLFieldsRepository(metaclass=Singleton):
await session.commit() await session.commit()
return model return model
async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]: async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
async with self.session() as session: async with self.session() as session:
try: try:
await session.execute(delete(DLFieldModel)) await session.execute(delete(DLFieldModel))
models: list[DLFieldModel] = [ models: list[DLFieldModel] = []
DLFieldModel(**item) if isinstance(item, dict) else item for item in items for item in items:
] if isinstance(item, dict):
models.append(DLFieldModel(**cast("dict[str, Any]", item)))
else:
models.append(item)
session.add_all(models) session.add_all(models)
await session.commit() await session.commit()
except Exception: except Exception:

View file

@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations") EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
async def get_all(self) -> list[DLFieldModel]: async def get_all(self) -> list[DLFieldModel]:
return await self._repo.list() return await self._repo.all()
async def get_all_serialized(self) -> list[dict[str, Any]]: async def get_all_serialized(self) -> list[dict[str, Any]]:
items = await self._repo.list() items = await self._repo.all()
return [DLField.model_validate(item).model_dump() for item in items] return [DLField.model_validate(item).model_dump() for item in items]
async def save(self, item: DLField | dict) -> DLFieldModel: async def save(self, item: DLField | dict) -> DLFieldModel:

View file

@ -204,7 +204,7 @@ class TestDLFieldsRepository:
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1}) await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0}) await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
items = await repo.list() items = await repo.all()
assert items[0].name == "c", "Lowest order should be first" assert items[0].name == "c", "Lowest order should be first"
assert items[1].name == "a", "Same order sorted alphabetically" assert items[1].name == "a", "Same order sorted alphabetically"
@ -243,6 +243,6 @@ class TestDLFieldsRepository:
assert len(result) == 2, "Should create 2 new fields" assert len(result) == 2, "Should create 2 new fields"
all_items = await repo.list() all_items = await repo.all()
assert len(all_items) == 2, "Should only have new fields" assert len(all_items) == 2, "Should only have new fields"
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items" assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"

View file

@ -11,20 +11,39 @@ from app.library.log import get_logger
from app.library.Singleton import Singleton from app.library.Singleton import Singleton
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger() LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
model = NotificationModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for NotificationModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
if isinstance(payload, NotificationModel):
return payload
return _model_from_payload(payload)
class NotificationsRepository(metaclass=Singleton): class NotificationsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None: async def run_migrations(self) -> None:
if self._migrated: if self._migrated:
@ -37,7 +56,7 @@ class NotificationsRepository(metaclass=Singleton):
def get_instance() -> NotificationsRepository: def get_instance() -> NotificationsRepository:
return NotificationsRepository() return NotificationsRepository()
async def list(self) -> list[NotificationModel]: async def all(self) -> list[NotificationModel]:
async with self.session() as session: async with self.session() as session:
result: Result[tuple[NotificationModel]] = await session.execute( result: Result[tuple[NotificationModel]] = await session.execute(
select(NotificationModel).order_by(NotificationModel.name.asc()) select(NotificationModel).order_by(NotificationModel.name.asc())
@ -92,11 +111,11 @@ class NotificationsRepository(metaclass=Singleton):
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1)) result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none() return result.scalar_one_or_none()
async def create(self, payload: NotificationModel | dict) -> NotificationModel: async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
async with self.session() as session: async with self.session() as session:
model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload model = _coerce_model(payload)
if model.id is not None: if model.id is not None:
model.id = None model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None: if await self.get_by_name(name=model.name) is not None:
msg: str = f"Notification target with name '{model.name}' already exists." msg: str = f"Notification target with name '{model.name}' already exists."

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio import asyncio
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any, cast
from app.features.notifications.models import NotificationModel from app.features.notifications.models import NotificationModel
from app.features.notifications.repository import NotificationsRepository from app.features.notifications.repository import NotificationsRepository
@ -64,10 +64,10 @@ class Notifications(metaclass=Singleton):
await self._repo.run_migrations() await self._repo.run_migrations()
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations") EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit") EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
async def list(self) -> list[NotificationModel]: async def all(self) -> list[NotificationModel]:
return await self._repo.list() return await self._repo.all()
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]: async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
return await self._repo.list_paginated(page, per_page) return await self._repo.list_paginated(page, per_page)
@ -92,8 +92,8 @@ class Notifications(metaclass=Singleton):
async def delete(self, identifier: int | str) -> NotificationModel: async def delete(self, identifier: int | str) -> NotificationModel:
return await self._repo.delete(identifier) return await self._repo.delete(identifier)
async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]: async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
targets = await self._repo.list() targets = await self._repo.all()
if len(targets) < 1: if len(targets) < 1:
return [] return []
@ -126,7 +126,7 @@ class Notifications(metaclass=Singleton):
if wait: if wait:
return await asyncio.gather(*tasks) return await asyncio.gather(*tasks)
return tasks return cast("list[dict | Awaitable[dict]]", tasks)
def emit(self, e: Event, _, **__) -> None: def emit(self, e: Event, _, **__) -> None:
if not NotificationEvents.is_valid(e.event): if not NotificationEvents.is_valid(e.event):
@ -271,7 +271,7 @@ class Notifications(metaclass=Singleton):
if not status: if not status:
msg = "Apprise failed to send notification." msg = "Apprise failed to send notification."
raise RuntimeError(msg) # noqa: TRY301 self._raise_apprise_error(msg)
except Exception as exc: except Exception as exc:
LOG.exception( LOG.exception(
"Failed to send Apprise notification for event '%s'.", "Failed to send Apprise notification for event '%s'.",
@ -288,6 +288,10 @@ class Notifications(metaclass=Singleton):
return {} return {}
@staticmethod
def _raise_apprise_error(msg: str) -> None:
raise RuntimeError(msg)
async def _send(self, target: Notification, ev: Event) -> dict: async def _send(self, target: Notification, ev: Event) -> dict:
try: try:
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name) LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)

View file

@ -35,7 +35,7 @@ class TestNotificationsRepository:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_list_empty(self, repo): async def test_list_empty(self, repo):
"""List returns empty when no notifications exist.""" """List returns empty when no notifications exist."""
notifications = await repo.list() notifications = await repo.all()
assert notifications == [], "Should return empty list when no notifications exist" assert notifications == [], "Should return empty list when no notifications exist"
@pytest.mark.asyncio @pytest.mark.asyncio

View file

@ -46,7 +46,7 @@ class Migration(FeatureMigration):
return return
inserted = 0 inserted = 0
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()} seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
for index, item in enumerate(items): for index, item in enumerate(items):
if not (normalized := self._normalize(item, index, seen_names)): if not (normalized := self._normalize(item, index, seen_names)):

View file

@ -30,6 +30,30 @@ if TYPE_CHECKING:
LOG = get_logger() LOG = get_logger()
def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
if isinstance(payload, dict):
data: dict[str, Any] = {}
for key, value in payload.items():
if not isinstance(key, str):
msg = "Preset payload keys must be strings."
raise TypeError(msg)
data[key] = value
return data
return {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
class PresetsRepository(metaclass=Singleton): class PresetsRepository(metaclass=Singleton):
SORT_FIELDS: dict[str, Any] = { SORT_FIELDS: dict[str, Any] = {
"id": PresetModel.id, "id": PresetModel.id,
@ -73,15 +97,15 @@ class PresetsRepository(metaclass=Singleton):
LOG.debug("Refreshing presets cache due to configuration update.") LOG.debug("Refreshing presets cache due to configuration update.")
await self._update_cache() await self._update_cache()
Services.get_instance().add(__class__.__name__, self) Services.get_instance().add(PresetsRepository.__name__, self)
EventBus.get_instance().subscribe( EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache") ).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
async def _update_cache(self) -> None: async def _update_cache(self) -> None:
from app.features.presets.service import Presets from app.features.presets.service import Presets
await Presets.get_instance().refresh_cache(await self.list()) await Presets.get_instance().refresh_cache(await self.all())
@staticmethod @staticmethod
def get_instance() -> PresetsRepository: def get_instance() -> PresetsRepository:
@ -97,7 +121,7 @@ class PresetsRepository(metaclass=Singleton):
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name) LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
config.default_preset = "default" config.default_preset = "default"
async def list(self) -> list[PresetModel]: async def all(self) -> list[PresetModel]:
async with self.session() as session: async with self.session() as session:
result: Result[tuple[PresetModel]] = await session.execute( result: Result[tuple[PresetModel]] = await session.execute(
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc()) select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
@ -238,24 +262,9 @@ class PresetsRepository(metaclass=Singleton):
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1)) result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none() return result.scalar_one_or_none()
async def create(self, payload: PresetModel | dict) -> PresetModel: async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
async with self.session() as session: async with self.session() as session:
data: dict[str, Any] data = _payload_data(payload)
if isinstance(payload, dict):
data = dict(payload)
else:
data = {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
data.pop("id", None) data.pop("id", None)
model = PresetModel(**data) model = PresetModel(**data)

View file

@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
def __init__(self, repo: PresetsRepository | None = None) -> None: def __init__(self, repo: PresetsRepository | None = None) -> None:
self._repo: PresetsRepository = repo or PresetsRepository.get_instance() self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
self._cache: list[tuple[int, str, Preset]] = [] self._cache: list[tuple[int, str, Preset]] = []
Services.get_instance().add(__class__.__name__, self) Services.get_instance().add(type(self).__name__, self)
@staticmethod @staticmethod
def get_instance() -> Presets: def get_instance() -> Presets:

View file

@ -57,7 +57,7 @@ class TestPresetsRepository:
await repo.create({"name": "A", "priority": 1}) await repo.create({"name": "A", "priority": 1})
await repo.create({"name": "C", "priority": 2}) await repo.create({"name": "C", "priority": 2})
items = await repo.list() items = await repo.all()
assert items[0].name == "c", "Highest priority should be first" assert items[0].name == "c", "Highest priority should be first"
assert items[1].name == "a", "Same priority should sort by name" assert items[1].name == "a", "Same priority should sort by name"

View file

@ -10,8 +10,6 @@ import os
import subprocess # qa: ignore import subprocess # qa: ignore
from pathlib import Path from pathlib import Path
import anyio
from app.features.streaming.types import FFProbeError from app.features.streaming.types import FFProbeError
from app.library.log import get_logger from app.library.log import get_logger
from app.library.Utils import timed_lru_cache from app.library.Utils import timed_lru_cache
@ -39,21 +37,26 @@ class FFStream:
def __repr__(self): def __repr__(self):
if "codec_long_name" not in self.__dict__: if "codec_long_name" not in self.__dict__:
self.codec_long_name = self.__dict__.get("codec_name", "") self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
index = self.__dict__.get("index", "?")
codec_type = self.__dict__.get("codec_type", "unknown")
codec_long_name = self.__dict__.get("codec_long_name", "")
if self.is_video(): if self.is_video():
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>" return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
if self.is_audio(): if self.is_audio():
return ( return (
f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), " f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
"{sample_rate}Hz> " f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
f"{self.__dict__.get('sample_rate')}Hz> "
) )
if self.is_subtitle() or self.is_attachment(): if self.is_subtitle() or self.is_attachment():
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>" return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
return f"<Stream: #{self.index} [{self.codec_type}]>" return f"<Stream: #{index} [{codec_type}]>"
def is_audio(self): def is_audio(self):
""" """
@ -252,14 +255,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
raise OSError(msg) raise OSError(msg)
try: try:
async with await anyio.open_file(os.devnull, "w") as tempf: proc = await asyncio.create_subprocess_exec(
await asyncio.create_subprocess_exec(
"ffprobe", "ffprobe",
"-h", "-h",
stdout=tempf, stdout=asyncio.subprocess.DEVNULL,
stderr=tempf, stderr=asyncio.subprocess.DEVNULL,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
) )
await proc.wait()
except FileNotFoundError as e: except FileNotFoundError as e:
msg = "ffprobe not found." msg = "ffprobe not found."
raise OSError(msg) from e raise OSError(msg) from e

View file

@ -39,7 +39,7 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0") m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD") m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
segmentSize: float = f"{self.duration:.6f}" segmentSize: str = f"{self.duration:.6f}"
splits: int = math.ceil(duration / self.duration) splits: int = math.ceil(duration / self.duration)
segmentParams: dict = {} segmentParams: dict = {}

View file

@ -14,7 +14,7 @@ class Playlist:
"The path where files are downloaded." "The path where files are downloaded."
async def make(self, file: Path) -> str: async def make(self, file: Path) -> str:
ref: str = Path(str(file.relative_to(self.download_path)).strip("/")) ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
try: try:
ff = await ffprobe(file) ff = await ffprobe(file)

View file

@ -1,7 +1,7 @@
# flake8: noqa: ARG002
from __future__ import annotations from __future__ import annotations
import os import os
import shutil
import subprocess import subprocess
import sys import sys
from functools import lru_cache from functools import lru_cache
@ -16,9 +16,13 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
values are dicts with keys "full" and "lp" booleans. values are dicts with keys "full" and "lp" booleans.
""" """
vainfo = shutil.which("vainfo")
if not vainfo:
return {}
try: try:
result: subprocess.CompletedProcess[str] = subprocess.run( result: subprocess.CompletedProcess[str] = subprocess.run(
["vainfo"], # noqa: S607 [vainfo],
capture_output=True, capture_output=True,
text=True, text=True,
check=False, check=False,
@ -87,9 +91,13 @@ def ffmpeg_encoders() -> set[str]:
""" """
from app.library.config import SUPPORTED_CODECS from app.library.config import SUPPORTED_CODECS
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
return set()
try: try:
result: subprocess.CompletedProcess[str] = subprocess.run( result: subprocess.CompletedProcess[str] = subprocess.run(
["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607 [ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
capture_output=True, capture_output=True,
text=True, text=True,
check=False, check=False,
@ -149,9 +157,11 @@ class _BaseBuilder:
codec_name: str codec_name: str
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]: def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return [] return []
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]: def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return [*args, "-codec:v", self.codec_name] return [*args, "-codec:v", self.codec_name]
@ -312,4 +322,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
"libx264": [], "libx264": [],
} }
return chains.get(codec, chains["libx264"]) return tuple(chains.get(codec, chains["libx264"]))

View file

@ -1,6 +1,6 @@
import asyncio import asyncio
import os import os
import subprocess # type: ignore import subprocess
import sys import sys
import tempfile import tempfile
from pathlib import Path from pathlib import Path
@ -178,6 +178,7 @@ class Segments:
) )
try: try:
assert proc.stdout is not None
while True: while True:
chunk: bytes = await proc.stdout.read(1024 * 64) chunk: bytes = await proc.stdout.read(1024 * 64)
if not chunk: if not chunk:
@ -240,7 +241,9 @@ class Segments:
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip()) return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
async def stream(self, file: Path, resp: web.StreamResponse) -> None: async def stream(self, file: Path, resp: web.StreamResponse) -> None:
codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec codec: str = self.vcodec
if Segments._cache_initialized and Segments._cached_vcodec:
codec = Segments._cached_vcodec
if not codec or codec not in SUPPORTED_CODECS: if not codec or codec not in SUPPORTED_CODECS:
codec: str = select_encoder(self.vcodec or "") codec: str = select_encoder(self.vcodec or "")

View file

@ -2,6 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any
import anyio import anyio
import pysubs2 import pysubs2
@ -48,7 +49,8 @@ def ms_to_timestamp(ms: int) -> str:
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}" return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
SubstationFormat.ms_to_timestamp = ms_to_timestamp _substation_format: Any = SubstationFormat
_substation_format.ms_to_timestamp = ms_to_timestamp
class Subtitle: class Subtitle:

View file

@ -24,6 +24,8 @@ THUMBNAIL_MISS_TTL = 3600.0
_LOCK = asyncio.Lock() _LOCK = asyncio.Lock()
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {} _IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
_SEM: asyncio.Semaphore | None = None _SEM: asyncio.Semaphore | None = None
_SEM_LIMIT: int | None = None _SEM_LIMIT: int | None = None

View file

@ -4,6 +4,7 @@ from pathlib import Path
from aiohttp import web from aiohttp import web
from aiohttp.web import Request, Response from aiohttp.web import Request, Response
from aiohttp.web_response import StreamResponse
from app.features.streaming.library.m3u8 import M3u8 from app.features.streaming.library.m3u8 import M3u8
from app.features.streaming.library.playlist import Playlist from app.features.streaming.library.playlist import Playlist
@ -32,7 +33,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
if not file: if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -83,8 +84,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
mode: str = request.match_info.get("mode") mode: str | None = request.match_info.get("mode")
if mode not in ["video", "subtitle"]: if mode not in ["video", "subtitle"]:
return web.json_response( return web.json_response(
@ -94,13 +95,14 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
if not file: if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
duration = request.query.get("duration", None) duration: float | None = None
duration_arg = request.query.get("duration", None)
if "subtitle" in mode: if "subtitle" in mode:
if not duration: if not duration_arg:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
duration = float(duration) duration = float(duration_arg)
base_path: str = config.base_path.rstrip("/") base_path: str = config.base_path.rstrip("/")
@ -124,6 +126,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
if "subtitle" in mode: if "subtitle" in mode:
if duration is None:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
text = await cls.make_subtitle(file=realFile, duration=duration) text = await cls.make_subtitle(file=realFile, duration=duration)
else: else:
text = await cls.make_stream(file=realFile) text = await cls.make_stream(file=realFile)
@ -149,7 +153,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream") @route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
async def segments_stream(request: Request, config: Config, app: web.Application) -> Response: async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
""" """
Get the segments. Get the segments.
@ -162,9 +166,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
segment: int = request.match_info.get("segment") segment: str | None = request.match_info.get("segment")
sd: int = request.query.get("sd") sd: str | None = request.query.get("sd")
vc: int = int(request.query.get("vc", 0)) vc: int = int(request.query.get("vc", 0))
ac: int = int(request.query.get("ac", 0)) ac: int = int(request.query.get("ac", 0))
@ -241,7 +245,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
if not file: if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -299,7 +303,7 @@ async def subtitles_manifest_get(request: Request, config: Config, app: web.Appl
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
if not file: if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -354,7 +358,7 @@ async def subtitles_track_get(request: Request, config: Config, app: web.Applica
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
source_format: str | None = request.match_info.get("source_format") source_format: str | None = request.match_info.get("source_format")
if not file: if not file:

View file

@ -4,6 +4,7 @@ from pathlib import Path
from typing import Any from typing import Any
import pytest import pytest
from aiohttp import web
from app.features.streaming.library.segments import Segments from app.features.streaming.library.segments import Segments
from app.tests.helpers import get_test_system_temp_root from app.tests.helpers import get_test_system_temp_root
@ -148,18 +149,18 @@ class _FakeProc:
self.killed = True self.killed = True
class _FakeResp: class _FakeResp(web.StreamResponse):
def __init__(self, fail_with: Exception | None = None) -> None: def __init__(self, fail_with: BaseException | None = None) -> None:
self.data: bytearray = bytearray() self.data: bytearray = bytearray()
self.eof = False self.eof = False
self._exc = fail_with self._exc = fail_with
async def write(self, data: bytes) -> None: async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
if self._exc: if self._exc:
raise self._exc raise self._exc
self.data.extend(data) self.data.extend(data)
async def write_eof(self) -> None: async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
self.eof = True self.eof = True

View file

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio import asyncio
from pathlib import Path from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock from unittest.mock import AsyncMock
import pytest import pytest
@ -260,7 +261,7 @@ async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
output = temp_dir / ".jpg" output = temp_dir / ".jpg"
media.write_text("video") media.write_text("video")
ff_info = FFProbeResult() ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "120.0"} ff_info.metadata = {"duration": "120.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()] ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
@ -321,7 +322,7 @@ async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
media1.write_text("video") media1.write_text("video")
media2.write_text("video") media2.write_text("video")
ff_info = FFProbeResult() ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "60.0"} ff_info.metadata = {"duration": "60.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()] ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info)) monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))

View file

@ -1,4 +1,3 @@
# flake8: noqa: ARG004
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
@ -12,6 +11,7 @@ if TYPE_CHECKING:
class BaseHandler: class BaseHandler:
@staticmethod @staticmethod
async def can_handle(task: HandleTask) -> bool: async def can_handle(task: HandleTask) -> bool:
_ = task
return False return False
@staticmethod @staticmethod
@ -24,6 +24,7 @@ class BaseHandler:
@staticmethod @staticmethod
def parse(url: str) -> Any | None: def parse(url: str) -> Any | None:
_ = url
return None return None
@staticmethod @staticmethod

View file

@ -64,7 +64,7 @@ class GenericTaskHandler(BaseHandler):
from app.features.tasks.definitions.utils import model_to_schema from app.features.tasks.definitions.utils import model_to_schema
repo = TaskDefinitionsRepository.get_instance() repo = TaskDefinitionsRepository.get_instance()
models = await repo.list() models = await repo.all()
cls._definitions = [model_to_schema(model) for model in models] cls._definitions = [model_to_schema(model) for model in models]
return cls._definitions return cls._definitions
@ -141,7 +141,8 @@ class GenericTaskHandler(BaseHandler):
return False return False
@staticmethod @staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
if not definition: if not definition:
return TaskFailure(message="No generic task definition matched the provided URL.") return TaskFailure(message="No generic task definition matched the provided URL.")

View file

@ -9,6 +9,7 @@ from app.features.tasks.definitions.results import HandleTask, TaskFailure, Task
from app.features.ytdlp.extractor import fetch_info from app.features.ytdlp.extractor import fetch_info
from app.features.ytdlp.utils import get_archive_id from app.features.ytdlp.utils import get_archive_id
from app.library.cache import Cache from app.library.cache import Cache
from app.library.config import Config
from app.library.log import get_logger from app.library.log import get_logger
from ._base_handler import BaseHandler from ._base_handler import BaseHandler
@ -160,12 +161,14 @@ class RssGenericHandler(BaseHandler):
return feed_url, items, real_count return feed_url, items, real_count
@staticmethod @staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure: async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
""" """
Extract items from an RSS/Atom feed. Extract items from an RSS/Atom feed.
Args: Args:
task (Task): The task containing the feed URL. task (Task): The task containing the feed URL.
config (Config | None): Optional handler configuration.
Returns: Returns:
TaskResult | TaskFailure: Extraction result with parsed items or failure information. TaskResult | TaskFailure: Extraction result with parsed items or failure information.

View file

@ -4,6 +4,7 @@ import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger from app.library.log import get_logger
from ._base_handler import BaseHandler from ._base_handler import BaseHandler
@ -177,7 +178,8 @@ class TverHandler(BaseHandler):
return feed_url, items, has_items return feed_url, items, has_items
@staticmethod @staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure: async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
series_id: str | None = TverHandler.parse(task.url) series_id: str | None = TverHandler.parse(task.url)
if not series_id: if not series_id:
return TaskFailure(message="Unrecognized Tver series URL.") return TaskFailure(message="Unrecognized Tver series URL.")
@ -208,10 +210,11 @@ class TverHandler(BaseHandler):
if not (url := entry.get("url")): if not (url := entry.get("url")):
continue continue
archive_id: str = entry.get("archive_id") archive_id: str | None = entry.get("archive_id")
task_items.append( metadata = entry.get("metadata", {})
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {})) if not isinstance(metadata, dict):
) metadata = {}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items}) return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -6,6 +6,7 @@ import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger from app.library.log import get_logger
from ._base_handler import BaseHandler from ._base_handler import BaseHandler
@ -92,7 +93,8 @@ class TwitchHandler(BaseHandler):
return feed_url, items, has_items return feed_url, items, has_items
@staticmethod @staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure: async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
handle_name: str | None = TwitchHandler.parse(task.url) handle_name: str | None = TwitchHandler.parse(task.url)
if not handle_name: if not handle_name:
return TaskFailure(message="Unrecognized Twitch channel URL.") return TaskFailure(message="Unrecognized Twitch channel URL.")
@ -122,7 +124,7 @@ class TwitchHandler(BaseHandler):
if not (url := entry.get("url")): if not (url := entry.get("url")):
continue continue
archive_id: str = entry.get("archive_id") archive_id: str | None = entry.get("archive_id")
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id)) task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items}) return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -6,6 +6,7 @@ import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger from app.library.log import get_logger
from ._base_handler import BaseHandler from ._base_handler import BaseHandler
@ -103,7 +104,8 @@ class YoutubeHandler(BaseHandler):
return feed_url, items, real_count return feed_url, items, real_count
@staticmethod @staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure: async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url) parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
if not parsed: if not parsed:
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.") return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
@ -133,7 +135,7 @@ class YoutubeHandler(BaseHandler):
if not (url := entry.get("url")): if not (url := entry.get("url")):
continue continue
archive_id: str = entry.get("archive_id") archive_id: str | None = entry.get("archive_id")
metadata: dict[str, Any] = {"published": entry.get("published")} metadata: dict[str, Any] = {"published": entry.get("published")}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata)) task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))

View file

@ -14,20 +14,23 @@ from app.library.Services import Services
from app.library.Singleton import Singleton from app.library.Singleton import Singleton
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger() LOG = get_logger()
class TaskDefinitionsRepository(metaclass=Singleton): class TaskDefinitionsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None: async def run_migrations(self) -> None:
if self._migrated: if self._migrated:
@ -51,12 +54,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
LOG.debug("Refreshing task definitions due to configuration update.") LOG.debug("Refreshing task definitions due to configuration update.")
await GenericTaskHandler.refresh_definitions(force=True) await GenericTaskHandler.refresh_definitions(force=True)
Services.get_instance().add(__class__.__name__, self) Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
EventBus.get_instance().subscribe( EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions") ).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
async def list(self) -> list[TaskDefinitionModel]: async def all(self) -> list[TaskDefinitionModel]:
async with self.session() as session: async with self.session() as session:
result: Result[tuple[TaskDefinitionModel]] = await session.execute( result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc()) select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
@ -115,9 +118,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel: async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
async with self.session() as session: async with self.session() as session:
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload model: TaskDefinitionModel = TaskDefinitionModel(**payload)
if model.id is not None: if model.id is not None:
model.id = None model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None: if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task definition with name '{model.name}' already exists." msg: str = f"Task definition with name '{model.name}' already exists."

View file

@ -48,10 +48,13 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple): if isinstance(ret, tuple):
return ret return ret
archive_file: Path = ret.get("file") archive_file = ret.get("file")
items: set[str] = ret.get("items", set()) items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
if len(items) < 1 or not archive_add(archive_file, list(items)): if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
return (True, "No new items to mark as downloaded.") return (True, "No new items to mark as downloaded.")
return (True, f"Task '{self.name}' items marked as downloaded.") return (True, f"Task '{self.name}' items marked as downloaded.")
@ -70,10 +73,13 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple): if isinstance(ret, tuple):
return ret return ret
archive_file: Path = ret.get("file") archive_file = ret.get("file")
items: set[str] = ret.get("items", set()) items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
if len(items) < 1 or not archive_delete(archive_file, list(items)): if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
return (True, "No items to remove from archive file.") return (True, "No items to remove from archive file.")
return (True, f"Removed '{self.name}' items from archive file.") return (True, f"Removed '{self.name}' items from archive file.")

View file

@ -8,6 +8,7 @@ import random
from datetime import UTC, datetime from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.handlers._base_handler import BaseHandler
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.tasks.models import TaskModel from app.features.tasks.models import TaskModel
from app.features.ytdlp.utils import archive_read from app.features.ytdlp.utils import archive_read
@ -27,7 +28,7 @@ LOG = get_logger()
class TaskHandle: class TaskHandle:
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None: def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
self._handlers: list[type] = [] self._handlers: list[type[BaseHandler]] = []
"The available handlers." "The available handlers."
self._repo: TasksRepository = tasks self._repo: TasksRepository = tasks
"The tasks manager." "The tasks manager."
@ -35,7 +36,7 @@ class TaskHandle:
"The scheduler." "The scheduler."
self._config: Config = config self._config: Config = config
"The configuration." "The configuration."
self._task_name: str = f"{__class__.__name__}._dispatcher" self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
"The task name for the scheduler." "The task name for the scheduler."
self._queued: dict[str, set[str]] = {} self._queued: dict[str, set[str]] = {}
"Queued archive IDs per handler." "Queued archive IDs per handler."
@ -45,11 +46,11 @@ class TaskHandle:
EventBus.get_instance().subscribe( EventBus.get_instance().subscribe(
Events.ITEM_ERROR, Events.ITEM_ERROR,
self._handle_item_error, self._handle_item_error,
f"{__class__.__name__}.item_error", f"{TaskHandle.__name__}.item_error",
) )
def load(self) -> None: def load(self) -> None:
self._handlers: list[type] = self._discover() self._handlers: list[type[BaseHandler]] = self._discover()
timer: str = self._config.tasks_handler_timer timer: str = self._config.tasks_handler_timer
try: try:
@ -72,16 +73,16 @@ class TaskHandle:
self._scheduler.add( self._scheduler.add(
timer=timer, timer=timer,
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"), func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
id=f"{__class__.__name__}._dispatcher", id=f"{TaskHandle.__name__}._dispatcher",
) )
async def _dispatcher(self): async def _dispatcher(self):
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []} s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
handler_groups: dict[str, list[tuple[HandleTask, type]]] = {} handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
dispatches: list[tuple[HandleTask, type, asyncio.Task[TaskResult | TaskFailure | None]]] = [] dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
tasks: list[TaskModel] = await self._repo.list() tasks: list[TaskModel] = await self._repo.all()
for task_model in tasks: for task_model in tasks:
task: HandleTask = HandleTask.model_validate(task_model) task: HandleTask = HandleTask.model_validate(task_model)
@ -100,7 +101,7 @@ class TaskHandle:
continue continue
try: try:
handler: type | None = await self._find_handler(task) handler: type[BaseHandler] | None = await self._find_handler(task)
if handler is None: if handler is None:
s["u"].append(task.name) s["u"].append(task.name)
continue continue
@ -181,7 +182,9 @@ class TaskHandle:
}, },
) )
async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None: async def _dispatch(
self, task: HandleTask, handler: type[BaseHandler], delay: float
) -> TaskResult | TaskFailure | None:
""" """
Dispatch a task after a random delay to avoid rate limiting. Dispatch a task after a random delay to avoid rate limiting.
@ -215,7 +218,7 @@ class TaskHandle:
exc_info=(type(exc), exc, exc.__traceback__), exc_info=(type(exc), exc, exc.__traceback__),
) )
async def _find_handler(self, task: HandleTask) -> type | None: async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
for cls in self._handlers: for cls in self._handlers:
try: try:
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task): if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
@ -239,9 +242,10 @@ class TaskHandle:
async def dispatch( async def dispatch(
self, self,
task: HandleTask, task: HandleTask,
handler: type | None = None, handler: type[BaseHandler] | None = None,
**kwargs, # noqa: ARG002 **kwargs,
) -> TaskResult | TaskFailure | None: ) -> TaskResult | TaskFailure | None:
_ = kwargs
""" """
Dispatch a task to the appropriate handler. Dispatch a task to the appropriate handler.
@ -582,11 +586,11 @@ class TaskHandle:
return TaskResult(items=list(extraction.items), metadata=combined_metadata) return TaskResult(items=list(extraction.items), metadata=combined_metadata)
def _discover(self) -> list[type]: def _discover(self) -> list[type[BaseHandler]]:
"""Discover all available task handlers.""" """Discover all available task handlers."""
import app.features.tasks.definitions.handlers as handlers_pkg import app.features.tasks.definitions.handlers as handlers_pkg
handlers: list[type] = [] handlers: list[type[BaseHandler]] = []
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__): for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
if module_name.startswith("_"): if module_name.startswith("_"):

View file

@ -8,6 +8,7 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
from app.features.tasks.definitions.schemas import ( from app.features.tasks.definitions.schemas import (
Definition, Definition,
EngineConfig, EngineConfig,
Parse,
RequestConfig, RequestConfig,
ResponseConfig, ResponseConfig,
TaskDefinition, TaskDefinition,
@ -30,10 +31,12 @@ def test_build_def_payload():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)"}, "link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"}, "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(), response=ResponseConfig(),
@ -54,7 +57,8 @@ def test_build_def_container():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"items": { "items": {
"selector": ".cards .card", "selector": ".cards .card",
"fields": { "fields": {
@ -62,7 +66,8 @@ def test_build_def_container():
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, "title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
}, },
} }
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(), response=ResponseConfig(),
@ -82,7 +87,8 @@ def test_build_def_json():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"items": { "items": {
"type": "jsonpath", "type": "jsonpath",
"selector": "items", "selector": "items",
@ -91,7 +97,8 @@ def test_build_def_json():
"title": {"type": "jsonpath", "expression": "title"}, "title": {"type": "jsonpath", "expression": "title"},
}, },
} }
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(type="json"), response=ResponseConfig(type="json"),
@ -112,11 +119,13 @@ def test_parse_items_basic():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None}, "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"}, "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"}, "id": {"type": "css", "expression": ".article", "attribute": "data-id"},
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(), response=ResponseConfig(),
@ -154,7 +163,8 @@ def test_parse_items_cards():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"items": { "items": {
"type": "css", "type": "css",
"selector": ".columns .card", "selector": ".columns .card",
@ -181,7 +191,8 @@ def test_parse_items_cards():
}, },
}, },
} }
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(), response=ResponseConfig(),
@ -249,7 +260,8 @@ def test_parse_items_json():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"items": { "items": {
"type": "jsonpath", "type": "jsonpath",
"selector": "entries", "selector": "entries",
@ -259,7 +271,8 @@ def test_parse_items_json():
"id": {"type": "jsonpath", "expression": "id"}, "id": {"type": "jsonpath", "expression": "id"},
}, },
} }
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(type="json"), response=ResponseConfig(type="json"),
@ -297,7 +310,8 @@ async def test_generic_task_handler_inspect(monkeypatch):
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"items": { "items": {
"type": "jsonpath", "type": "jsonpath",
"selector": "items", "selector": "items",
@ -306,7 +320,8 @@ async def test_generic_task_handler_inspect(monkeypatch):
"title": {"type": "jsonpath", "expression": "title"}, "title": {"type": "jsonpath", "expression": "title"},
}, },
} }
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(type="json"), response=ResponseConfig(type="json"),
@ -351,7 +366,8 @@ def test_parse_items_json_list():
created_at=datetime.now(), created_at=datetime.now(),
updated_at=datetime.now(), updated_at=datetime.now(),
definition=Definition( definition=Definition(
parse={ parse=Parse.model_validate(
{
"items": { "items": {
"type": "jsonpath", "type": "jsonpath",
"selector": "[]", "selector": "[]",
@ -360,7 +376,8 @@ def test_parse_items_json_list():
"title": {"type": "jsonpath", "expression": "title"}, "title": {"type": "jsonpath", "expression": "title"},
}, },
} }
}, }
),
engine=EngineConfig(), engine=EngineConfig(),
request=RequestConfig(), request=RequestConfig(),
response=ResponseConfig(type="json"), response=ResponseConfig(type="json"),

View file

@ -69,7 +69,7 @@ class TestTaskDefinitionsRepository:
await repo.create(_sample_definition("Alpha", priority=2)) await repo.create(_sample_definition("Alpha", priority=2))
await repo.create(_sample_definition("Beta", priority=1)) await repo.create(_sample_definition("Beta", priority=1))
items = await repo.list() items = await repo.all()
assert len(items) == 2, "Should return two task definitions" assert len(items) == 2, "Should return two task definitions"
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name" assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"

View file

@ -1,9 +1,17 @@
from typing import Any from typing import Any, Literal, overload
from app.features.tasks.definitions.models import TaskDefinitionModel from app.features.tasks.definitions.models import TaskDefinitionModel
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary: def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
""" """
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema. Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.

View file

@ -10,20 +10,39 @@ from app.library.log import get_logger
from app.library.Singleton import Singleton from app.library.Singleton import Singleton
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncGenerator from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger() LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
model = TaskModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for TaskModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
if isinstance(payload, TaskModel):
return payload
return _model_from_payload(payload)
class TasksRepository(metaclass=Singleton): class TasksRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False self._migrated = False
self.session = session or get_session self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None: async def run_migrations(self) -> None:
if self._migrated: if self._migrated:
@ -38,7 +57,7 @@ class TasksRepository(metaclass=Singleton):
def get_instance() -> TasksRepository: def get_instance() -> TasksRepository:
return TasksRepository() return TasksRepository()
async def list(self) -> list[TaskModel]: async def all(self) -> list[TaskModel]:
async with self.session() as session: async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc())) result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
return list(result.scalars().all()) return list(result.scalars().all())
@ -90,7 +109,7 @@ class TasksRepository(metaclass=Singleton):
"""Get all enabled tasks.""" """Get all enabled tasks."""
async with self.session() as session: async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute( result: Result[tuple[TaskModel]] = await session.execute(
select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712 select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
) )
return list(result.scalars().all()) return list(result.scalars().all())
@ -102,11 +121,11 @@ class TasksRepository(metaclass=Singleton):
) )
return list(result.scalars().all()) return list(result.scalars().all())
async def create(self, payload: TaskModel | dict) -> TaskModel: async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
async with self.session() as session: async with self.session() as session:
model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload model = _coerce_model(payload)
if model.id is not None: if model.id is not None:
model.id = None model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None: if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task with name '{model.name}' already exists." msg: str = f"Task with name '{model.name}' already exists."

View file

@ -705,8 +705,9 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n' xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
if info.get("uploader"): if info.get("uploader"):
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n" xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
if info.get("tags", []): tags = info.get("tags", [])
for tag in info.get("tags", []): if isinstance(tags, list):
for tag in tags:
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n" xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
if info.get("year"): if info.get("year"):
xml_content += f" <year>{info.get('year')}</year>\n" xml_content += f" <year>{info.get('year')}</year>\n"

View file

@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
pass pass
async def _load_tasks(self) -> None: async def _load_tasks(self) -> None:
tasks = await self._repo.list() tasks = await self._repo.all()
for task in tasks: for task in tasks:
if not task.timer or not task.enabled: if not task.timer or not task.enabled:
@ -134,9 +134,11 @@ class Tasks(metaclass=Singleton):
task_id = task.id task_id = task.id
task_name = task.name task_name = task.name
try: try:
if not (task := await self._repo.get(task_id)): current_task = await self._repo.get(task_id)
if not current_task:
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name}) LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
return return
task = current_task
if not task.enabled: if not task.enabled:
LOG.debug( LOG.debug(

View file

@ -260,7 +260,7 @@ class TestTasksRepository:
await repo.create({"name": "Alice", "url": "https://example.com"}) await repo.create({"name": "Alice", "url": "https://example.com"})
await repo.create({"name": "Bob", "url": "https://example.com"}) await repo.create({"name": "Bob", "url": "https://example.com"})
items = await repo.list() items = await repo.all()
assert items[0].name == "Alice", "Should be alphabetically first" assert items[0].name == "Alice", "Should be alphabetically first"
assert items[1].name == "Bob", "Should be alphabetically second" assert items[1].name == "Bob", "Should be alphabetically second"

View file

@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any
import pytest import pytest
import pytest_asyncio import pytest_asyncio
@ -41,13 +42,13 @@ def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request: def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
request = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace()) mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
async def _json() -> object: async def _json() -> object:
return payload return payload
request.json = _json # type: ignore[attr-defined] mock_request.json = _json
return request return mock_request
class _Notify: class _Notify:
@ -85,7 +86,7 @@ async def test_add_requires_timer_without_handler(repo) -> None:
assert response.status == web.HTTPBadRequest.status_code assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body assert b"requires a timer" in response.body
assert await repo.list() == [] assert await repo.all() == []
@pytest.mark.asyncio @pytest.mark.asyncio
@ -109,7 +110,7 @@ async def test_add_all_or_nothing(repo) -> None:
assert response.status == web.HTTPBadRequest.status_code assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body assert b"requires a timer" in response.body
assert await repo.list() == [] assert await repo.all() == []
@pytest.mark.asyncio @pytest.mark.asyncio
@ -123,7 +124,7 @@ async def test_add_allows_handler_only(repo) -> None:
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True)) response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
assert response.status == web.HTTPOk.status_code assert response.status == web.HTTPOk.status_code
items = await repo.list() items = await repo.all()
assert len(items) == 1 assert len(items) == 1
assert items[0].name == "Handler Only" assert items[0].name == "Handler Only"

View file

@ -120,7 +120,7 @@ class ExtractorPool(metaclass=Singleton):
def attach(self, app: web.Application) -> None: def attach(self, app: web.Application) -> None:
"""Attach the extractor pool to the application (no-op for now).""" """Attach the extractor pool to the application (no-op for now)."""
app.on_shutdown.append(self.shutdown) app.on_shutdown.append(self.shutdown)
Services.get_instance().add(__class__.__name__, self) Services.get_instance().add(type(self).__name__, self)
def _ensure_initialized(self, config: ExtractorConfig) -> None: def _ensure_initialized(self, config: ExtractorConfig) -> None:
""" """

View file

@ -286,7 +286,7 @@ async def convert(request: Request) -> Response:
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
try: try:
response = {"opts": {}, "output_template": None, "download_path": None} response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
data = arg_converter(args, dumps=True) data = arg_converter(args, dumps=True)
@ -384,6 +384,8 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
if cache.has(key) and not request.query.get("force", False): if cache.has(key) and not request.query.get("force", False):
data: Any | None = cache.get(key) data: Any | None = cache.get(key)
if data is None:
data = {}
data["_cached"] = { data["_cached"] = {
"status": "hit", "status": "hit",
"preset": preset, "preset": preset,
@ -393,7 +395,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(), "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
"expires": data.get("_cached", {}).get("expires", time.time() + 300), "expires": data.get("_cached", {}).get("expires", time.time() + 300),
} }
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
ytdlp_opts: dict = opts.get_all() ytdlp_opts: dict = opts.get_all()
@ -450,7 +452,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
cache.set(key=key, value=data, ttl=300) cache.set(key=key, value=data, ttl=300)
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
except Exception as e: except Exception as e:
LOG.exception( LOG.exception(
"Failed to get video info for '%s': %s.", "Failed to get video info for '%s': %s.",
@ -486,7 +488,7 @@ async def get_options() -> Response:
""" """
from app.features.ytdlp.ytdlp import ytdlp_options from app.features.ytdlp.ytdlp import ytdlp_options
return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code) return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids") @route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
@ -508,7 +510,11 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
response = [] response = []
for i, url in enumerate(data): for i, url in enumerate(data):
dct = {"index": i, "url": url} dct: dict[str, Any] = {"index": i, "url": url}
if not isinstance(url, str):
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
response.append(dct)
continue
try: try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls) await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
dct.update(get_archive_id(url)) dct.update(get_archive_id(url))

View file

@ -122,6 +122,7 @@ class TestExtractInfo:
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING {}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
) )
assert result is not None
assert result["id"] == "test123" assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"] assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
@ -155,6 +156,7 @@ class TestExtractInfo:
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING {}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
) )
assert result is not None
assert result["id"] == "test123" assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"] assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen

View file

@ -1,5 +1,6 @@
import importlib import importlib
from pathlib import Path from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, Mock, patch from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
@ -219,7 +220,8 @@ class TestYTDLP:
"""Test _delete_downloaded_files skips cleanup when _interrupted is True.""" """Test _delete_downloaded_files skips cleanup when _interrupted is True."""
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None): with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
ytdlp = YTDLP(params={}) ytdlp = YTDLP(params={})
ytdlp.to_screen = Mock() mock_obj: Any = ytdlp
mock_obj.to_screen = Mock()
# Set interrupted flag # Set interrupted flag
ytdlp._interrupted = True ytdlp._interrupted = True
@ -229,7 +231,7 @@ class TestYTDLP:
# Should not call super method # Should not call super method
mock_super_delete.assert_not_called() mock_super_delete.assert_not_called()
# Should show message # Should show message
ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.") mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
# Should return None # Should return None
assert result is None assert result is None

View file

@ -776,7 +776,7 @@ class TestYTDLPCli:
from app.features.ytdlp.ytdlp_opts import YTDLPCli from app.features.ytdlp.ytdlp_opts import YTDLPCli
with pytest.raises(ValueError, match="Expected Item instance"): with pytest.raises(ValueError, match="Expected Item instance"):
YTDLPCli(item="not an item") # type: ignore YTDLPCli(item="not an item")
@patch("app.features.presets.service.Presets") @patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.Config") @patch("app.features.ytdlp.ytdlp_opts.Config")

View file

@ -46,7 +46,8 @@ class TestLogWrapper:
def test_add_target_type_validation(self) -> None: def test_add_target_type_validation(self) -> None:
lw = LogWrapper() lw = LogWrapper()
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"): with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
lw.add_target(123) # type: ignore[arg-type] bad: Any = 123
lw.add_target(bad)
def test_add_target_names(self) -> None: def test_add_target_names(self) -> None:
lw = LogWrapper() lw = LogWrapper()
@ -128,7 +129,7 @@ class TestExtractYtdlpLogs:
def test_extract_ytdlp_logs_with_filters(self): def test_extract_ytdlp_logs_with_filters(self):
"""Test log extraction with filters.""" """Test log extraction with filters."""
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"] logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
filters = [re.compile(r"ERROR")] filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
result = extract_ytdlp_logs(logs, filters) result = extract_ytdlp_logs(logs, filters)
assert result == ["ERROR: Failed"] assert result == ["ERROR: Failed"]
@ -312,9 +313,12 @@ class TestGetThumbnail:
def test_non_list(self): def test_non_list(self):
"""Test that None is returned for non-list input.""" """Test that None is returned for non-list input."""
assert get_thumbnail(None) is None bad_none: Any = None
assert get_thumbnail("not a list") is None bad_str: Any = "not a list"
assert get_thumbnail({"not": "list"}) is None bad_dict: Any = {"not": "list"}
assert get_thumbnail(bad_none) is None
assert get_thumbnail(bad_str) is None
assert get_thumbnail(bad_dict) is None
def test_thumbnail_preference(self): def test_thumbnail_preference(self):
"""Test that the thumbnail with highest preference is returned.""" """Test that the thumbnail with highest preference is returned."""
@ -349,6 +353,7 @@ class TestGetThumbnail:
] ]
result = get_thumbnail(thumbnails) result = get_thumbnail(thumbnails)
assert result is not None
assert result["url"] == "with_pref.jpg" assert result["url"] == "with_pref.jpg"
def test_all_equal(self): def test_all_equal(self):
@ -365,12 +370,15 @@ class TestGetThumbnail:
class TestGetExtras: class TestGetExtras:
def test_none(self): def test_none(self):
"""Test that empty dict is returned for None input.""" """Test that empty dict is returned for None input."""
assert get_extras(None) == {} bad: Any = None
assert get_extras(bad) == {}
def test_non_dict(self): def test_non_dict(self):
"""Test that empty dict is returned for non-dict input.""" """Test that empty dict is returned for non-dict input."""
assert get_extras("not a dict") == {} bad_str: Any = "not a dict"
assert get_extras([]) == {} bad_list: Any = []
assert get_extras(bad_str) == {}
assert get_extras(bad_list) == {}
def test_extracts_video_information(self): def test_extracts_video_information(self):
"""Test extracting information from a video entry.""" """Test extracting information from a video entry."""

View file

@ -101,12 +101,12 @@ class LogWrapper:
name (str|None): The name of the logging target. Defaults to None. name (str|None): The name of the logging target. Defaults to None.
""" """
if not isinstance(target, logging.Logger | Callable): if not isinstance(target, logging.Logger) and not callable(target):
msg = "Target must be a logging.Logger instance or a callable." msg = "Target must be a logging.Logger instance or a callable."
raise TypeError(msg) raise TypeError(msg)
if name is None: if name is None:
name = target.name if isinstance(target, logging.Logger) else target.__name__ name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
self.targets.append( self.targets.append(
LogTarget( LogTarget(
@ -134,8 +134,9 @@ class LogWrapper:
if target.logger: if target.logger:
log_kwargs = {**kwargs} log_kwargs = {**kwargs}
log_kwargs.setdefault("stacklevel", 3) log_kwargs.setdefault("stacklevel", 3)
if isinstance(target.target, logging.Logger):
target.target.log(level, msg, *args, **log_kwargs) target.target.log(level, msg, *args, **log_kwargs)
else: elif callable(target.target):
target.target(level, msg, *args, **kwargs) target.target(level, msg, *args, **kwargs)
def debug(self, msg, *args, **kwargs): def debug(self, msg, *args, **kwargs):
@ -179,13 +180,17 @@ def arg_converter(
create_parser = yt_dlp.options.create_parser create_parser = yt_dlp.options.create_parser
def _default_opts(args: str): def _default_opts(args: str | list[str]):
patched_parser = create_parser() patched_parser = create_parser()
def patched_create_parser():
return patched_parser
try: try:
yt_dlp.options.create_parser = lambda: patched_parser yt_dlp.options.__dict__["create_parser"] = patched_create_parser
return yt_dlp.parse_options(args) return yt_dlp.parse_options(args)
finally: finally:
yt_dlp.options.create_parser = create_parser yt_dlp.options.__dict__["create_parser"] = create_parser
apply_ytdlp_patches() apply_ytdlp_patches()
@ -238,7 +243,7 @@ def arg_converter(
return diff return diff
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]: def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
""" """
Extract yt-dlp log lines matching built-in filters plus any extras. Extract yt-dlp log lines matching built-in filters plus any extras.
@ -332,7 +337,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
return _DATA.YTDLP_INFO_CLS return _DATA.YTDLP_INFO_CLS
def get_thumbnail(thumbnails: list) -> str | None: def get_thumbnail(thumbnails: list) -> dict | None:
""" """
Extract thumbnail URL from a yt-dlp entry. Extract thumbnail URL from a yt-dlp entry.
@ -340,7 +345,7 @@ def get_thumbnail(thumbnails: list) -> str | None:
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry. thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
Returns: Returns:
str | None: The thumbnail URL if available, otherwise None. dict | None: The best thumbnail dict if available, otherwise None.
""" """
if not thumbnails or not isinstance(thumbnails, list): if not thumbnails or not isinstance(thumbnails, list):
@ -381,7 +386,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
extras[f"playlist_{property}"] = val extras[f"playlist_{property}"] = val
if thumbnail := get_thumbnail(entry.get("thumbnails", [])): if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
extras["thumbnail"] = thumbnail.get("url") extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
elif thumbnail := entry.get("thumbnail"): elif thumbnail := entry.get("thumbnail"):
extras["thumbnail"] = thumbnail extras["thumbnail"] = thumbnail
@ -439,7 +444,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
} }
""" """
idDict: dict[str, None] = { idDict: dict[str, str | None] = {
"id": None, "id": None,
"ie_key": None, "ie_key": None,
"archive_id": None, "archive_id": None,

View file

@ -50,12 +50,12 @@ class ARGSMerger:
""" """
return str(self) return str(self)
def as_dict(self) -> dict: def as_dict(self) -> list[str]:
""" """
Get all the options as a dict. Get all the options as a shell argument list.
Returns: Returns:
dict: The options as a dict list[str]: The options as shell arguments.
""" """
return shlex.split(shlex.join(self.args)) return shlex.split(shlex.join(self.args))

View file

@ -29,7 +29,7 @@ class BackgroundWorker(metaclass=Singleton):
"The queue to hold the tasks." "The queue to hold the tasks."
self.running = True self.running = True
"Whether the background worker is running or not." "Whether the background worker is running or not."
self.thread: threading.Thread = None self.thread: threading.Thread | None = None
"The thread that runs the background worker." "The thread that runs the background worker."
@staticmethod @staticmethod
@ -49,6 +49,7 @@ class BackgroundWorker(metaclass=Singleton):
try: try:
LOG.debug("Stopping background worker thread.") LOG.debug("Stopping background worker thread.")
self.queue.put((CloseThread, (), {})) self.queue.put((CloseThread, (), {}))
if self.thread:
self.thread.join(timeout=5) self.thread.join(timeout=5)
LOG.debug("Background worker thread has been shut down.") LOG.debug("Background worker thread has been shut down.")
except Exception as e: except Exception as e:

View file

@ -1,14 +1,17 @@
import base64 import base64
import hmac import hmac
import inspect import inspect
from collections.abc import Awaitable
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Any, cast
import anyio import anyio
from aiohttp import web from aiohttp import web
from aiohttp.web import Request, RequestHandler, Response from aiohttp.typedefs import Handler, Middleware
from aiohttp.web import Request, Response
from aiohttp.web_log import AccessLogger from aiohttp.web_log import AccessLogger
from aiohttp.web_request import BaseRequest
from aiohttp.web_response import StreamResponse
from app.library.log import get_logger from app.library.log import get_logger
from app.library.Services import Services from app.library.Services import Services
@ -24,12 +27,12 @@ LOG = get_logger("http")
class HttpAccessLogger(AccessLogger): class HttpAccessLogger(AccessLogger):
def log(self, request: Request, response: Response, time: float) -> None: def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
try: try:
fmt_info = self._format_line(request, response, time) fmt_info = self._format_line(request, response, time)
values: list[object] = [] values: list[object] = []
extra: dict[str, object] = {"elapsed_ms": round(time * 1000.0, 2)} extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
for key, value in fmt_info: for key, value in fmt_info:
values.append(value) values.append(value)
@ -37,7 +40,7 @@ class HttpAccessLogger(AccessLogger):
extra[key] = value extra[key] = value
else: else:
parent, child = key parent, child = key
group = extra.get(parent, {}) group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment]
if not isinstance(group, dict): if not isinstance(group, dict):
group = {} group = {}
group[child] = value group[child] = value
@ -177,7 +180,7 @@ class HttpAPI:
registered_options.append(route.path) registered_options.append(route.path)
@staticmethod @staticmethod
def basic_auth(username: str, password: str) -> Awaitable: def basic_auth(username: str, password: str) -> Middleware:
""" """
Middleware to handle basic authentication. Middleware to handle basic authentication.
@ -191,7 +194,7 @@ class HttpAPI:
""" """
@web.middleware @web.middleware
async def auth_handler(request: Request, handler: RequestHandler) -> Response: async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
# if OPTIONS request, skip auth # if OPTIONS request, skip auth
if "OPTIONS" == request.method: if "OPTIONS" == request.method:
return await handler(request) return await handler(request)
@ -204,7 +207,7 @@ class HttpAPI:
auth_cookie = request.cookies.get("auth") auth_cookie = request.cookies.get("auth")
if auth_cookie is not None: if auth_cookie is not None:
try: try:
data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key) data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
if data is not None: if data is not None:
data = base64.b64encode(data.encode()).decode() data = base64.b64encode(data.encode()).decode()
auth_header = f"Basic {data}" auth_header = f"Basic {data}"
@ -250,7 +253,7 @@ class HttpAPI:
}, },
) )
response: Response = await handler(request) response: StreamResponse = await handler(request)
contentType: str | None = response.headers.get("content-type", None) contentType: str | None = response.headers.get("content-type", None)
if contentType and not contentType.startswith("text/html"): if contentType and not contentType.startswith("text/html"):
@ -261,7 +264,7 @@ class HttpAPI:
"auth", "auth",
encrypt_data( encrypt_data(
f"{user_name}:{user_password}", f"{user_name}:{user_password}",
key=Config.get_instance().secret_key, key=cast("bytes", Config.get_instance().secret_key),
), ),
max_age=60 * 60 * 24 * 7, max_age=60 * 60 * 24 * 7,
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"), expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
@ -280,9 +283,9 @@ class HttpAPI:
return auth_handler return auth_handler
@staticmethod @staticmethod
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable: def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
@web.middleware @web.middleware
async def middleware_handler(request: Request, handler: RequestHandler) -> Response: async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
static_path = str(app.router["download_static"].url_for(filename="")) static_path = str(app.router["download_static"].url_for(filename=""))
if request.path.startswith(static_path): if request.path.startswith(static_path):
realFile, status = get_file( realFile, status = get_file(
@ -301,12 +304,13 @@ class HttpAPI:
}, },
) )
response: Response = await handler(request) response: StreamResponse = await handler(request)
contentType: str = str(response.headers.get("content-type", "")) contentType: str = str(response.headers.get("content-type", ""))
if contentType.startswith("text/html") and getattr(response, "_path", None): if contentType.startswith("text/html") and getattr(response, "_path", None):
rewrite_path: str = base_path.rstrip("/") rewrite_path: str = base_path.rstrip("/")
async with await anyio.open_file(response._path, "rb") as f: response_path = cast("Any", response)._path
async with await anyio.open_file(response_path, "rb") as f:
content = await f.read() content = await f.read()
content: str = ( content: str = (
content.decode("utf-8") content.decode("utf-8")
@ -339,7 +343,7 @@ class HttpAPI:
return middleware_handler return middleware_handler
async def _handle(self, handler: RequestHandler, request: Request) -> Response: async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
""" """
Call the handler with the request and return the response. Call the handler with the request and return the response.

View file

@ -104,19 +104,20 @@ class HttpSocket:
} }
) )
self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit") self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
@staticmethod @staticmethod
def ws_event(func): # type: ignore def ws_event(func):
""" """
Decorator to mark a method as a socket event. Decorator to mark a method as a socket event.
""" """
@functools.wraps(func) # type: ignore @functools.wraps(func)
async def wrapper(*args, **kwargs): async def wrapper(*args, **kwargs):
return await func(*args, **kwargs) # type: ignore return await func(*args, **kwargs)
wrapper._ws_event = func.__name__ # type: ignore wrapper_with_event: Any = wrapper
wrapper_with_event._ws_event = func.__name__
return wrapper return wrapper
async def on_shutdown(self, _: web.Application): async def on_shutdown(self, _: web.Application):
@ -134,9 +135,11 @@ class HttpSocket:
if not (data and data.data): if not (data and data.data):
return return
await Services.get_instance().get("queue").add(item=Item.format(data.data)) queue = Services.get_instance().get("queue")
if queue is not None:
await queue.add(item=Item.format(data.data))
self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add") self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
load_modules(self.rootPath, self.rootPath / "routes" / "socket") load_modules(self.rootPath, self.rootPath / "routes" / "socket")

View file

@ -93,7 +93,7 @@ class Item:
bool: True if the item has extras, False otherwise. bool: True if the item has extras, False otherwise.
""" """
return self.extras and len(self.extras) > 0 return bool(self.extras and len(self.extras) > 0)
def add_extras(self, key: str, value: Any) -> None: def add_extras(self, key: str, value: Any) -> None:
""" """
@ -117,7 +117,7 @@ class Item:
bool: True if the item has command options for yt, False otherwise. bool: True if the item has command options for yt, False otherwise.
""" """
return self.cli and len(self.cli) > 2 return bool(self.cli and len(self.cli) > 2)
@staticmethod @staticmethod
def _default_preset() -> str: def _default_preset() -> str:
@ -161,7 +161,7 @@ class Item:
Item: The formatted item. Item: The formatted item.
""" """
url: str = item.get("url") url = item.get("url")
if not url or not isinstance(url, str): if not url or not isinstance(url, str):
msg = "url param is required." msg = "url param is required."
@ -173,7 +173,7 @@ class Item:
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url): if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
url = f"https://www.youtube.com/watch?v={url}" url = f"https://www.youtube.com/watch?v={url}"
data: dict[str, str] = {"url": url} data: dict[str, Any] = {"url": url}
preset: str | None = item.get("preset") preset: str | None = item.get("preset")
if preset and isinstance(preset, str) and preset != Item._default_preset(): if preset and isinstance(preset, str) and preset != Item._default_preset():
@ -185,24 +185,28 @@ class Item:
data["preset"] = preset data["preset"] = preset
if item.get("folder") and isinstance(item.get("folder"), str): folder = item.get("folder")
data["folder"] = item.get("folder") if isinstance(folder, str) and folder:
data["folder"] = folder
if item.get("cookies") and isinstance(item.get("cookies"), str): cookies = item.get("cookies")
data["cookies"] = item.get("cookies") if isinstance(cookies, str) and cookies:
data["cookies"] = cookies
if item.get("template") and isinstance(item.get("template"), str): template = item.get("template")
data["template"] = item.get("template") if isinstance(template, str) and template:
data["template"] = template
if "auto_start" in item and isinstance(item.get("auto_start"), bool): if isinstance(item.get("auto_start"), bool):
data["auto_start"] = bool(item.get("auto_start")) data["auto_start"] = item["auto_start"]
extras: dict | None = item.get("extras") extras: dict | None = item.get("extras")
if extras and isinstance(extras, dict) and len(extras) > 0: if isinstance(extras, dict) and len(extras) > 0:
data["extras"] = extras data["extras"] = extras
if item.get("requeued") and isinstance(item.get("requeued"), bool): requeued = item.get("requeued")
data["requeued"] = item.get("requeued") if isinstance(requeued, bool):
data["requeued"] = requeued
cli: str | None = item.get("cli") cli: str | None = item.get("cli")
if cli and len(cli) > 2: if cli and len(cli) > 2:
@ -574,8 +578,10 @@ class ItemDTO:
self.is_archivable = bool(self._archive_file) self.is_archivable = bool(self._archive_file)
if not self.is_archivable: if not self.is_archivable:
self.is_archived = False self.is_archived = False
else: elif self.archive_id and self._archive_file:
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0 self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
else:
self.is_archived = False
def get_archive_file(self) -> str | None: def get_archive_file(self) -> str | None:
""" """

View file

@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
class Packages: class Packages:
def __init__(self, env: str | None, file: str | None, upgrade: bool = False): def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
from_env: list[str] = env.split() if env else [] from_env: list[str] = env.split() if env else []
from_file = [] from_file = []

View file

@ -1,4 +1,6 @@
import asyncio import asyncio
from collections.abc import Callable
from typing import Any
from aiocron import Cron from aiocron import Cron
from aiohttp import web from aiohttp import web
@ -66,7 +68,7 @@ class Scheduler(metaclass=Singleton):
if data and data.data: if data and data.data:
self.add(**data.data) self.add(**data.data)
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add") EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
def get_all(self) -> dict[str, Cron]: def get_all(self) -> dict[str, Cron]:
"""Return the jobs.""" """Return the jobs."""
@ -99,7 +101,7 @@ class Scheduler(metaclass=Singleton):
return id in self._jobs return id in self._jobs
def add( def add(
self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
) -> str: ) -> str:
""" """
Add a job to the schedule. Add a job to the schedule.

View file

@ -1,4 +1,5 @@
import inspect import inspect
from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
@ -117,7 +118,7 @@ class Services(metaclass=Singleton):
return candidates[0] return candidates[0]
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]: def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
sig: inspect.Signature = inspect.signature(handler) sig: inspect.Signature = inspect.signature(handler)
try: try:
@ -169,10 +170,10 @@ class Services(metaclass=Singleton):
return resolved return resolved
async def handle_async(self, handler: callable, **kwargs) -> Any: async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs) resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return await handler(**resolved) return await handler(**resolved)
def handle_sync(self, handler: callable, **kwargs) -> Any: def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs) resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return handler(**resolved) return handler(**resolved)

View file

@ -75,7 +75,7 @@ class TerminalSessionManager(metaclass=Singleton):
self._cleanup_job_id = Scheduler.get_instance().add( self._cleanup_job_id = Scheduler.get_instance().add(
timer=CLEANUP_SCHEDULE, timer=CLEANUP_SCHEDULE,
func=self.cleanup, func=self.cleanup,
id=f"{__class__.__name__}.{__class__.cleanup.__name__}", id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
) )
async def on_startup(self, _: web.Application) -> None: async def on_startup(self, _: web.Application) -> None:

View file

@ -80,7 +80,7 @@ class UpdateChecker(metaclass=Singleton):
self._notify.subscribe( self._notify.subscribe(
event=Events.STARTED, event=Events.STARTED,
callback=event_handler, callback=event_handler,
name=f"{__class__.__name__}.{__class__.attach.__name__}", name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
) )
self._schedule_check() self._schedule_check()
@ -108,7 +108,7 @@ class UpdateChecker(metaclass=Singleton):
self._job_id = self._scheduler.add( self._job_id = self._scheduler.add(
timer=timer, timer=timer,
func=lambda: asyncio.create_task(self.check_for_updates()), func=lambda: asyncio.create_task(self.check_for_updates()),
id=f"{__class__.__name__}.{self.check_for_updates.__name__}", id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
) )
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]: async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:

View file

@ -40,7 +40,8 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
class FileLogFormatter(logging.Formatter): class FileLogFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802 def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
@ -78,7 +79,8 @@ class JsonLogFormatter(logging.Formatter):
return json.dumps(data, ensure_ascii=False, default=str) return json.dumps(data, ensure_ascii=False, default=str)
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802 def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
@staticmethod @staticmethod
@ -210,7 +212,7 @@ class JsonLogFormatter(logging.Formatter):
return source return source
def timed_lru_cache(ttl_seconds: int, max_size: int = 128): def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
""" """
Decorator that applies an LRU cache with a time-to-live (TTL) to a function. Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
Supports both synchronous and asynchronous functions. Supports both synchronous and asynchronous functions.
@ -280,8 +282,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache)) return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
async_wrapper.cache_clear = cache_clear async_wrapper_cache: Any = async_wrapper
async_wrapper.cache_info = cache_info async_wrapper_cache.cache_clear = cache_clear
async_wrapper_cache.cache_info = cache_info
return async_wrapper return async_wrapper
# For sync functions, use the original implementation # For sync functions, use the original implementation
@ -309,8 +312,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
return result return result
# expose cache_clear, cache_info # expose cache_clear, cache_info
sync_wrapper.cache_clear = cached.cache_clear sync_wrapper_cache: Any = sync_wrapper
sync_wrapper.cache_info = cached.cache_info sync_wrapper_cache.cache_clear = cached.cache_clear
sync_wrapper_cache.cache_info = cached.cache_info
return sync_wrapper return sync_wrapper
return decorator return decorator
@ -381,7 +385,12 @@ def _is_safe_key(key: Any) -> bool:
def merge_dict( def merge_dict(
source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None source: dict,
destination: dict,
max_depth: int = 50,
max_list_size: int = 10000,
_depth: int = 0,
_seen: set | None = None,
) -> dict: ) -> dict:
""" """
Merge data from source into destination. Merge data from source into destination.
@ -472,7 +481,7 @@ def merge_dict(
return destination_copy return destination_copy
def check_id(file: Path) -> bool | str: def check_id(file: Path) -> bool | Path:
""" """
Check if we are able to get an id from the file name. Check if we are able to get an id from the file name.
if so check if any video file with the same id exists. if so check if any video file with the same id exists.
@ -491,6 +500,8 @@ def check_id(file: Path) -> bool | str:
return False return False
id: str | None = match.groupdict().get("id") id: str | None = match.groupdict().get("id")
if id is None:
return False
try: try:
if not file.parent.exists(): if not file.parent.exists():
@ -545,9 +556,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
from yarl import URL from yarl import URL
parsed_url = URL(url) parsed_url = URL(url)
except ValueError: except ValueError as exc:
msg = "Invalid URL." msg = "Invalid URL."
raise ValueError(msg) # noqa: B904 raise ValueError(msg) from exc
# Check allowed schemes # Check allowed schemes
if parsed_url.scheme not in ["http", "https"]: if parsed_url.scheme not in ["http", "https"]:
@ -596,7 +607,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
@timed_lru_cache(ttl_seconds=60, max_size=256) @timed_lru_cache(ttl_seconds=60, max_size=256)
def get_file_sidecar(file: Path | None = None) -> dict[dict]: def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
""" """
Get sidecar files for the given file. Get sidecar files for the given file.
@ -925,7 +936,7 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
download_path = Path(download_path) download_path = Path(download_path)
try: try:
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False)) realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
if realFile.exists(): if realFile.exists():
return (realFile, 200) return (realFile, 200)
except Exception as e: except Exception as e:
@ -936,11 +947,11 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
) )
return (Path(file), 404) return (Path(file), 404)
possibleFile: bool | str = check_id(file=realFile) possibleFile: bool | Path = check_id(file=realFile)
if not possibleFile: if not isinstance(possibleFile, Path):
return (realFile, 404) return (realFile, 404)
return (Path(possibleFile), 302) return (possibleFile, 302)
def encrypt_data(data: str, key: bytes) -> str: def encrypt_data(data: str, key: bytes) -> str:
@ -962,7 +973,7 @@ def encrypt_data(data: str, key: bytes) -> str:
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode() return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
def decrypt_data(data: str, key: bytes) -> str: def decrypt_data(data: str, key: bytes) -> str | None:
""" """
Decrypts AES-GCM encrypted data Decrypts AES-GCM encrypted data
@ -975,8 +986,8 @@ def decrypt_data(data: str, key: bytes) -> str:
""" """
try: try:
data = base64.urlsafe_b64decode(data) raw: bytes = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = data[:12], data[12:-16], data[-16:] iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
cipher = AES.new(key, AES.MODE_GCM, nonce=iv) cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
plaintext = cipher.decrypt_and_verify(ciphertext, tag) plaintext = cipher.decrypt_and_verify(ciphertext, tag)
return plaintext.decode() return plaintext.decode()
@ -987,7 +998,7 @@ def decrypt_data(data: str, key: bytes) -> str:
def get( def get(
data: dict | list, data: dict | list,
path: str | list | None = None, path: str | list | None = None,
default: any = None, default: Any = None,
separator=".", separator=".",
): ):
""" """
@ -1111,7 +1122,7 @@ def get_files(
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__}, extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
exc_info=True, exc_info=True,
) )
return [] return [], 0
try: try:
dir_path.relative_to(base_path) dir_path.relative_to(base_path)
@ -1122,7 +1133,7 @@ def get_files(
base_path, base_path,
extra={"path": str(dir_path), "base_path": str(base_path)}, extra={"path": str(dir_path), "base_path": str(base_path)},
) )
return [] return [], 0
if not str(dir_path).startswith(str(base_path)): if not str(dir_path).startswith(str(base_path)):
LOG.warning( LOG.warning(
@ -1131,7 +1142,7 @@ def get_files(
base_path, base_path,
extra={"path": str(dir_path), "base_path": str(base_path)}, extra={"path": str(dir_path), "base_path": str(base_path)},
) )
return [] return [], 0
if not dir_path.is_dir(): if not dir_path.is_dir():
LOG.warning( LOG.warning(
@ -1139,7 +1150,7 @@ def get_files(
dir_path, dir_path,
extra={"path": str(dir_path), "base_path": str(base_path)}, extra={"path": str(dir_path), "base_path": str(base_path)},
) )
return [] return [], 0
contents: list = [] contents: list = []
for file in dir_path.iterdir(): for file in dir_path.iterdir():
@ -1298,7 +1309,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
try: try:
from http.cookiejar import MozillaCookieJar from http.cookiejar import MozillaCookieJar
cookies = MozillaCookieJar(str(file), None, None) cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
cookies.load() cookies.load()
return (True, cookies) return (True, cookies)

View file

@ -1,6 +1,20 @@
import sys
from typing import Any from typing import Any
def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
if key not in data or data[key] is None:
return False, None
return True, data[key]
def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
if key not in data:
return False
del data[key]
return True
def get_value(value: Any) -> Any: def get_value(value: Any) -> Any:
""" """
Return the result of calling `value` if it is callable, else return `value`. Return the result of calling `value` if it is callable, else return `value`.
@ -91,14 +105,19 @@ def ag(
return val return val
return get_value(default) return get_value(default)
key = path key = path
if isinstance(array, dict) and key in array and array[key] is not None: if isinstance(array, dict):
return array[key] found, value = _dict_value(array, key)
if found:
return value
if not isinstance(key, str) or separator not in key: if not isinstance(key, str) or separator not in key:
return get_value(default) return get_value(default)
current = array current = array
for segment in key.split(separator): for segment in key.split(separator):
if isinstance(current, dict) and segment in current and current[segment] is not None: if isinstance(current, dict):
current = current[segment] found, value = _dict_value(current, segment)
if not found:
return get_value(default)
current = value
elif isinstance(current, list): elif isinstance(current, list):
try: try:
idx = int(segment) idx = int(segment)
@ -150,7 +169,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
except TypeError: except TypeError:
return False return False
if isinstance(data, dict): if isinstance(data, dict):
if path in data and data[path] is not None: found, _ = _dict_value(data, path)
if found:
return True return True
elif isinstance(data, list) and isinstance(path, int): elif isinstance(data, list) and isinstance(path, int):
return 0 <= path < len(data) and data[path] is not None return 0 <= path < len(data) and data[path] is not None
@ -158,8 +178,11 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
segments = path_str.split(separator) segments = path_str.split(separator)
current = data current = data
for seg in segments: for seg in segments:
if isinstance(current, dict) and seg in current and current[seg] is not None: if isinstance(current, dict):
current = current[seg] found, value = _dict_value(current, seg)
if not found:
return False
current = value
elif isinstance(current, list): elif isinstance(current, list):
try: try:
idx = int(seg) idx = int(seg)
@ -189,17 +212,16 @@ def ag_delete(
The modified structure. The modified structure.
""" """
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if not isinstance(data, dict | list): if not isinstance(data, dict | list):
try: try:
data = vars(data) data = vars(data)
except TypeError: except TypeError:
return data # type: ignore return data # type: ignore
if isinstance(data, dict) and path in data: if isinstance(path, list | tuple):
del data[path] for p in path:
ag_delete(data, p, separator)
return data
if isinstance(data, dict) and _dict_delete(data, path):
return data return data
if isinstance(data, list) and isinstance(path, int): if isinstance(data, list) and isinstance(path, int):
if 0 <= path < len(data): if 0 <= path < len(data):
@ -210,8 +232,12 @@ def ag_delete(
last = segments.pop() last = segments.pop()
current = data current = data
for seg in segments: for seg in segments:
if isinstance(current, dict) and seg in current: if isinstance(current, dict):
current = current[seg] found, value = _dict_value(current, seg)
if not found:
current = None
break
current = value
elif isinstance(current, list): elif isinstance(current, list):
try: try:
idx = int(seg) idx = int(seg)
@ -228,8 +254,8 @@ def ag_delete(
break break
if current is None: if current is None:
return data return data
if isinstance(current, dict) and last in current: if isinstance(current, dict):
del current[last] _dict_delete(current, last)
elif isinstance(current, list): elif isinstance(current, list):
try: try:
idx = int(last) idx = int(last)
@ -246,10 +272,12 @@ def run_tests():
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}} assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
def test_ag_set_overwrite_error(): def test_ag_set_overwrite_error():
error = ""
try: try:
ag_set({"a": 1}, "a.b", 2) ag_set({"a": 1}, "a.b", 2)
except RuntimeError as e: except RuntimeError as e:
assert "Cannot set value at path" in str(e) # noqa: PT017 error = str(e)
assert "Cannot set value at path" in error
def test_ag_get_value_callable_and_value(): def test_ag_get_value_callable_and_value():
assert get_value(5) == 5 assert get_value(5) == 5
@ -320,9 +348,9 @@ def run_tests():
]: ]:
try: try:
test() test()
print(f"PASS: {test.__name__}") # noqa: T201 sys.stdout.write(f"PASS: {test.__name__}\n")
except AssertionError: except AssertionError:
print(f"FAIL {test.__name__}") # noqa: T201 sys.stdout.write(f"FAIL {test.__name__}\n")
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -38,7 +38,7 @@ class Cache(metaclass=ThreadSafe):
Scheduler.get_instance().add( Scheduler.get_instance().add(
timer="* * * * *", timer="* * * * *",
func=self.cleanup, func=self.cleanup,
id=f"{__class__.__name__}.{__class__.cleanup.__name__}", id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
) )
def set(self, key: str, value: Any, ttl: float | None = None) -> None: def set(self, key: str, value: Any, ttl: float | None = None) -> None:

View file

@ -3,7 +3,7 @@ from __future__ import annotations
import http.cookiejar import http.cookiejar
from abc import ABC from abc import ABC
from collections.abc import Callable from collections.abc import Callable
from typing import Any, ClassVar from typing import Any, ClassVar, cast
from yt_dlp.networking.common import ( from yt_dlp.networking.common import (
_REQUEST_HANDLERS, _REQUEST_HANDLERS,
@ -223,11 +223,13 @@ class CFSolverRH(RequestHandler, ABC):
try: try:
response: Response = director.send(request) response: Response = director.send(request)
except HTTPError as error: except HTTPError as error:
if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers): if error.response and is_cf_challenge(
getattr(error.response, "status", None), cast("Any", error.response.headers)
):
return self._retry_with_clearance(request, error.response, director) return self._retry_with_clearance(request, error.response, director)
raise raise
if is_cf_challenge(getattr(response, "status", None), response.headers): if is_cf_challenge(getattr(response, "status", None), cast("Any", response.headers)):
return self._retry_with_clearance(request, response, director) return self._retry_with_clearance(request, response, director)
return response return response

View file

@ -1,16 +1,18 @@
# flake8: noqa: S310
from __future__ import annotations from __future__ import annotations
import json import json
import time import time
import urllib.request import urllib.request
from typing import Any from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse from urllib.parse import urlparse
from app.library.log import get_logger from app.library.log import get_logger
from .cache import Cache from .cache import Cache
if TYPE_CHECKING:
from collections.abc import Mapping
CACHE: Cache = Cache() CACHE: Cache = Cache()
LOG = get_logger() LOG = get_logger()
@ -57,7 +59,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
if user_agent: if user_agent:
payload.setdefault("headers", {})["User-Agent"] = user_agent payload.setdefault("headers", {})["User-Agent"] = user_agent
req = urllib.request.Request( req = urllib.request.Request( # noqa: S310
endpoint, endpoint,
data=json.dumps(payload).encode("utf-8"), data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"}, headers={"Content-Type": "application/json"},
@ -68,7 +70,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
"Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint} "Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint}
) )
start_time = time.time() start_time = time.time()
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: # noqa: S310
result = json.loads(resp.read().decode("utf-8")) result = json.loads(resp.read().decode("utf-8"))
if "ok" != result.get("status"): if "ok" != result.get("status"):
@ -98,7 +100,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
return CACHE.get(host) return CACHE.get(host)
def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool: def is_cf_challenge(status: int | None, headers: Mapping[str, Any] | None) -> bool:
""" """
Determine whether a response indicates a Cloudflare challenge. Determine whether a response indicates a Cloudflare challenge.
""" """

View file

@ -2,6 +2,7 @@ import logging
import multiprocessing import multiprocessing
import os import os
import re import re
import shutil
import sys import sys
import time import time
from logging.handlers import TimedRotatingFileHandler from logging.handlers import TimedRotatingFileHandler
@ -30,7 +31,7 @@ APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = (
if TYPE_CHECKING: if TYPE_CHECKING:
from subprocess import CompletedProcess from subprocess import CompletedProcess
SUPPORTED_CODECS: tuple[str] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264") SUPPORTED_CODECS: tuple[str, ...] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
"Supported encoder names in order of preference." "Supported encoder names in order of preference."
@ -179,7 +180,7 @@ class Config(metaclass=Singleton):
file_logging: bool = True file_logging: bool = True
"Enable file logging." "Enable file logging."
secret_key: str secret_key: str | bytes
"The secret key to use for the application." "The secret key to use for the application."
tasks_handler_timer: str = "15 */1 * * *" tasks_handler_timer: str = "15 */1 * * *"
@ -377,7 +378,7 @@ class Config(metaclass=Singleton):
LOG = get_logger() LOG = get_logger()
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config") self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
envFile: str = Path(self.config_path) / ".env" envFile: Path = Path(self.config_path) / ".env"
if envFile.exists(): if envFile.exists():
LOG.info("Loading environment variables from '%s'.", envFile) LOG.info("Loading environment variables from '%s'.", envFile)
@ -388,7 +389,7 @@ class Config(metaclass=Singleton):
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str( self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
Path(baseDefaultPath) / "var" / "downloads" Path(baseDefaultPath) / "var" / "downloads"
) )
self.app_path = Path(__file__).parent.parent.absolute() self.app_path = str(Path(__file__).parent.parent.absolute())
for k, v in self._get_attributes().items(): for k, v in self._get_attributes().items():
if k.startswith("_") or k in self._manual_vars: if k.startswith("_") or k in self._manual_vars:
@ -501,7 +502,7 @@ class Config(metaclass=Singleton):
handler.setFormatter(formatter) handler.setFormatter(formatter)
logging.getLogger().addHandler(handler) logging.getLogger().addHandler(handler)
key_file: str = Path(self.config_path) / "secret.key" key_file: Path = Path(self.config_path) / "secret.key"
if key_file.exists() and key_file.stat().st_size > 2: if key_file.exists() and key_file.stat().st_size > 2:
with open(key_file, "rb") as f: with open(key_file, "rb") as f:
@ -511,7 +512,7 @@ class Config(metaclass=Singleton):
with open(key_file, "wb") as f: with open(key_file, "wb") as f:
f.write(self.secret_key) f.write(self.secret_key)
self.started = time.time() self.started = int(time.time())
for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS: for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS:
logging.getLogger(_tool).setLevel(_level) logging.getLogger(_tool).setLevel(_level)
@ -545,7 +546,7 @@ class Config(metaclass=Singleton):
def _get_attributes(self) -> dict: def _get_attributes(self) -> dict:
attrs: dict = {} attrs: dict = {}
vClass: str = self.__class__ vClass: type = self.__class__
for attribute in vClass.__dict__: for attribute in vClass.__dict__:
if attribute.startswith("_"): if attribute.startswith("_"):
@ -601,7 +602,7 @@ class Config(metaclass=Singleton):
dict[str, str]: The replacer variables. dict[str, str]: The replacer variables.
""" """
keys: tuple[str] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file") keys: tuple[str, ...] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
return {k: getattr(self, k) for k in keys} return {k: getattr(self, k) for k in keys}
@staticmethod @staticmethod
@ -619,15 +620,20 @@ class Config(metaclass=Singleton):
This is used to set the version to the latest git tag. This is used to set the version to the latest git tag.
""" """
LOG = get_logger() LOG = get_logger()
git_path: str = Path(__file__).parent / ".." / ".." / ".git" git_path: Path = Path(__file__).parent / ".." / ".." / ".git"
if not git_path.exists(): if not git_path.exists():
return return
try: try:
import subprocess import subprocess
git = shutil.which("git")
if not git:
LOG.warning("Git executable was not found.")
return
branch_result: CompletedProcess[str] = subprocess.run( branch_result: CompletedProcess[str] = subprocess.run(
["git", "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607 [git, "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=os.path.dirname(git_path), cwd=os.path.dirname(git_path),
capture_output=True, capture_output=True,
text=True, text=True,
@ -645,7 +651,7 @@ class Config(metaclass=Singleton):
return return
commit_result: CompletedProcess[str] = subprocess.run( commit_result: CompletedProcess[str] = subprocess.run(
["git", "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], # noqa: S607 [git, "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"],
cwd=os.path.dirname(git_path), cwd=os.path.dirname(git_path),
capture_output=True, capture_output=True,
text=True, text=True,

View file

@ -39,6 +39,14 @@ if TYPE_CHECKING:
class Download: class Download:
update_task: asyncio.Task | None = None update_task: asyncio.Task | None = None
@staticmethod
def _raise_no_formats(msg: str) -> None:
raise ValueError(msg)
@staticmethod
def _raise_interrupted() -> None:
raise SystemExit(130)
def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None): def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None):
""" """
Initialize download task. Initialize download task.
@ -51,7 +59,7 @@ class Download:
""" """
config: Config = Config.get_instance() config: Config = Config.get_instance()
self.download_dir: str = info.download_dir self.download_dir: str = info.download_dir or ""
self.temp_dir: str | None = info.temp_dir self.temp_dir: str | None = info.temp_dir
self.template: str | None = info.template self.template: str | None = info.template
self.template_chapter: str | None = info.template_chapter self.template_chapter: str | None = info.template_chapter
@ -98,6 +106,16 @@ class Download:
params: dict[str, Any] = {} params: dict[str, Any] = {}
download_skipped = False download_skipped = False
hook_handlers = self._hook_handlers
if hook_handlers is None:
msg = "_hook_handlers must be initialized before _download(). Call start() first."
raise RuntimeError(msg)
status_queue = self.status_queue
if status_queue is None:
msg = "status_queue must be initialized before _download(). Call start() first."
raise RuntimeError(msg)
try: try:
params = ( params = (
self.info.get_ytdlp_opts() self.info.get_ytdlp_opts()
@ -125,9 +143,9 @@ class Download:
params.update( params.update(
{ {
"progress_hooks": [self._hook_handlers.progress_hook], "progress_hooks": [hook_handlers.progress_hook],
"postprocessor_hooks": [self._hook_handlers.postprocessor_hook], "postprocessor_hooks": [hook_handlers.postprocessor_hook],
"post_hooks": [self._hook_handlers.post_hook], "post_hooks": [hook_handlers.post_hook],
} }
) )
@ -137,7 +155,10 @@ class Download:
if self.info.cookies: if self.info.cookies:
try: try:
cookie_file = Path(self._temp_manager.temp_path or self.temp_dir) / f"cookie_{self.info._id}.txt" cookie_file = (
Path(self._temp_manager.temp_path or self.temp_dir or self.download_dir)
/ f"cookie_{self.info._id}.txt"
)
self.logger.debug( self.logger.debug(
f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.", f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.",
extra={ extra={
@ -173,7 +194,9 @@ class Download:
f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info." f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info."
) )
self.info_dict = None self.info_dict = None
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default: elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and (
(preset := self.info.get_preset()) is not None and preset.default
):
self.logger.debug( self.logger.debug(
"Removing download archive option for generic extractor on '%s'.", "Removing download archive option for generic extractor on '%s'.",
self.info.url, self.info.url,
@ -267,7 +290,7 @@ class Download:
if filtered_logs := extract_ytdlp_logs(self.logs): if filtered_logs := extract_ytdlp_logs(self.logs):
msg += " " + ", ".join(filtered_logs) msg += " " + ", ".join(filtered_logs)
raise ValueError(msg) # noqa: TRY301 self._raise_no_formats(msg)
self.logger.info( self.logger.info(
f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.", f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.",
@ -327,11 +350,11 @@ class Download:
def mark_cancelled(*_) -> None: def mark_cancelled(*_) -> None:
cls._interrupted = True cls._interrupted = True
cls.to_screen("[info] Interrupt received, exiting cleanly...") cls.to_screen("[info] Interrupt received, exiting cleanly...")
raise SystemExit(130) # noqa: TRY301 self._raise_interrupted()
signal.signal(signal.SIGUSR1, mark_cancelled) signal.signal(signal.SIGUSR1, mark_cancelled)
self.status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped}) status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
self.logger.debug( self.logger.debug(
@ -386,7 +409,7 @@ class Download:
) )
ret = cls.download(url_list=[self.info.url]) ret = cls.download(url_list=[self.info.url])
self.status_queue.put( status_queue.put(
{ {
"id": self.id, "id": self.id,
"status": "finished" if 0 == ret else "error", "status": "finished" if 0 == ret else "error",
@ -407,7 +430,7 @@ class Download:
} }
}, },
) )
self.status_queue.put( status_queue.put(
{ {
"id": self.id, "id": self.id,
"status": "skip", "status": "skip",
@ -432,7 +455,7 @@ class Download:
} }
}, },
) )
self.status_queue.put( status_queue.put(
{ {
"id": self.id, "id": self.id,
"status": "error", "status": "error",
@ -442,7 +465,7 @@ class Download:
} }
) )
finally: finally:
self.status_queue.put(Terminator()) status_queue.put(Terminator())
if cookie_file and cookie_file.exists(): if cookie_file and cookie_file.exists():
try: try:
cookie_file.unlink() cookie_file.unlink()
@ -499,24 +522,24 @@ class Download:
Process exit code or None Process exit code or None
""" """
self.status_queue = Config.get_manager().Queue() status_queue: Any = Config.get_manager().Queue()
self.status_queue = status_queue
if temp_path := self._temp_manager.create_temp_path(): temp_path = self._temp_manager.create_temp_path()
self.info.temp_path = str(temp_path)
self._status_tracker = StatusTracker( self._status_tracker = StatusTracker(
info=self.info, info=self.info,
download_id=self.id, download_id=self.id,
download_dir=self.download_dir, download_dir=self.download_dir,
temp_path=temp_path, temp_path=temp_path,
status_queue=self.status_queue, status_queue=status_queue,
logger=self.logger, logger=self.logger,
debug=self.debug, debug=self.debug,
) )
self._hook_handlers = HookHandlers( self._hook_handlers = HookHandlers(
download_id=self.id, download_id=self.id,
status_queue=self.status_queue, status_queue=status_queue,
logger=self.logger, logger=self.logger,
debug=self.debug, debug=self.debug,
) )
@ -529,7 +552,8 @@ class Download:
EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info) EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info)
asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}") asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}")
ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join) proc = self._process_manager.proc
ret = await asyncio.get_running_loop().run_in_executor(None, proc.join) if proc else None
if self._status_tracker.final_update: if self._status_tracker.final_update:
return ret return ret

View file

@ -4,7 +4,7 @@ import time
import uuid import uuid
from numbers import Number from numbers import Number
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, Any
import yt_dlp.utils import yt_dlp.utils
@ -67,7 +67,7 @@ async def add_item(
already=None, already=None,
logs: list | None = None, logs: list | None = None,
yt_params: dict | None = None, yt_params: dict | None = None,
) -> dict[str, str]: ) -> dict[str, Any]:
""" """
Route an entry to the appropriate processor based on type. Route an entry to the appropriate processor based on type.
@ -102,7 +102,7 @@ async def add_item(
async def add( async def add(
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
) -> dict[str, str]: ) -> dict[str, Any]:
""" """
Add an item to the download queue. Add an item to the download queue.
@ -204,7 +204,7 @@ async def add(
) )
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"): if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
dlInfo.info.msg += f" Found in archive '{archive_file}'." dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{archive_file}'."
await queue.done.put(dlInfo) await queue.done.put(dlInfo)
@ -306,7 +306,10 @@ async def add(
new_archive_id: str | None = None new_archive_id: str | None = None
if entry.get("extractor_key") and entry.get("id"): if entry.get("extractor_key") and entry.get("id"):
new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}" _extractor_key = entry.get("extractor_key")
_entry_id = entry.get("id")
if isinstance(_extractor_key, str) and _entry_id is not None:
new_archive_id = f"{_extractor_key.lower()} {_entry_id!s}"
if new_archive_id != archive_id: if new_archive_id != archive_id:
extra_ids.append(new_archive_id) extra_ids.append(new_archive_id)
@ -336,7 +339,7 @@ async def add(
) )
) )
dlInfo.info.msg += f" Found in archive '{_archive_file}'." dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{_archive_file}'."
await queue.done.put(dlInfo) await queue.done.put(dlInfo)
queue._notify.emit( queue._notify.emit(
@ -374,7 +377,7 @@ async def add(
if condition.extras.get("ignore_download", False): if condition.extras.get("ignore_download", False):
extra_msg: str = "" extra_msg: str = ""
if _archive_file and not condition.extras.get("no_archive", False): if _archive_file and not condition.extras.get("no_archive", False) and archive_id:
archive_add(_archive_file, [archive_id]) archive_add(_archive_file, [archive_id])
extra_msg = f" and added to archive file '{_archive_file}'" extra_msg = f" and added to archive file '{_archive_file}'"
@ -385,7 +388,7 @@ async def add(
if not store_type: if not store_type:
dlInfo = Download( dlInfo = Download(
info=ItemDTO( info=ItemDTO(
id=entry.get("id"), id=str(entry.get("id") or item.url),
title=_name, title=_name,
url=item.url, url=item.url,
preset=item.preset, preset=item.preset,
@ -401,6 +404,9 @@ async def add(
LOG.info(log_message) LOG.info(log_message)
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message) queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
if dlInfo is None:
msg = "Download info is not available."
raise RuntimeError(msg)
queue._notify.emit( queue._notify.emit(
Events.ITEM_MOVED, Events.ITEM_MOVED,
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},

View file

@ -121,8 +121,8 @@ async def process_playlist(
max_downloads: int = -1 max_downloads: int = -1
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all() ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int): if isinstance(ytdlp_opts.get("max_downloads"), int):
max_downloads: int = ytdlp_opts.get("max_downloads") max_downloads = ytdlp_opts["max_downloads"]
results: list[dict[str, str]] = [] results: list[dict[str, str]] = []
for i, etr in enumerate(entries, start=1): for i, etr in enumerate(entries, start=1):

View file

@ -99,15 +99,19 @@ class ProcessManager:
if not self.running(): if not self.running():
return False return False
procId: int | None = self.proc.ident proc = self.proc
if proc is None:
return False
procId: int | None = proc.ident
try: try:
self.logger.info( self.logger.info(
"Stopping download process PID=%s.", "Stopping download process PID=%s.",
self.proc.pid, proc.pid,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"process_ident": procId, "process_ident": procId,
"is_live": self.is_live, "is_live": self.is_live,
} }
@ -117,25 +121,25 @@ class ProcessManager:
if self.is_live: if self.is_live:
self.logger.debug( self.logger.debug(
"Requesting graceful live cancellation for PID=%s.", "Requesting graceful live cancellation for PID=%s.",
self.proc.pid, proc.pid,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"is_live": self.is_live, "is_live": self.is_live,
} }
}, },
) )
self.cancel_event.set() self.cancel_event.set()
if wait_for_process_with_timeout(self.proc, 10): if wait_for_process_with_timeout(proc, 10):
self.logger.debug( self.logger.debug(
"Download process PID=%s stopped gracefully.", "Download process PID=%s stopped gracefully.",
self.proc.pid, proc.pid,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"is_live": self.is_live, "is_live": self.is_live,
} }
}, },
@ -143,43 +147,43 @@ class ProcessManager:
return True return True
self.logger.warning( self.logger.warning(
"Download process PID=%s did not respond to live cancellation; forcing termination.", "Download process PID=%s did not respond to live cancellation; forcing termination.",
self.proc.pid, proc.pid,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"is_live": self.is_live, "is_live": self.is_live,
"force": True, "force": True,
} }
}, },
) )
elif self.proc.pid and "posix" == os.name: elif proc.pid and "posix" == os.name:
signal_name = "SIGUSR1" signal_name = "SIGUSR1"
try: try:
self.logger.debug( self.logger.debug(
"Sending %s signal to download process PID=%s.", "Sending %s signal to download process PID=%s.",
signal_name, signal_name,
self.proc.pid, proc.pid,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"signal": signal_name, "signal": signal_name,
} }
}, },
) )
os.kill(self.proc.pid, signal.SIGUSR1) os.kill(proc.pid, signal.SIGUSR1)
if wait_for_process_with_timeout(self.proc, 5): if wait_for_process_with_timeout(proc, 5):
self.logger.debug( self.logger.debug(
"Download process PID=%s stopped gracefully.", "Download process PID=%s stopped gracefully.",
self.proc.pid, proc.pid,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"signal": signal_name, "signal": signal_name,
} }
}, },
@ -187,12 +191,12 @@ class ProcessManager:
return True return True
self.logger.warning( self.logger.warning(
"Download process PID=%s did not respond to %s; forcing termination.", "Download process PID=%s did not respond to %s; forcing termination.",
self.proc.pid, proc.pid,
signal_name, signal_name,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"signal": signal_name, "signal": signal_name,
"force": True, "force": True,
} }
@ -203,51 +207,49 @@ class ProcessManager:
self.logger.debug( self.logger.debug(
"Failed to send %s signal to download process PID=%s. %s", "Failed to send %s signal to download process PID=%s. %s",
signal_name, signal_name,
self.proc.pid, proc.pid,
e, e,
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid,
"signal": signal_name, "signal": signal_name,
"exception_type": type(e).__name__, "exception_type": type(e).__name__,
} }
}, },
) )
if self.proc.is_alive(): if proc.is_alive():
self.logger.info( self.logger.info(
"Terminating download process PID=%s.", "Terminating download process PID=%s.",
self.proc.pid, proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}}, extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
) )
self.proc.terminate() proc.terminate()
if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2): if not wait_for_process_with_timeout(proc, 1 if self.is_live else 2):
self.logger.warning( self.logger.warning(
"Download process PID=%s did not terminate; killing forcefully.", "Download process PID=%s did not terminate; killing forcefully.",
self.proc.pid, proc.pid,
extra={ extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}
},
) )
self.proc.kill() proc.kill()
wait_for_process_with_timeout(self.proc, 1) wait_for_process_with_timeout(proc, 1)
self.logger.info( self.logger.info(
"Download process PID=%s stopped.", "Download process PID=%s stopped.",
self.proc.pid, proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid}}, extra={"download": {"download_id": self.download_id, "process_id": proc.pid}},
) )
return True return True
except Exception as e: except Exception as e:
self.logger.exception( self.logger.exception(
f"Failed to stop download process PID={self.proc.pid}, ident={procId}.", f"Failed to stop download process PID={proc.pid if proc else None}, ident={procId}.",
extra={ extra={
"download": { "download": {
"download_id": self.download_id, "download_id": self.download_id,
"process_id": self.proc.pid, "process_id": proc.pid if proc else None,
"process_ident": procId, "process_ident": procId,
"is_live": self.is_live, "is_live": self.is_live,
"exception_type": type(e).__name__, "exception_type": type(e).__name__,
@ -271,11 +273,12 @@ class ProcessManager:
return False return False
self.cancel_in_progress = True self.cancel_in_progress = True
procId: int | None = self.proc.ident proc = self.proc
procId: int | None = proc.ident if proc else None
if not procId: if not procId:
if self.proc: if proc:
self.proc.close() proc.close()
self.proc = None self.proc = None
self.logger.warning("Attempted to close download process, but it is not running.") self.logger.warning("Attempted to close download process, but it is not running.")
return False return False
@ -293,13 +296,14 @@ class ProcessManager:
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
if self.proc.is_alive(): current = self.proc
if current is not None and current.is_alive():
self.logger.debug( self.logger.debug(
"Waiting for download process PID='%s' to close.", "Waiting for download process PID='%s' to close.",
procId, procId,
extra={"download": {"download_id": self.download_id, "process_ident": procId}}, extra={"download": {"download_id": self.download_id, "process_ident": procId}},
) )
await loop.run_in_executor(None, self.proc.join) await loop.run_in_executor(None, current.join)
self.logger.debug( self.logger.debug(
"Download process PID='%s' closed.", "Download process PID='%s' closed.",
procId, procId,

View file

@ -53,7 +53,7 @@ class DownloadQueue(metaclass=Singleton):
async def event_handler(_, __): async def event_handler(_, __):
await self.initialize() await self.initialize()
self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}") self._notify.subscribe(Events.STARTED, event_handler, f"{DownloadQueue.__name__}.initialize")
Scheduler.get_instance().add( Scheduler.get_instance().add(
timer="* * * * *", timer="* * * * *",
@ -689,7 +689,7 @@ class DownloadQueue(metaclass=Singleton):
) )
return {"deleted": deleted_count} return {"deleted": deleted_count}
async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: async def get(self, mode: str = "all") -> dict[str, dict[str, ItemDTO]]:
""" """
Get the download queue and the download history. Get the download queue and the download history.

View file

@ -55,7 +55,7 @@ class StatusTracker:
self.final_update = False self.final_update = False
self._terminator_sent: bool = False self._terminator_sent: bool = False
self._candidate_filepath: Path | None = None self._candidate_filepath: Path | None = None
self.update_task: asyncio.Task | None = None self.update_task: asyncio.Future[Any] | None = None
self._last_progress_time: float = 0.0 self._last_progress_time: float = 0.0
self._pending_progress: bool = False self._pending_progress: bool = False
self._progress_interval: float = 0.5 self._progress_interval: float = 0.5
@ -256,7 +256,7 @@ class StatusTracker:
self.info.percent = status["downloaded_bytes"] / total * 100 self.info.percent = status["downloaded_bytes"] / total * 100
except ZeroDivisionError: except ZeroDivisionError:
self.info.percent = 0 self.info.percent = 0
self.info.total_bytes = total self.info.total_bytes = int(total)
self.info.speed = status.get("speed") self.info.speed = status.get("speed")
self.info.eta = status.get("eta") self.info.eta = status.get("eta")
@ -280,8 +280,9 @@ class StatusTracker:
""" """
while True: while True:
try: try:
self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get) update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
status = await self.update_task self.update_task = asyncio.ensure_future(update_task)
status = await update_task
if status is None or isinstance(status, Terminator): if status is None or isinstance(status, Terminator):
self._flush_progress() self._flush_progress()
return return

View file

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio import asyncio
import functools import functools
import importlib.util
import threading import threading
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Literal, cast, overload from typing import Any, Literal, cast, overload
@ -98,12 +99,7 @@ def _normalize_proxy(proxy: str | dict[str, str] | None) -> str | None:
@functools.lru_cache(maxsize=1) @functools.lru_cache(maxsize=1)
def _curl_available() -> bool: def _curl_available() -> bool:
try: return importlib.util.find_spec("httpx_curl_cffi") is not None
import httpx_curl_cffi # noqa: F401
return True
except Exception:
return False
def resolve_curl_transport(use_curl: bool = True) -> bool: def resolve_curl_transport(use_curl: bool = True) -> bool:
@ -121,7 +117,7 @@ def _build_async_curl_transport(
from httpx_curl_cffi import AsyncCurlTransport from httpx_curl_cffi import AsyncCurlTransport
return AsyncCurlTransport( return AsyncCurlTransport(
impersonate=curl_impersonate, impersonate=cast("Any", curl_impersonate),
default_headers=curl_default_headers, default_headers=curl_default_headers,
) )
@ -308,10 +304,7 @@ def get_async_client(
curl_default_headers=curl_default_headers, curl_default_headers=curl_default_headers,
) )
client = httpx.AsyncClient( client = httpx.AsyncClient(
transport=cast( transport=_get_transport(enable_cf, is_async=True, transport=transport),
"httpx.AsyncBaseTransport",
_get_transport(enable_cf, is_async=True, transport=transport),
),
proxy=cast("Any", proxy), proxy=cast("Any", proxy),
) )
Globals.SHARED_ASYNC_CLIENTS[key] = client Globals.SHARED_ASYNC_CLIENTS[key] = client
@ -330,10 +323,7 @@ def get_sync_client(
return Globals.SHARED_SYNC_CLIENTS[key] return Globals.SHARED_SYNC_CLIENTS[key]
client = httpx.Client( client = httpx.Client(
transport=cast( transport=_get_transport(enable_cf, is_async=False, transport=None),
"httpx.BaseTransport",
_get_transport(enable_cf, is_async=False, transport=None),
),
proxy=cast("Any", proxy), proxy=cast("Any", proxy),
) )
Globals.SHARED_SYNC_CLIENTS[key] = client Globals.SHARED_SYNC_CLIENTS[key] = client

View file

@ -1,4 +1,3 @@
# flake8: noqa: S608
""" """
This module is a fork of the Caribou library This module is a fork of the Caribou library
(https://pypi.org/project/caribou/) modified to work for our specific use case. (https://pypi.org/project/caribou/) modified to work for our specific use case.
@ -9,10 +8,11 @@ import datetime
import glob import glob
import os.path import os.path
import traceback import traceback
from collections.abc import AsyncIterator, Iterable, Sequence from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.machinery import ModuleSpec, SourceFileLoader
from importlib.util import module_from_spec, spec_from_loader from importlib.util import module_from_spec, spec_from_loader
from typing import TYPE_CHECKING from pathlib import Path
from typing import TYPE_CHECKING, Any
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.ext.asyncio import AsyncConnection
@ -44,7 +44,9 @@ class InvalidNameError(Error):
@contextlib.asynccontextmanager @contextlib.asynccontextmanager
async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator: async def execute(
conn: AsyncConnection, sql: str, params: Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None
) -> AsyncIterator:
params = {} if params is None else params params = {} if params is None else params
result = await conn.execute(text(sql), params) result = await conn.execute(text(sql), params)
try: try:
@ -126,12 +128,12 @@ class Database:
self.version_table: str = version_table self.version_table: str = version_table
self._owns_connection = bool(isinstance(db_url, str)) self._owns_connection = isinstance(db_url, str)
if self._owns_connection: if isinstance(db_url, str):
self.conn: AsyncConnection | None = None self.conn: AsyncConnection | None = None
self.db_url: str = db_url self.db_url: str = db_url
else: else:
self.conn: AsyncConnection = db_url self.conn = db_url
async def __aenter__(self): async def __aenter__(self):
if self._owns_connection and self.conn is None: if self._owns_connection and self.conn is None:
@ -183,7 +185,8 @@ class Database:
async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None: async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
assert self.conn assert self.conn
if target_version not in (0, "0"): if target_version not in (0, "0"):
_assert_migration_exists(migrations, target_version) _assert_migration_exists(migrations, str(target_version))
target = str(target_version)
migrations.sort(key=lambda x: x.get_version(), reverse=True) migrations.sort(key=lambda x: x.get_version(), reverse=True)
database_version: str | None = await self.get_version() database_version: str | None = await self.get_version()
@ -192,7 +195,7 @@ class Database:
current_version: str = migration.get_version() current_version: str = migration.get_version()
if database_version is not None and current_version > database_version: if database_version is not None and current_version > database_version:
continue continue
if current_version <= target_version: if current_version <= target:
break break
await migration.downgrade(self.conn) await migration.downgrade(self.conn)
next_version: str | int = 0 next_version: str | int = 0
@ -210,14 +213,14 @@ class Database:
assert self.conn assert self.conn
if not await self.is_version_controlled(): if not await self.is_version_controlled():
return None return None
sql: str = f"SELECT version FROM {self.version_table}" sql: str = f"SELECT version FROM {self.version_table}" # noqa: S608
async with execute(self.conn, sql) as result: async with execute(self.conn, sql) as result:
rows = result.fetchall() rows = result.fetchall()
return rows[0][0] if rows else "0" return rows[0][0] if rows else "0"
async def update_version(self, version: str) -> None: async def update_version(self, version: str) -> None:
assert self.conn assert self.conn
sql: str = f"UPDATE {self.version_table} SET version = :version" sql: str = f"UPDATE {self.version_table} SET version = :version" # noqa: S608
async with transaction(self.conn): async with transaction(self.conn):
await self.conn.execute(text(sql), {"version": version}) await self.conn.execute(text(sql), {"version": version})
@ -226,7 +229,7 @@ class Database:
sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """ sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
async with transaction(self.conn): async with transaction(self.conn):
await self.conn.execute(text(sql)) await self.conn.execute(text(sql))
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) # noqa: S608
def __repr__(self) -> str: def __repr__(self) -> str:
return f'Database("{self.db_url if self._owns_connection else "external_connection"}")' return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
@ -238,7 +241,7 @@ def _assert_migration_exists(migrations: Iterable[Migration], version: str | int
raise Error(msg) raise Error(msg)
def load_migrations(directory: str) -> list[Migration]: def load_migrations(directory: str | Path) -> list[Migration]:
"""Return the migrations contained in the given directory.""" """Return the migrations contained in the given directory."""
directory = str(directory) directory = str(directory)
if not os.path.exists(directory) or not os.path.isdir(directory): if not os.path.exists(directory) or not os.path.isdir(directory):
@ -247,7 +250,7 @@ def load_migrations(directory: str) -> list[Migration]:
return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))] return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None: async def upgrade(db_url: AsyncConnection | str, migration_dir: str | Path, version: str | None = None) -> None:
""" """
Upgrade the given database with the migrations contained in the Upgrade the given database with the migrations contained in the
migrations directory. If a version is not specified, upgrade migrations directory. If a version is not specified, upgrade
@ -260,7 +263,7 @@ async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: st
await db.upgrade(load_migrations(migration_dir), version) await db.upgrade(load_migrations(migration_dir), version)
async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None: async def downgrade(db_url: str | AsyncConnection, migration_dir: str | Path, version: str) -> None:
""" """
Downgrade the database to the given version with the migrations Downgrade the database to the given version with the migrations
contained in the given migration directory. contained in the given migration directory.

View file

@ -2,14 +2,16 @@ import asyncio
import contextlib import contextlib
import json import json
import os import os
from collections.abc import Iterable from collections.abc import Iterable, Mapping
from dataclasses import fields from dataclasses import fields
from datetime import UTC, datetime from datetime import UTC, datetime
from email.utils import formatdate from email.utils import formatdate
from typing import Any
from urllib.parse import quote_plus from urllib.parse import quote_plus
from aiohttp import web from aiohttp import web
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.engine.row import RowMapping
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.ext.asyncio.engine import AsyncConnection from sqlalchemy.ext.asyncio.engine import AsyncConnection
@ -36,13 +38,13 @@ def _memory_db_url(db_path: str) -> str:
return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true" return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true"
def _decode_row(row: dict) -> ItemDTO: def _decode_row(row: Mapping[str, Any] | RowMapping) -> ItemDTO:
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC)
data = json.loads(row["data"]) data = json.loads(row["data"])
data.pop("_id", None) data.pop("_id", None)
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"] item._id = row["id"]
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) item.datetime = formatdate(row_date.timestamp())
return item return item
@ -149,8 +151,8 @@ class SqliteStore(metaclass=ThreadSafe):
return self._sessionmaker return self._sessionmaker
async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]: async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
await self.get_connection() conn = await self.get_connection()
result = await self._conn.execute( result = await conn.execute(
text( text(
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC' 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC'
), ),
@ -163,12 +165,12 @@ class SqliteStore(metaclass=ThreadSafe):
async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool: async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
return await self.get(type_value, key=key, url=url) is not None return await self.get(type_value, key=key, url=url) is not None
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> bool: async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> ItemDTO | None:
if not key and not url: if not key and not url:
msg = "key or url must be provided." msg = "key or url must be provided."
raise KeyError(msg) raise KeyError(msg)
await self.get_connection() conn = await self.get_connection()
clauses: list[str] = [] clauses: list[str] = []
params: dict[str, str] = {"type_value": type_value} params: dict[str, str] = {"type_value": type_value}
@ -186,7 +188,7 @@ class SqliteStore(metaclass=ThreadSafe):
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608 f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608
) )
result = await self._conn.execute(text(query), params) result = await conn.execute(text(query), params)
row = result.mappings().first() row = result.mappings().first()
if not row: if not row:
@ -195,8 +197,8 @@ class SqliteStore(metaclass=ThreadSafe):
return _decode_row(row) return _decode_row(row)
async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None: async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
await self.get_connection() conn = await self.get_connection()
result = await self._conn.execute( result = await conn.execute(
text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'), text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'),
{"type_value": type_value, "id": id}, {"type_value": type_value, "id": id},
) )
@ -212,24 +214,24 @@ class SqliteStore(metaclass=ThreadSafe):
if not ids_list: if not ids_list:
return [] return []
await self.get_connection() conn = await self.get_connection()
placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list))) placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list)))
params: dict[str, str] = {"type_value": type_value} params: dict[str, str] = {"type_value": type_value}
params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)}) params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)})
result = await self._conn.execute( result = await conn.execute(
text( text(
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608 f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608
), ),
params, params,
) )
rows = result.mappings().all() rows = list(result.mappings().all())
order = {item_id: index for index, item_id in enumerate(ids_list)} order = {item_id: index for index, item_id in enumerate(ids_list)}
rows.sort(key=lambda row: order.get(row["id"], len(ids_list))) rows.sort(key=lambda row: order.get(row["id"], len(ids_list)))
return [(row["id"], _decode_row(row)) for row in rows] return [(row["id"], _decode_row(row)) for row in rows]
async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]: async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]:
await self.get_connection() conn = await self.get_connection()
params: dict[str, str] = {"type_value": type_value} params: dict[str, str] = {"type_value": type_value}
where_clauses = ['"type" = :type_value'] where_clauses = ['"type" = :type_value']
@ -239,7 +241,7 @@ class SqliteStore(metaclass=ThreadSafe):
params.update(status_params) params.update(status_params)
query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608 query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608
result = await self._conn.execute(text(query), params) result = await conn.execute(text(query), params)
rows = result.mappings().all() rows = result.mappings().all()
return [(row["id"], _decode_row(row)) for row in rows] return [(row["id"], _decode_row(row)) for row in rows]
@ -254,7 +256,7 @@ class SqliteStore(metaclass=ThreadSafe):
if not kwargs: if not kwargs:
return None return None
await self.get_connection() conn = await self.get_connection()
clauses: list[str] = [] clauses: list[str] = []
params: dict[str, str | float | int] = {"type_value": type_value} params: dict[str, str | float | int] = {"type_value": type_value}
@ -321,7 +323,7 @@ class SqliteStore(metaclass=ThreadSafe):
where_clause: str = " OR ".join(f"({clause})" for clause in clauses) where_clause: str = " OR ".join(f"({clause})" for clause in clauses)
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608 query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608
result = await self._conn.execute(text(query), params) result = await conn.execute(text(query), params)
row = result.mappings().first() row = result.mappings().first()
if not row: if not row:
@ -332,7 +334,7 @@ class SqliteStore(metaclass=ThreadSafe):
return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None
async def count(self, type_value: str, status_filter: str | None = None) -> int: async def count(self, type_value: str, status_filter: str | None = None) -> int:
await self.get_connection() conn = await self.get_connection()
where_clauses: list[str] = ['"type" = :type_value'] where_clauses: list[str] = ['"type" = :type_value']
params: dict[str, str] = {"type_value": type_value} params: dict[str, str] = {"type_value": type_value}
@ -344,7 +346,7 @@ class SqliteStore(metaclass=ThreadSafe):
where_clause: str = " AND ".join(where_clauses) where_clause: str = " AND ".join(where_clauses)
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
result = await self._conn.execute(text(count_query), params) result = await conn.execute(text(count_query), params)
row = result.mappings().first() row = result.mappings().first()
return row["count"] if row else 0 return row["count"] if row else 0
@ -357,7 +359,7 @@ class SqliteStore(metaclass=ThreadSafe):
order: str, order: str,
status_filter: str | None = None, status_filter: str | None = None,
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
await self.get_connection() conn = await self.get_connection()
where_clauses: list[str] = ['"type" = :type_value'] where_clauses: list[str] = ['"type" = :type_value']
params: dict[str, str | int] = {"type_value": type_value} params: dict[str, str | int] = {"type_value": type_value}
@ -369,7 +371,7 @@ class SqliteStore(metaclass=ThreadSafe):
where_clause: str = " AND ".join(where_clauses) where_clause: str = " AND ".join(where_clauses)
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
result = await self._conn.execute(text(count_query), params) result = await conn.execute(text(count_query), params)
count_row = result.mappings().first() count_row = result.mappings().first()
total_items = count_row["count"] if count_row else 0 total_items = count_row["count"] if count_row else 0
@ -384,7 +386,7 @@ class SqliteStore(metaclass=ThreadSafe):
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608 query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608
result = await self._conn.execute(text(query), params) result = await conn.execute(text(query), params)
rows = result.mappings().all() rows = result.mappings().all()
items = [(row["id"], _decode_row(row)) for row in rows] items = [(row["id"], _decode_row(row)) for row in rows]
@ -396,15 +398,15 @@ class SqliteStore(metaclass=ThreadSafe):
await self._upsert_now(type_value, item) await self._upsert_now(type_value, item)
async def delete(self, type_value: str, key: str) -> None: async def delete(self, type_value: str, key: str) -> None:
await self.get_connection() conn = await self.get_connection()
await self._conn.execute( await conn.execute(
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
{"type_value": type_value, "key": key}, {"type_value": type_value, "key": key},
) )
await self._conn.commit() await conn.commit()
async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int: async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
await self.get_connection() conn = await self.get_connection()
keys_list: list[str] = list(keys) keys_list: list[str] = list(keys)
if not keys_list: if not keys_list:
return 0 return 0
@ -412,15 +414,15 @@ class SqliteStore(metaclass=ThreadSafe):
params: dict[str, str] = {"type_value": type_value} params: dict[str, str] = {"type_value": type_value}
params.update({f"key_{i}": key for i, key in enumerate(keys_list)}) params.update({f"key_{i}": key for i, key in enumerate(keys_list)})
result = await self._conn.execute( result = await conn.execute(
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
params, params,
) )
await self._conn.commit() await conn.commit()
return result.rowcount if result else 0 return result.rowcount if result else 0
async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int: async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int:
await self.get_connection() conn = await self.get_connection()
params: dict[str, str] = {"type_value": type_value} params: dict[str, str] = {"type_value": type_value}
where_clauses = ['"type" = :type_value'] where_clauses = ['"type" = :type_value']
@ -429,11 +431,11 @@ class SqliteStore(metaclass=ThreadSafe):
where_clauses.append(status_clause) where_clauses.append(status_clause)
params.update(status_params) params.update(status_params)
result = await self._conn.execute( result = await conn.execute(
text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608 text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608
params, params,
) )
await self._conn.commit() await conn.commit()
return result.rowcount if result else 0 return result.rowcount if result else 0
async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None: async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
@ -484,6 +486,9 @@ class SqliteStore(metaclass=ThreadSafe):
async def _enqueue(self, op: _Op) -> None: async def _enqueue(self, op: _Op) -> None:
self._ensure_worker() self._ensure_worker()
if self._queue is None:
msg = "Queue was not initialized by _ensure_worker."
raise RuntimeError(msg)
await self._queue.put(op) await self._queue.put(op)
def _ensure_worker(self) -> None: def _ensure_worker(self) -> None:
@ -492,10 +497,14 @@ class SqliteStore(metaclass=ThreadSafe):
self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer") self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
async def _writer(self) -> None: async def _writer(self) -> None:
queue = self._queue
if queue is None:
msg = "Writer started before queue was initialized."
raise RuntimeError(msg)
while True: while True:
op = await self._queue.get() op = await queue.get()
if isinstance(op.op, Terminator): if isinstance(op.op, Terminator):
self._queue.task_done() queue.task_done()
break break
try: try:
async with self._lock: async with self._lock:
@ -507,34 +516,34 @@ class SqliteStore(metaclass=ThreadSafe):
extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__}, extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__},
) )
finally: finally:
self._queue.task_done() queue.task_done()
await asyncio.sleep(self._flush_interval) await asyncio.sleep(self._flush_interval)
async def _apply(self, op: _Op) -> None: async def _apply(self, op: _Op) -> None:
await self.get_connection() conn = await self.get_connection()
if op.op == "upsert" and op.item: if op.op == "upsert" and op.item:
await self._upsert_now_conn(self._conn, op.type_value, op.item) await self._upsert_now_conn(conn, op.type_value, op.item)
await self._conn.commit() await conn.commit()
elif op.op == "delete" and op.key: elif op.op == "delete" and op.key:
await self._conn.execute( await conn.execute(
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
{"type_value": op.type_value, "key": op.key}, {"type_value": op.type_value, "key": op.key},
) )
await self._conn.commit() await conn.commit()
elif op.op == "bulk_delete" and op.keys: elif op.op == "bulk_delete" and op.keys:
placeholders = ",".join(f":key_{i}" for i in range(len(op.keys))) placeholders = ",".join(f":key_{i}" for i in range(len(op.keys)))
params: dict[str, str] = {"type_value": op.type_value} params: dict[str, str] = {"type_value": op.type_value}
params.update({f"key_{i}": key for i, key in enumerate(op.keys)}) params.update({f"key_{i}": key for i, key in enumerate(op.keys)})
await self._conn.execute( await conn.execute(
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
params, params,
) )
await self._conn.commit() await conn.commit()
async def _upsert_now(self, type_value: str, item: ItemDTO) -> None: async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
await self.get_connection() conn = await self.get_connection()
await self._upsert_now_conn(self._conn, type_value, item) await self._upsert_now_conn(conn, type_value, item)
await self._conn.commit() await conn.commit()
async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None: async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None:
"""Helper to upsert using an existing connection.""" """Helper to upsert using an existing connection."""
@ -560,7 +569,7 @@ class SqliteStore(metaclass=ThreadSafe):
async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None: async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None:
"""Execute raw SQL query (for testing purposes).""" """Execute raw SQL query (for testing purposes)."""
await self.get_connection() conn = await self.get_connection()
if isinstance(params, tuple): if isinstance(params, tuple):
# Convert positional params to dict for SQLAlchemy # Convert positional params to dict for SQLAlchemy
# Assuming queries use ? placeholders, we need to count them # Assuming queries use ? placeholders, we need to count them
@ -575,16 +584,16 @@ class SqliteStore(metaclass=ThreadSafe):
# Replace ? with :p0, :p1, etc. # Replace ? with :p0, :p1, etc.
for i in range(len(params)): for i in range(len(params)):
query = query.replace("?", f":p{i}", 1) query = query.replace("?", f":p{i}", 1)
await self._conn.execute(text(query), param_dict) await conn.execute(text(query), param_dict)
elif isinstance(params, dict): elif isinstance(params, dict):
await self._conn.execute(text(query), params) await conn.execute(text(query), params)
else: else:
await self._conn.execute(text(query)) await conn.execute(text(query))
await self._conn.commit() await conn.commit()
async def fetch_raw(self, query: str, params: dict | tuple | None = None): async def fetch_raw(self, query: str, params: dict | tuple | None = None):
"""Fetch results from raw SQL query (for testing purposes).""" """Fetch results from raw SQL query (for testing purposes)."""
await self.get_connection() conn = await self.get_connection()
if isinstance(params, tuple): if isinstance(params, tuple):
# Convert positional params to dict for SQLAlchemy # Convert positional params to dict for SQLAlchemy
placeholders = query.count("?") placeholders = query.count("?")
@ -596,11 +605,11 @@ class SqliteStore(metaclass=ThreadSafe):
param_dict = {f"p{i}": params[i] for i in range(len(params))} param_dict = {f"p{i}": params[i] for i in range(len(params))}
for i in range(len(params)): for i in range(len(params)):
query = query.replace("?", f":p{i}", 1) query = query.replace("?", f":p{i}", 1)
result = await self._conn.execute(text(query), param_dict) result = await conn.execute(text(query), param_dict)
elif isinstance(params, dict): elif isinstance(params, dict):
result = await self._conn.execute(text(query), params) result = await conn.execute(text(query), params)
else: else:
result = await self._conn.execute(text(query)) result = await conn.execute(text(query))
return result.mappings().all() return result.mappings().all()
async def get_connection(self) -> AsyncConnection: async def get_connection(self) -> AsyncConnection:

View file

@ -1,14 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# flake8: noqa: S310 T201
import argparse import argparse
import http.client
import os import os
import pathlib import pathlib
import socket import socket
import sys import sys
import time import time
import urllib.request
import webbrowser import webbrowser
from urllib.parse import urlsplit
if __name__ == "__main__": if __name__ == "__main__":
from multiprocessing import freeze_support from multiprocessing import freeze_support
@ -26,7 +26,7 @@ APP_ROOT = str((pathlib.Path(__file__).parent / "..").resolve())
if APP_ROOT not in sys.path: if APP_ROOT not in sys.path:
sys.path.insert(0, APP_ROOT) sys.path.insert(0, APP_ROOT)
import platformdirs # type: ignore import platformdirs
def set_env(): def set_env():
@ -55,18 +55,27 @@ def open_browser_when_ready(url: str, timeout: float = 5.0) -> None:
import threading import threading
def wait_then_open(): def wait_then_open():
parsed = urlsplit(url)
path = parsed.path or "/"
deadline = time.time() + timeout deadline = time.time() + timeout
while time.time() < deadline: while time.time() < deadline:
try: try:
with urllib.request.urlopen(url, timeout=1) as response: conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 80, timeout=1)
if 200 == response.status: try:
conn.request("GET", path)
response = conn.getresponse()
if response.status == 200:
os.environ.pop("LD_LIBRARY_PATH", None) os.environ.pop("LD_LIBRARY_PATH", None)
webbrowser.open_new(url) webbrowser.open_new(url)
return return
finally:
conn.close()
except Exception: except Exception:
time.sleep(0.5) time.sleep(0.5)
print(f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.") sys.stderr.write(
f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.\n"
)
threading.Thread(target=wait_then_open, daemon=True).start() threading.Thread(target=wait_then_open, daemon=True).start()
@ -115,7 +124,7 @@ def main():
set_env() set_env()
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH")) / ".env" env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH", "")) / ".env"
port = None port = None
if env_file.exists(): if env_file.exists():

View file

@ -199,18 +199,16 @@ class Main:
) )
HTTP_LOGGER_CLASS = HttpAccessLogger HTTP_LOGGER_CLASS = HttpAccessLogger
run_args = { web.run_app(
"host": host, self._app,
"port": port, host=host,
"loop": asyncio.get_event_loop(), port=port,
"access_log": HTTP_LOGGER, loop=asyncio.get_event_loop(),
"print": started, access_log=HTTP_LOGGER,
"handle_signals": cb is None, print=started,
} handle_signals=cb is None,
if HTTP_LOGGER_CLASS is not None: access_log_class=HTTP_LOGGER_CLASS if HTTP_LOGGER_CLASS is not None else web.AccessLogger,
run_args["access_log_class"] = HTTP_LOGGER_CLASS )
web.run_app(self._app, **run_args)
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -35,7 +35,7 @@ async def get_ffprobe(request: Request, config: Config, encoder: Encoder, app: w
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
if not file: if not file:
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
@ -76,7 +76,7 @@ async def get_file_info(request: Request, config: Config, encoder: Encoder, app:
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str | None = request.match_info.get("file")
if not file: if not file:
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
@ -137,7 +137,7 @@ async def file_browser(request: Request, config: Config, encoder: Encoder) -> Re
Response: The response object. Response: The response object.
""" """
req_path: str = request.match_info.get("path") req_path: str | None = request.match_info.get("path")
req_path: str = "/" if not req_path else unquote_plus(req_path) req_path: str = "/" if not req_path else unquote_plus(req_path)
raw_req: str = (req_path or "").strip() raw_req: str = (req_path or "").strip()
@ -461,7 +461,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
record(path, ok=False, error=str(e), action=action, extra={"item": params}) record(path, ok=False, error=str(e), action=action, extra={"item": params})
continue continue
else: else:
extra_info: dict[str, Path] = {"new_path": renamed} extra_info: dict[str, Any] = {"new_path": renamed}
if sidecar_renamed: if sidecar_renamed:
extra_info["sidecar_count"] = len(sidecar_renamed) extra_info["sidecar_count"] = len(sidecar_renamed)
record(path, ok=True, action=action, extra=extra_info) record(path, ok=True, action=action, extra=extra_info)
@ -591,7 +591,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
record(path, ok=False, error=str(e), action=action, extra={"item": params}) record(path, ok=False, error=str(e), action=action, extra={"item": params})
continue continue
else: else:
extra_info: dict[str, Path] = {"new_path": moved} extra_info: dict[str, Any] = {"new_path": moved}
if sidecar_moved: if sidecar_moved:
extra_info["sidecar_count"] = len(sidecar_moved) extra_info["sidecar_count"] = len(sidecar_moved)
@ -617,7 +617,7 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
if not json or not isinstance(json, list): if not json or not isinstance(json, list):
return web.json_response({"error": "Invalid parameters."}, status=400) return web.json_response({"error": "Invalid parameters."}, status=400)
files: list[str] = [] files: list[Path] = []
for f in json: for f in json:
if not isinstance(f, str): if not isinstance(f, str):
continue continue
@ -627,12 +627,10 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
continue continue
files.append(ref) files.append(ref)
sc: list[dict] = get_file_sidecar(ref) sc: dict[str, list[dict[str, Any]]] = get_file_sidecar(ref)
if sc: if sc:
for side in sc: for value in sc.values():
for scf in sc[side]: files.extend(scf["file"] for scf in value if isinstance(scf, dict) and "file" in scf)
if isinstance(scf, dict) and "file" in scf:
files.append(scf["file"]) # noqa: PERF401
if not files: if not files:
return web.json_response({"error": "No valid files."}, status=400) return web.json_response({"error": "No valid files."}, status=400)

View file

@ -95,7 +95,7 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response:
cache.set(cache_key, dct, ttl=3600) cache.set(cache_key, dct, ttl=3600)
return web.Response(**dct) return web.Response(body=dct["body"], headers=dct["headers"])
except Exception: except Exception:
LOG.exception( LOG.exception(
"Failed to request doc '%s' from '%s'.", "Failed to request doc '%s' from '%s'.",

View file

@ -1,5 +1,5 @@
from aiohttp import web from aiohttp import web
from aiohttp.web import Request, Response from aiohttp.web import Request, Response, StreamResponse
from app.library.config import Config from app.library.config import Config
from app.library.log import get_logger from app.library.log import get_logger
@ -10,7 +10,7 @@ LOG = get_logger()
@route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static") @route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static")
async def download_file(request: Request, config: Config, app: web.Application) -> Response: async def download_file(request: Request, config: Config, app: web.Application) -> StreamResponse:
""" """
Serve download files. Serve download files.

View file

@ -1,9 +1,10 @@
import asyncio import asyncio
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any from typing import Any
from aiohttp import web from aiohttp import web
from aiohttp.web import Request, Response from aiohttp.web import Request, Response
from aiohttp.web_response import StreamResponse
from app.features.presets.schemas import Preset from app.features.presets.schemas import Preset
from app.features.presets.service import Presets from app.features.presets.service import Presets
@ -20,10 +21,6 @@ from app.library.log import get_logger
from app.library.router import route from app.library.router import route
from app.library.Utils import calc_download_path, get_file_sidecar, rename_file from app.library.Utils import calc_download_path, get_file_sidecar, rename_file
if TYPE_CHECKING:
from library.downloads import Download
LOG = get_logger() LOG = get_logger()
@ -280,7 +277,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
Response: The response object. Response: The response object.
""" """
id: str = request.match_info.get("id") id: str | None = request.match_info.get("id")
if not id: if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
@ -314,7 +311,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
@route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail") @route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail")
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> Response: async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> StreamResponse:
if not (id := request.match_info.get("id")): if not (id := request.match_info.get("id")):
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
@ -458,7 +455,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
Response: The response object. Response: The response object.
""" """
id: str = request.match_info.get("id") id: str | None = request.match_info.get("id")
if not id: if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)

View file

@ -116,8 +116,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp
safe_backend: str = _safe_url(backend) safe_backend: str = _safe_url(backend)
await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24) await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24)
else: else:
backend = await cache.aget(CACHE_KEY_BING) cached_backend = await cache.aget(CACHE_KEY_BING)
safe_backend = _safe_url(backend if isinstance(backend, str) else None) backend = cached_backend if isinstance(cached_backend, str) else ""
safe_backend = _safe_url(backend)
if not isinstance(backend, str) or not backend: if not isinstance(backend, str) or not backend:
return web.json_response( return web.json_response(

View file

@ -1,6 +1,7 @@
import asyncio import asyncio
import json import json
import os import os
from collections.abc import Awaitable, Callable
from pathlib import Path from pathlib import Path
from aiohttp import web from aiohttp import web
@ -110,7 +111,7 @@ async def _read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict:
return {"logs": [], "next_offset": None, "end_is_reached": True} return {"logs": [], "next_offset": None, "end_is_reached": True}
async def _tail_log(file: Path, emitter: callable, sleep_time: float = 0.5): async def _tail_log(file: Path, emitter: Callable[[dict], Awaitable[None]], sleep_time: float = 0.5):
""" """
Live tail. Live tail.

View file

@ -2,6 +2,7 @@ import asyncio
import os import os
import time import time
from pathlib import Path from pathlib import Path
from typing import Any
from aiohttp import web from aiohttp import web
from aiohttp.web import Request, Response from aiohttp.web import Request, Response
@ -489,7 +490,7 @@ async def system_limits(queue: DownloadQueue, config: Config, encoder: Encoder)
for download in [*active_non_live, *queued_non_live]: for download in [*active_non_live, *queued_non_live]:
extractor = (download.info.get_extractor() or "unknown").lower() extractor = (download.info.get_extractor() or "unknown").lower()
entry = per_extractor.setdefault( entry: dict[str, Any] = per_extractor.setdefault(
extractor, extractor,
{ {
"name": extractor, "name": extractor,

View file

@ -15,7 +15,7 @@ from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve()) APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve())
if APP_ROOT not in sys.path: if APP_ROOT not in sys.path:
@ -69,7 +69,7 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args() return parser.parse_args()
async def connect_db(db_file: str) -> AsyncConnection: async def connect_db(db_file: Path) -> tuple[AsyncConnection, AsyncEngine]:
"""Create async SQLAlchemy connection.""" """Create async SQLAlchemy connection."""
engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}") engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}")
conn = await engine.connect() conn = await engine.connect()

View file

@ -1,3 +1,4 @@
from typing import Any
from unittest.mock import MagicMock from unittest.mock import MagicMock
import pytest import pytest
@ -91,7 +92,7 @@ class TestAgSet:
def test_ag_set_error_on_non_dict_final(self): def test_ag_set_error_on_non_dict_final(self):
"""Test error when final target is not a dict.""" """Test error when final target is not a dict."""
data = "not_a_dict" data: Any = "not_a_dict"
with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"): with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"):
ag_set(data, "key", "value") ag_set(data, "key", "value")

View file

@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import http.cookiejar import http.cookiejar
from typing import Any
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
import pytest import pytest
@ -12,10 +13,10 @@ try:
YTDLP_AVAILABLE = True YTDLP_AVAILABLE = True
except ImportError: except ImportError:
YTDLP_AVAILABLE = False YTDLP_AVAILABLE = False
Request = Mock Request: Any = Mock
Response = Mock Response: Any = Mock
RequestDirector = Mock RequestDirector: Any = Mock
HTTPError = Exception HTTPError: Any = Exception
pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available") pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")

View file

@ -1,5 +1,6 @@
from pathlib import Path from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, Mock, patch from unittest.mock import AsyncMock, Mock, patch
import pytest import pytest
@ -24,7 +25,7 @@ class TestConditionIgnoreMatching:
second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10) second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10)
service = Conditions.get_instance() service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first, second])) service._repo = Mock(all=AsyncMock(return_value=[first, second]))
matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"]) matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"])
@ -36,7 +37,7 @@ class TestConditionIgnoreMatching:
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10) second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
service = Conditions.get_instance() service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first, second])) service._repo = Mock(all=AsyncMock(return_value=[first, second]))
matched = await service.match(info={"duration": 60}, ignore_conditions=["123"]) matched = await service.match(info={"duration": 60}, ignore_conditions=["123"])
@ -48,7 +49,7 @@ class TestConditionIgnoreMatching:
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10) second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
service = Conditions.get_instance() service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first, second])) service._repo = Mock(all=AsyncMock(return_value=[first, second]))
matched = await service.match(info={"duration": 60}, ignore_conditions=[123]) matched = await service.match(info={"duration": 60}, ignore_conditions=[123])
@ -59,7 +60,7 @@ class TestConditionIgnoreMatching:
first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20) first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20)
service = Conditions.get_instance() service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first])) service._repo = Mock(all=AsyncMock(return_value=[first]))
matched = await service.match(info={"duration": 60}, ignore_conditions=["*"]) matched = await service.match(info={"duration": 60}, ignore_conditions=["*"])
@ -78,10 +79,11 @@ class TestConditionIgnorePropagation:
preset="default", preset="default",
extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]}, extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]},
) )
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={}))) mock_item: Any = item
item.get_archive_id = Mock(return_value=None) mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
item.get_archive_file = Mock(return_value=None) mock_item.get_archive_id = Mock(return_value=None)
item.is_archived = Mock(return_value=False) mock_item.get_archive_file = Mock(return_value=None)
mock_item.is_archived = Mock(return_value=False)
matcher = Mock(match=AsyncMock(return_value=None)) matcher = Mock(match=AsyncMock(return_value=None))
entry = {"id": "video-1", "duration": 60} entry = {"id": "video-1", "duration": 60}
@ -113,10 +115,11 @@ class TestConditionIgnorePropagation:
preset="default", preset="default",
extras={"ignore_conditions": [123, "Primary", True, " * "]}, extras={"ignore_conditions": [123, "Primary", True, " * "]},
) )
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={}))) mock_item: Any = item
item.get_archive_id = Mock(return_value=None) mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
item.get_archive_file = Mock(return_value=None) mock_item.get_archive_id = Mock(return_value=None)
item.is_archived = Mock(return_value=False) mock_item.get_archive_file = Mock(return_value=None)
mock_item.is_archived = Mock(return_value=False)
matcher = Mock(match=AsyncMock(return_value=None)) matcher = Mock(match=AsyncMock(return_value=None))
entry = {"id": "video-1", "duration": 60} entry = {"id": "video-1", "duration": 60}
@ -135,7 +138,7 @@ class TestConditionIgnorePropagation:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_playlist_keeps_parent_ignore(self) -> None: async def test_playlist_keeps_parent_ignore(self) -> None:
captured: dict[str, object] = {} captured: dict[str, Any] = {}
class FakeItem: class FakeItem:
def __init__(self) -> None: def __init__(self) -> None:
@ -158,7 +161,8 @@ class TestConditionIgnorePropagation:
} }
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")): with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
result = await process_playlist(queue=queue, entry=entry, item=FakeItem()) fake: Any = FakeItem()
result = await process_playlist(queue=queue, entry=entry, item=fake)
assert result == {"status": "ok"} assert result == {"status": "ok"}
assert captured["url"] == "https://example.com/v/1" assert captured["url"] == "https://example.com/v/1"
@ -209,7 +213,8 @@ class TestConditionIgnorePropagation:
} }
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")): with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
result = await process_playlist(queue=queue, entry=entry, item=FakeItem()) fake: Any = FakeItem()
result = await process_playlist(queue=queue, entry=entry, item=fake)
assert result == {"status": "ok"} assert result == {"status": "ok"}
assert queue.add.await_count == 2 assert queue.add.await_count == 2

View file

@ -4,10 +4,13 @@ from collections import OrderedDict
from dataclasses import asdict from dataclasses import asdict
from datetime import UTC, datetime from datetime import UTC, datetime
from email.utils import formatdate from email.utils import formatdate
from typing import Any
from unittest.mock import AsyncMock, Mock from unittest.mock import AsyncMock, Mock
import pytest import pytest
from app.library.DataStore import DataStore, StoreType from app.library.DataStore import DataStore, StoreType
from app.library.downloads import Download
from app.library.ItemDTO import ItemDTO from app.library.ItemDTO import ItemDTO
from app.library.operations import Operation from app.library.operations import Operation
from app.library.sqlite_store import SqliteStore from app.library.sqlite_store import SqliteStore
@ -62,17 +65,18 @@ async def make_db(data: int = 0) -> SqliteStore:
return ins return ins
class StubDownload: class StubDownload(Download):
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False): def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False) -> None:
self.info = info self.info = info
self._started: bool = started self.id = info._id
self._cancelled: bool = cancelled self._stub_started: bool = started
self._stub_cancelled: bool = cancelled
def started(self) -> bool: def started(self) -> bool:
return self._started return self._stub_started
def is_cancelled(self) -> bool: def is_cancelled(self) -> bool:
return self._cancelled return self._stub_cancelled
async def make_store_async(store_type: StoreType) -> DataStore: async def make_store_async(store_type: StoreType) -> DataStore:
@ -357,7 +361,8 @@ class TestDataStore:
def __init__(self): def __init__(self):
self.info = None self.info = None
store._dict["broken"] = BrokenDownload() broken: Any = BrokenDownload()
store._dict["broken"] = broken
# Should still find the valid item # Should still find the valid item
result = await store.get_item(title="Video 1") result = await store.get_item(title="Video 1")
@ -960,10 +965,10 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1") item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1" item1._id = "id1"
item1.filesize = 1000 setattr(item1, "filesize", 1000)
item2 = make_item(id="vid2", title="Video 2") item2 = make_item(id="vid2", title="Video 2")
item2._id = "id2" item2._id = "id2"
item2.filesize = 2000 setattr(item2, "filesize", 2000)
await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item1))
await store.put(StubDownload(info=item2)) await store.put(StubDownload(info=item2))
@ -986,10 +991,10 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1") item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1" item1._id = "id1"
item1.filesize = 1000 setattr(item1, "filesize", 1000)
item2 = make_item(id="vid2", title="Video 2") item2 = make_item(id="vid2", title="Video 2")
item2._id = "id2" item2._id = "id2"
item2.filesize = 2000 setattr(item2, "filesize", 2000)
await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item1))
await store.put(StubDownload(info=item2)) await store.put(StubDownload(info=item2))
@ -1007,7 +1012,7 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1") item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1" item1._id = "id1"
item1.filesize = 1000 setattr(item1, "filesize", 1000)
await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item1))
@ -1033,7 +1038,7 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1") item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1" item1._id = "id1"
item1.filesize = 1000 setattr(item1, "filesize", 1000)
await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item1))
@ -1084,7 +1089,7 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1") item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1" item1._id = "id1"
item1.description = None setattr(item1, "description", None)
await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item1))

View file

@ -307,7 +307,8 @@ class TestDownloadFlow:
postprocessor_hook=Mock(), postprocessor_hook=Mock(),
post_hook=Mock(), post_hook=Mock(),
) )
download.info.get_ytdlp_opts = Mock( download_any: Any = download.info
download_any.get_ytdlp_opts = Mock(
return_value=Mock( return_value=Mock(
add=Mock( add=Mock(
return_value=Mock( return_value=Mock(
@ -379,7 +380,8 @@ class TestDownloadFlow:
postprocessor_hook=Mock(), postprocessor_hook=Mock(),
post_hook=Mock(), post_hook=Mock(),
) )
download.info.get_ytdlp_opts = Mock( download_any2: Any = download.info
download_any2.get_ytdlp_opts = Mock(
return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={})))) return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={}))))
) )
@ -481,7 +483,8 @@ class TestDownloadFlow:
) )
queue.put(Terminator()) queue.put(Terminator())
download._download = fake_download download_mock: Any = download
download_mock._download = fake_download
class InlineProcess: class InlineProcess:
def __init__(self, target): def __init__(self, target):
@ -578,7 +581,8 @@ class TestDownloadFlow:
queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)}) queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)})
queue.put(Terminator()) queue.put(Terminator())
download._download = fake_download download_mock: Any = download
download_mock._download = fake_download
class InlineProcess: class InlineProcess:
def __init__(self, target): def __init__(self, target):
@ -1028,7 +1032,7 @@ class TestStatusTracker:
"debug": False, "debug": False,
} }
def test_init_sets_attributes(self, mock_config: dict) -> None: def test_init_sets_attributes(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
assert st.id == "test-id", "Should set download ID" assert st.id == "test-id", "Should set download ID"
assert st.info == mock_config["info"], "Should set info reference" assert st.info == mock_config["info"], "Should set info reference"
@ -1036,7 +1040,7 @@ class TestStatusTracker:
assert st.final_update is False, "Should initialize final_update as False" assert st.final_update is False, "Should initialize final_update as False"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_ignores_bad_id(self, mock_config: dict) -> None: async def test_status_ignores_bad_id(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "wrong-id", "status": "downloading"} status = {"id": "wrong-id", "status": "downloading"}
@ -1044,14 +1048,14 @@ class TestStatusTracker:
assert st.info.status != "downloading", "Should not update status for wrong ID" assert st.info.status != "downloading", "Should not update status for wrong ID"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_ignores_short(self, mock_config: dict) -> None: async def test_status_ignores_short(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "test-id"} status = {"id": "test-id"}
await st.process_status_update(status) await st.process_status_update(status)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_process_status_update_sets_status(self, mock_config: dict) -> None: async def test_process_status_update_sets_status(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000} status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000}
@ -1059,7 +1063,7 @@ class TestStatusTracker:
assert st.info.status == "downloading", "Should update info status" assert st.info.status == "downloading", "Should update info status"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_sets_skipped(self, mock_config: dict) -> None: async def test_status_sets_skipped(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "download_skipped": True} status = {"id": "test-id", "status": "downloading", "download_skipped": True}
@ -1067,7 +1071,7 @@ class TestStatusTracker:
assert st.info.download_skipped is True, "Should update download_skipped from status queue" assert st.info.download_skipped is True, "Should update download_skipped from status queue"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None: async def test_process_status_update_sets_tmpfilename(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"} status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"}
@ -1075,7 +1079,7 @@ class TestStatusTracker:
assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename" assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_sets_percent(self, mock_config: dict) -> None: async def test_status_sets_percent(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = { status = {
"id": "test-id", "id": "test-id",
@ -1090,7 +1094,7 @@ class TestStatusTracker:
assert st.info.percent == 50.0, "Should calculate percent correctly" assert st.info.percent == 50.0, "Should calculate percent correctly"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_uses_estimate(self, mock_config: dict) -> None: async def test_status_uses_estimate(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = { status = {
"id": "test-id", "id": "test-id",
@ -1104,7 +1108,7 @@ class TestStatusTracker:
assert st.info.percent == 30.0, "Should calculate percent from estimate" assert st.info.percent == 30.0, "Should calculate percent from estimate"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_percent(self, mock_config: dict) -> None: async def test_status_percent(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = { status = {
"id": "test-id", "id": "test-id",
@ -1117,7 +1121,7 @@ class TestStatusTracker:
assert st.info.percent == 50.0, "Should calculate percent correctly with valid total" assert st.info.percent == 50.0, "Should calculate percent correctly with valid total"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_sets_speed_eta(self, mock_config: dict) -> None: async def test_status_sets_speed_eta(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60} status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60}
@ -1126,7 +1130,7 @@ class TestStatusTracker:
assert st.info.eta == 60, "Should set eta" assert st.info.eta == 60, "Should set eta"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_process_status_update_sets_error(self, mock_config: dict) -> None: async def test_process_status_update_sets_error(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "error", "error": "Download failed"} status = {"id": "test-id", "status": "error", "error": "Download failed"}
@ -1135,7 +1139,7 @@ class TestStatusTracker:
assert st.info.error == "Download failed", "Should set error message" assert st.info.error == "Download failed", "Should set error message"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None: async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
test_file = tmp_path / "test.mp4" test_file = tmp_path / "test.mp4"
test_file.write_text("test content") test_file.write_text("test content")
@ -1148,20 +1152,20 @@ class TestStatusTracker:
assert st.info.filename == "test.mp4", "Should set relative filename" assert st.info.filename == "test.mp4", "Should set relative filename"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None: async def test_drain_queue_processes_remaining_updates(self, mock_config: dict[str, Any]) -> None:
queue = DummyQueue() queue = DummyQueue()
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100}) queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100})
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200}) queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200})
queue.put(Terminator()) queue.put(Terminator())
config = {**mock_config, "status_queue": queue} config: dict[str, Any] = {**mock_config, "status_queue": queue}
st = StatusTracker(**config) st = StatusTracker(**config)
await st.drain_queue(max_iterations=10) await st.drain_queue(max_iterations=10)
assert st.info.downloaded_bytes == 200, "Should process all queued updates" assert st.info.downloaded_bytes == 200, "Should process all queued updates"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None: async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
test_file = tmp_path / "test.mp4" test_file = tmp_path / "test.mp4"
test_file.write_text("test content") test_file.write_text("test content")
@ -1169,25 +1173,25 @@ class TestStatusTracker:
queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)}) queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)})
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999}) queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999})
config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)} config: dict[str, Any] = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
st = StatusTracker(**config) st = StatusTracker(**config)
await st.drain_queue(max_iterations=10) await st.drain_queue(max_iterations=10)
assert st.final_update is True, "Should stop draining after final update" assert st.final_update is True, "Should stop draining after final update"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_drain_queue_skips_invalid(self, mock_config: dict) -> None: async def test_drain_queue_skips_invalid(self, mock_config: dict[str, Any]) -> None:
queue = DummyQueue() queue = DummyQueue()
queue.put({"id": "test-id", "status": "downloading"}) queue.put({"id": "test-id", "status": "downloading"})
queue.put(None) queue.put(None)
config = {**mock_config, "status_queue": queue} config: dict[str, Any] = {**mock_config, "status_queue": queue}
st = StatusTracker(**config) st = StatusTracker(**config)
await st.drain_queue(max_iterations=5) await st.drain_queue(max_iterations=5)
assert st.info.status == "downloading", "valid updates should still be processed" assert st.info.status == "downloading", "valid updates should still be processed"
def test_cancel_update_task(self, mock_config: dict) -> None: def test_cancel_update_task(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
st.update_task = Mock() st.update_task = Mock()
st.update_task.done = Mock(return_value=False) st.update_task.done = Mock(return_value=False)
@ -1196,15 +1200,15 @@ class TestStatusTracker:
st.cancel_update_task() st.cancel_update_task()
st.update_task.cancel.assert_called_once() st.update_task.cancel.assert_called_once()
def test_cancel_update_task_noop(self, mock_config: dict) -> None: def test_cancel_update_task_noop(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
st.cancel_update_task() st.cancel_update_task()
assert st.update_task is None, "missing tasks should be ignored" assert st.update_task is None, "missing tasks should be ignored"
def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None: def test_put_terminator_adds_to_queue(self, mock_config: dict[str, Any]) -> None:
queue = DummyQueue() queue = DummyQueue()
config = {**mock_config, "status_queue": queue} config: dict[str, Any] = {**mock_config, "status_queue": queue}
st = StatusTracker(**config) st = StatusTracker(**config)
st.put_terminator() st.put_terminator()
@ -1212,7 +1216,7 @@ class TestStatusTracker:
assert isinstance(queue.items[0], Terminator), "Should add Terminator instance" assert isinstance(queue.items[0], Terminator), "Should add Terminator instance"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_progress_emits_item_progress(self, mock_config: dict) -> None: async def test_progress_emits_item_progress(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
st.info.status = "downloading" st.info.status = "downloading"
calls: list = [] calls: list = []
@ -1233,7 +1237,7 @@ class TestStatusTracker:
assert "options" not in payload assert "options" not in payload
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_status_change_emits_item_updated(self, mock_config: dict) -> None: async def test_status_change_emits_item_updated(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
st.info.status = "started" st.info.status = "started"
calls: list = [] calls: list = []
@ -1249,7 +1253,7 @@ class TestStatusTracker:
assert updated_calls[0][1]["data"] is st.info assert updated_calls[0][1]["data"] is st.info
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_progress_throttled(self, mock_config: dict) -> None: async def test_progress_throttled(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
st.info.status = "downloading" st.info.status = "downloading"
st._progress_interval = 0.5 st._progress_interval = 0.5
@ -1272,7 +1276,7 @@ class TestStatusTracker:
assert st._pending_progress is True assert st._pending_progress is True
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_flush_on_status_change(self, mock_config: dict) -> None: async def test_flush_on_status_change(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config) st = StatusTracker(**mock_config)
st.info.status = "downloading" st.info.status = "downloading"
st._progress_interval = 10.0 st._progress_interval = 10.0
@ -1345,23 +1349,29 @@ class TestQueueManager:
auto_start=True, auto_start=True,
) )
@staticmethod
def _any_video_item() -> Any:
return TestQueueManager._video_item()
def test_live_queue_caps_visible_items(self) -> None: def test_live_queue_caps_visible_items(self) -> None:
queue_manager = object.__new__(DownloadQueue) queue_manager = object.__new__(DownloadQueue)
items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
queue_manager.queue = self.LiveStore(items) queue_manager.queue = self.LiveStore(items)
queue_manager.pool = Mock() queue_manager.pool = Mock()
queue_manager.pool.get_active_downloads.return_value = {} queue_manager.pool.get_active_downloads.return_value = {}
snapshot = DownloadQueue.live_queue(queue_manager, limit=2) snapshot = DownloadQueue.live_queue(queue_manager, limit=2)
assert list(snapshot["queue"].keys()) == ["id0", "id1"] queue_view = snapshot["queue"]
assert isinstance(queue_view, dict)
assert list(queue_view.keys()) == ["id0", "id1"]
assert snapshot["queue_count"] == 5 assert snapshot["queue_count"] == 5
assert snapshot["queue_loaded"] == 2 assert snapshot["queue_loaded"] == 2
assert snapshot["queue_limit"] == 2 assert snapshot["queue_limit"] == 2
def test_live_queue_keeps_active(self) -> None: def test_live_queue_keeps_active(self) -> None:
queue_manager = object.__new__(DownloadQueue) queue_manager = object.__new__(DownloadQueue)
items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
queue_manager.queue = self.LiveStore(items) queue_manager.queue = self.LiveStore(items)
queue_manager.pool = Mock() queue_manager.pool = Mock()
queue_manager.pool.get_active_downloads.return_value = { queue_manager.pool.get_active_downloads.return_value = {
@ -1371,7 +1381,9 @@ class TestQueueManager:
snapshot = DownloadQueue.live_queue(queue_manager, limit=1) snapshot = DownloadQueue.live_queue(queue_manager, limit=1)
assert list(snapshot["queue"].keys()) == ["id3", "id4"] queue_view = snapshot["queue"]
assert isinstance(queue_view, dict)
assert list(queue_view.keys()) == ["id3", "id4"]
assert snapshot["queue_count"] == 5 assert snapshot["queue_count"] == 5
assert snapshot["queue_loaded"] == 2 assert snapshot["queue_loaded"] == 2
assert snapshot["queue_limit"] == 1 assert snapshot["queue_limit"] == 1
@ -1388,7 +1400,7 @@ class TestQueueManager:
result = await add_video( result = await add_video(
queue=self._video_queue(), queue=self._video_queue(),
item=self._video_item(), item=self._any_video_item(),
entry={ entry={
"id": "live-id", "id": "live-id",
"title": "Live stream", "title": "Live stream",
@ -1420,7 +1432,7 @@ class TestQueueManager:
"formats": [{"format_id": "18"}], "formats": [{"format_id": "18"}],
} }
result = await add_video(queue=self._video_queue(), item=self._video_item(), entry=entry) result = await add_video(queue=self._video_queue(), item=self._any_video_item(), entry=entry)
assert result == {"status": "ok"} assert result == {"status": "ok"}
assert seen == [entry] assert seen == [entry]

View file

@ -1,6 +1,7 @@
import json import json
from datetime import date from datetime import date
from pathlib import Path from pathlib import Path
from typing import Any
from unittest.mock import MagicMock from unittest.mock import MagicMock
import pytest import pytest
@ -132,7 +133,8 @@ class TestEncoder:
return True return True
return original_isinstance(obj, cls) return original_isinstance(obj, cls)
builtins.isinstance = mock_isinstance builtins_any: Any = builtins
builtins_any.isinstance = mock_isinstance
try: try:
result = self.encoder.default(mock_daterange) result = self.encoder.default(mock_daterange)

View file

@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
from typing import Any
from unittest.mock import Mock from unittest.mock import Mock
import pytest import pytest
@ -7,7 +8,7 @@ import pytest
from app.yt_dlp_plugins.extractor import generic_browser from app.yt_dlp_plugins.extractor import generic_browser
def _make_ie(config: dict[str, str | None] | None = None) -> generic_browser.GenericBrowserIE: def _make_ie(config: dict[str, str | None] | None = None) -> Any:
ie = object.__new__(generic_browser.GenericBrowserIE) ie = object.__new__(generic_browser.GenericBrowserIE)
values = config or {} values = config or {}
ie._configuration_arg = lambda name, default: [values.get(name)] ie._configuration_arg = lambda name, default: [values.get(name)]
@ -250,6 +251,7 @@ def test_extract_network_formats_playlist_entries_keep_own_urls() -> None:
{"title": "Title", "webpage_url": "https://example.com/page", "original_url": "https://example.com/page"}, {"title": "Title", "webpage_url": "https://example.com/page", "original_url": "https://example.com/page"},
) )
assert result is not None
assert result["_type"] == "playlist" assert result["_type"] == "playlist"
assert result["entries"][0]["url"] == "https://cdn.example/1.mp3" assert result["entries"][0]["url"] == "https://cdn.example/1.mp3"
assert result["entries"][0]["webpage_url"] == "https://cdn.example/1.mp3" assert result["entries"][0]["webpage_url"] == "https://cdn.example/1.mp3"

View file

@ -36,7 +36,8 @@ def _make_download(
) -> SimpleNamespace: ) -> SimpleNamespace:
base_dir = download_dir or "/downloads" base_dir = download_dir or "/downloads"
original_post_init = ItemDTO.__post_init__ original_post_init = ItemDTO.__post_init__
ItemDTO.__post_init__ = lambda self: None cls: Any = ItemDTO
cls.__post_init__ = lambda self: None
try: try:
item = ItemDTO( item = ItemDTO(

View file

@ -73,5 +73,5 @@ async def test_bg_log_redact(caplog: pytest.LogCaptureFixture, monkeypatch: pyte
) )
assert "apitoken=secret" not in record.getMessage() assert "apitoken=secret" not in record.getMessage()
assert "user:pass@" not in record.getMessage() assert "user:pass@" not in record.getMessage()
assert record.url == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted" assert getattr(record, "url", None) == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted"
assert record.exception_type == "RuntimeError" assert getattr(record, "exception_type", None) == "RuntimeError"

View file

@ -386,7 +386,7 @@ class TestItemAddExtras:
def test_add_extras_none(self): def test_add_extras_none(self):
"""Test adding extras when extras is None.""" """Test adding extras when extras is None."""
item = Item(url="https://example.com") item = Item(url="https://example.com")
item.extras = None setattr(item, "extras", None)
item.add_extras("key1", "value1") item.add_extras("key1", "value1")

View file

@ -1,6 +1,7 @@
import asyncio import asyncio
import sys import sys
import types import types
from collections.abc import Callable
from typing import Any from typing import Any
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
@ -21,7 +22,7 @@ if "aiocron" not in sys.modules:
def uuid(self) -> str: def uuid(self) -> str:
return "stub-uuid" return "stub-uuid"
aiocron_stub.Cron = _CronImportStub setattr(aiocron_stub, "Cron", _CronImportStub)
sys.modules["aiocron"] = aiocron_stub sys.modules["aiocron"] = aiocron_stub
@ -36,7 +37,7 @@ class DummyCron:
self, self,
*, *,
spec: str, spec: str,
func: callable, func: Callable,
args: tuple = (), args: tuple = (),
kwargs: dict | None = None, kwargs: dict | None = None,
uuid: str | None = None, uuid: str | None = None,
@ -231,7 +232,7 @@ class TestScheduler:
self, self,
*_, *_,
spec: str, spec: str,
func: callable, func: Callable,
args: tuple = (), args: tuple = (),
kwargs: dict | None = None, kwargs: dict | None = None,
uuid: str | None = None, uuid: str | None = None,
@ -266,7 +267,7 @@ class TestScheduler:
self, self,
*_, *_,
spec: str, spec: str,
func: callable, func: Callable,
args: tuple = (), args: tuple = (),
kwargs: dict | None = None, kwargs: dict | None = None,
uuid: str | None = None, uuid: str | None = None,

View file

@ -1,4 +1,5 @@
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from typing import Any
from unittest.mock import AsyncMock, patch from unittest.mock import AsyncMock, patch
import pytest import pytest
@ -182,12 +183,14 @@ async def test_paginate_order_and_bounds():
encoded = itm.json() encoded = itm.json()
created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S") created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S")
_list.append(itm) _list.append(itm)
assert store._conn is not None
await store._conn.execute( await store._conn.execute(
text( text(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)' 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)'
), ),
{"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at}, {"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at},
) )
assert store._conn is not None
await store._conn.commit() await store._conn.commit()
items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC") items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC")
@ -208,7 +211,9 @@ async def test_get_by_id():
item = make_item(99) item = make_item(99)
await store.enqueue_upsert("queue", item) await store.enqueue_upsert("queue", item)
await store.flush() await store.flush()
assert item._id == (await store.get_by_id("queue", item._id))._id fetched = await store.get_by_id("queue", item._id)
assert fetched is not None
assert item._id == fetched._id
await store.close() await store.close()
@ -358,7 +363,8 @@ async def test_on_shutdown_closes_connection():
await store.enqueue_upsert("queue", make_item(1)) await store.enqueue_upsert("queue", make_item(1))
await store.flush() await store.flush()
await store.on_shutdown(None) app: Any = None
await store.on_shutdown(app)
assert store._conn is None assert store._conn is None
# Note: on_shutdown already closes the connection, so no need to call close() again # Note: on_shutdown already closes the connection, so no need to call close() again

View file

@ -1,6 +1,6 @@
import asyncio import asyncio
import copy import copy
import importlib import importlib.util
import json import json
import logging import logging
import re import re
@ -10,6 +10,7 @@ import uuid
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
import pytest import pytest
@ -861,16 +862,20 @@ class TestMergeDict:
def test_merge_dict_type_validation(self): def test_merge_dict_type_validation(self):
"""Test that non-dict parameters are properly rejected.""" """Test that non-dict parameters are properly rejected."""
# Test with non-dict source # Test with non-dict source
bad_src: Any = "not_a_dict"
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"): with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
merge_dict("not_a_dict", {"key": "value"}) merge_dict(bad_src, {"key": "value"})
# Test with non-dict destination # Test with non-dict destination
bad_dst: Any = "not_a_dict"
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"): with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
merge_dict({"key": "value"}, "not_a_dict") merge_dict({"key": "value"}, bad_dst)
# Test with both non-dict # Test with both non-dict
bad_src2: Any = "not_a_dict"
bad_dst2: Any = ["also_not_dict"]
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"): with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
merge_dict("not_a_dict", ["also_not_dict"]) merge_dict(bad_src2, bad_dst2)
def test_merge_dict_immutability(self): def test_merge_dict_immutability(self):
"""Test that original dictionaries are not modified (immutability).""" """Test that original dictionaries are not modified (immutability)."""
@ -1084,7 +1089,8 @@ class TestEncryptDecrypt:
assert encrypted != data assert encrypted != data
assert isinstance(encrypted, str) assert isinstance(encrypted, str)
decrypted: str = decrypt_data(encrypted, key) decrypted = decrypt_data(encrypted, key)
assert decrypted is not None
assert decrypted == data assert decrypted == data
def test_encrypt_empty_string(self): def test_encrypt_empty_string(self):
@ -1093,7 +1099,8 @@ class TestEncryptDecrypt:
data: str = "" data: str = ""
encrypted: str = encrypt_data(data, key) encrypted: str = encrypt_data(data, key)
decrypted: str = decrypt_data(encrypted, key) decrypted = decrypt_data(encrypted, key)
assert decrypted is not None
assert decrypted == data assert decrypted == data
@ -1489,7 +1496,8 @@ class TestGet:
def test_get_list_access(self): def test_get_list_access(self):
"""Test getting from list.""" """Test getting from list."""
data = ["item0", "item1", "item2"] data = ["item0", "item1", "item2"]
result = get(data, 1) key: Any = 1
result = get(data, key)
assert result == "item1" assert result == "item1"
def test_get_empty_path(self): def test_get_empty_path(self):

View file

@ -10,9 +10,8 @@ import logging
import os import os
from pathlib import Path from pathlib import Path
from library.PackageInstaller import PackageInstaller, Packages
from app.library.log import get_logger from app.library.log import get_logger
from app.library.PackageInstaller import PackageInstaller, Packages
LOG = get_logger() LOG = get_logger()

View file

@ -1,4 +1,5 @@
import base64 import base64
import importlib.util
import os import os
import re import re
import time import time
@ -190,12 +191,7 @@ class RemoteBrowserUnavailableError(ExtractorError):
class PlaywrightDriver: class PlaywrightDriver:
@staticmethod @staticmethod
def is_available() -> bool: def is_available() -> bool:
try: return importlib.util.find_spec("playwright.sync_api") is not None
import playwright.sync_api # noqa: F401
return True
except ImportError:
return False
@staticmethod @staticmethod
def connect(ws_url: str, timeout: int | None = None): def connect(ws_url: str, timeout: int | None = None):
@ -336,15 +332,11 @@ class PlaywrightDriver:
class SeleniumDriver: class SeleniumDriver:
@staticmethod @staticmethod
def is_available() -> bool: def is_available() -> bool:
try: return importlib.util.find_spec("selenium.webdriver") is not None
import selenium.webdriver # noqa: F401
return True
except ImportError:
return False
@staticmethod @staticmethod
def connect(ws_url: str, timeout: int | None = None): # noqa: ARG004 def connect(ws_url: str, timeout: int | None = None):
_ = timeout
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
@ -441,6 +433,10 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
_failed: bool = False _failed: bool = False
_remote_browser_failures: dict[str, str] = {} _remote_browser_failures: dict[str, str] = {}
_url: str = "" _url: str = ""
__wrapped__: Any
def _fallback_extract(self, url: str) -> dict[str, Any]:
return self.__wrapped__._real_extract(self, url)
def _get_config(self, name: str, env_name: str) -> str | None: def _get_config(self, name: str, env_name: str) -> str | None:
value = self._configuration_arg(name, [None])[0] value = self._configuration_arg(name, [None])[0]
@ -473,7 +469,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
self._url = url self._url = url
if not (browser_url := self._get_config("url", "YTP_BROWSER_URL")) or self._failed: if not (browser_url := self._get_config("url", "YTP_BROWSER_URL")) or self._failed:
return self.__wrapped__._real_extract(self, url) return self._fallback_extract(url)
safe_url = self._safe_url(browser_url) safe_url = self._safe_url(browser_url)
@ -496,7 +492,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
message = str(e) message = str(e)
self._failed = True self._failed = True
self.report_warning(f"Remote browser unavailable: {message}, marking as failed.", video_id) self.report_warning(f"Remote browser unavailable: {message}, marking as failed.", video_id)
return self.__wrapped__._real_extract(self, url) return self._fallback_extract(url)
try: try:
self.report_extraction(url) self.report_extraction(url)
@ -513,17 +509,18 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
requests = self._merge_requests(session.get_requests(), session.get_media_requests()) requests = self._merge_requests(session.get_requests(), session.get_media_requests())
self.write_debug(f"Captured {len(requests)} network requests") self.write_debug(f"Captured {len(requests)} network requests")
if self._downloader.params.get("dump_intermediate_pages"): downloader = self._downloader
if downloader and downloader.params.get("dump_intermediate_pages"):
self.to_screen(f"Browser content dump for: {url}") self.to_screen(f"Browser content dump for: {url}")
self.to_screen(base64.b64encode(webpage.encode("utf-8")).decode("ascii")) self.to_screen(base64.b64encode(webpage.encode("utf-8")).decode("ascii"))
if self._downloader.params.get("write_pages"): if downloader and downloader.params.get("write_pages"):
filename = _request_dump_filename(url, video_id, None, self._downloader.params.get("trim_file_name")) filename = _request_dump_filename(url, video_id, None, downloader.params.get("trim_file_name"))
self.to_screen(f"Saving request to {filename}") self.to_screen(f"Saving request to {filename}")
with open(filename, "w", encoding="utf-8") as f: with open(filename, "w", encoding="utf-8") as f:
f.write(webpage) f.write(webpage)
if self._downloader.params.get("debug_printtraffic"): if downloader and downloader.params.get("debug_printtraffic"):
self.to_screen(f"[browser] {url}") self.to_screen(f"[browser] {url}")
self.to_screen(f"[browser] Captured '{len(requests)}' network requests") self.to_screen(f"[browser] Captured '{len(requests)}' network requests")
for req in requests: for req in requests:
@ -599,9 +596,9 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
self, requests: list[dict], video_id: str, base_info: dict[str, Any] self, requests: list[dict], video_id: str, base_info: dict[str, Any]
) -> dict[str, Any] | None: ) -> dict[str, Any] | None:
candidates = self._pick_network_candidates(requests) candidates = self._pick_network_candidates(requests)
formats = [] formats: list[dict[str, Any]] = []
direct_formats = [] direct_formats: list[dict[str, Any]] = []
subtitles = {} subtitles: dict[str, Any] = {}
source_counts = {} source_counts = {}
has_manifest_formats = False has_manifest_formats = False
@ -674,7 +671,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
self.report_warning( self.report_warning(
"Generic browser extractor found no media formats. falling back to generic extractor.", video_id "Generic browser extractor found no media formats. falling back to generic extractor.", video_id
) )
return self.__wrapped__._real_extract(self, self._url) return self._fallback_extract(self._url)
if not has_manifest_formats and len(direct_formats) > 1: if not has_manifest_formats and len(direct_formats) > 1:
base_title = (base_info.get("title") or "").strip() or video_id base_title = (base_info.get("title") or "").strip() or video_id
@ -698,7 +695,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
) )
return {"_type": "playlist", "entries": entries} return {"_type": "playlist", "entries": entries}
result = {"formats": formats, "direct": True} result: dict[str, Any] = {"formats": formats, "direct": True}
if subtitles: if subtitles:
result["subtitles"] = subtitles result["subtitles"] = subtitles
if formats and formats[0].get("url"): if formats and formats[0].get("url"):
@ -733,8 +730,12 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
has_media_header_ext = header_ext and header_ext in MEDIA_EXTENSIONS has_media_header_ext = header_ext and header_ext in MEDIA_EXTENSIONS
resource_type = entry.get("resourceType") resource_type = entry.get("resourceType")
if resource_type and resource_type.lower() not in MEDIA_RESOURCE_TYPES: # noqa: SIM102 if (
if not has_media_ext and not has_media_header_ext: resource_type
and resource_type.lower() not in MEDIA_RESOURCE_TYPES
and not has_media_ext
and not has_media_header_ext
):
continue continue
if not ext and not header_ext: if not ext and not header_ext:
@ -779,7 +780,8 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"):
) )
def _get_timeout_ms(self) -> int | None: def _get_timeout_ms(self) -> int | None:
socket_timeout = self._downloader.params.get("socket_timeout") downloader = self._downloader
socket_timeout = downloader.params.get("socket_timeout") if downloader else None
if isinstance(socket_timeout, (int, float)) and socket_timeout > 0: if isinstance(socket_timeout, (int, float)) and socket_timeout > 0:
return int(socket_timeout * 1000) return int(socket_timeout * 1000)
return None return None

View file

@ -119,7 +119,8 @@ class NFOMakerPP(PostProcessor):
def pp_key(cls) -> str: def pp_key(cls) -> str:
return "NFOMaker" return "NFOMaker"
def run(self, info: dict | None = None) -> tuple[list, dict]: def run(self, information: Any) -> tuple[list, dict[str, Any]]:
info = information
if "fail" == self.mode: if "fail" == self.mode:
msg = "Intentionally failing due to mode=fail." msg = "Intentionally failing due to mode=fail."
raise Exception(msg) raise Exception(msg)

Some files were not shown because too many files have changed in this diff Show more