From 36319472342d6704e4627a6e4bbef7741c8a3c4e Mon Sep 17 00:00:00 2001 From: arabcoders Date: Sun, 14 Jun 2026 10:43:58 +0300 Subject: [PATCH] chore: use ty as type checker. --- app/features/conditions/repository.py | 39 +- app/features/conditions/router.py | 6 + app/features/conditions/service.py | 20 +- .../conditions/tests/test_conditions.py | 11 +- app/features/core/models.py | 8 +- app/features/dl_fields/repository.py | 44 +- app/features/dl_fields/service.py | 4 +- .../tests/test_dl_fields_repository.py | 4 +- app/features/notifications/repository.py | 33 +- app/features/notifications/service.py | 20 +- .../tests/test_notifications_repository.py | 2 +- app/features/presets/migration.py | 2 +- app/features/presets/repository.py | 51 +- app/features/presets/service.py | 2 +- .../presets/tests/test_presets_repository.py | 2 +- app/features/streaming/library/ffprobe.py | 35 +- app/features/streaming/library/m3u8.py | 2 +- app/features/streaming/library/playlist.py | 2 +- .../streaming/library/segment_encoders.py | 18 +- app/features/streaming/library/segments.py | 7 +- app/features/streaming/library/subtitle.py | 4 +- app/features/streaming/library/thumbnail.py | 2 + app/features/streaming/router.py | 30 +- app/features/streaming/tests/test_segments.py | 9 +- .../streaming/tests/test_thumbnail.py | 5 +- .../definitions/handlers/_base_handler.py | 3 +- .../tasks/definitions/handlers/generic.py | 5 +- .../tasks/definitions/handlers/rss.py | 5 +- .../tasks/definitions/handlers/tver.py | 13 +- .../tasks/definitions/handlers/twitch.py | 6 +- .../tasks/definitions/handlers/youtube.py | 6 +- app/features/tasks/definitions/repository.py | 19 +- app/features/tasks/definitions/results.py | 18 +- app/features/tasks/definitions/service.py | 34 +- .../tests/test_generic_task_handler.py | 177 ++-- .../tests/test_task_definitions.py | 2 +- app/features/tasks/definitions/utils.py | 10 +- app/features/tasks/repository.py | 35 +- app/features/tasks/router.py | 5 +- app/features/tasks/service.py | 6 +- .../tasks/tests/test_tasks_repository.py | 2 +- app/features/tasks/tests/test_tasks_router.py | 13 +- app/features/ytdlp/extractor.py | 2 +- app/features/ytdlp/router.py | 16 +- .../ytdlp/tests/test_ytdlp_extractor.py | 2 + app/features/ytdlp/tests/test_ytdlp_module.py | 6 +- app/features/ytdlp/tests/test_ytdlp_opts.py | 2 +- app/features/ytdlp/tests/test_ytdlp_utils.py | 24 +- app/features/ytdlp/utils.py | 27 +- app/features/ytdlp/ytdlp_opts.py | 6 +- app/library/BackgroundWorker.py | 5 +- app/library/HttpAPI.py | 34 +- app/library/HttpSocket.py | 17 +- app/library/ItemDTO.py | 38 +- app/library/PackageInstaller.py | 2 +- app/library/Scheduler.py | 6 +- app/library/Services.py | 7 +- app/library/TerminalSessionManager.py | 2 +- app/library/UpdateChecker.py | 4 +- app/library/Utils.py | 61 +- app/library/ag_utils.py | 68 +- app/library/cache.py | 2 +- app/library/cf_solver_handler.py | 8 +- app/library/cf_solver_shared.py | 12 +- app/library/config.py | 28 +- app/library/downloads/core.py | 62 +- app/library/downloads/item_adder.py | 26 +- app/library/downloads/playlist_processor.py | 4 +- app/library/downloads/process_manager.py | 86 +- app/library/downloads/queue_manager.py | 4 +- app/library/downloads/status_tracker.py | 9 +- app/library/httpx_client.py | 20 +- app/library/migrate.py | 33 +- app/library/sqlite_store.py | 117 ++- app/local.py | 23 +- app/main.py | 22 +- app/routes/api/browser.py | 20 +- app/routes/api/docs.py | 2 +- app/routes/api/download.py | 4 +- app/routes/api/history.py | 13 +- app/routes/api/images.py | 5 +- app/routes/api/logs.py | 3 +- app/routes/api/system.py | 3 +- app/scripts/seed_db.py | 4 +- app/tests/test_ag_utils.py | 3 +- app/tests/test_cf_solver_handler.py | 9 +- app/tests/test_condition_ignore.py | 35 +- app/tests/test_datastore.py | 33 +- app/tests/test_download.py | 84 +- app/tests/test_encoder.py | 4 +- app/tests/test_generic_browser.py | 4 +- app/tests/test_history_routes.py | 3 +- app/tests/test_images_routes.py | 4 +- app/tests/test_itemdto.py | 2 +- app/tests/test_scheduler.py | 9 +- app/tests/test_sqlite_store.py | 10 +- app/tests/test_utils.py | 22 +- app/upgrader.py | 3 +- .../extractor/generic_browser.py | 58 +- app/yt_dlp_plugins/postprocessor/nfo_maker.py | 3 +- pyproject.toml | 3 + ui/app/composables/useAppSocket.ts | 2 +- ui/app/spa-loading-template.html | 2 +- ui/app/utils/ytdlp.ts | 4 +- ui/bun.lock | 972 ++++++++---------- ui/package.json | 18 +- uv.lock | 70 +- 107 files changed, 1563 insertions(+), 1354 deletions(-) diff --git a/app/features/conditions/repository.py b/app/features/conditions/repository.py index 4d1a3082..5b41b29e 100644 --- a/app/features/conditions/repository.py +++ b/app/features/conditions/repository.py @@ -6,13 +6,16 @@ from app.features.conditions.migration import Migration from app.library.Singleton import Singleton if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Iterable + from collections.abc import Callable, Iterable + from contextlib import AbstractAsyncContextManager from sqlalchemy.engine.result import Result from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.selectable import Select + SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]] + from sqlalchemy import delete, func, or_, select from app.features.conditions.models import ConditionModel @@ -22,10 +25,26 @@ from app.library.log import get_logger LOG = get_logger() +def _model_from_payload(payload: dict[str, Any]) -> ConditionModel: + model = ConditionModel() + for key, value in payload.items(): + if not hasattr(model, key): + msg = f"'{key}' is an invalid keyword argument for ConditionModel" + raise TypeError(msg) + setattr(model, key, value) + return model + + +def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel: + if isinstance(payload, ConditionModel): + return payload + return _model_from_payload(payload) + + class ConditionsRepository(metaclass=Singleton): - def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + def __init__(self, session: SessionFactory | None = None) -> None: self._migrated = False - self.session: AsyncGenerator[AsyncSession] = session or get_session + self.session: SessionFactory = session or get_session async def run_migrations(self) -> None: if self._migrated: @@ -38,7 +57,7 @@ class ConditionsRepository(metaclass=Singleton): def get_instance() -> ConditionsRepository: return ConditionsRepository() - async def list(self) -> list[ConditionModel]: + async def all(self) -> list[ConditionModel]: async with self.session() as session: result: Result[tuple[ConditionModel]] = await session.execute( select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc()) @@ -93,11 +112,11 @@ class ConditionsRepository(metaclass=Singleton): result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1)) return result.scalar_one_or_none() - async def create(self, payload: ConditionModel | dict) -> ConditionModel: + async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel: async with self.session() as session: - model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload + model = _coerce_model(payload) if model.id is not None: - model.id = None + model.id = None # ty: ignore if await self.get_by_name(name=model.name) is not None: msg: str = f"Condition with name '{model.name}' already exists." @@ -160,13 +179,11 @@ class ConditionsRepository(metaclass=Singleton): await session.commit() return model - async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]: + async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]: async with self.session() as session: try: await session.execute(delete(ConditionModel)) - models: list[ConditionModel] = [ - ConditionModel(**item) if isinstance(item, dict) else item for item in items - ] + models: list[ConditionModel] = [_coerce_model(item) for item in items] session.add_all(models) await session.commit() except Exception: diff --git a/app/features/conditions/router.py b/app/features/conditions/router.py index a0bd7fa2..dd22397c 100644 --- a/app/features/conditions/router.py +++ b/app/features/conditions/router.py @@ -131,6 +131,12 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf status=web.HTTPInternalServerError.status_code, ) + if not isinstance(data, dict): + return web.json_response( + data={"error": "Failed to extract video info."}, + status=web.HTTPInternalServerError.status_code, + ) + try: from app.features.ytdlp.mini_filter import match_str diff --git a/app/features/conditions/service.py b/app/features/conditions/service.py index 1a94342d..564b6435 100644 --- a/app/features/conditions/service.py +++ b/app/features/conditions/service.py @@ -1,10 +1,10 @@ from collections.abc import Iterable -from numbers import Number from aiohttp import web from app.features.conditions.models import ConditionModel from app.features.conditions.repository import ConditionsRepository +from app.features.conditions.schemas import Condition from app.features.ytdlp.mini_filter import match_str from app.library.Events import EventBus, Events from app.library.log import get_logger @@ -13,7 +13,7 @@ from app.library.Singleton import Singleton LOG = get_logger() -def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tuple[set[str], bool]: +def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]: ignored: set[str] = set() ignore_all = False @@ -21,7 +21,7 @@ def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tu return ignored, ignore_all for value in ignore_conditions: - if isinstance(value, bool) or not isinstance(value, (str, Number)): + if isinstance(value, bool) or not isinstance(value, (str, int, float)): continue identifier = str(value).strip() @@ -55,7 +55,7 @@ class Conditions(metaclass=Singleton): EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations") async def get_all(self) -> list[ConditionModel]: - return await self._repo.list() + return await self._repo.all() async def save(self, item: ConditionModel | dict) -> ConditionModel: """ @@ -80,7 +80,7 @@ class Conditions(metaclass=Singleton): if item.id is None or 0 == item.id: model = await repo.create(item) else: - model = await repo.update(item.id, item.serialize()) + model = await repo.update(item.id, Condition.model_validate(item).model_dump()) except KeyError as exc: raise ValueError(str(exc)) from exc @@ -113,13 +113,15 @@ class Conditions(metaclass=Singleton): repo = self._repo return await repo.get(identifier) - async def match(self, info: dict, ignore_conditions: Iterable[str | Number] | None = None) -> ConditionModel | None: + async def match( + self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None + ) -> ConditionModel | None: """ Check if any condition matches the info dict. Args: info (dict): The info dict to check. - ignore_conditions (Iterable[str | Number] | None): Condition ids or names to skip for this match. + ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match. Returns: Condition|None: The condition if found, None otherwise. @@ -133,7 +135,7 @@ class Conditions(metaclass=Singleton): return None repo = self._repo - items: list[ConditionModel] = await repo.list() + items: list[ConditionModel] = await repo.all() if len(items) < 1: return None @@ -187,7 +189,7 @@ class Conditions(metaclass=Singleton): if not info or not isinstance(info, dict) or len(info) < 1: return None - if not (item := await self.get(identifier)) or not item.enabled or not item.filter: + if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter: return None return item if match_str(item.filter, info) else None diff --git a/app/features/conditions/tests/test_conditions.py b/app/features/conditions/tests/test_conditions.py index 0586404f..4a612404 100644 --- a/app/features/conditions/tests/test_conditions.py +++ b/app/features/conditions/tests/test_conditions.py @@ -6,6 +6,7 @@ import pytest import pytest_asyncio from types import SimpleNamespace import pytest +from typing import Any from aiohttp import web from aiohttp.test_utils import make_mocked_request @@ -39,13 +40,13 @@ async def repo(): def _json_request(path: str, payload: object) -> web.Request: - request = make_mocked_request("POST", path) + mock_request: Any = make_mocked_request("POST", path) async def _json() -> object: return payload - request.json = _json # type: ignore[attr-defined] - return request + mock_request.json = _json + return mock_request class TestAllowInternalUrlsScope: @@ -208,7 +209,7 @@ class TestConditionsRepository: await repo.create({"name": "A", "filter": "test", "priority": 1}) await repo.create({"name": "C", "filter": "test", "priority": 2}) - items = await repo.list() + items = await repo.all() assert items[0].name == "C", "Highest priority should be first" assert items[1].name == "A", "Same priority sorted alphabetically" @@ -240,6 +241,6 @@ class TestConditionsRepository: assert len(result) == 2, "Should create 2 new conditions" - all_items = await repo.list() + all_items = await repo.all() assert len(all_items) == 2, "Should only have new conditions" assert all_items[0].name in ["New 1", "New 2"], "Should only have new items" diff --git a/app/features/core/models.py b/app/features/core/models.py index a09d8160..3900165e 100644 --- a/app/features/core/models.py +++ b/app/features/core/models.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import UTC, datetime from sqlalchemy import DateTime as SQLADateTime -from sqlalchemy import TypeDecorator +from sqlalchemy import Dialect, TypeDecorator from sqlalchemy.orm import DeclarativeBase @@ -20,16 +20,18 @@ class UTCDateTime(TypeDecorator): impl = SQLADateTime cache_ok = True - def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None: + def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None: """Convert datetime to UTC before storing.""" + _ = dialect if value is not None: if value.tzinfo is None: return value.replace(tzinfo=UTC) return value.astimezone(UTC).replace(tzinfo=None) return value - def process_result_value(self, value: datetime | None, _dialect) -> datetime | None: + def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None: """Ensure datetime is timezone-aware (UTC) when loading.""" + _ = dialect if value is not None and value.tzinfo is None: return value.replace(tzinfo=UTC) return value diff --git a/app/features/dl_fields/repository.py b/app/features/dl_fields/repository.py index e1489277..e62337c5 100644 --- a/app/features/dl_fields/repository.py +++ b/app/features/dl_fields/repository.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from sqlalchemy import delete, func, or_, select @@ -11,20 +11,39 @@ from app.library.log import get_logger from app.library.Singleton import Singleton if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Iterable + from collections.abc import Callable, Iterable + from contextlib import AbstractAsyncContextManager from sqlalchemy.engine.result import Result from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.selectable import Select + SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]] + LOG = get_logger() +def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel: + model = DLFieldModel() + for key, value in payload.items(): + if not hasattr(model, key): + msg = f"'{key}' is an invalid keyword argument for DLFieldModel" + raise TypeError(msg) + setattr(model, key, value) + return model + + +def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel: + if isinstance(payload, DLFieldModel): + return payload + return _model_from_payload(payload) + + class DLFieldsRepository(metaclass=Singleton): - def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + def __init__(self, session: SessionFactory | None = None) -> None: self._migrated = False - self.session: AsyncGenerator[AsyncSession] = session or get_session + self.session: SessionFactory = session or get_session async def run_migrations(self) -> None: if self._migrated: @@ -37,7 +56,7 @@ class DLFieldsRepository(metaclass=Singleton): def get_instance() -> DLFieldsRepository: return DLFieldsRepository() - async def list(self) -> list[DLFieldModel]: + async def all(self) -> list[DLFieldModel]: async with self.session() as session: result: Result[tuple[DLFieldModel]] = await session.execute( select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc()) @@ -90,9 +109,9 @@ class DLFieldsRepository(metaclass=Singleton): result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1)) return result.scalar_one_or_none() - async def create(self, payload: DLFieldModel | dict) -> DLFieldModel: + async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel: async with self.session() as session: - model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload + model = _coerce_model(payload) if await self.get_by_name(name=model.name) is not None: msg: str = f"DL field with name '{model.name}' already exists." @@ -153,13 +172,16 @@ class DLFieldsRepository(metaclass=Singleton): await session.commit() return model - async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]: + async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]: async with self.session() as session: try: await session.execute(delete(DLFieldModel)) - models: list[DLFieldModel] = [ - DLFieldModel(**item) if isinstance(item, dict) else item for item in items - ] + models: list[DLFieldModel] = [] + for item in items: + if isinstance(item, dict): + models.append(DLFieldModel(**cast("dict[str, Any]", item))) + else: + models.append(item) session.add_all(models) await session.commit() except Exception: diff --git a/app/features/dl_fields/service.py b/app/features/dl_fields/service.py index a019c8ab..6e58807b 100644 --- a/app/features/dl_fields/service.py +++ b/app/features/dl_fields/service.py @@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton): EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations") async def get_all(self) -> list[DLFieldModel]: - return await self._repo.list() + return await self._repo.all() async def get_all_serialized(self) -> list[dict[str, Any]]: - items = await self._repo.list() + items = await self._repo.all() return [DLField.model_validate(item).model_dump() for item in items] async def save(self, item: DLField | dict) -> DLFieldModel: diff --git a/app/features/dl_fields/tests/test_dl_fields_repository.py b/app/features/dl_fields/tests/test_dl_fields_repository.py index 7b2de6aa..c1f77bd9 100644 --- a/app/features/dl_fields/tests/test_dl_fields_repository.py +++ b/app/features/dl_fields/tests/test_dl_fields_repository.py @@ -204,7 +204,7 @@ class TestDLFieldsRepository: await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1}) await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0}) - items = await repo.list() + items = await repo.all() assert items[0].name == "c", "Lowest order should be first" assert items[1].name == "a", "Same order sorted alphabetically" @@ -243,6 +243,6 @@ class TestDLFieldsRepository: assert len(result) == 2, "Should create 2 new fields" - all_items = await repo.list() + all_items = await repo.all() assert len(all_items) == 2, "Should only have new fields" assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items" diff --git a/app/features/notifications/repository.py b/app/features/notifications/repository.py index 4fae9f3a..7e64c2db 100644 --- a/app/features/notifications/repository.py +++ b/app/features/notifications/repository.py @@ -11,20 +11,39 @@ from app.library.log import get_logger from app.library.Singleton import Singleton if TYPE_CHECKING: - from collections.abc import AsyncGenerator + from collections.abc import Callable + from contextlib import AbstractAsyncContextManager from sqlalchemy.engine.result import Result from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.selectable import Select + SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]] + LOG = get_logger() +def _model_from_payload(payload: dict[str, Any]) -> NotificationModel: + model = NotificationModel() + for key, value in payload.items(): + if not hasattr(model, key): + msg = f"'{key}' is an invalid keyword argument for NotificationModel" + raise TypeError(msg) + setattr(model, key, value) + return model + + +def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel: + if isinstance(payload, NotificationModel): + return payload + return _model_from_payload(payload) + + class NotificationsRepository(metaclass=Singleton): - def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + def __init__(self, session: SessionFactory | None = None) -> None: self._migrated = False - self.session: AsyncGenerator[AsyncSession] = session or get_session + self.session: SessionFactory = session or get_session async def run_migrations(self) -> None: if self._migrated: @@ -37,7 +56,7 @@ class NotificationsRepository(metaclass=Singleton): def get_instance() -> NotificationsRepository: return NotificationsRepository() - async def list(self) -> list[NotificationModel]: + async def all(self) -> list[NotificationModel]: async with self.session() as session: result: Result[tuple[NotificationModel]] = await session.execute( select(NotificationModel).order_by(NotificationModel.name.asc()) @@ -92,11 +111,11 @@ class NotificationsRepository(metaclass=Singleton): result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1)) return result.scalar_one_or_none() - async def create(self, payload: NotificationModel | dict) -> NotificationModel: + async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel: async with self.session() as session: - model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload + model = _coerce_model(payload) if model.id is not None: - model.id = None + model.id = None # ty: ignore if await self.get_by_name(name=model.name) is not None: msg: str = f"Notification target with name '{model.name}' already exists." diff --git a/app/features/notifications/service.py b/app/features/notifications/service.py index 80ab1d35..26b6bdd8 100644 --- a/app/features/notifications/service.py +++ b/app/features/notifications/service.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from app.features.notifications.models import NotificationModel from app.features.notifications.repository import NotificationsRepository @@ -64,10 +64,10 @@ class Notifications(metaclass=Singleton): await self._repo.run_migrations() EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations") - EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit") + EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit") - async def list(self) -> list[NotificationModel]: - return await self._repo.list() + async def all(self) -> list[NotificationModel]: + return await self._repo.all() async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]: return await self._repo.list_paginated(page, per_page) @@ -92,8 +92,8 @@ class Notifications(metaclass=Singleton): async def delete(self, identifier: int | str) -> NotificationModel: return await self._repo.delete(identifier) - async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]: - targets = await self._repo.list() + async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]: + targets = await self._repo.all() if len(targets) < 1: return [] @@ -126,7 +126,7 @@ class Notifications(metaclass=Singleton): if wait: return await asyncio.gather(*tasks) - return tasks + return cast("list[dict | Awaitable[dict]]", tasks) def emit(self, e: Event, _, **__) -> None: if not NotificationEvents.is_valid(e.event): @@ -271,7 +271,7 @@ class Notifications(metaclass=Singleton): if not status: msg = "Apprise failed to send notification." - raise RuntimeError(msg) # noqa: TRY301 + self._raise_apprise_error(msg) except Exception as exc: LOG.exception( "Failed to send Apprise notification for event '%s'.", @@ -288,6 +288,10 @@ class Notifications(metaclass=Singleton): return {} + @staticmethod + def _raise_apprise_error(msg: str) -> None: + raise RuntimeError(msg) + async def _send(self, target: Notification, ev: Event) -> dict: try: LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name) diff --git a/app/features/notifications/tests/test_notifications_repository.py b/app/features/notifications/tests/test_notifications_repository.py index 0940ed4e..f2e7fbd6 100644 --- a/app/features/notifications/tests/test_notifications_repository.py +++ b/app/features/notifications/tests/test_notifications_repository.py @@ -35,7 +35,7 @@ class TestNotificationsRepository: @pytest.mark.asyncio async def test_list_empty(self, repo): """List returns empty when no notifications exist.""" - notifications = await repo.list() + notifications = await repo.all() assert notifications == [], "Should return empty list when no notifications exist" @pytest.mark.asyncio diff --git a/app/features/presets/migration.py b/app/features/presets/migration.py index 1b76e730..06bd42e8 100644 --- a/app/features/presets/migration.py +++ b/app/features/presets/migration.py @@ -46,7 +46,7 @@ class Migration(FeatureMigration): return inserted = 0 - seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()} + seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()} for index, item in enumerate(items): if not (normalized := self._normalize(item, index, seen_names)): diff --git a/app/features/presets/repository.py b/app/features/presets/repository.py index 90b039f0..b6a83c4d 100644 --- a/app/features/presets/repository.py +++ b/app/features/presets/repository.py @@ -30,6 +30,30 @@ if TYPE_CHECKING: LOG = get_logger() +def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]: + if isinstance(payload, dict): + data: dict[str, Any] = {} + for key, value in payload.items(): + if not isinstance(key, str): + msg = "Preset payload keys must be strings." + raise TypeError(msg) + data[key] = value + return data + + return { + "name": payload.name, + "description": payload.description, + "folder": payload.folder, + "template": payload.template, + "cookies": payload.cookies, + "cli": payload.cli, + "default": payload.default, + "priority": payload.priority, + "created_at": payload.created_at, + "updated_at": payload.updated_at, + } + + class PresetsRepository(metaclass=Singleton): SORT_FIELDS: dict[str, Any] = { "id": PresetModel.id, @@ -73,15 +97,15 @@ class PresetsRepository(metaclass=Singleton): LOG.debug("Refreshing presets cache due to configuration update.") await self._update_cache() - Services.get_instance().add(__class__.__name__, self) + Services.get_instance().add(PresetsRepository.__name__, self) EventBus.get_instance().subscribe( - Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" + Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations" ).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache") async def _update_cache(self) -> None: from app.features.presets.service import Presets - await Presets.get_instance().refresh_cache(await self.list()) + await Presets.get_instance().refresh_cache(await self.all()) @staticmethod def get_instance() -> PresetsRepository: @@ -97,7 +121,7 @@ class PresetsRepository(metaclass=Singleton): LOG.error("Default preset '%s' not found, using 'default' preset.", default_name) config.default_preset = "default" - async def list(self) -> list[PresetModel]: + async def all(self) -> list[PresetModel]: async with self.session() as session: result: Result[tuple[PresetModel]] = await session.execute( select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc()) @@ -238,24 +262,9 @@ class PresetsRepository(metaclass=Singleton): result: Result[tuple[PresetModel]] = await session.execute(query.limit(1)) return result.scalar_one_or_none() - async def create(self, payload: PresetModel | dict) -> PresetModel: + async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel: async with self.session() as session: - data: dict[str, Any] - if isinstance(payload, dict): - data = dict(payload) - else: - data = { - "name": payload.name, - "description": payload.description, - "folder": payload.folder, - "template": payload.template, - "cookies": payload.cookies, - "cli": payload.cli, - "default": payload.default, - "priority": payload.priority, - "created_at": payload.created_at, - "updated_at": payload.updated_at, - } + data = _payload_data(payload) data.pop("id", None) model = PresetModel(**data) diff --git a/app/features/presets/service.py b/app/features/presets/service.py index 9d483ba5..b77b28a9 100644 --- a/app/features/presets/service.py +++ b/app/features/presets/service.py @@ -12,7 +12,7 @@ class Presets(metaclass=Singleton): def __init__(self, repo: PresetsRepository | None = None) -> None: self._repo: PresetsRepository = repo or PresetsRepository.get_instance() self._cache: list[tuple[int, str, Preset]] = [] - Services.get_instance().add(__class__.__name__, self) + Services.get_instance().add(type(self).__name__, self) @staticmethod def get_instance() -> Presets: diff --git a/app/features/presets/tests/test_presets_repository.py b/app/features/presets/tests/test_presets_repository.py index 5b26de4a..e4290c92 100644 --- a/app/features/presets/tests/test_presets_repository.py +++ b/app/features/presets/tests/test_presets_repository.py @@ -57,7 +57,7 @@ class TestPresetsRepository: await repo.create({"name": "A", "priority": 1}) await repo.create({"name": "C", "priority": 2}) - items = await repo.list() + items = await repo.all() assert items[0].name == "c", "Highest priority should be first" assert items[1].name == "a", "Same priority should sort by name" diff --git a/app/features/streaming/library/ffprobe.py b/app/features/streaming/library/ffprobe.py index c87ea467..38158710 100644 --- a/app/features/streaming/library/ffprobe.py +++ b/app/features/streaming/library/ffprobe.py @@ -10,8 +10,6 @@ import os import subprocess # qa: ignore from pathlib import Path -import anyio - from app.features.streaming.types import FFProbeError from app.library.log import get_logger from app.library.Utils import timed_lru_cache @@ -39,21 +37,26 @@ class FFStream: def __repr__(self): if "codec_long_name" not in self.__dict__: - self.codec_long_name = self.__dict__.get("codec_name", "") + self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "") + + index = self.__dict__.get("index", "?") + codec_type = self.__dict__.get("codec_type", "unknown") + codec_long_name = self.__dict__.get("codec_long_name", "") if self.is_video(): - return f"" + return f"" if self.is_audio(): return ( - f" " + f" " ) if self.is_subtitle() or self.is_attachment(): - return f"" + return f"" - return f"" + return f"" def is_audio(self): """ @@ -252,14 +255,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult: raise OSError(msg) try: - async with await anyio.open_file(os.devnull, "w") as tempf: - await asyncio.create_subprocess_exec( - "ffprobe", - "-h", - stdout=tempf, - stderr=tempf, - creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, - ) + proc = await asyncio.create_subprocess_exec( + "ffprobe", + "-h", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, + ) + await proc.wait() except FileNotFoundError as e: msg = "ffprobe not found." raise OSError(msg) from e diff --git a/app/features/streaming/library/m3u8.py b/app/features/streaming/library/m3u8.py index 818065b3..434139f0 100644 --- a/app/features/streaming/library/m3u8.py +++ b/app/features/streaming/library/m3u8.py @@ -39,7 +39,7 @@ class M3u8: m3u8.append("#EXT-X-MEDIA-SEQUENCE:0") m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD") - segmentSize: float = f"{self.duration:.6f}" + segmentSize: str = f"{self.duration:.6f}" splits: int = math.ceil(duration / self.duration) segmentParams: dict = {} diff --git a/app/features/streaming/library/playlist.py b/app/features/streaming/library/playlist.py index d2f0554b..e5e394ee 100644 --- a/app/features/streaming/library/playlist.py +++ b/app/features/streaming/library/playlist.py @@ -14,7 +14,7 @@ class Playlist: "The path where files are downloaded." async def make(self, file: Path) -> str: - ref: str = Path(str(file.relative_to(self.download_path)).strip("/")) + ref: Path = Path(str(file.relative_to(self.download_path)).strip("/")) try: ff = await ffprobe(file) diff --git a/app/features/streaming/library/segment_encoders.py b/app/features/streaming/library/segment_encoders.py index af943a0b..ea6664bb 100644 --- a/app/features/streaming/library/segment_encoders.py +++ b/app/features/streaming/library/segment_encoders.py @@ -1,7 +1,7 @@ -# flake8: noqa: ARG002 from __future__ import annotations import os +import shutil import subprocess import sys from functools import lru_cache @@ -16,9 +16,13 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]: Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and values are dicts with keys "full" and "lp" booleans. """ + vainfo = shutil.which("vainfo") + if not vainfo: + return {} + try: result: subprocess.CompletedProcess[str] = subprocess.run( - ["vainfo"], # noqa: S607 + [vainfo], capture_output=True, text=True, check=False, @@ -87,9 +91,13 @@ def ffmpeg_encoders() -> set[str]: """ from app.library.config import SUPPORTED_CODECS + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + return set() + try: result: subprocess.CompletedProcess[str] = subprocess.run( - ["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607 + [ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"], capture_output=True, text=True, check=False, @@ -149,9 +157,11 @@ class _BaseBuilder: codec_name: str def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]: + _ = ctx return [] def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]: + _ = ctx return [*args, "-codec:v", self.codec_name] @@ -312,4 +322,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]: "libx264": [], } - return chains.get(codec, chains["libx264"]) + return tuple(chains.get(codec, chains["libx264"])) diff --git a/app/features/streaming/library/segments.py b/app/features/streaming/library/segments.py index 46720adb..fcf28c03 100644 --- a/app/features/streaming/library/segments.py +++ b/app/features/streaming/library/segments.py @@ -1,6 +1,6 @@ import asyncio import os -import subprocess # type: ignore +import subprocess import sys import tempfile from pathlib import Path @@ -178,6 +178,7 @@ class Segments: ) try: + assert proc.stdout is not None while True: chunk: bytes = await proc.stdout.read(1024 * 64) if not chunk: @@ -240,7 +241,9 @@ class Segments: return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip()) async def stream(self, file: Path, resp: web.StreamResponse) -> None: - codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec + codec: str = self.vcodec + if Segments._cache_initialized and Segments._cached_vcodec: + codec = Segments._cached_vcodec if not codec or codec not in SUPPORTED_CODECS: codec: str = select_encoder(self.vcodec or "") diff --git a/app/features/streaming/library/subtitle.py b/app/features/streaming/library/subtitle.py index 464e66cc..acfbb85b 100644 --- a/app/features/streaming/library/subtitle.py +++ b/app/features/streaming/library/subtitle.py @@ -2,6 +2,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path +from typing import Any import anyio import pysubs2 @@ -48,7 +49,8 @@ def ms_to_timestamp(ms: int) -> str: return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}" -SubstationFormat.ms_to_timestamp = ms_to_timestamp +_substation_format: Any = SubstationFormat +_substation_format.ms_to_timestamp = ms_to_timestamp class Subtitle: diff --git a/app/features/streaming/library/thumbnail.py b/app/features/streaming/library/thumbnail.py index e73e7806..e4de26e1 100644 --- a/app/features/streaming/library/thumbnail.py +++ b/app/features/streaming/library/thumbnail.py @@ -24,6 +24,8 @@ THUMBNAIL_MISS_TTL = 3600.0 _LOCK = asyncio.Lock() _IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {} + + _SEM: asyncio.Semaphore | None = None _SEM_LIMIT: int | None = None diff --git a/app/features/streaming/router.py b/app/features/streaming/router.py index e29741f0..34244b13 100644 --- a/app/features/streaming/router.py +++ b/app/features/streaming/router.py @@ -4,6 +4,7 @@ from pathlib import Path from aiohttp import web from aiohttp.web import Request, Response +from aiohttp.web_response import StreamResponse from app.features.streaming.library.m3u8 import M3u8 from app.features.streaming.library.playlist import Playlist @@ -32,7 +33,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application Response: The response object. """ - file: str = request.match_info.get("file") + file: str | None = request.match_info.get("file") if not file: return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) @@ -83,8 +84,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) -> Response: The response object. """ - file: str = request.match_info.get("file") - mode: str = request.match_info.get("mode") + file: str | None = request.match_info.get("file") + mode: str | None = request.match_info.get("mode") if mode not in ["video", "subtitle"]: return web.json_response( @@ -94,13 +95,14 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) -> if not file: return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) - duration = request.query.get("duration", None) + duration: float | None = None + duration_arg = request.query.get("duration", None) if "subtitle" in mode: - if not duration: + if not duration_arg: return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code) - duration = float(duration) + duration = float(duration_arg) base_path: str = config.base_path.rstrip("/") @@ -124,6 +126,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) -> return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status) if "subtitle" in mode: + if duration is None: + return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code) text = await cls.make_subtitle(file=realFile, duration=duration) else: text = await cls.make_stream(file=realFile) @@ -149,7 +153,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) -> @route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream") -async def segments_stream(request: Request, config: Config, app: web.Application) -> Response: +async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse: """ Get the segments. @@ -162,9 +166,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application Response: The response object. """ - file: str = request.match_info.get("file") - segment: int = request.match_info.get("segment") - sd: int = request.query.get("sd") + file: str | None = request.match_info.get("file") + segment: str | None = request.match_info.get("segment") + sd: str | None = request.query.get("sd") vc: int = int(request.query.get("vc", 0)) ac: int = int(request.query.get("ac", 0)) @@ -241,7 +245,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application) Response: The response object. """ - file: str = request.match_info.get("file") + file: str | None = request.match_info.get("file") if not file: return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) @@ -299,7 +303,7 @@ async def subtitles_manifest_get(request: Request, config: Config, app: web.Appl Response: The response object. """ - file: str = request.match_info.get("file") + file: str | None = request.match_info.get("file") if not file: return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code) @@ -354,7 +358,7 @@ async def subtitles_track_get(request: Request, config: Config, app: web.Applica Response: The response object. """ - file: str = request.match_info.get("file") + file: str | None = request.match_info.get("file") source_format: str | None = request.match_info.get("source_format") if not file: diff --git a/app/features/streaming/tests/test_segments.py b/app/features/streaming/tests/test_segments.py index 7f9d46fa..05a5ca78 100644 --- a/app/features/streaming/tests/test_segments.py +++ b/app/features/streaming/tests/test_segments.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any import pytest +from aiohttp import web from app.features.streaming.library.segments import Segments from app.tests.helpers import get_test_system_temp_root @@ -148,18 +149,18 @@ class _FakeProc: self.killed = True -class _FakeResp: - def __init__(self, fail_with: Exception | None = None) -> None: +class _FakeResp(web.StreamResponse): + def __init__(self, fail_with: BaseException | None = None) -> None: self.data: bytearray = bytearray() self.eof = False self._exc = fail_with - async def write(self, data: bytes) -> None: + async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None: if self._exc: raise self._exc self.data.extend(data) - async def write_eof(self) -> None: + async def write_eof(self, *_args: Any, **_kwargs: Any) -> None: self.eof = True diff --git a/app/features/streaming/tests/test_thumbnail.py b/app/features/streaming/tests/test_thumbnail.py index 932c2c66..99a69031 100644 --- a/app/features/streaming/tests/test_thumbnail.py +++ b/app/features/streaming/tests/test_thumbnail.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio from pathlib import Path +from typing import Any from unittest.mock import AsyncMock import pytest @@ -260,7 +261,7 @@ async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None: output = temp_dir / ".jpg" media.write_text("video") - ff_info = FFProbeResult() + ff_info: Any = FFProbeResult() ff_info.metadata = {"duration": "120.0"} ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()] @@ -321,7 +322,7 @@ async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None: media1.write_text("video") media2.write_text("video") - ff_info = FFProbeResult() + ff_info: Any = FFProbeResult() ff_info.metadata = {"duration": "60.0"} ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()] monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info)) diff --git a/app/features/tasks/definitions/handlers/_base_handler.py b/app/features/tasks/definitions/handlers/_base_handler.py index 57b05c4c..19082e91 100644 --- a/app/features/tasks/definitions/handlers/_base_handler.py +++ b/app/features/tasks/definitions/handlers/_base_handler.py @@ -1,4 +1,3 @@ -# flake8: noqa: ARG004 from typing import TYPE_CHECKING, Any from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult @@ -12,6 +11,7 @@ if TYPE_CHECKING: class BaseHandler: @staticmethod async def can_handle(task: HandleTask) -> bool: + _ = task return False @staticmethod @@ -24,6 +24,7 @@ class BaseHandler: @staticmethod def parse(url: str) -> Any | None: + _ = url return None @staticmethod diff --git a/app/features/tasks/definitions/handlers/generic.py b/app/features/tasks/definitions/handlers/generic.py index 37dcf1a2..9e6fca57 100644 --- a/app/features/tasks/definitions/handlers/generic.py +++ b/app/features/tasks/definitions/handlers/generic.py @@ -64,7 +64,7 @@ class GenericTaskHandler(BaseHandler): from app.features.tasks.definitions.utils import model_to_schema repo = TaskDefinitionsRepository.get_instance() - models = await repo.list() + models = await repo.all() cls._definitions = [model_to_schema(model) for model in models] return cls._definitions @@ -141,7 +141,8 @@ class GenericTaskHandler(BaseHandler): return False @staticmethod - async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: + _ = config definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) if not definition: return TaskFailure(message="No generic task definition matched the provided URL.") diff --git a/app/features/tasks/definitions/handlers/rss.py b/app/features/tasks/definitions/handlers/rss.py index 5a99b619..3d74bd9f 100644 --- a/app/features/tasks/definitions/handlers/rss.py +++ b/app/features/tasks/definitions/handlers/rss.py @@ -9,6 +9,7 @@ from app.features.tasks.definitions.results import HandleTask, TaskFailure, Task from app.features.ytdlp.extractor import fetch_info from app.features.ytdlp.utils import get_archive_id from app.library.cache import Cache +from app.library.config import Config from app.library.log import get_logger from ._base_handler import BaseHandler @@ -160,12 +161,14 @@ class RssGenericHandler(BaseHandler): return feed_url, items, real_count @staticmethod - async def extract(task: HandleTask) -> TaskResult | TaskFailure: + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: + _ = config """ Extract items from an RSS/Atom feed. Args: task (Task): The task containing the feed URL. + config (Config | None): Optional handler configuration. Returns: TaskResult | TaskFailure: Extraction result with parsed items or failure information. diff --git a/app/features/tasks/definitions/handlers/tver.py b/app/features/tasks/definitions/handlers/tver.py index 9a89bdc3..fb9242ef 100644 --- a/app/features/tasks/definitions/handlers/tver.py +++ b/app/features/tasks/definitions/handlers/tver.py @@ -4,6 +4,7 @@ import httpx from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.utils import get_archive_id +from app.library.config import Config from app.library.log import get_logger from ._base_handler import BaseHandler @@ -177,7 +178,8 @@ class TverHandler(BaseHandler): return feed_url, items, has_items @staticmethod - async def extract(task: HandleTask) -> TaskResult | TaskFailure: + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: + _ = config series_id: str | None = TverHandler.parse(task.url) if not series_id: return TaskFailure(message="Unrecognized Tver series URL.") @@ -208,10 +210,11 @@ class TverHandler(BaseHandler): if not (url := entry.get("url")): continue - archive_id: str = entry.get("archive_id") - task_items.append( - TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {})) - ) + archive_id: str | None = entry.get("archive_id") + metadata = entry.get("metadata", {}) + if not isinstance(metadata, dict): + metadata = {} + task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata)) return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items}) diff --git a/app/features/tasks/definitions/handlers/twitch.py b/app/features/tasks/definitions/handlers/twitch.py index d61e26a6..03ee82d7 100644 --- a/app/features/tasks/definitions/handlers/twitch.py +++ b/app/features/tasks/definitions/handlers/twitch.py @@ -6,6 +6,7 @@ import httpx from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.utils import get_archive_id +from app.library.config import Config from app.library.log import get_logger from ._base_handler import BaseHandler @@ -92,7 +93,8 @@ class TwitchHandler(BaseHandler): return feed_url, items, has_items @staticmethod - async def extract(task: HandleTask) -> TaskResult | TaskFailure: + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: + _ = config handle_name: str | None = TwitchHandler.parse(task.url) if not handle_name: return TaskFailure(message="Unrecognized Twitch channel URL.") @@ -122,7 +124,7 @@ class TwitchHandler(BaseHandler): if not (url := entry.get("url")): continue - archive_id: str = entry.get("archive_id") + archive_id: str | None = entry.get("archive_id") task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id)) return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items}) diff --git a/app/features/tasks/definitions/handlers/youtube.py b/app/features/tasks/definitions/handlers/youtube.py index 971c8749..43f8c462 100644 --- a/app/features/tasks/definitions/handlers/youtube.py +++ b/app/features/tasks/definitions/handlers/youtube.py @@ -6,6 +6,7 @@ import httpx from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.ytdlp.utils import get_archive_id +from app.library.config import Config from app.library.log import get_logger from ._base_handler import BaseHandler @@ -103,7 +104,8 @@ class YoutubeHandler(BaseHandler): return feed_url, items, real_count @staticmethod - async def extract(task: HandleTask) -> TaskResult | TaskFailure: + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: + _ = config parsed: dict[str, str] | None = YoutubeHandler.parse(task.url) if not parsed: return TaskFailure(message="Unrecognized YouTube channel or playlist URL.") @@ -133,7 +135,7 @@ class YoutubeHandler(BaseHandler): if not (url := entry.get("url")): continue - archive_id: str = entry.get("archive_id") + archive_id: str | None = entry.get("archive_id") metadata: dict[str, Any] = {"published": entry.get("published")} task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata)) diff --git a/app/features/tasks/definitions/repository.py b/app/features/tasks/definitions/repository.py index 2813ff53..2d0ca828 100644 --- a/app/features/tasks/definitions/repository.py +++ b/app/features/tasks/definitions/repository.py @@ -14,20 +14,23 @@ from app.library.Services import Services from app.library.Singleton import Singleton if TYPE_CHECKING: - from collections.abc import AsyncGenerator + from collections.abc import Callable + from contextlib import AbstractAsyncContextManager from sqlalchemy.engine.result import Result from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.selectable import Select + SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]] + LOG = get_logger() class TaskDefinitionsRepository(metaclass=Singleton): - def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + def __init__(self, session: SessionFactory | None = None) -> None: self._migrated = False - self.session: AsyncGenerator[AsyncSession] = session or get_session + self.session: SessionFactory = session or get_session async def run_migrations(self) -> None: if self._migrated: @@ -51,12 +54,12 @@ class TaskDefinitionsRepository(metaclass=Singleton): LOG.debug("Refreshing task definitions due to configuration update.") await GenericTaskHandler.refresh_definitions(force=True) - Services.get_instance().add(__class__.__name__, self) + Services.get_instance().add(TaskDefinitionsRepository.__name__, self) EventBus.get_instance().subscribe( - Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" + Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations" ).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions") - async def list(self) -> list[TaskDefinitionModel]: + async def all(self) -> list[TaskDefinitionModel]: async with self.session() as session: result: Result[tuple[TaskDefinitionModel]] = await session.execute( select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc()) @@ -115,9 +118,9 @@ class TaskDefinitionsRepository(metaclass=Singleton): async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel: async with self.session() as session: - model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload + model: TaskDefinitionModel = TaskDefinitionModel(**payload) if model.id is not None: - model.id = None + model.id = None # ty: ignore if await self.get_by_name(name=model.name) is not None: msg: str = f"Task definition with name '{model.name}' already exists." diff --git a/app/features/tasks/definitions/results.py b/app/features/tasks/definitions/results.py index 3520b8e6..fb07efff 100644 --- a/app/features/tasks/definitions/results.py +++ b/app/features/tasks/definitions/results.py @@ -48,10 +48,13 @@ class HandleTask(TaskSchema): if isinstance(ret, tuple): return ret - archive_file: Path = ret.get("file") - items: set[str] = ret.get("items", set()) + archive_file = ret.get("file") + items = ret.get("items", set()) + if not isinstance(archive_file, Path) or not isinstance(items, set): + return (False, "Failed to get archive information.") + archive_items = [item for item in items if isinstance(item, str)] - if len(items) < 1 or not archive_add(archive_file, list(items)): + if len(archive_items) < 1 or not archive_add(archive_file, archive_items): return (True, "No new items to mark as downloaded.") return (True, f"Task '{self.name}' items marked as downloaded.") @@ -70,10 +73,13 @@ class HandleTask(TaskSchema): if isinstance(ret, tuple): return ret - archive_file: Path = ret.get("file") - items: set[str] = ret.get("items", set()) + archive_file = ret.get("file") + items = ret.get("items", set()) + if not isinstance(archive_file, Path) or not isinstance(items, set): + return (False, "Failed to get archive information.") + archive_items = [item for item in items if isinstance(item, str)] - if len(items) < 1 or not archive_delete(archive_file, list(items)): + if len(archive_items) < 1 or not archive_delete(archive_file, archive_items): return (True, "No items to remove from archive file.") return (True, f"Removed '{self.name}' items from archive file.") diff --git a/app/features/tasks/definitions/service.py b/app/features/tasks/definitions/service.py index a0dd5f78..f0b7f000 100644 --- a/app/features/tasks/definitions/service.py +++ b/app/features/tasks/definitions/service.py @@ -8,6 +8,7 @@ import random from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from app.features.tasks.definitions.handlers._base_handler import BaseHandler from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.features.tasks.models import TaskModel from app.features.ytdlp.utils import archive_read @@ -27,7 +28,7 @@ LOG = get_logger() class TaskHandle: def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None: - self._handlers: list[type] = [] + self._handlers: list[type[BaseHandler]] = [] "The available handlers." self._repo: TasksRepository = tasks "The tasks manager." @@ -35,7 +36,7 @@ class TaskHandle: "The scheduler." self._config: Config = config "The configuration." - self._task_name: str = f"{__class__.__name__}._dispatcher" + self._task_name: str = f"{TaskHandle.__name__}._dispatcher" "The task name for the scheduler." self._queued: dict[str, set[str]] = {} "Queued archive IDs per handler." @@ -45,11 +46,11 @@ class TaskHandle: EventBus.get_instance().subscribe( Events.ITEM_ERROR, self._handle_item_error, - f"{__class__.__name__}.item_error", + f"{TaskHandle.__name__}.item_error", ) def load(self) -> None: - self._handlers: list[type] = self._discover() + self._handlers: list[type[BaseHandler]] = self._discover() timer: str = self._config.tasks_handler_timer try: @@ -72,16 +73,16 @@ class TaskHandle: self._scheduler.add( timer=timer, func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"), - id=f"{__class__.__name__}._dispatcher", + id=f"{TaskHandle.__name__}._dispatcher", ) async def _dispatcher(self): s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []} - handler_groups: dict[str, list[tuple[HandleTask, type]]] = {} - dispatches: list[tuple[HandleTask, type, asyncio.Task[TaskResult | TaskFailure | None]]] = [] + handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {} + dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = [] - tasks: list[TaskModel] = await self._repo.list() + tasks: list[TaskModel] = await self._repo.all() for task_model in tasks: task: HandleTask = HandleTask.model_validate(task_model) @@ -100,7 +101,7 @@ class TaskHandle: continue try: - handler: type | None = await self._find_handler(task) + handler: type[BaseHandler] | None = await self._find_handler(task) if handler is None: s["u"].append(task.name) continue @@ -181,7 +182,9 @@ class TaskHandle: }, ) - async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None: + async def _dispatch( + self, task: HandleTask, handler: type[BaseHandler], delay: float + ) -> TaskResult | TaskFailure | None: """ Dispatch a task after a random delay to avoid rate limiting. @@ -215,7 +218,7 @@ class TaskHandle: exc_info=(type(exc), exc, exc.__traceback__), ) - async def _find_handler(self, task: HandleTask) -> type | None: + async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None: for cls in self._handlers: try: if await Services.get_instance().handle_async(handler=cls.can_handle, task=task): @@ -239,9 +242,10 @@ class TaskHandle: async def dispatch( self, task: HandleTask, - handler: type | None = None, - **kwargs, # noqa: ARG002 + handler: type[BaseHandler] | None = None, + **kwargs, ) -> TaskResult | TaskFailure | None: + _ = kwargs """ Dispatch a task to the appropriate handler. @@ -582,11 +586,11 @@ class TaskHandle: return TaskResult(items=list(extraction.items), metadata=combined_metadata) - def _discover(self) -> list[type]: + def _discover(self) -> list[type[BaseHandler]]: """Discover all available task handlers.""" import app.features.tasks.definitions.handlers as handlers_pkg - handlers: list[type] = [] + handlers: list[type[BaseHandler]] = [] for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__): if module_name.startswith("_"): diff --git a/app/features/tasks/definitions/tests/test_generic_task_handler.py b/app/features/tasks/definitions/tests/test_generic_task_handler.py index d06b3d67..310ba85e 100644 --- a/app/features/tasks/definitions/tests/test_generic_task_handler.py +++ b/app/features/tasks/definitions/tests/test_generic_task_handler.py @@ -8,6 +8,7 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult from app.features.tasks.definitions.schemas import ( Definition, EngineConfig, + Parse, RequestConfig, ResponseConfig, TaskDefinition, @@ -30,10 +31,12 @@ def test_build_def_payload(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "link": {"type": "css", "expression": ".article a.link::attr(href)"}, - "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, - }, + parse=Parse.model_validate( + { + "link": {"type": "css", "expression": ".article a.link::attr(href)"}, + "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, + } + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(), @@ -54,15 +57,17 @@ def test_build_def_container(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "items": { - "selector": ".cards .card", - "fields": { - "link": {"type": "css", "expression": ".card-header a", "attribute": "href"}, - "title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, - }, + parse=Parse.model_validate( + { + "items": { + "selector": ".cards .card", + "fields": { + "link": {"type": "css", "expression": ".card-header a", "attribute": "href"}, + "title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, + }, + } } - }, + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(), @@ -82,16 +87,18 @@ def test_build_def_json(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "items": { - "type": "jsonpath", - "selector": "items", - "fields": { - "link": {"type": "jsonpath", "expression": "url"}, - "title": {"type": "jsonpath", "expression": "title"}, - }, + parse=Parse.model_validate( + { + "items": { + "type": "jsonpath", + "selector": "items", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } } - }, + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(type="json"), @@ -112,11 +119,13 @@ def test_parse_items_basic(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None}, - "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, - "id": {"type": "css", "expression": ".article", "attribute": "data-id"}, - }, + parse=Parse.model_validate( + { + "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None}, + "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, + "id": {"type": "css", "expression": ".article", "attribute": "data-id"}, + } + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(), @@ -154,34 +163,36 @@ def test_parse_items_cards(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "items": { - "type": "css", - "selector": ".columns .card", - "fields": { - "link": { - "type": "css", - "expression": ".card-header a[href]", - "attribute": "href", + parse=Parse.model_validate( + { + "items": { + "type": "css", + "selector": ".columns .card", + "fields": { + "link": { + "type": "css", + "expression": ".card-header a[href]", + "attribute": "href", + }, + "title": { + "type": "css", + "expression": ".card-header a[href]", + "attribute": "text", + }, + "poet": { + "type": "css", + "expression": "footer .card-footer-item:first-child a", + "attribute": "text", + }, + "category": { + "type": "css", + "expression": "footer .card-footer-item:nth-child(2) a", + "attribute": "text", + }, }, - "title": { - "type": "css", - "expression": ".card-header a[href]", - "attribute": "text", - }, - "poet": { - "type": "css", - "expression": "footer .card-footer-item:first-child a", - "attribute": "text", - }, - "category": { - "type": "css", - "expression": "footer .card-footer-item:nth-child(2) a", - "attribute": "text", - }, - }, + } } - }, + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(), @@ -249,17 +260,19 @@ def test_parse_items_json(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "items": { - "type": "jsonpath", - "selector": "entries", - "fields": { - "link": {"type": "jsonpath", "expression": "url"}, - "title": {"type": "jsonpath", "expression": "title"}, - "id": {"type": "jsonpath", "expression": "id"}, - }, + parse=Parse.model_validate( + { + "items": { + "type": "jsonpath", + "selector": "entries", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + "id": {"type": "jsonpath", "expression": "id"}, + }, + } } - }, + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(type="json"), @@ -297,16 +310,18 @@ async def test_generic_task_handler_inspect(monkeypatch): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "items": { - "type": "jsonpath", - "selector": "items", - "fields": { - "link": {"type": "jsonpath", "expression": "url"}, - "title": {"type": "jsonpath", "expression": "title"}, - }, + parse=Parse.model_validate( + { + "items": { + "type": "jsonpath", + "selector": "items", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } } - }, + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(type="json"), @@ -351,16 +366,18 @@ def test_parse_items_json_list(): created_at=datetime.now(), updated_at=datetime.now(), definition=Definition( - parse={ - "items": { - "type": "jsonpath", - "selector": "[]", - "fields": { - "link": {"type": "jsonpath", "expression": "url"}, - "title": {"type": "jsonpath", "expression": "title"}, - }, + parse=Parse.model_validate( + { + "items": { + "type": "jsonpath", + "selector": "[]", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } } - }, + ), engine=EngineConfig(), request=RequestConfig(), response=ResponseConfig(type="json"), diff --git a/app/features/tasks/definitions/tests/test_task_definitions.py b/app/features/tasks/definitions/tests/test_task_definitions.py index 5a3d23ff..6108748c 100644 --- a/app/features/tasks/definitions/tests/test_task_definitions.py +++ b/app/features/tasks/definitions/tests/test_task_definitions.py @@ -69,7 +69,7 @@ class TestTaskDefinitionsRepository: await repo.create(_sample_definition("Alpha", priority=2)) await repo.create(_sample_definition("Beta", priority=1)) - items = await repo.list() + items = await repo.all() assert len(items) == 2, "Should return two task definitions" assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name" diff --git a/app/features/tasks/definitions/utils.py b/app/features/tasks/definitions/utils.py index c1fba3d8..2209e17e 100644 --- a/app/features/tasks/definitions/utils.py +++ b/app/features/tasks/definitions/utils.py @@ -1,9 +1,17 @@ -from typing import Any +from typing import Any, Literal, overload from app.features.tasks.definitions.models import TaskDefinitionModel from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary +@overload +def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ... + + +@overload +def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ... + + def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary: """ Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema. diff --git a/app/features/tasks/repository.py b/app/features/tasks/repository.py index 66ca0afc..baef0000 100644 --- a/app/features/tasks/repository.py +++ b/app/features/tasks/repository.py @@ -10,20 +10,39 @@ from app.library.log import get_logger from app.library.Singleton import Singleton if TYPE_CHECKING: - from collections.abc import AsyncGenerator + from collections.abc import Callable + from contextlib import AbstractAsyncContextManager from sqlalchemy.engine.result import Result from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.selectable import Select + SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]] + LOG = get_logger() +def _model_from_payload(payload: dict[str, Any]) -> TaskModel: + model = TaskModel() + for key, value in payload.items(): + if not hasattr(model, key): + msg = f"'{key}' is an invalid keyword argument for TaskModel" + raise TypeError(msg) + setattr(model, key, value) + return model + + +def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel: + if isinstance(payload, TaskModel): + return payload + return _model_from_payload(payload) + + class TasksRepository(metaclass=Singleton): - def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + def __init__(self, session: SessionFactory | None = None) -> None: self._migrated = False - self.session = session or get_session + self.session: SessionFactory = session or get_session async def run_migrations(self) -> None: if self._migrated: @@ -38,7 +57,7 @@ class TasksRepository(metaclass=Singleton): def get_instance() -> TasksRepository: return TasksRepository() - async def list(self) -> list[TaskModel]: + async def all(self) -> list[TaskModel]: async with self.session() as session: result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc())) return list(result.scalars().all()) @@ -90,7 +109,7 @@ class TasksRepository(metaclass=Singleton): """Get all enabled tasks.""" async with self.session() as session: result: Result[tuple[TaskModel]] = await session.execute( - select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712 + select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc()) ) return list(result.scalars().all()) @@ -102,11 +121,11 @@ class TasksRepository(metaclass=Singleton): ) return list(result.scalars().all()) - async def create(self, payload: TaskModel | dict) -> TaskModel: + async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel: async with self.session() as session: - model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload + model = _coerce_model(payload) if model.id is not None: - model.id = None + model.id = None # ty: ignore if await self.get_by_name(name=model.name) is not None: msg: str = f"Task with name '{model.name}' already exists." diff --git a/app/features/tasks/router.py b/app/features/tasks/router.py index cada50ef..3e2ffd59 100644 --- a/app/features/tasks/router.py +++ b/app/features/tasks/router.py @@ -705,8 +705,9 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config, xml_content += f' {NFOMakerPP._escape_text(info.get("id"))}\n' if info.get("uploader"): xml_content += f" {NFOMakerPP._escape_text(info.get('uploader'))}\n" - if info.get("tags", []): - for tag in info.get("tags", []): + tags = info.get("tags", []) + if isinstance(tags, list): + for tag in tags: xml_content += f" {NFOMakerPP._escape_text(tag)}\n" if info.get("year"): xml_content += f" {info.get('year')}\n" diff --git a/app/features/tasks/service.py b/app/features/tasks/service.py index 53554d02..705806a6 100644 --- a/app/features/tasks/service.py +++ b/app/features/tasks/service.py @@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton): pass async def _load_tasks(self) -> None: - tasks = await self._repo.list() + tasks = await self._repo.all() for task in tasks: if not task.timer or not task.enabled: @@ -134,9 +134,11 @@ class Tasks(metaclass=Singleton): task_id = task.id task_name = task.name try: - if not (task := await self._repo.get(task_id)): + current_task = await self._repo.get(task_id) + if not current_task: LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name}) return + task = current_task if not task.enabled: LOG.debug( diff --git a/app/features/tasks/tests/test_tasks_repository.py b/app/features/tasks/tests/test_tasks_repository.py index 221cd66e..a7898903 100644 --- a/app/features/tasks/tests/test_tasks_repository.py +++ b/app/features/tasks/tests/test_tasks_repository.py @@ -260,7 +260,7 @@ class TestTasksRepository: await repo.create({"name": "Alice", "url": "https://example.com"}) await repo.create({"name": "Bob", "url": "https://example.com"}) - items = await repo.list() + items = await repo.all() assert items[0].name == "Alice", "Should be alphabetically first" assert items[1].name == "Bob", "Should be alphabetically second" diff --git a/app/features/tasks/tests/test_tasks_router.py b/app/features/tasks/tests/test_tasks_router.py index d3064427..ae9fc037 100644 --- a/app/features/tasks/tests/test_tasks_router.py +++ b/app/features/tasks/tests/test_tasks_router.py @@ -1,6 +1,7 @@ from __future__ import annotations from types import SimpleNamespace +from typing import Any import pytest import pytest_asyncio @@ -41,13 +42,13 @@ def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None: def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request: - request = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace()) + mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace()) async def _json() -> object: return payload - request.json = _json # type: ignore[attr-defined] - return request + mock_request.json = _json + return mock_request class _Notify: @@ -85,7 +86,7 @@ async def test_add_requires_timer_without_handler(repo) -> None: assert response.status == web.HTTPBadRequest.status_code assert b"requires a timer" in response.body - assert await repo.list() == [] + assert await repo.all() == [] @pytest.mark.asyncio @@ -109,7 +110,7 @@ async def test_add_all_or_nothing(repo) -> None: assert response.status == web.HTTPBadRequest.status_code assert b"requires a timer" in response.body - assert await repo.list() == [] + assert await repo.all() == [] @pytest.mark.asyncio @@ -123,7 +124,7 @@ async def test_add_allows_handler_only(repo) -> None: response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True)) assert response.status == web.HTTPOk.status_code - items = await repo.list() + items = await repo.all() assert len(items) == 1 assert items[0].name == "Handler Only" diff --git a/app/features/ytdlp/extractor.py b/app/features/ytdlp/extractor.py index 70f1b9bc..08a994d7 100644 --- a/app/features/ytdlp/extractor.py +++ b/app/features/ytdlp/extractor.py @@ -120,7 +120,7 @@ class ExtractorPool(metaclass=Singleton): def attach(self, app: web.Application) -> None: """Attach the extractor pool to the application (no-op for now).""" app.on_shutdown.append(self.shutdown) - Services.get_instance().add(__class__.__name__, self) + Services.get_instance().add(type(self).__name__, self) def _ensure_initialized(self, config: ExtractorConfig) -> None: """ diff --git a/app/features/ytdlp/router.py b/app/features/ytdlp/router.py index a4771524..11d4b6ef 100644 --- a/app/features/ytdlp/router.py +++ b/app/features/ytdlp/router.py @@ -286,7 +286,7 @@ async def convert(request: Request) -> Response: return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code) try: - response = {"opts": {}, "output_template": None, "download_path": None} + response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None} data = arg_converter(args, dumps=True) @@ -384,6 +384,8 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response: if cache.has(key) and not request.query.get("force", False): data: Any | None = cache.get(key) + if data is None: + data = {} data["_cached"] = { "status": "hit", "preset": preset, @@ -393,7 +395,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response: "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(), "expires": data.get("_cached", {}).get("expires", time.time() + 300), } - return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) + return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) ytdlp_opts: dict = opts.get_all() @@ -450,7 +452,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response: cache.set(key=key, value=data, ttl=300) - return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) + return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code) except Exception as e: LOG.exception( "Failed to get video info for '%s': %s.", @@ -486,7 +488,7 @@ async def get_options() -> Response: """ from app.features.ytdlp.ytdlp import ytdlp_options - return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code) + return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code) @route("POST", "api/yt-dlp/archive_id/", "get_archive_ids") @@ -508,7 +510,11 @@ async def get_archive_ids(request: Request, config: Config) -> Response: response = [] for i, url in enumerate(data): - dct = {"index": i, "url": url} + dct: dict[str, Any] = {"index": i, "url": url} + if not isinstance(url, str): + dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."}) + response.append(dct) + continue try: await asyncio.to_thread(validate_url, url, config.allow_internal_urls) dct.update(get_archive_id(url)) diff --git a/app/features/ytdlp/tests/test_ytdlp_extractor.py b/app/features/ytdlp/tests/test_ytdlp_extractor.py index c7fec874..3df1cd78 100644 --- a/app/features/ytdlp/tests/test_ytdlp_extractor.py +++ b/app/features/ytdlp/tests/test_ytdlp_extractor.py @@ -122,6 +122,7 @@ class TestExtractInfo: {}, "https://example.com/video", debug=True, capture_logs=logging.WARNING ) + assert result is not None assert result["id"] == "test123" assert logs == ["[generic_browser] Browser fallback warning"] assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen @@ -155,6 +156,7 @@ class TestExtractInfo: {}, "https://example.com/video", debug=False, capture_logs=logging.WARNING ) + assert result is not None assert result["id"] == "test123" assert logs == ["[generic_browser] Browser fallback warning"] assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen diff --git a/app/features/ytdlp/tests/test_ytdlp_module.py b/app/features/ytdlp/tests/test_ytdlp_module.py index 0e84c7df..fb9e9fc4 100644 --- a/app/features/ytdlp/tests/test_ytdlp_module.py +++ b/app/features/ytdlp/tests/test_ytdlp_module.py @@ -1,5 +1,6 @@ import importlib from pathlib import Path +from typing import Any from unittest.mock import MagicMock, Mock, patch import pytest @@ -219,7 +220,8 @@ class TestYTDLP: """Test _delete_downloaded_files skips cleanup when _interrupted is True.""" with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None): ytdlp = YTDLP(params={}) - ytdlp.to_screen = Mock() + mock_obj: Any = ytdlp + mock_obj.to_screen = Mock() # Set interrupted flag ytdlp._interrupted = True @@ -229,7 +231,7 @@ class TestYTDLP: # Should not call super method mock_super_delete.assert_not_called() # Should show message - ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.") + mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.") # Should return None assert result is None diff --git a/app/features/ytdlp/tests/test_ytdlp_opts.py b/app/features/ytdlp/tests/test_ytdlp_opts.py index a9ed5f62..a4a3c5b0 100644 --- a/app/features/ytdlp/tests/test_ytdlp_opts.py +++ b/app/features/ytdlp/tests/test_ytdlp_opts.py @@ -776,7 +776,7 @@ class TestYTDLPCli: from app.features.ytdlp.ytdlp_opts import YTDLPCli with pytest.raises(ValueError, match="Expected Item instance"): - YTDLPCli(item="not an item") # type: ignore + YTDLPCli(item="not an item") @patch("app.features.presets.service.Presets") @patch("app.features.ytdlp.ytdlp_opts.Config") diff --git a/app/features/ytdlp/tests/test_ytdlp_utils.py b/app/features/ytdlp/tests/test_ytdlp_utils.py index f47c617c..19c14458 100644 --- a/app/features/ytdlp/tests/test_ytdlp_utils.py +++ b/app/features/ytdlp/tests/test_ytdlp_utils.py @@ -46,7 +46,8 @@ class TestLogWrapper: def test_add_target_type_validation(self) -> None: lw = LogWrapper() with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"): - lw.add_target(123) # type: ignore[arg-type] + bad: Any = 123 + lw.add_target(bad) def test_add_target_names(self) -> None: lw = LogWrapper() @@ -128,7 +129,7 @@ class TestExtractYtdlpLogs: def test_extract_ytdlp_logs_with_filters(self): """Test log extraction with filters.""" logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"] - filters = [re.compile(r"ERROR")] + filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")] result = extract_ytdlp_logs(logs, filters) assert result == ["ERROR: Failed"] @@ -312,9 +313,12 @@ class TestGetThumbnail: def test_non_list(self): """Test that None is returned for non-list input.""" - assert get_thumbnail(None) is None - assert get_thumbnail("not a list") is None - assert get_thumbnail({"not": "list"}) is None + bad_none: Any = None + bad_str: Any = "not a list" + bad_dict: Any = {"not": "list"} + assert get_thumbnail(bad_none) is None + assert get_thumbnail(bad_str) is None + assert get_thumbnail(bad_dict) is None def test_thumbnail_preference(self): """Test that the thumbnail with highest preference is returned.""" @@ -349,6 +353,7 @@ class TestGetThumbnail: ] result = get_thumbnail(thumbnails) + assert result is not None assert result["url"] == "with_pref.jpg" def test_all_equal(self): @@ -365,12 +370,15 @@ class TestGetThumbnail: class TestGetExtras: def test_none(self): """Test that empty dict is returned for None input.""" - assert get_extras(None) == {} + bad: Any = None + assert get_extras(bad) == {} def test_non_dict(self): """Test that empty dict is returned for non-dict input.""" - assert get_extras("not a dict") == {} - assert get_extras([]) == {} + bad_str: Any = "not a dict" + bad_list: Any = [] + assert get_extras(bad_str) == {} + assert get_extras(bad_list) == {} def test_extracts_video_information(self): """Test extracting information from a video entry.""" diff --git a/app/features/ytdlp/utils.py b/app/features/ytdlp/utils.py index 8785eb33..dd64c0f4 100644 --- a/app/features/ytdlp/utils.py +++ b/app/features/ytdlp/utils.py @@ -101,12 +101,12 @@ class LogWrapper: name (str|None): The name of the logging target. Defaults to None. """ - if not isinstance(target, logging.Logger | Callable): + if not isinstance(target, logging.Logger) and not callable(target): msg = "Target must be a logging.Logger instance or a callable." raise TypeError(msg) if name is None: - name = target.name if isinstance(target, logging.Logger) else target.__name__ + name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable") self.targets.append( LogTarget( @@ -134,8 +134,9 @@ class LogWrapper: if target.logger: log_kwargs = {**kwargs} log_kwargs.setdefault("stacklevel", 3) + if isinstance(target.target, logging.Logger): target.target.log(level, msg, *args, **log_kwargs) - else: + elif callable(target.target): target.target(level, msg, *args, **kwargs) def debug(self, msg, *args, **kwargs): @@ -179,13 +180,17 @@ def arg_converter( create_parser = yt_dlp.options.create_parser - def _default_opts(args: str): + def _default_opts(args: str | list[str]): patched_parser = create_parser() + + def patched_create_parser(): + return patched_parser + try: - yt_dlp.options.create_parser = lambda: patched_parser + yt_dlp.options.__dict__["create_parser"] = patched_create_parser return yt_dlp.parse_options(args) finally: - yt_dlp.options.create_parser = create_parser + yt_dlp.options.__dict__["create_parser"] = create_parser apply_ytdlp_patches() @@ -238,7 +243,7 @@ def arg_converter( return diff -def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]: +def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]: """ Extract yt-dlp log lines matching built-in filters plus any extras. @@ -332,7 +337,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP: return _DATA.YTDLP_INFO_CLS -def get_thumbnail(thumbnails: list) -> str | None: +def get_thumbnail(thumbnails: list) -> dict | None: """ Extract thumbnail URL from a yt-dlp entry. @@ -340,7 +345,7 @@ def get_thumbnail(thumbnails: list) -> str | None: thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry. Returns: - str | None: The thumbnail URL if available, otherwise None. + dict | None: The best thumbnail dict if available, otherwise None. """ if not thumbnails or not isinstance(thumbnails, list): @@ -381,7 +386,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict: extras[f"playlist_{property}"] = val if thumbnail := get_thumbnail(entry.get("thumbnails", [])): - extras["thumbnail"] = thumbnail.get("url") + extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail elif thumbnail := entry.get("thumbnail"): extras["thumbnail"] = thumbnail @@ -439,7 +444,7 @@ def get_archive_id(url: str) -> dict[str, str | None]: } """ - idDict: dict[str, None] = { + idDict: dict[str, str | None] = { "id": None, "ie_key": None, "archive_id": None, diff --git a/app/features/ytdlp/ytdlp_opts.py b/app/features/ytdlp/ytdlp_opts.py index 80234459..f20cc83b 100644 --- a/app/features/ytdlp/ytdlp_opts.py +++ b/app/features/ytdlp/ytdlp_opts.py @@ -50,12 +50,12 @@ class ARGSMerger: """ return str(self) - def as_dict(self) -> dict: + def as_dict(self) -> list[str]: """ - Get all the options as a dict. + Get all the options as a shell argument list. Returns: - dict: The options as a dict + list[str]: The options as shell arguments. """ return shlex.split(shlex.join(self.args)) diff --git a/app/library/BackgroundWorker.py b/app/library/BackgroundWorker.py index 7a59ed34..80c07d9b 100644 --- a/app/library/BackgroundWorker.py +++ b/app/library/BackgroundWorker.py @@ -29,7 +29,7 @@ class BackgroundWorker(metaclass=Singleton): "The queue to hold the tasks." self.running = True "Whether the background worker is running or not." - self.thread: threading.Thread = None + self.thread: threading.Thread | None = None "The thread that runs the background worker." @staticmethod @@ -49,7 +49,8 @@ class BackgroundWorker(metaclass=Singleton): try: LOG.debug("Stopping background worker thread.") self.queue.put((CloseThread, (), {})) - self.thread.join(timeout=5) + if self.thread: + self.thread.join(timeout=5) LOG.debug("Background worker thread has been shut down.") except Exception as e: LOG.exception( diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 609ed7e4..59d46ae3 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -1,14 +1,17 @@ import base64 import hmac import inspect -from collections.abc import Awaitable from datetime import UTC, datetime, timedelta from pathlib import Path +from typing import Any, cast import anyio from aiohttp import web -from aiohttp.web import Request, RequestHandler, Response +from aiohttp.typedefs import Handler, Middleware +from aiohttp.web import Request, Response from aiohttp.web_log import AccessLogger +from aiohttp.web_request import BaseRequest +from aiohttp.web_response import StreamResponse from app.library.log import get_logger from app.library.Services import Services @@ -24,12 +27,12 @@ LOG = get_logger("http") class HttpAccessLogger(AccessLogger): - def log(self, request: Request, response: Response, time: float) -> None: + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: try: fmt_info = self._format_line(request, response, time) values: list[object] = [] - extra: dict[str, object] = {"elapsed_ms": round(time * 1000.0, 2)} + extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)} for key, value in fmt_info: values.append(value) @@ -37,7 +40,7 @@ class HttpAccessLogger(AccessLogger): extra[key] = value else: parent, child = key - group = extra.get(parent, {}) + group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment] if not isinstance(group, dict): group = {} group[child] = value @@ -177,7 +180,7 @@ class HttpAPI: registered_options.append(route.path) @staticmethod - def basic_auth(username: str, password: str) -> Awaitable: + def basic_auth(username: str, password: str) -> Middleware: """ Middleware to handle basic authentication. @@ -191,7 +194,7 @@ class HttpAPI: """ @web.middleware - async def auth_handler(request: Request, handler: RequestHandler) -> Response: + async def auth_handler(request: Request, handler: Handler) -> StreamResponse: # if OPTIONS request, skip auth if "OPTIONS" == request.method: return await handler(request) @@ -204,7 +207,7 @@ class HttpAPI: auth_cookie = request.cookies.get("auth") if auth_cookie is not None: try: - data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key) + data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key)) if data is not None: data = base64.b64encode(data.encode()).decode() auth_header = f"Basic {data}" @@ -250,7 +253,7 @@ class HttpAPI: }, ) - response: Response = await handler(request) + response: StreamResponse = await handler(request) contentType: str | None = response.headers.get("content-type", None) if contentType and not contentType.startswith("text/html"): @@ -261,7 +264,7 @@ class HttpAPI: "auth", encrypt_data( f"{user_name}:{user_password}", - key=Config.get_instance().secret_key, + key=cast("bytes", Config.get_instance().secret_key), ), max_age=60 * 60 * 24 * 7, expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"), @@ -280,9 +283,9 @@ class HttpAPI: return auth_handler @staticmethod - def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable: + def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware: @web.middleware - async def middleware_handler(request: Request, handler: RequestHandler) -> Response: + async def middleware_handler(request: Request, handler: Handler) -> StreamResponse: static_path = str(app.router["download_static"].url_for(filename="")) if request.path.startswith(static_path): realFile, status = get_file( @@ -301,12 +304,13 @@ class HttpAPI: }, ) - response: Response = await handler(request) + response: StreamResponse = await handler(request) contentType: str = str(response.headers.get("content-type", "")) if contentType.startswith("text/html") and getattr(response, "_path", None): rewrite_path: str = base_path.rstrip("/") - async with await anyio.open_file(response._path, "rb") as f: + response_path = cast("Any", response)._path + async with await anyio.open_file(response_path, "rb") as f: content = await f.read() content: str = ( content.decode("utf-8") @@ -339,7 +343,7 @@ class HttpAPI: return middleware_handler - async def _handle(self, handler: RequestHandler, request: Request) -> Response: + async def _handle(self, handler: Handler, request: Request) -> StreamResponse: """ Call the handler with the request and return the response. diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index aed12564..127af2df 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -104,19 +104,20 @@ class HttpSocket: } ) - self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit") + self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit") @staticmethod - def ws_event(func): # type: ignore + def ws_event(func): """ Decorator to mark a method as a socket event. """ - @functools.wraps(func) # type: ignore + @functools.wraps(func) async def wrapper(*args, **kwargs): - return await func(*args, **kwargs) # type: ignore + return await func(*args, **kwargs) - wrapper._ws_event = func.__name__ # type: ignore + wrapper_with_event: Any = wrapper + wrapper_with_event._ws_event = func.__name__ return wrapper async def on_shutdown(self, _: web.Application): @@ -134,9 +135,11 @@ class HttpSocket: if not (data and data.data): return - await Services.get_instance().get("queue").add(item=Item.format(data.data)) + queue = Services.get_instance().get("queue") + if queue is not None: + await queue.add(item=Item.format(data.data)) - self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add") + self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add") load_modules(self.rootPath, self.rootPath / "routes" / "socket") diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index e768d66c..cbb522f0 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -93,7 +93,7 @@ class Item: bool: True if the item has extras, False otherwise. """ - return self.extras and len(self.extras) > 0 + return bool(self.extras and len(self.extras) > 0) def add_extras(self, key: str, value: Any) -> None: """ @@ -117,7 +117,7 @@ class Item: bool: True if the item has command options for yt, False otherwise. """ - return self.cli and len(self.cli) > 2 + return bool(self.cli and len(self.cli) > 2) @staticmethod def _default_preset() -> str: @@ -161,7 +161,7 @@ class Item: Item: The formatted item. """ - url: str = item.get("url") + url = item.get("url") if not url or not isinstance(url, str): msg = "url param is required." @@ -173,7 +173,7 @@ class Item: if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url): url = f"https://www.youtube.com/watch?v={url}" - data: dict[str, str] = {"url": url} + data: dict[str, Any] = {"url": url} preset: str | None = item.get("preset") if preset and isinstance(preset, str) and preset != Item._default_preset(): @@ -185,24 +185,28 @@ class Item: data["preset"] = preset - if item.get("folder") and isinstance(item.get("folder"), str): - data["folder"] = item.get("folder") + folder = item.get("folder") + if isinstance(folder, str) and folder: + data["folder"] = folder - if item.get("cookies") and isinstance(item.get("cookies"), str): - data["cookies"] = item.get("cookies") + cookies = item.get("cookies") + if isinstance(cookies, str) and cookies: + data["cookies"] = cookies - if item.get("template") and isinstance(item.get("template"), str): - data["template"] = item.get("template") + template = item.get("template") + if isinstance(template, str) and template: + data["template"] = template - if "auto_start" in item and isinstance(item.get("auto_start"), bool): - data["auto_start"] = bool(item.get("auto_start")) + if isinstance(item.get("auto_start"), bool): + data["auto_start"] = item["auto_start"] extras: dict | None = item.get("extras") - if extras and isinstance(extras, dict) and len(extras) > 0: + if isinstance(extras, dict) and len(extras) > 0: data["extras"] = extras - if item.get("requeued") and isinstance(item.get("requeued"), bool): - data["requeued"] = item.get("requeued") + requeued = item.get("requeued") + if isinstance(requeued, bool): + data["requeued"] = requeued cli: str | None = item.get("cli") if cli and len(cli) > 2: @@ -574,8 +578,10 @@ class ItemDTO: self.is_archivable = bool(self._archive_file) if not self.is_archivable: self.is_archived = False - else: + elif self.archive_id and self._archive_file: self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0 + else: + self.is_archived = False def get_archive_file(self) -> str | None: """ diff --git a/app/library/PackageInstaller.py b/app/library/PackageInstaller.py index 2008a8b6..84ed62e0 100644 --- a/app/library/PackageInstaller.py +++ b/app/library/PackageInstaller.py @@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]: class Packages: - def __init__(self, env: str | None, file: str | None, upgrade: bool = False): + def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False): from_env: list[str] = env.split() if env else [] from_file = [] diff --git a/app/library/Scheduler.py b/app/library/Scheduler.py index 947697d6..a579f3f8 100644 --- a/app/library/Scheduler.py +++ b/app/library/Scheduler.py @@ -1,4 +1,6 @@ import asyncio +from collections.abc import Callable +from typing import Any from aiocron import Cron from aiohttp import web @@ -66,7 +68,7 @@ class Scheduler(metaclass=Singleton): if data and data.data: self.add(**data.data) - EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add") + EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add") def get_all(self) -> dict[str, Cron]: """Return the jobs.""" @@ -99,7 +101,7 @@ class Scheduler(metaclass=Singleton): return id in self._jobs def add( - self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None + self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None ) -> str: """ Add a job to the schedule. diff --git a/app/library/Services.py b/app/library/Services.py index d67d9d9c..cd49e6f7 100644 --- a/app/library/Services.py +++ b/app/library/Services.py @@ -1,4 +1,5 @@ import inspect +from collections.abc import Callable from dataclasses import dataclass from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints @@ -117,7 +118,7 @@ class Services(metaclass=Singleton): return candidates[0] - def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]: + def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]: sig: inspect.Signature = inspect.signature(handler) try: @@ -169,10 +170,10 @@ class Services(metaclass=Singleton): return resolved - async def handle_async(self, handler: callable, **kwargs) -> Any: + async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any: resolved: dict[str, Any] = self._build_call_args(handler, kwargs) return await handler(**resolved) - def handle_sync(self, handler: callable, **kwargs) -> Any: + def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any: resolved: dict[str, Any] = self._build_call_args(handler, kwargs) return handler(**resolved) diff --git a/app/library/TerminalSessionManager.py b/app/library/TerminalSessionManager.py index 8ae5c783..f6901702 100644 --- a/app/library/TerminalSessionManager.py +++ b/app/library/TerminalSessionManager.py @@ -75,7 +75,7 @@ class TerminalSessionManager(metaclass=Singleton): self._cleanup_job_id = Scheduler.get_instance().add( timer=CLEANUP_SCHEDULE, func=self.cleanup, - id=f"{__class__.__name__}.{__class__.cleanup.__name__}", + id=f"{type(self).__name__}.{type(self).cleanup.__name__}", ) async def on_startup(self, _: web.Application) -> None: diff --git a/app/library/UpdateChecker.py b/app/library/UpdateChecker.py index 06ac6308..825a3d3b 100644 --- a/app/library/UpdateChecker.py +++ b/app/library/UpdateChecker.py @@ -80,7 +80,7 @@ class UpdateChecker(metaclass=Singleton): self._notify.subscribe( event=Events.STARTED, callback=event_handler, - name=f"{__class__.__name__}.{__class__.attach.__name__}", + name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}", ) self._schedule_check() @@ -108,7 +108,7 @@ class UpdateChecker(metaclass=Singleton): self._job_id = self._scheduler.add( timer=timer, func=lambda: asyncio.create_task(self.check_for_updates()), - id=f"{__class__.__name__}.{self.check_for_updates.__name__}", + id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}", ) async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]: diff --git a/app/library/Utils.py b/app/library/Utils.py index 1a753445..c8c1c28a 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -40,7 +40,8 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c") class FileLogFormatter(logging.Formatter): - def formatTime(self, record, datefmt=None): # noqa: ARG002, N802 + def formatTime(self, record, datefmt=None): # noqa: N802 + _ = datefmt return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") @@ -78,7 +79,8 @@ class JsonLogFormatter(logging.Formatter): return json.dumps(data, ensure_ascii=False, default=str) - def formatTime(self, record, datefmt=None): # noqa: ARG002, N802 + def formatTime(self, record, datefmt=None): # noqa: N802 + _ = datefmt return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") @staticmethod @@ -210,7 +212,7 @@ class JsonLogFormatter(logging.Formatter): return source -def timed_lru_cache(ttl_seconds: int, max_size: int = 128): +def timed_lru_cache(ttl_seconds: float, max_size: int = 128): """ Decorator that applies an LRU cache with a time-to-live (TTL) to a function. Supports both synchronous and asynchronous functions. @@ -280,8 +282,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128): return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache)) - async_wrapper.cache_clear = cache_clear - async_wrapper.cache_info = cache_info + async_wrapper_cache: Any = async_wrapper + async_wrapper_cache.cache_clear = cache_clear + async_wrapper_cache.cache_info = cache_info return async_wrapper # For sync functions, use the original implementation @@ -309,8 +312,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128): return result # expose cache_clear, cache_info - sync_wrapper.cache_clear = cached.cache_clear - sync_wrapper.cache_info = cached.cache_info + sync_wrapper_cache: Any = sync_wrapper + sync_wrapper_cache.cache_clear = cached.cache_clear + sync_wrapper_cache.cache_info = cached.cache_info return sync_wrapper return decorator @@ -381,7 +385,12 @@ def _is_safe_key(key: Any) -> bool: def merge_dict( - source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None + source: dict, + destination: dict, + max_depth: int = 50, + max_list_size: int = 10000, + _depth: int = 0, + _seen: set | None = None, ) -> dict: """ Merge data from source into destination. @@ -472,7 +481,7 @@ def merge_dict( return destination_copy -def check_id(file: Path) -> bool | str: +def check_id(file: Path) -> bool | Path: """ Check if we are able to get an id from the file name. if so check if any video file with the same id exists. @@ -491,6 +500,8 @@ def check_id(file: Path) -> bool | str: return False id: str | None = match.groupdict().get("id") + if id is None: + return False try: if not file.parent.exists(): @@ -545,9 +556,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool: from yarl import URL parsed_url = URL(url) - except ValueError: + except ValueError as exc: msg = "Invalid URL." - raise ValueError(msg) # noqa: B904 + raise ValueError(msg) from exc # Check allowed schemes if parsed_url.scheme not in ["http", "https"]: @@ -596,7 +607,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool: @timed_lru_cache(ttl_seconds=60, max_size=256) -def get_file_sidecar(file: Path | None = None) -> dict[dict]: +def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]: """ Get sidecar files for the given file. @@ -925,7 +936,7 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]: download_path = Path(download_path) try: - realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False)) + realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False)) if realFile.exists(): return (realFile, 200) except Exception as e: @@ -936,11 +947,11 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]: ) return (Path(file), 404) - possibleFile: bool | str = check_id(file=realFile) - if not possibleFile: + possibleFile: bool | Path = check_id(file=realFile) + if not isinstance(possibleFile, Path): return (realFile, 404) - return (Path(possibleFile), 302) + return (possibleFile, 302) def encrypt_data(data: str, key: bytes) -> str: @@ -962,7 +973,7 @@ def encrypt_data(data: str, key: bytes) -> str: return base64.urlsafe_b64encode(iv + ciphertext + tag).decode() -def decrypt_data(data: str, key: bytes) -> str: +def decrypt_data(data: str, key: bytes) -> str | None: """ Decrypts AES-GCM encrypted data @@ -975,8 +986,8 @@ def decrypt_data(data: str, key: bytes) -> str: """ try: - data = base64.urlsafe_b64decode(data) - iv, ciphertext, tag = data[:12], data[12:-16], data[-16:] + raw: bytes = base64.urlsafe_b64decode(data) + iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:] cipher = AES.new(key, AES.MODE_GCM, nonce=iv) plaintext = cipher.decrypt_and_verify(ciphertext, tag) return plaintext.decode() @@ -987,7 +998,7 @@ def decrypt_data(data: str, key: bytes) -> str: def get( data: dict | list, path: str | list | None = None, - default: any = None, + default: Any = None, separator=".", ): """ @@ -1111,7 +1122,7 @@ def get_files( extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__}, exc_info=True, ) - return [] + return [], 0 try: dir_path.relative_to(base_path) @@ -1122,7 +1133,7 @@ def get_files( base_path, extra={"path": str(dir_path), "base_path": str(base_path)}, ) - return [] + return [], 0 if not str(dir_path).startswith(str(base_path)): LOG.warning( @@ -1131,7 +1142,7 @@ def get_files( base_path, extra={"path": str(dir_path), "base_path": str(base_path)}, ) - return [] + return [], 0 if not dir_path.is_dir(): LOG.warning( @@ -1139,7 +1150,7 @@ def get_files( dir_path, extra={"path": str(dir_path), "base_path": str(base_path)}, ) - return [] + return [], 0 contents: list = [] for file in dir_path.iterdir(): @@ -1298,7 +1309,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]: try: from http.cookiejar import MozillaCookieJar - cookies = MozillaCookieJar(str(file), None, None) + cookies = MozillaCookieJar(str(file), delayload=False, policy=None) cookies.load() return (True, cookies) diff --git a/app/library/ag_utils.py b/app/library/ag_utils.py index 796b875c..91116c2e 100644 --- a/app/library/ag_utils.py +++ b/app/library/ag_utils.py @@ -1,6 +1,20 @@ +import sys from typing import Any +def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]: + if key not in data or data[key] is None: + return False, None + return True, data[key] + + +def _dict_delete(data: dict[Any, Any], key: Any) -> bool: + if key not in data: + return False + del data[key] + return True + + def get_value(value: Any) -> Any: """ Return the result of calling `value` if it is callable, else return `value`. @@ -91,14 +105,19 @@ def ag( return val return get_value(default) key = path - if isinstance(array, dict) and key in array and array[key] is not None: - return array[key] + if isinstance(array, dict): + found, value = _dict_value(array, key) + if found: + return value if not isinstance(key, str) or separator not in key: return get_value(default) current = array for segment in key.split(separator): - if isinstance(current, dict) and segment in current and current[segment] is not None: - current = current[segment] + if isinstance(current, dict): + found, value = _dict_value(current, segment) + if not found: + return get_value(default) + current = value elif isinstance(current, list): try: idx = int(segment) @@ -150,7 +169,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa except TypeError: return False if isinstance(data, dict): - if path in data and data[path] is not None: + found, _ = _dict_value(data, path) + if found: return True elif isinstance(data, list) and isinstance(path, int): return 0 <= path < len(data) and data[path] is not None @@ -158,8 +178,11 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa segments = path_str.split(separator) current = data for seg in segments: - if isinstance(current, dict) and seg in current and current[seg] is not None: - current = current[seg] + if isinstance(current, dict): + found, value = _dict_value(current, seg) + if not found: + return False + current = value elif isinstance(current, list): try: idx = int(seg) @@ -189,17 +212,16 @@ def ag_delete( The modified structure. """ - if isinstance(path, list | tuple): - for p in path: - ag_delete(data, p, separator) - return data if not isinstance(data, dict | list): try: data = vars(data) except TypeError: return data # type: ignore - if isinstance(data, dict) and path in data: - del data[path] + if isinstance(path, list | tuple): + for p in path: + ag_delete(data, p, separator) + return data + if isinstance(data, dict) and _dict_delete(data, path): return data if isinstance(data, list) and isinstance(path, int): if 0 <= path < len(data): @@ -210,8 +232,12 @@ def ag_delete( last = segments.pop() current = data for seg in segments: - if isinstance(current, dict) and seg in current: - current = current[seg] + if isinstance(current, dict): + found, value = _dict_value(current, seg) + if not found: + current = None + break + current = value elif isinstance(current, list): try: idx = int(seg) @@ -228,8 +254,8 @@ def ag_delete( break if current is None: return data - if isinstance(current, dict) and last in current: - del current[last] + if isinstance(current, dict): + _dict_delete(current, last) elif isinstance(current, list): try: idx = int(last) @@ -246,10 +272,12 @@ def run_tests(): assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}} def test_ag_set_overwrite_error(): + error = "" try: ag_set({"a": 1}, "a.b", 2) except RuntimeError as e: - assert "Cannot set value at path" in str(e) # noqa: PT017 + error = str(e) + assert "Cannot set value at path" in error def test_ag_get_value_callable_and_value(): assert get_value(5) == 5 @@ -320,9 +348,9 @@ def run_tests(): ]: try: test() - print(f"PASS: {test.__name__}") # noqa: T201 + sys.stdout.write(f"PASS: {test.__name__}\n") except AssertionError: - print(f"FAIL {test.__name__}") # noqa: T201 + sys.stdout.write(f"FAIL {test.__name__}\n") if __name__ == "__main__": diff --git a/app/library/cache.py b/app/library/cache.py index 84104570..4ad54c5a 100644 --- a/app/library/cache.py +++ b/app/library/cache.py @@ -38,7 +38,7 @@ class Cache(metaclass=ThreadSafe): Scheduler.get_instance().add( timer="* * * * *", func=self.cleanup, - id=f"{__class__.__name__}.{__class__.cleanup.__name__}", + id=f"{type(self).__name__}.{type(self).cleanup.__name__}", ) def set(self, key: str, value: Any, ttl: float | None = None) -> None: diff --git a/app/library/cf_solver_handler.py b/app/library/cf_solver_handler.py index fd2bca9e..198175c4 100644 --- a/app/library/cf_solver_handler.py +++ b/app/library/cf_solver_handler.py @@ -3,7 +3,7 @@ from __future__ import annotations import http.cookiejar from abc import ABC from collections.abc import Callable -from typing import Any, ClassVar +from typing import Any, ClassVar, cast from yt_dlp.networking.common import ( _REQUEST_HANDLERS, @@ -223,11 +223,13 @@ class CFSolverRH(RequestHandler, ABC): try: response: Response = director.send(request) except HTTPError as error: - if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers): + if error.response and is_cf_challenge( + getattr(error.response, "status", None), cast("Any", error.response.headers) + ): return self._retry_with_clearance(request, error.response, director) raise - if is_cf_challenge(getattr(response, "status", None), response.headers): + if is_cf_challenge(getattr(response, "status", None), cast("Any", response.headers)): return self._retry_with_clearance(request, response, director) return response diff --git a/app/library/cf_solver_shared.py b/app/library/cf_solver_shared.py index 66797887..965b4c59 100644 --- a/app/library/cf_solver_shared.py +++ b/app/library/cf_solver_shared.py @@ -1,16 +1,18 @@ -# flake8: noqa: S310 from __future__ import annotations import json import time import urllib.request -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse from app.library.log import get_logger from .cache import Cache +if TYPE_CHECKING: + from collections.abc import Mapping + CACHE: Cache = Cache() LOG = get_logger() @@ -57,7 +59,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d if user_agent: payload.setdefault("headers", {})["User-Agent"] = user_agent - req = urllib.request.Request( + req = urllib.request.Request( # noqa: S310 endpoint, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, @@ -68,7 +70,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d "Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint} ) start_time = time.time() - with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: + with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: # noqa: S310 result = json.loads(resp.read().decode("utf-8")) if "ok" != result.get("status"): @@ -98,7 +100,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d return CACHE.get(host) -def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool: +def is_cf_challenge(status: int | None, headers: Mapping[str, Any] | None) -> bool: """ Determine whether a response indicates a Cloudflare challenge. """ diff --git a/app/library/config.py b/app/library/config.py index 5b995f2d..8ca81398 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -2,6 +2,7 @@ import logging import multiprocessing import os import re +import shutil import sys import time from logging.handlers import TimedRotatingFileHandler @@ -30,7 +31,7 @@ APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = ( if TYPE_CHECKING: from subprocess import CompletedProcess -SUPPORTED_CODECS: tuple[str] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264") +SUPPORTED_CODECS: tuple[str, ...] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264") "Supported encoder names in order of preference." @@ -179,7 +180,7 @@ class Config(metaclass=Singleton): file_logging: bool = True "Enable file logging." - secret_key: str + secret_key: str | bytes "The secret key to use for the application." tasks_handler_timer: str = "15 */1 * * *" @@ -377,7 +378,7 @@ class Config(metaclass=Singleton): LOG = get_logger() self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config") - envFile: str = Path(self.config_path) / ".env" + envFile: Path = Path(self.config_path) / ".env" if envFile.exists(): LOG.info("Loading environment variables from '%s'.", envFile) @@ -388,7 +389,7 @@ class Config(metaclass=Singleton): self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str( Path(baseDefaultPath) / "var" / "downloads" ) - self.app_path = Path(__file__).parent.parent.absolute() + self.app_path = str(Path(__file__).parent.parent.absolute()) for k, v in self._get_attributes().items(): if k.startswith("_") or k in self._manual_vars: @@ -501,7 +502,7 @@ class Config(metaclass=Singleton): handler.setFormatter(formatter) logging.getLogger().addHandler(handler) - key_file: str = Path(self.config_path) / "secret.key" + key_file: Path = Path(self.config_path) / "secret.key" if key_file.exists() and key_file.stat().st_size > 2: with open(key_file, "rb") as f: @@ -511,7 +512,7 @@ class Config(metaclass=Singleton): with open(key_file, "wb") as f: f.write(self.secret_key) - self.started = time.time() + self.started = int(time.time()) for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS: logging.getLogger(_tool).setLevel(_level) @@ -545,7 +546,7 @@ class Config(metaclass=Singleton): def _get_attributes(self) -> dict: attrs: dict = {} - vClass: str = self.__class__ + vClass: type = self.__class__ for attribute in vClass.__dict__: if attribute.startswith("_"): @@ -601,7 +602,7 @@ class Config(metaclass=Singleton): dict[str, str]: The replacer variables. """ - keys: tuple[str] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file") + keys: tuple[str, ...] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file") return {k: getattr(self, k) for k in keys} @staticmethod @@ -619,15 +620,20 @@ class Config(metaclass=Singleton): This is used to set the version to the latest git tag. """ LOG = get_logger() - git_path: str = Path(__file__).parent / ".." / ".." / ".git" + git_path: Path = Path(__file__).parent / ".." / ".." / ".git" if not git_path.exists(): return try: import subprocess + git = shutil.which("git") + if not git: + LOG.warning("Git executable was not found.") + return + branch_result: CompletedProcess[str] = subprocess.run( - ["git", "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607 + [git, "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], cwd=os.path.dirname(git_path), capture_output=True, text=True, @@ -645,7 +651,7 @@ class Config(metaclass=Singleton): return commit_result: CompletedProcess[str] = subprocess.run( - ["git", "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], # noqa: S607 + [git, "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], cwd=os.path.dirname(git_path), capture_output=True, text=True, diff --git a/app/library/downloads/core.py b/app/library/downloads/core.py index cdea25cf..46b02db5 100644 --- a/app/library/downloads/core.py +++ b/app/library/downloads/core.py @@ -39,6 +39,14 @@ if TYPE_CHECKING: class Download: update_task: asyncio.Task | None = None + @staticmethod + def _raise_no_formats(msg: str) -> None: + raise ValueError(msg) + + @staticmethod + def _raise_interrupted() -> None: + raise SystemExit(130) + def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None): """ Initialize download task. @@ -51,7 +59,7 @@ class Download: """ config: Config = Config.get_instance() - self.download_dir: str = info.download_dir + self.download_dir: str = info.download_dir or "" self.temp_dir: str | None = info.temp_dir self.template: str | None = info.template self.template_chapter: str | None = info.template_chapter @@ -98,6 +106,16 @@ class Download: params: dict[str, Any] = {} download_skipped = False + hook_handlers = self._hook_handlers + if hook_handlers is None: + msg = "_hook_handlers must be initialized before _download(). Call start() first." + raise RuntimeError(msg) + + status_queue = self.status_queue + if status_queue is None: + msg = "status_queue must be initialized before _download(). Call start() first." + raise RuntimeError(msg) + try: params = ( self.info.get_ytdlp_opts() @@ -125,9 +143,9 @@ class Download: params.update( { - "progress_hooks": [self._hook_handlers.progress_hook], - "postprocessor_hooks": [self._hook_handlers.postprocessor_hook], - "post_hooks": [self._hook_handlers.post_hook], + "progress_hooks": [hook_handlers.progress_hook], + "postprocessor_hooks": [hook_handlers.postprocessor_hook], + "post_hooks": [hook_handlers.post_hook], } ) @@ -137,7 +155,10 @@ class Download: if self.info.cookies: try: - cookie_file = Path(self._temp_manager.temp_path or self.temp_dir) / f"cookie_{self.info._id}.txt" + cookie_file = ( + Path(self._temp_manager.temp_path or self.temp_dir or self.download_dir) + / f"cookie_{self.info._id}.txt" + ) self.logger.debug( f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.", extra={ @@ -173,7 +194,9 @@ class Download: f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info." ) self.info_dict = None - elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default: + elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and ( + (preset := self.info.get_preset()) is not None and preset.default + ): self.logger.debug( "Removing download archive option for generic extractor on '%s'.", self.info.url, @@ -267,7 +290,7 @@ class Download: if filtered_logs := extract_ytdlp_logs(self.logs): msg += " " + ", ".join(filtered_logs) - raise ValueError(msg) # noqa: TRY301 + self._raise_no_formats(msg) self.logger.info( f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.", @@ -327,11 +350,11 @@ class Download: def mark_cancelled(*_) -> None: cls._interrupted = True cls.to_screen("[info] Interrupt received, exiting cleanly...") - raise SystemExit(130) # noqa: TRY301 + self._raise_interrupted() signal.signal(signal.SIGUSR1, mark_cancelled) - self.status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped}) + status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped}) if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: self.logger.debug( @@ -386,7 +409,7 @@ class Download: ) ret = cls.download(url_list=[self.info.url]) - self.status_queue.put( + status_queue.put( { "id": self.id, "status": "finished" if 0 == ret else "error", @@ -407,7 +430,7 @@ class Download: } }, ) - self.status_queue.put( + status_queue.put( { "id": self.id, "status": "skip", @@ -432,7 +455,7 @@ class Download: } }, ) - self.status_queue.put( + status_queue.put( { "id": self.id, "status": "error", @@ -442,7 +465,7 @@ class Download: } ) finally: - self.status_queue.put(Terminator()) + status_queue.put(Terminator()) if cookie_file and cookie_file.exists(): try: cookie_file.unlink() @@ -499,24 +522,24 @@ class Download: Process exit code or None """ - self.status_queue = Config.get_manager().Queue() + status_queue: Any = Config.get_manager().Queue() + self.status_queue = status_queue - if temp_path := self._temp_manager.create_temp_path(): - self.info.temp_path = str(temp_path) + temp_path = self._temp_manager.create_temp_path() self._status_tracker = StatusTracker( info=self.info, download_id=self.id, download_dir=self.download_dir, temp_path=temp_path, - status_queue=self.status_queue, + status_queue=status_queue, logger=self.logger, debug=self.debug, ) self._hook_handlers = HookHandlers( download_id=self.id, - status_queue=self.status_queue, + status_queue=status_queue, logger=self.logger, debug=self.debug, ) @@ -529,7 +552,8 @@ class Download: EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info) asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}") - ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join) + proc = self._process_manager.proc + ret = await asyncio.get_running_loop().run_in_executor(None, proc.join) if proc else None if self._status_tracker.final_update: return ret diff --git a/app/library/downloads/item_adder.py b/app/library/downloads/item_adder.py index 4eb1b683..d83728c2 100644 --- a/app/library/downloads/item_adder.py +++ b/app/library/downloads/item_adder.py @@ -4,7 +4,7 @@ import time import uuid from numbers import Number from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import yt_dlp.utils @@ -67,7 +67,7 @@ async def add_item( already=None, logs: list | None = None, yt_params: dict | None = None, -) -> dict[str, str]: +) -> dict[str, Any]: """ Route an entry to the appropriate processor based on type. @@ -102,7 +102,7 @@ async def add_item( async def add( queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None -) -> dict[str, str]: +) -> dict[str, Any]: """ Add an item to the download queue. @@ -204,7 +204,7 @@ async def add( ) if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"): - dlInfo.info.msg += f" Found in archive '{archive_file}'." + dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{archive_file}'." await queue.done.put(dlInfo) @@ -306,9 +306,12 @@ async def add( new_archive_id: str | None = None if entry.get("extractor_key") and entry.get("id"): - new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}" - if new_archive_id != archive_id: - extra_ids.append(new_archive_id) + _extractor_key = entry.get("extractor_key") + _entry_id = entry.get("id") + if isinstance(_extractor_key, str) and _entry_id is not None: + new_archive_id = f"{_extractor_key.lower()} {_entry_id!s}" + if new_archive_id != archive_id: + extra_ids.append(new_archive_id) if len(extra_ids) > 0: archive_ids: list[str] = archive_read(_archive_file, extra_ids) @@ -336,7 +339,7 @@ async def add( ) ) - dlInfo.info.msg += f" Found in archive '{_archive_file}'." + dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{_archive_file}'." await queue.done.put(dlInfo) queue._notify.emit( @@ -374,7 +377,7 @@ async def add( if condition.extras.get("ignore_download", False): extra_msg: str = "" - if _archive_file and not condition.extras.get("no_archive", False): + if _archive_file and not condition.extras.get("no_archive", False) and archive_id: archive_add(_archive_file, [archive_id]) extra_msg = f" and added to archive file '{_archive_file}'" @@ -385,7 +388,7 @@ async def add( if not store_type: dlInfo = Download( info=ItemDTO( - id=entry.get("id"), + id=str(entry.get("id") or item.url), title=_name, url=item.url, preset=item.preset, @@ -401,6 +404,9 @@ async def add( LOG.info(log_message) queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message) + if dlInfo is None: + msg = "Download info is not available." + raise RuntimeError(msg) queue._notify.emit( Events.ITEM_MOVED, data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, diff --git a/app/library/downloads/playlist_processor.py b/app/library/downloads/playlist_processor.py index 4a8a0dfa..3c7af995 100644 --- a/app/library/downloads/playlist_processor.py +++ b/app/library/downloads/playlist_processor.py @@ -121,8 +121,8 @@ async def process_playlist( max_downloads: int = -1 ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all() - if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int): - max_downloads: int = ytdlp_opts.get("max_downloads") + if isinstance(ytdlp_opts.get("max_downloads"), int): + max_downloads = ytdlp_opts["max_downloads"] results: list[dict[str, str]] = [] for i, etr in enumerate(entries, start=1): diff --git a/app/library/downloads/process_manager.py b/app/library/downloads/process_manager.py index 851fe181..e6b06789 100644 --- a/app/library/downloads/process_manager.py +++ b/app/library/downloads/process_manager.py @@ -99,15 +99,19 @@ class ProcessManager: if not self.running(): return False - procId: int | None = self.proc.ident + proc = self.proc + if proc is None: + return False + + procId: int | None = proc.ident try: self.logger.info( "Stopping download process PID=%s.", - self.proc.pid, + proc.pid, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "process_ident": procId, "is_live": self.is_live, } @@ -117,25 +121,25 @@ class ProcessManager: if self.is_live: self.logger.debug( "Requesting graceful live cancellation for PID=%s.", - self.proc.pid, + proc.pid, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "is_live": self.is_live, } }, ) self.cancel_event.set() - if wait_for_process_with_timeout(self.proc, 10): + if wait_for_process_with_timeout(proc, 10): self.logger.debug( "Download process PID=%s stopped gracefully.", - self.proc.pid, + proc.pid, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "is_live": self.is_live, } }, @@ -143,43 +147,43 @@ class ProcessManager: return True self.logger.warning( "Download process PID=%s did not respond to live cancellation; forcing termination.", - self.proc.pid, + proc.pid, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "is_live": self.is_live, "force": True, } }, ) - elif self.proc.pid and "posix" == os.name: + elif proc.pid and "posix" == os.name: signal_name = "SIGUSR1" try: self.logger.debug( "Sending %s signal to download process PID=%s.", signal_name, - self.proc.pid, + proc.pid, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "signal": signal_name, } }, ) - os.kill(self.proc.pid, signal.SIGUSR1) + os.kill(proc.pid, signal.SIGUSR1) - if wait_for_process_with_timeout(self.proc, 5): + if wait_for_process_with_timeout(proc, 5): self.logger.debug( "Download process PID=%s stopped gracefully.", - self.proc.pid, + proc.pid, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "signal": signal_name, } }, @@ -187,12 +191,12 @@ class ProcessManager: return True self.logger.warning( "Download process PID=%s did not respond to %s; forcing termination.", - self.proc.pid, + proc.pid, signal_name, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "signal": signal_name, "force": True, } @@ -203,51 +207,49 @@ class ProcessManager: self.logger.debug( "Failed to send %s signal to download process PID=%s. %s", signal_name, - self.proc.pid, + proc.pid, e, extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid, "signal": signal_name, "exception_type": type(e).__name__, } }, ) - if self.proc.is_alive(): + if proc.is_alive(): self.logger.info( "Terminating download process PID=%s.", - self.proc.pid, - extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}}, + proc.pid, + extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}}, ) - self.proc.terminate() + proc.terminate() - if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2): + if not wait_for_process_with_timeout(proc, 1 if self.is_live else 2): self.logger.warning( "Download process PID=%s did not terminate; killing forcefully.", - self.proc.pid, - extra={ - "download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True} - }, + proc.pid, + extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}}, ) - self.proc.kill() - wait_for_process_with_timeout(self.proc, 1) + proc.kill() + wait_for_process_with_timeout(proc, 1) self.logger.info( "Download process PID=%s stopped.", - self.proc.pid, - extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid}}, + proc.pid, + extra={"download": {"download_id": self.download_id, "process_id": proc.pid}}, ) return True except Exception as e: self.logger.exception( - f"Failed to stop download process PID={self.proc.pid}, ident={procId}.", + f"Failed to stop download process PID={proc.pid if proc else None}, ident={procId}.", extra={ "download": { "download_id": self.download_id, - "process_id": self.proc.pid, + "process_id": proc.pid if proc else None, "process_ident": procId, "is_live": self.is_live, "exception_type": type(e).__name__, @@ -271,11 +273,12 @@ class ProcessManager: return False self.cancel_in_progress = True - procId: int | None = self.proc.ident + proc = self.proc + procId: int | None = proc.ident if proc else None if not procId: - if self.proc: - self.proc.close() + if proc: + proc.close() self.proc = None self.logger.warning("Attempted to close download process, but it is not running.") return False @@ -293,13 +296,14 @@ class ProcessManager: loop = asyncio.get_running_loop() - if self.proc.is_alive(): + current = self.proc + if current is not None and current.is_alive(): self.logger.debug( "Waiting for download process PID='%s' to close.", procId, extra={"download": {"download_id": self.download_id, "process_ident": procId}}, ) - await loop.run_in_executor(None, self.proc.join) + await loop.run_in_executor(None, current.join) self.logger.debug( "Download process PID='%s' closed.", procId, diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py index 53fd3b22..af506550 100644 --- a/app/library/downloads/queue_manager.py +++ b/app/library/downloads/queue_manager.py @@ -53,7 +53,7 @@ class DownloadQueue(metaclass=Singleton): async def event_handler(_, __): await self.initialize() - self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}") + self._notify.subscribe(Events.STARTED, event_handler, f"{DownloadQueue.__name__}.initialize") Scheduler.get_instance().add( timer="* * * * *", @@ -689,7 +689,7 @@ class DownloadQueue(metaclass=Singleton): ) return {"deleted": deleted_count} - async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: + async def get(self, mode: str = "all") -> dict[str, dict[str, ItemDTO]]: """ Get the download queue and the download history. diff --git a/app/library/downloads/status_tracker.py b/app/library/downloads/status_tracker.py index 704a9a17..1f158085 100644 --- a/app/library/downloads/status_tracker.py +++ b/app/library/downloads/status_tracker.py @@ -55,7 +55,7 @@ class StatusTracker: self.final_update = False self._terminator_sent: bool = False self._candidate_filepath: Path | None = None - self.update_task: asyncio.Task | None = None + self.update_task: asyncio.Future[Any] | None = None self._last_progress_time: float = 0.0 self._pending_progress: bool = False self._progress_interval: float = 0.5 @@ -256,7 +256,7 @@ class StatusTracker: self.info.percent = status["downloaded_bytes"] / total * 100 except ZeroDivisionError: self.info.percent = 0 - self.info.total_bytes = total + self.info.total_bytes = int(total) self.info.speed = status.get("speed") self.info.eta = status.get("eta") @@ -280,8 +280,9 @@ class StatusTracker: """ while True: try: - self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get) - status = await self.update_task + update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get) + self.update_task = asyncio.ensure_future(update_task) + status = await update_task if status is None or isinstance(status, Terminator): self._flush_progress() return diff --git a/app/library/httpx_client.py b/app/library/httpx_client.py index 87e9ebc3..12c2d82c 100644 --- a/app/library/httpx_client.py +++ b/app/library/httpx_client.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import functools +import importlib.util import threading from dataclasses import dataclass from typing import Any, Literal, cast, overload @@ -98,12 +99,7 @@ def _normalize_proxy(proxy: str | dict[str, str] | None) -> str | None: @functools.lru_cache(maxsize=1) def _curl_available() -> bool: - try: - import httpx_curl_cffi # noqa: F401 - - return True - except Exception: - return False + return importlib.util.find_spec("httpx_curl_cffi") is not None def resolve_curl_transport(use_curl: bool = True) -> bool: @@ -121,7 +117,7 @@ def _build_async_curl_transport( from httpx_curl_cffi import AsyncCurlTransport return AsyncCurlTransport( - impersonate=curl_impersonate, + impersonate=cast("Any", curl_impersonate), default_headers=curl_default_headers, ) @@ -308,10 +304,7 @@ def get_async_client( curl_default_headers=curl_default_headers, ) client = httpx.AsyncClient( - transport=cast( - "httpx.AsyncBaseTransport", - _get_transport(enable_cf, is_async=True, transport=transport), - ), + transport=_get_transport(enable_cf, is_async=True, transport=transport), proxy=cast("Any", proxy), ) Globals.SHARED_ASYNC_CLIENTS[key] = client @@ -330,10 +323,7 @@ def get_sync_client( return Globals.SHARED_SYNC_CLIENTS[key] client = httpx.Client( - transport=cast( - "httpx.BaseTransport", - _get_transport(enable_cf, is_async=False, transport=None), - ), + transport=_get_transport(enable_cf, is_async=False, transport=None), proxy=cast("Any", proxy), ) Globals.SHARED_SYNC_CLIENTS[key] = client diff --git a/app/library/migrate.py b/app/library/migrate.py index decc9c3a..16bfc67f 100644 --- a/app/library/migrate.py +++ b/app/library/migrate.py @@ -1,4 +1,3 @@ -# flake8: noqa: S608 """ This module is a fork of the Caribou library (https://pypi.org/project/caribou/) modified to work for our specific use case. @@ -9,10 +8,11 @@ import datetime import glob import os.path import traceback -from collections.abc import AsyncIterator, Iterable, Sequence +from collections.abc import AsyncIterator, Iterable, Mapping, Sequence from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.util import module_from_spec, spec_from_loader -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Any from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncConnection @@ -44,7 +44,9 @@ class InvalidNameError(Error): @contextlib.asynccontextmanager -async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator: +async def execute( + conn: AsyncConnection, sql: str, params: Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None +) -> AsyncIterator: params = {} if params is None else params result = await conn.execute(text(sql), params) try: @@ -126,12 +128,12 @@ class Database: self.version_table: str = version_table - self._owns_connection = bool(isinstance(db_url, str)) - if self._owns_connection: + self._owns_connection = isinstance(db_url, str) + if isinstance(db_url, str): self.conn: AsyncConnection | None = None self.db_url: str = db_url else: - self.conn: AsyncConnection = db_url + self.conn = db_url async def __aenter__(self): if self._owns_connection and self.conn is None: @@ -183,7 +185,8 @@ class Database: async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None: assert self.conn if target_version not in (0, "0"): - _assert_migration_exists(migrations, target_version) + _assert_migration_exists(migrations, str(target_version)) + target = str(target_version) migrations.sort(key=lambda x: x.get_version(), reverse=True) database_version: str | None = await self.get_version() @@ -192,7 +195,7 @@ class Database: current_version: str = migration.get_version() if database_version is not None and current_version > database_version: continue - if current_version <= target_version: + if current_version <= target: break await migration.downgrade(self.conn) next_version: str | int = 0 @@ -210,14 +213,14 @@ class Database: assert self.conn if not await self.is_version_controlled(): return None - sql: str = f"SELECT version FROM {self.version_table}" + sql: str = f"SELECT version FROM {self.version_table}" # noqa: S608 async with execute(self.conn, sql) as result: rows = result.fetchall() return rows[0][0] if rows else "0" async def update_version(self, version: str) -> None: assert self.conn - sql: str = f"UPDATE {self.version_table} SET version = :version" + sql: str = f"UPDATE {self.version_table} SET version = :version" # noqa: S608 async with transaction(self.conn): await self.conn.execute(text(sql), {"version": version}) @@ -226,7 +229,7 @@ class Database: sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """ async with transaction(self.conn): await self.conn.execute(text(sql)) - await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) + await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) # noqa: S608 def __repr__(self) -> str: return f'Database("{self.db_url if self._owns_connection else "external_connection"}")' @@ -238,7 +241,7 @@ def _assert_migration_exists(migrations: Iterable[Migration], version: str | int raise Error(msg) -def load_migrations(directory: str) -> list[Migration]: +def load_migrations(directory: str | Path) -> list[Migration]: """Return the migrations contained in the given directory.""" directory = str(directory) if not os.path.exists(directory) or not os.path.isdir(directory): @@ -247,7 +250,7 @@ def load_migrations(directory: str) -> list[Migration]: return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))] -async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None: +async def upgrade(db_url: AsyncConnection | str, migration_dir: str | Path, version: str | None = None) -> None: """ Upgrade the given database with the migrations contained in the migrations directory. If a version is not specified, upgrade @@ -260,7 +263,7 @@ async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: st await db.upgrade(load_migrations(migration_dir), version) -async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None: +async def downgrade(db_url: str | AsyncConnection, migration_dir: str | Path, version: str) -> None: """ Downgrade the database to the given version with the migrations contained in the given migration directory. diff --git a/app/library/sqlite_store.py b/app/library/sqlite_store.py index ceaba09b..12a2f914 100644 --- a/app/library/sqlite_store.py +++ b/app/library/sqlite_store.py @@ -2,14 +2,16 @@ import asyncio import contextlib import json import os -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from dataclasses import fields from datetime import UTC, datetime from email.utils import formatdate +from typing import Any from urllib.parse import quote_plus from aiohttp import web from sqlalchemy import text +from sqlalchemy.engine.row import RowMapping from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.ext.asyncio.engine import AsyncConnection @@ -36,13 +38,13 @@ def _memory_db_url(db_path: str) -> str: return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true" -def _decode_row(row: dict) -> ItemDTO: - row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 +def _decode_row(row: Mapping[str, Any] | RowMapping) -> ItemDTO: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC) data = json.loads(row["data"]) data.pop("_id", None) item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) item._id = row["id"] - item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + item.datetime = formatdate(row_date.timestamp()) return item @@ -149,8 +151,8 @@ class SqliteStore(metaclass=ThreadSafe): return self._sessionmaker async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]: - await self.get_connection() - result = await self._conn.execute( + conn = await self.get_connection() + result = await conn.execute( text( 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC' ), @@ -163,12 +165,12 @@ class SqliteStore(metaclass=ThreadSafe): async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool: return await self.get(type_value, key=key, url=url) is not None - async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> bool: + async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> ItemDTO | None: if not key and not url: msg = "key or url must be provided." raise KeyError(msg) - await self.get_connection() + conn = await self.get_connection() clauses: list[str] = [] params: dict[str, str] = {"type_value": type_value} @@ -186,7 +188,7 @@ class SqliteStore(metaclass=ThreadSafe): f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608 ) - result = await self._conn.execute(text(query), params) + result = await conn.execute(text(query), params) row = result.mappings().first() if not row: @@ -195,8 +197,8 @@ class SqliteStore(metaclass=ThreadSafe): return _decode_row(row) async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None: - await self.get_connection() - result = await self._conn.execute( + conn = await self.get_connection() + result = await conn.execute( text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'), {"type_value": type_value, "id": id}, ) @@ -212,24 +214,24 @@ class SqliteStore(metaclass=ThreadSafe): if not ids_list: return [] - await self.get_connection() + conn = await self.get_connection() placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list))) params: dict[str, str] = {"type_value": type_value} params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)}) - result = await self._conn.execute( + result = await conn.execute( text( f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608 ), params, ) - rows = result.mappings().all() + rows = list(result.mappings().all()) order = {item_id: index for index, item_id in enumerate(ids_list)} rows.sort(key=lambda row: order.get(row["id"], len(ids_list))) return [(row["id"], _decode_row(row)) for row in rows] async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]: - await self.get_connection() + conn = await self.get_connection() params: dict[str, str] = {"type_value": type_value} where_clauses = ['"type" = :type_value'] @@ -239,7 +241,7 @@ class SqliteStore(metaclass=ThreadSafe): params.update(status_params) query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608 - result = await self._conn.execute(text(query), params) + result = await conn.execute(text(query), params) rows = result.mappings().all() return [(row["id"], _decode_row(row)) for row in rows] @@ -254,7 +256,7 @@ class SqliteStore(metaclass=ThreadSafe): if not kwargs: return None - await self.get_connection() + conn = await self.get_connection() clauses: list[str] = [] params: dict[str, str | float | int] = {"type_value": type_value} @@ -321,7 +323,7 @@ class SqliteStore(metaclass=ThreadSafe): where_clause: str = " OR ".join(f"({clause})" for clause in clauses) query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608 - result = await self._conn.execute(text(query), params) + result = await conn.execute(text(query), params) row = result.mappings().first() if not row: @@ -332,7 +334,7 @@ class SqliteStore(metaclass=ThreadSafe): return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None async def count(self, type_value: str, status_filter: str | None = None) -> int: - await self.get_connection() + conn = await self.get_connection() where_clauses: list[str] = ['"type" = :type_value'] params: dict[str, str] = {"type_value": type_value} @@ -344,7 +346,7 @@ class SqliteStore(metaclass=ThreadSafe): where_clause: str = " AND ".join(where_clauses) count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - result = await self._conn.execute(text(count_query), params) + result = await conn.execute(text(count_query), params) row = result.mappings().first() return row["count"] if row else 0 @@ -357,7 +359,7 @@ class SqliteStore(metaclass=ThreadSafe): order: str, status_filter: str | None = None, ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: - await self.get_connection() + conn = await self.get_connection() where_clauses: list[str] = ['"type" = :type_value'] params: dict[str, str | int] = {"type_value": type_value} @@ -369,7 +371,7 @@ class SqliteStore(metaclass=ThreadSafe): where_clause: str = " AND ".join(where_clauses) count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - result = await self._conn.execute(text(count_query), params) + result = await conn.execute(text(count_query), params) count_row = result.mappings().first() total_items = count_row["count"] if count_row else 0 @@ -384,7 +386,7 @@ class SqliteStore(metaclass=ThreadSafe): query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608 - result = await self._conn.execute(text(query), params) + result = await conn.execute(text(query), params) rows = result.mappings().all() items = [(row["id"], _decode_row(row)) for row in rows] @@ -396,15 +398,15 @@ class SqliteStore(metaclass=ThreadSafe): await self._upsert_now(type_value, item) async def delete(self, type_value: str, key: str) -> None: - await self.get_connection() - await self._conn.execute( + conn = await self.get_connection() + await conn.execute( text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), {"type_value": type_value, "key": key}, ) - await self._conn.commit() + await conn.commit() async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int: - await self.get_connection() + conn = await self.get_connection() keys_list: list[str] = list(keys) if not keys_list: return 0 @@ -412,15 +414,15 @@ class SqliteStore(metaclass=ThreadSafe): params: dict[str, str] = {"type_value": type_value} params.update({f"key_{i}": key for i, key in enumerate(keys_list)}) - result = await self._conn.execute( + result = await conn.execute( text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 params, ) - await self._conn.commit() + await conn.commit() return result.rowcount if result else 0 async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int: - await self.get_connection() + conn = await self.get_connection() params: dict[str, str] = {"type_value": type_value} where_clauses = ['"type" = :type_value'] @@ -429,11 +431,11 @@ class SqliteStore(metaclass=ThreadSafe): where_clauses.append(status_clause) params.update(status_params) - result = await self._conn.execute( + result = await conn.execute( text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608 params, ) - await self._conn.commit() + await conn.commit() return result.rowcount if result else 0 async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None: @@ -484,6 +486,9 @@ class SqliteStore(metaclass=ThreadSafe): async def _enqueue(self, op: _Op) -> None: self._ensure_worker() + if self._queue is None: + msg = "Queue was not initialized by _ensure_worker." + raise RuntimeError(msg) await self._queue.put(op) def _ensure_worker(self) -> None: @@ -492,10 +497,14 @@ class SqliteStore(metaclass=ThreadSafe): self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer") async def _writer(self) -> None: + queue = self._queue + if queue is None: + msg = "Writer started before queue was initialized." + raise RuntimeError(msg) while True: - op = await self._queue.get() + op = await queue.get() if isinstance(op.op, Terminator): - self._queue.task_done() + queue.task_done() break try: async with self._lock: @@ -507,34 +516,34 @@ class SqliteStore(metaclass=ThreadSafe): extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__}, ) finally: - self._queue.task_done() + queue.task_done() await asyncio.sleep(self._flush_interval) async def _apply(self, op: _Op) -> None: - await self.get_connection() + conn = await self.get_connection() if op.op == "upsert" and op.item: - await self._upsert_now_conn(self._conn, op.type_value, op.item) - await self._conn.commit() + await self._upsert_now_conn(conn, op.type_value, op.item) + await conn.commit() elif op.op == "delete" and op.key: - await self._conn.execute( + await conn.execute( text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), {"type_value": op.type_value, "key": op.key}, ) - await self._conn.commit() + await conn.commit() elif op.op == "bulk_delete" and op.keys: placeholders = ",".join(f":key_{i}" for i in range(len(op.keys))) params: dict[str, str] = {"type_value": op.type_value} params.update({f"key_{i}": key for i, key in enumerate(op.keys)}) - await self._conn.execute( + await conn.execute( text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 params, ) - await self._conn.commit() + await conn.commit() async def _upsert_now(self, type_value: str, item: ItemDTO) -> None: - await self.get_connection() - await self._upsert_now_conn(self._conn, type_value, item) - await self._conn.commit() + conn = await self.get_connection() + await self._upsert_now_conn(conn, type_value, item) + await conn.commit() async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None: """Helper to upsert using an existing connection.""" @@ -560,7 +569,7 @@ class SqliteStore(metaclass=ThreadSafe): async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None: """Execute raw SQL query (for testing purposes).""" - await self.get_connection() + conn = await self.get_connection() if isinstance(params, tuple): # Convert positional params to dict for SQLAlchemy # Assuming queries use ? placeholders, we need to count them @@ -575,16 +584,16 @@ class SqliteStore(metaclass=ThreadSafe): # Replace ? with :p0, :p1, etc. for i in range(len(params)): query = query.replace("?", f":p{i}", 1) - await self._conn.execute(text(query), param_dict) + await conn.execute(text(query), param_dict) elif isinstance(params, dict): - await self._conn.execute(text(query), params) + await conn.execute(text(query), params) else: - await self._conn.execute(text(query)) - await self._conn.commit() + await conn.execute(text(query)) + await conn.commit() async def fetch_raw(self, query: str, params: dict | tuple | None = None): """Fetch results from raw SQL query (for testing purposes).""" - await self.get_connection() + conn = await self.get_connection() if isinstance(params, tuple): # Convert positional params to dict for SQLAlchemy placeholders = query.count("?") @@ -596,11 +605,11 @@ class SqliteStore(metaclass=ThreadSafe): param_dict = {f"p{i}": params[i] for i in range(len(params))} for i in range(len(params)): query = query.replace("?", f":p{i}", 1) - result = await self._conn.execute(text(query), param_dict) + result = await conn.execute(text(query), param_dict) elif isinstance(params, dict): - result = await self._conn.execute(text(query), params) + result = await conn.execute(text(query), params) else: - result = await self._conn.execute(text(query)) + result = await conn.execute(text(query)) return result.mappings().all() async def get_connection(self) -> AsyncConnection: diff --git a/app/local.py b/app/local.py index f66d00fa..a75eafc2 100644 --- a/app/local.py +++ b/app/local.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 -# flake8: noqa: S310 T201 import argparse +import http.client import os import pathlib import socket import sys import time -import urllib.request import webbrowser +from urllib.parse import urlsplit if __name__ == "__main__": from multiprocessing import freeze_support @@ -26,7 +26,7 @@ APP_ROOT = str((pathlib.Path(__file__).parent / "..").resolve()) if APP_ROOT not in sys.path: sys.path.insert(0, APP_ROOT) -import platformdirs # type: ignore +import platformdirs def set_env(): @@ -55,18 +55,27 @@ def open_browser_when_ready(url: str, timeout: float = 5.0) -> None: import threading def wait_then_open(): + parsed = urlsplit(url) + path = parsed.path or "/" deadline = time.time() + timeout while time.time() < deadline: try: - with urllib.request.urlopen(url, timeout=1) as response: - if 200 == response.status: + conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 80, timeout=1) + try: + conn.request("GET", path) + response = conn.getresponse() + if response.status == 200: os.environ.pop("LD_LIBRARY_PATH", None) webbrowser.open_new(url) return + finally: + conn.close() except Exception: time.sleep(0.5) - print(f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.") + sys.stderr.write( + f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.\n" + ) threading.Thread(target=wait_then_open, daemon=True).start() @@ -115,7 +124,7 @@ def main(): set_env() - env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH")) / ".env" + env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH", "")) / ".env" port = None if env_file.exists(): diff --git a/app/main.py b/app/main.py index 4ba39dcd..6a02a9ea 100644 --- a/app/main.py +++ b/app/main.py @@ -199,18 +199,16 @@ class Main: ) HTTP_LOGGER_CLASS = HttpAccessLogger - run_args = { - "host": host, - "port": port, - "loop": asyncio.get_event_loop(), - "access_log": HTTP_LOGGER, - "print": started, - "handle_signals": cb is None, - } - if HTTP_LOGGER_CLASS is not None: - run_args["access_log_class"] = HTTP_LOGGER_CLASS - - web.run_app(self._app, **run_args) + web.run_app( + self._app, + host=host, + port=port, + loop=asyncio.get_event_loop(), + access_log=HTTP_LOGGER, + print=started, + handle_signals=cb is None, + access_log_class=HTTP_LOGGER_CLASS if HTTP_LOGGER_CLASS is not None else web.AccessLogger, + ) if __name__ == "__main__": diff --git a/app/routes/api/browser.py b/app/routes/api/browser.py index fb3d8a01..f8b06b25 100644 --- a/app/routes/api/browser.py +++ b/app/routes/api/browser.py @@ -35,7 +35,7 @@ async def get_ffprobe(request: Request, config: Config, encoder: Encoder, app: w Response: The response object. """ - file: str = request.match_info.get("file") + file: str | None = request.match_info.get("file") if not file: return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) @@ -76,7 +76,7 @@ async def get_file_info(request: Request, config: Config, encoder: Encoder, app: Response: The response object. """ - file: str = request.match_info.get("file") + file: str | None = request.match_info.get("file") if not file: return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code) @@ -137,7 +137,7 @@ async def file_browser(request: Request, config: Config, encoder: Encoder) -> Re Response: The response object. """ - req_path: str = request.match_info.get("path") + req_path: str | None = request.match_info.get("path") req_path: str = "/" if not req_path else unquote_plus(req_path) raw_req: str = (req_path or "").strip() @@ -461,7 +461,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n record(path, ok=False, error=str(e), action=action, extra={"item": params}) continue else: - extra_info: dict[str, Path] = {"new_path": renamed} + extra_info: dict[str, Any] = {"new_path": renamed} if sidecar_renamed: extra_info["sidecar_count"] = len(sidecar_renamed) record(path, ok=True, action=action, extra=extra_info) @@ -591,7 +591,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n record(path, ok=False, error=str(e), action=action, extra={"item": params}) continue else: - extra_info: dict[str, Path] = {"new_path": moved} + extra_info: dict[str, Any] = {"new_path": moved} if sidecar_moved: extra_info["sidecar_count"] = len(sidecar_moved) @@ -617,7 +617,7 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache): if not json or not isinstance(json, list): return web.json_response({"error": "Invalid parameters."}, status=400) - files: list[str] = [] + files: list[Path] = [] for f in json: if not isinstance(f, str): continue @@ -627,12 +627,10 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache): continue files.append(ref) - sc: list[dict] = get_file_sidecar(ref) + sc: dict[str, list[dict[str, Any]]] = get_file_sidecar(ref) if sc: - for side in sc: - for scf in sc[side]: - if isinstance(scf, dict) and "file" in scf: - files.append(scf["file"]) # noqa: PERF401 + for value in sc.values(): + files.extend(scf["file"] for scf in value if isinstance(scf, dict) and "file" in scf) if not files: return web.json_response({"error": "No valid files."}, status=400) diff --git a/app/routes/api/docs.py b/app/routes/api/docs.py index 4f04fdab..c66af10f 100644 --- a/app/routes/api/docs.py +++ b/app/routes/api/docs.py @@ -95,7 +95,7 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response: cache.set(cache_key, dct, ttl=3600) - return web.Response(**dct) + return web.Response(body=dct["body"], headers=dct["headers"]) except Exception: LOG.exception( "Failed to request doc '%s' from '%s'.", diff --git a/app/routes/api/download.py b/app/routes/api/download.py index a6d2cacb..cb13d4c2 100644 --- a/app/routes/api/download.py +++ b/app/routes/api/download.py @@ -1,5 +1,5 @@ from aiohttp import web -from aiohttp.web import Request, Response +from aiohttp.web import Request, Response, StreamResponse from app.library.config import Config from app.library.log import get_logger @@ -10,7 +10,7 @@ LOG = get_logger() @route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static") -async def download_file(request: Request, config: Config, app: web.Application) -> Response: +async def download_file(request: Request, config: Config, app: web.Application) -> StreamResponse: """ Serve download files. diff --git a/app/routes/api/history.py b/app/routes/api/history.py index d2df5242..b5e74139 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -1,9 +1,10 @@ import asyncio from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any from aiohttp import web from aiohttp.web import Request, Response +from aiohttp.web_response import StreamResponse from app.features.presets.schemas import Preset from app.features.presets.service import Presets @@ -20,10 +21,6 @@ from app.library.log import get_logger from app.library.router import route from app.library.Utils import calc_download_path, get_file_sidecar, rename_file -if TYPE_CHECKING: - from library.downloads import Download - - LOG = get_logger() @@ -280,7 +277,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: The response object. """ - id: str = request.match_info.get("id") + id: str | None = request.match_info.get("id") if not id: return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) @@ -314,7 +311,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) -> @route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail") -async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> Response: +async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> StreamResponse: if not (id := request.match_info.get("id")): return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) @@ -458,7 +455,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder, Response: The response object. """ - id: str = request.match_info.get("id") + id: str | None = request.match_info.get("id") if not id: return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) diff --git a/app/routes/api/images.py b/app/routes/api/images.py index 5dac7d20..4111248c 100644 --- a/app/routes/api/images.py +++ b/app/routes/api/images.py @@ -116,8 +116,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp safe_backend: str = _safe_url(backend) await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24) else: - backend = await cache.aget(CACHE_KEY_BING) - safe_backend = _safe_url(backend if isinstance(backend, str) else None) + cached_backend = await cache.aget(CACHE_KEY_BING) + backend = cached_backend if isinstance(cached_backend, str) else "" + safe_backend = _safe_url(backend) if not isinstance(backend, str) or not backend: return web.json_response( diff --git a/app/routes/api/logs.py b/app/routes/api/logs.py index 1c363e3b..72f5224f 100644 --- a/app/routes/api/logs.py +++ b/app/routes/api/logs.py @@ -1,6 +1,7 @@ import asyncio import json import os +from collections.abc import Awaitable, Callable from pathlib import Path from aiohttp import web @@ -110,7 +111,7 @@ async def _read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict: return {"logs": [], "next_offset": None, "end_is_reached": True} -async def _tail_log(file: Path, emitter: callable, sleep_time: float = 0.5): +async def _tail_log(file: Path, emitter: Callable[[dict], Awaitable[None]], sleep_time: float = 0.5): """ Live tail. diff --git a/app/routes/api/system.py b/app/routes/api/system.py index acd323f1..bd830225 100644 --- a/app/routes/api/system.py +++ b/app/routes/api/system.py @@ -2,6 +2,7 @@ import asyncio import os import time from pathlib import Path +from typing import Any from aiohttp import web from aiohttp.web import Request, Response @@ -489,7 +490,7 @@ async def system_limits(queue: DownloadQueue, config: Config, encoder: Encoder) for download in [*active_non_live, *queued_non_live]: extractor = (download.info.get_extractor() or "unknown").lower() - entry = per_extractor.setdefault( + entry: dict[str, Any] = per_extractor.setdefault( extractor, { "name": extractor, diff --git a/app/scripts/seed_db.py b/app/scripts/seed_db.py index 7ba72a74..3bd5fa9b 100644 --- a/app/scripts/seed_db.py +++ b/app/scripts/seed_db.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import TYPE_CHECKING from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve()) if APP_ROOT not in sys.path: @@ -69,7 +69,7 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -async def connect_db(db_file: str) -> AsyncConnection: +async def connect_db(db_file: Path) -> tuple[AsyncConnection, AsyncEngine]: """Create async SQLAlchemy connection.""" engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}") conn = await engine.connect() diff --git a/app/tests/test_ag_utils.py b/app/tests/test_ag_utils.py index 4f36d437..c3c1b327 100644 --- a/app/tests/test_ag_utils.py +++ b/app/tests/test_ag_utils.py @@ -1,3 +1,4 @@ +from typing import Any from unittest.mock import MagicMock import pytest @@ -91,7 +92,7 @@ class TestAgSet: def test_ag_set_error_on_non_dict_final(self): """Test error when final target is not a dict.""" - data = "not_a_dict" + data: Any = "not_a_dict" with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"): ag_set(data, "key", "value") diff --git a/app/tests/test_cf_solver_handler.py b/app/tests/test_cf_solver_handler.py index 53ba8907..9c5b49ff 100644 --- a/app/tests/test_cf_solver_handler.py +++ b/app/tests/test_cf_solver_handler.py @@ -1,6 +1,7 @@ from __future__ import annotations import http.cookiejar +from typing import Any from unittest.mock import Mock, patch import pytest @@ -12,10 +13,10 @@ try: YTDLP_AVAILABLE = True except ImportError: YTDLP_AVAILABLE = False - Request = Mock - Response = Mock - RequestDirector = Mock - HTTPError = Exception + Request: Any = Mock + Response: Any = Mock + RequestDirector: Any = Mock + HTTPError: Any = Exception pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available") diff --git a/app/tests/test_condition_ignore.py b/app/tests/test_condition_ignore.py index 2962df80..0655211f 100644 --- a/app/tests/test_condition_ignore.py +++ b/app/tests/test_condition_ignore.py @@ -1,5 +1,6 @@ from pathlib import Path from types import SimpleNamespace +from typing import Any from unittest.mock import AsyncMock, Mock, patch import pytest @@ -24,7 +25,7 @@ class TestConditionIgnoreMatching: second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10) service = Conditions.get_instance() - service._repo = Mock(list=AsyncMock(return_value=[first, second])) + service._repo = Mock(all=AsyncMock(return_value=[first, second])) matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"]) @@ -36,7 +37,7 @@ class TestConditionIgnoreMatching: second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10) service = Conditions.get_instance() - service._repo = Mock(list=AsyncMock(return_value=[first, second])) + service._repo = Mock(all=AsyncMock(return_value=[first, second])) matched = await service.match(info={"duration": 60}, ignore_conditions=["123"]) @@ -48,7 +49,7 @@ class TestConditionIgnoreMatching: second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10) service = Conditions.get_instance() - service._repo = Mock(list=AsyncMock(return_value=[first, second])) + service._repo = Mock(all=AsyncMock(return_value=[first, second])) matched = await service.match(info={"duration": 60}, ignore_conditions=[123]) @@ -59,7 +60,7 @@ class TestConditionIgnoreMatching: first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20) service = Conditions.get_instance() - service._repo = Mock(list=AsyncMock(return_value=[first])) + service._repo = Mock(all=AsyncMock(return_value=[first])) matched = await service.match(info={"duration": 60}, ignore_conditions=["*"]) @@ -78,10 +79,11 @@ class TestConditionIgnorePropagation: preset="default", extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]}, ) - item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={}))) - item.get_archive_id = Mock(return_value=None) - item.get_archive_file = Mock(return_value=None) - item.is_archived = Mock(return_value=False) + mock_item: Any = item + mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={}))) + mock_item.get_archive_id = Mock(return_value=None) + mock_item.get_archive_file = Mock(return_value=None) + mock_item.is_archived = Mock(return_value=False) matcher = Mock(match=AsyncMock(return_value=None)) entry = {"id": "video-1", "duration": 60} @@ -113,10 +115,11 @@ class TestConditionIgnorePropagation: preset="default", extras={"ignore_conditions": [123, "Primary", True, " * "]}, ) - item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={}))) - item.get_archive_id = Mock(return_value=None) - item.get_archive_file = Mock(return_value=None) - item.is_archived = Mock(return_value=False) + mock_item: Any = item + mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={}))) + mock_item.get_archive_id = Mock(return_value=None) + mock_item.get_archive_file = Mock(return_value=None) + mock_item.is_archived = Mock(return_value=False) matcher = Mock(match=AsyncMock(return_value=None)) entry = {"id": "video-1", "duration": 60} @@ -135,7 +138,7 @@ class TestConditionIgnorePropagation: @pytest.mark.asyncio async def test_playlist_keeps_parent_ignore(self) -> None: - captured: dict[str, object] = {} + captured: dict[str, Any] = {} class FakeItem: def __init__(self) -> None: @@ -158,7 +161,8 @@ class TestConditionIgnorePropagation: } with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")): - result = await process_playlist(queue=queue, entry=entry, item=FakeItem()) + fake: Any = FakeItem() + result = await process_playlist(queue=queue, entry=entry, item=fake) assert result == {"status": "ok"} assert captured["url"] == "https://example.com/v/1" @@ -209,7 +213,8 @@ class TestConditionIgnorePropagation: } with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")): - result = await process_playlist(queue=queue, entry=entry, item=FakeItem()) + fake: Any = FakeItem() + result = await process_playlist(queue=queue, entry=entry, item=fake) assert result == {"status": "ok"} assert queue.add.await_count == 2 diff --git a/app/tests/test_datastore.py b/app/tests/test_datastore.py index d44120d0..8c5c1c09 100644 --- a/app/tests/test_datastore.py +++ b/app/tests/test_datastore.py @@ -4,10 +4,13 @@ from collections import OrderedDict from dataclasses import asdict from datetime import UTC, datetime from email.utils import formatdate +from typing import Any from unittest.mock import AsyncMock, Mock + import pytest from app.library.DataStore import DataStore, StoreType +from app.library.downloads import Download from app.library.ItemDTO import ItemDTO from app.library.operations import Operation from app.library.sqlite_store import SqliteStore @@ -62,17 +65,18 @@ async def make_db(data: int = 0) -> SqliteStore: return ins -class StubDownload: - def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False): +class StubDownload(Download): + def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False) -> None: self.info = info - self._started: bool = started - self._cancelled: bool = cancelled + self.id = info._id + self._stub_started: bool = started + self._stub_cancelled: bool = cancelled def started(self) -> bool: - return self._started + return self._stub_started def is_cancelled(self) -> bool: - return self._cancelled + return self._stub_cancelled async def make_store_async(store_type: StoreType) -> DataStore: @@ -357,7 +361,8 @@ class TestDataStore: def __init__(self): self.info = None - store._dict["broken"] = BrokenDownload() + broken: Any = BrokenDownload() + store._dict["broken"] = broken # Should still find the valid item result = await store.get_item(title="Video 1") @@ -960,10 +965,10 @@ class TestDataStoreOperations: item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - item1.filesize = 1000 + setattr(item1, "filesize", 1000) item2 = make_item(id="vid2", title="Video 2") item2._id = "id2" - item2.filesize = 2000 + setattr(item2, "filesize", 2000) await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item2)) @@ -986,10 +991,10 @@ class TestDataStoreOperations: item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - item1.filesize = 1000 + setattr(item1, "filesize", 1000) item2 = make_item(id="vid2", title="Video 2") item2._id = "id2" - item2.filesize = 2000 + setattr(item2, "filesize", 2000) await store.put(StubDownload(info=item1)) await store.put(StubDownload(info=item2)) @@ -1007,7 +1012,7 @@ class TestDataStoreOperations: item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - item1.filesize = 1000 + setattr(item1, "filesize", 1000) await store.put(StubDownload(info=item1)) @@ -1033,7 +1038,7 @@ class TestDataStoreOperations: item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - item1.filesize = 1000 + setattr(item1, "filesize", 1000) await store.put(StubDownload(info=item1)) @@ -1084,7 +1089,7 @@ class TestDataStoreOperations: item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - item1.description = None + setattr(item1, "description", None) await store.put(StubDownload(info=item1)) diff --git a/app/tests/test_download.py b/app/tests/test_download.py index a0614e70..2276347a 100644 --- a/app/tests/test_download.py +++ b/app/tests/test_download.py @@ -307,7 +307,8 @@ class TestDownloadFlow: postprocessor_hook=Mock(), post_hook=Mock(), ) - download.info.get_ytdlp_opts = Mock( + download_any: Any = download.info + download_any.get_ytdlp_opts = Mock( return_value=Mock( add=Mock( return_value=Mock( @@ -379,7 +380,8 @@ class TestDownloadFlow: postprocessor_hook=Mock(), post_hook=Mock(), ) - download.info.get_ytdlp_opts = Mock( + download_any2: Any = download.info + download_any2.get_ytdlp_opts = Mock( return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={})))) ) @@ -481,7 +483,8 @@ class TestDownloadFlow: ) queue.put(Terminator()) - download._download = fake_download + download_mock: Any = download + download_mock._download = fake_download class InlineProcess: def __init__(self, target): @@ -578,7 +581,8 @@ class TestDownloadFlow: queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)}) queue.put(Terminator()) - download._download = fake_download + download_mock: Any = download + download_mock._download = fake_download class InlineProcess: def __init__(self, target): @@ -1028,7 +1032,7 @@ class TestStatusTracker: "debug": False, } - def test_init_sets_attributes(self, mock_config: dict) -> None: + def test_init_sets_attributes(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) assert st.id == "test-id", "Should set download ID" assert st.info == mock_config["info"], "Should set info reference" @@ -1036,7 +1040,7 @@ class TestStatusTracker: assert st.final_update is False, "Should initialize final_update as False" @pytest.mark.asyncio - async def test_status_ignores_bad_id(self, mock_config: dict) -> None: + async def test_status_ignores_bad_id(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "wrong-id", "status": "downloading"} @@ -1044,14 +1048,14 @@ class TestStatusTracker: assert st.info.status != "downloading", "Should not update status for wrong ID" @pytest.mark.asyncio - async def test_status_ignores_short(self, mock_config: dict) -> None: + async def test_status_ignores_short(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "test-id"} await st.process_status_update(status) @pytest.mark.asyncio - async def test_process_status_update_sets_status(self, mock_config: dict) -> None: + async def test_process_status_update_sets_status(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000} @@ -1059,7 +1063,7 @@ class TestStatusTracker: assert st.info.status == "downloading", "Should update info status" @pytest.mark.asyncio - async def test_status_sets_skipped(self, mock_config: dict) -> None: + async def test_status_sets_skipped(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "test-id", "status": "downloading", "download_skipped": True} @@ -1067,7 +1071,7 @@ class TestStatusTracker: assert st.info.download_skipped is True, "Should update download_skipped from status queue" @pytest.mark.asyncio - async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None: + async def test_process_status_update_sets_tmpfilename(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"} @@ -1075,7 +1079,7 @@ class TestStatusTracker: assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename" @pytest.mark.asyncio - async def test_status_sets_percent(self, mock_config: dict) -> None: + async def test_status_sets_percent(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = { "id": "test-id", @@ -1090,7 +1094,7 @@ class TestStatusTracker: assert st.info.percent == 50.0, "Should calculate percent correctly" @pytest.mark.asyncio - async def test_status_uses_estimate(self, mock_config: dict) -> None: + async def test_status_uses_estimate(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = { "id": "test-id", @@ -1104,7 +1108,7 @@ class TestStatusTracker: assert st.info.percent == 30.0, "Should calculate percent from estimate" @pytest.mark.asyncio - async def test_status_percent(self, mock_config: dict) -> None: + async def test_status_percent(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = { "id": "test-id", @@ -1117,7 +1121,7 @@ class TestStatusTracker: assert st.info.percent == 50.0, "Should calculate percent correctly with valid total" @pytest.mark.asyncio - async def test_status_sets_speed_eta(self, mock_config: dict) -> None: + async def test_status_sets_speed_eta(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60} @@ -1126,7 +1130,7 @@ class TestStatusTracker: assert st.info.eta == 60, "Should set eta" @pytest.mark.asyncio - async def test_process_status_update_sets_error(self, mock_config: dict) -> None: + async def test_process_status_update_sets_error(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) status = {"id": "test-id", "status": "error", "error": "Download failed"} @@ -1135,7 +1139,7 @@ class TestStatusTracker: assert st.info.error == "Download failed", "Should set error message" @pytest.mark.asyncio - async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None: + async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None: test_file = tmp_path / "test.mp4" test_file.write_text("test content") @@ -1148,20 +1152,20 @@ class TestStatusTracker: assert st.info.filename == "test.mp4", "Should set relative filename" @pytest.mark.asyncio - async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None: + async def test_drain_queue_processes_remaining_updates(self, mock_config: dict[str, Any]) -> None: queue = DummyQueue() queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100}) queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200}) queue.put(Terminator()) - config = {**mock_config, "status_queue": queue} + config: dict[str, Any] = {**mock_config, "status_queue": queue} st = StatusTracker(**config) await st.drain_queue(max_iterations=10) assert st.info.downloaded_bytes == 200, "Should process all queued updates" @pytest.mark.asyncio - async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None: + async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None: test_file = tmp_path / "test.mp4" test_file.write_text("test content") @@ -1169,25 +1173,25 @@ class TestStatusTracker: queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)}) queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999}) - config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)} + config: dict[str, Any] = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)} st = StatusTracker(**config) await st.drain_queue(max_iterations=10) assert st.final_update is True, "Should stop draining after final update" @pytest.mark.asyncio - async def test_drain_queue_skips_invalid(self, mock_config: dict) -> None: + async def test_drain_queue_skips_invalid(self, mock_config: dict[str, Any]) -> None: queue = DummyQueue() queue.put({"id": "test-id", "status": "downloading"}) queue.put(None) - config = {**mock_config, "status_queue": queue} + config: dict[str, Any] = {**mock_config, "status_queue": queue} st = StatusTracker(**config) await st.drain_queue(max_iterations=5) assert st.info.status == "downloading", "valid updates should still be processed" - def test_cancel_update_task(self, mock_config: dict) -> None: + def test_cancel_update_task(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) st.update_task = Mock() st.update_task.done = Mock(return_value=False) @@ -1196,15 +1200,15 @@ class TestStatusTracker: st.cancel_update_task() st.update_task.cancel.assert_called_once() - def test_cancel_update_task_noop(self, mock_config: dict) -> None: + def test_cancel_update_task_noop(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) st.cancel_update_task() assert st.update_task is None, "missing tasks should be ignored" - def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None: + def test_put_terminator_adds_to_queue(self, mock_config: dict[str, Any]) -> None: queue = DummyQueue() - config = {**mock_config, "status_queue": queue} + config: dict[str, Any] = {**mock_config, "status_queue": queue} st = StatusTracker(**config) st.put_terminator() @@ -1212,7 +1216,7 @@ class TestStatusTracker: assert isinstance(queue.items[0], Terminator), "Should add Terminator instance" @pytest.mark.asyncio - async def test_progress_emits_item_progress(self, mock_config: dict) -> None: + async def test_progress_emits_item_progress(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) st.info.status = "downloading" calls: list = [] @@ -1233,7 +1237,7 @@ class TestStatusTracker: assert "options" not in payload @pytest.mark.asyncio - async def test_status_change_emits_item_updated(self, mock_config: dict) -> None: + async def test_status_change_emits_item_updated(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) st.info.status = "started" calls: list = [] @@ -1249,7 +1253,7 @@ class TestStatusTracker: assert updated_calls[0][1]["data"] is st.info @pytest.mark.asyncio - async def test_progress_throttled(self, mock_config: dict) -> None: + async def test_progress_throttled(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) st.info.status = "downloading" st._progress_interval = 0.5 @@ -1272,7 +1276,7 @@ class TestStatusTracker: assert st._pending_progress is True @pytest.mark.asyncio - async def test_flush_on_status_change(self, mock_config: dict) -> None: + async def test_flush_on_status_change(self, mock_config: dict[str, Any]) -> None: st = StatusTracker(**mock_config) st.info.status = "downloading" st._progress_interval = 10.0 @@ -1345,23 +1349,29 @@ class TestQueueManager: auto_start=True, ) + @staticmethod + def _any_video_item() -> Any: + return TestQueueManager._video_item() + def test_live_queue_caps_visible_items(self) -> None: queue_manager = object.__new__(DownloadQueue) - items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} + items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} queue_manager.queue = self.LiveStore(items) queue_manager.pool = Mock() queue_manager.pool.get_active_downloads.return_value = {} snapshot = DownloadQueue.live_queue(queue_manager, limit=2) - assert list(snapshot["queue"].keys()) == ["id0", "id1"] + queue_view = snapshot["queue"] + assert isinstance(queue_view, dict) + assert list(queue_view.keys()) == ["id0", "id1"] assert snapshot["queue_count"] == 5 assert snapshot["queue_loaded"] == 2 assert snapshot["queue_limit"] == 2 def test_live_queue_keeps_active(self) -> None: queue_manager = object.__new__(DownloadQueue) - items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} + items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} queue_manager.queue = self.LiveStore(items) queue_manager.pool = Mock() queue_manager.pool.get_active_downloads.return_value = { @@ -1371,7 +1381,9 @@ class TestQueueManager: snapshot = DownloadQueue.live_queue(queue_manager, limit=1) - assert list(snapshot["queue"].keys()) == ["id3", "id4"] + queue_view = snapshot["queue"] + assert isinstance(queue_view, dict) + assert list(queue_view.keys()) == ["id3", "id4"] assert snapshot["queue_count"] == 5 assert snapshot["queue_loaded"] == 2 assert snapshot["queue_limit"] == 1 @@ -1388,7 +1400,7 @@ class TestQueueManager: result = await add_video( queue=self._video_queue(), - item=self._video_item(), + item=self._any_video_item(), entry={ "id": "live-id", "title": "Live stream", @@ -1420,7 +1432,7 @@ class TestQueueManager: "formats": [{"format_id": "18"}], } - result = await add_video(queue=self._video_queue(), item=self._video_item(), entry=entry) + result = await add_video(queue=self._video_queue(), item=self._any_video_item(), entry=entry) assert result == {"status": "ok"} assert seen == [entry] diff --git a/app/tests/test_encoder.py b/app/tests/test_encoder.py index 68f74746..79d11305 100644 --- a/app/tests/test_encoder.py +++ b/app/tests/test_encoder.py @@ -1,6 +1,7 @@ import json from datetime import date from pathlib import Path +from typing import Any from unittest.mock import MagicMock import pytest @@ -132,7 +133,8 @@ class TestEncoder: return True return original_isinstance(obj, cls) - builtins.isinstance = mock_isinstance + builtins_any: Any = builtins + builtins_any.isinstance = mock_isinstance try: result = self.encoder.default(mock_daterange) diff --git a/app/tests/test_generic_browser.py b/app/tests/test_generic_browser.py index 0ef657e8..8a80dfdb 100644 --- a/app/tests/test_generic_browser.py +++ b/app/tests/test_generic_browser.py @@ -1,5 +1,6 @@ from __future__ import annotations +from typing import Any from unittest.mock import Mock import pytest @@ -7,7 +8,7 @@ import pytest from app.yt_dlp_plugins.extractor import generic_browser -def _make_ie(config: dict[str, str | None] | None = None) -> generic_browser.GenericBrowserIE: +def _make_ie(config: dict[str, str | None] | None = None) -> Any: ie = object.__new__(generic_browser.GenericBrowserIE) values = config or {} ie._configuration_arg = lambda name, default: [values.get(name)] @@ -250,6 +251,7 @@ def test_extract_network_formats_playlist_entries_keep_own_urls() -> None: {"title": "Title", "webpage_url": "https://example.com/page", "original_url": "https://example.com/page"}, ) + assert result is not None assert result["_type"] == "playlist" assert result["entries"][0]["url"] == "https://cdn.example/1.mp3" assert result["entries"][0]["webpage_url"] == "https://cdn.example/1.mp3" diff --git a/app/tests/test_history_routes.py b/app/tests/test_history_routes.py index 31d2f543..f28b3d0f 100644 --- a/app/tests/test_history_routes.py +++ b/app/tests/test_history_routes.py @@ -36,7 +36,8 @@ def _make_download( ) -> SimpleNamespace: base_dir = download_dir or "/downloads" original_post_init = ItemDTO.__post_init__ - ItemDTO.__post_init__ = lambda self: None + cls: Any = ItemDTO + cls.__post_init__ = lambda self: None try: item = ItemDTO( diff --git a/app/tests/test_images_routes.py b/app/tests/test_images_routes.py index 9557a0ee..b3a8c674 100644 --- a/app/tests/test_images_routes.py +++ b/app/tests/test_images_routes.py @@ -73,5 +73,5 @@ async def test_bg_log_redact(caplog: pytest.LogCaptureFixture, monkeypatch: pyte ) assert "apitoken=secret" not in record.getMessage() assert "user:pass@" not in record.getMessage() - assert record.url == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted" - assert record.exception_type == "RuntimeError" + assert getattr(record, "url", None) == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted" + assert getattr(record, "exception_type", None) == "RuntimeError" diff --git a/app/tests/test_itemdto.py b/app/tests/test_itemdto.py index a7c63498..78a308c7 100644 --- a/app/tests/test_itemdto.py +++ b/app/tests/test_itemdto.py @@ -386,7 +386,7 @@ class TestItemAddExtras: def test_add_extras_none(self): """Test adding extras when extras is None.""" item = Item(url="https://example.com") - item.extras = None + setattr(item, "extras", None) item.add_extras("key1", "value1") diff --git a/app/tests/test_scheduler.py b/app/tests/test_scheduler.py index dfa5ef24..91642bee 100644 --- a/app/tests/test_scheduler.py +++ b/app/tests/test_scheduler.py @@ -1,6 +1,7 @@ import asyncio import sys import types +from collections.abc import Callable from typing import Any from unittest.mock import MagicMock, patch @@ -21,7 +22,7 @@ if "aiocron" not in sys.modules: def uuid(self) -> str: return "stub-uuid" - aiocron_stub.Cron = _CronImportStub + setattr(aiocron_stub, "Cron", _CronImportStub) sys.modules["aiocron"] = aiocron_stub @@ -36,7 +37,7 @@ class DummyCron: self, *, spec: str, - func: callable, + func: Callable, args: tuple = (), kwargs: dict | None = None, uuid: str | None = None, @@ -231,7 +232,7 @@ class TestScheduler: self, *_, spec: str, - func: callable, + func: Callable, args: tuple = (), kwargs: dict | None = None, uuid: str | None = None, @@ -266,7 +267,7 @@ class TestScheduler: self, *_, spec: str, - func: callable, + func: Callable, args: tuple = (), kwargs: dict | None = None, uuid: str | None = None, diff --git a/app/tests/test_sqlite_store.py b/app/tests/test_sqlite_store.py index 6039ae3f..0d7673d6 100644 --- a/app/tests/test_sqlite_store.py +++ b/app/tests/test_sqlite_store.py @@ -1,4 +1,5 @@ from datetime import UTC, datetime, timedelta +from typing import Any from unittest.mock import AsyncMock, patch import pytest @@ -182,12 +183,14 @@ async def test_paginate_order_and_bounds(): encoded = itm.json() created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S") _list.append(itm) + assert store._conn is not None await store._conn.execute( text( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)' ), {"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at}, ) + assert store._conn is not None await store._conn.commit() items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC") @@ -208,7 +211,9 @@ async def test_get_by_id(): item = make_item(99) await store.enqueue_upsert("queue", item) await store.flush() - assert item._id == (await store.get_by_id("queue", item._id))._id + fetched = await store.get_by_id("queue", item._id) + assert fetched is not None + assert item._id == fetched._id await store.close() @@ -358,7 +363,8 @@ async def test_on_shutdown_closes_connection(): await store.enqueue_upsert("queue", make_item(1)) await store.flush() - await store.on_shutdown(None) + app: Any = None + await store.on_shutdown(app) assert store._conn is None # Note: on_shutdown already closes the connection, so no need to call close() again diff --git a/app/tests/test_utils.py b/app/tests/test_utils.py index 01052374..4dd9e5cb 100644 --- a/app/tests/test_utils.py +++ b/app/tests/test_utils.py @@ -1,6 +1,6 @@ import asyncio import copy -import importlib +import importlib.util import json import logging import re @@ -10,6 +10,7 @@ import uuid from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -861,16 +862,20 @@ class TestMergeDict: def test_merge_dict_type_validation(self): """Test that non-dict parameters are properly rejected.""" # Test with non-dict source + bad_src: Any = "not_a_dict" with pytest.raises(TypeError, match="Both source and destination must be dictionaries"): - merge_dict("not_a_dict", {"key": "value"}) + merge_dict(bad_src, {"key": "value"}) # Test with non-dict destination + bad_dst: Any = "not_a_dict" with pytest.raises(TypeError, match="Both source and destination must be dictionaries"): - merge_dict({"key": "value"}, "not_a_dict") + merge_dict({"key": "value"}, bad_dst) # Test with both non-dict + bad_src2: Any = "not_a_dict" + bad_dst2: Any = ["also_not_dict"] with pytest.raises(TypeError, match="Both source and destination must be dictionaries"): - merge_dict("not_a_dict", ["also_not_dict"]) + merge_dict(bad_src2, bad_dst2) def test_merge_dict_immutability(self): """Test that original dictionaries are not modified (immutability).""" @@ -1084,7 +1089,8 @@ class TestEncryptDecrypt: assert encrypted != data assert isinstance(encrypted, str) - decrypted: str = decrypt_data(encrypted, key) + decrypted = decrypt_data(encrypted, key) + assert decrypted is not None assert decrypted == data def test_encrypt_empty_string(self): @@ -1093,7 +1099,8 @@ class TestEncryptDecrypt: data: str = "" encrypted: str = encrypt_data(data, key) - decrypted: str = decrypt_data(encrypted, key) + decrypted = decrypt_data(encrypted, key) + assert decrypted is not None assert decrypted == data @@ -1489,7 +1496,8 @@ class TestGet: def test_get_list_access(self): """Test getting from list.""" data = ["item0", "item1", "item2"] - result = get(data, 1) + key: Any = 1 + result = get(data, key) assert result == "item1" def test_get_empty_path(self): diff --git a/app/upgrader.py b/app/upgrader.py index da6f308e..32bf83d6 100644 --- a/app/upgrader.py +++ b/app/upgrader.py @@ -10,9 +10,8 @@ import logging import os from pathlib import Path -from library.PackageInstaller import PackageInstaller, Packages - from app.library.log import get_logger +from app.library.PackageInstaller import PackageInstaller, Packages LOG = get_logger() diff --git a/app/yt_dlp_plugins/extractor/generic_browser.py b/app/yt_dlp_plugins/extractor/generic_browser.py index b67e00e7..f4cdd6bd 100644 --- a/app/yt_dlp_plugins/extractor/generic_browser.py +++ b/app/yt_dlp_plugins/extractor/generic_browser.py @@ -1,4 +1,5 @@ import base64 +import importlib.util import os import re import time @@ -190,12 +191,7 @@ class RemoteBrowserUnavailableError(ExtractorError): class PlaywrightDriver: @staticmethod def is_available() -> bool: - try: - import playwright.sync_api # noqa: F401 - - return True - except ImportError: - return False + return importlib.util.find_spec("playwright.sync_api") is not None @staticmethod def connect(ws_url: str, timeout: int | None = None): @@ -336,15 +332,11 @@ class PlaywrightDriver: class SeleniumDriver: @staticmethod def is_available() -> bool: - try: - import selenium.webdriver # noqa: F401 - - return True - except ImportError: - return False + return importlib.util.find_spec("selenium.webdriver") is not None @staticmethod - def connect(ws_url: str, timeout: int | None = None): # noqa: ARG004 + def connect(ws_url: str, timeout: int | None = None): + _ = timeout from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By @@ -441,6 +433,10 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): _failed: bool = False _remote_browser_failures: dict[str, str] = {} _url: str = "" + __wrapped__: Any + + def _fallback_extract(self, url: str) -> dict[str, Any]: + return self.__wrapped__._real_extract(self, url) def _get_config(self, name: str, env_name: str) -> str | None: value = self._configuration_arg(name, [None])[0] @@ -473,7 +469,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): self._url = url if not (browser_url := self._get_config("url", "YTP_BROWSER_URL")) or self._failed: - return self.__wrapped__._real_extract(self, url) + return self._fallback_extract(url) safe_url = self._safe_url(browser_url) @@ -496,7 +492,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): message = str(e) self._failed = True self.report_warning(f"Remote browser unavailable: {message}, marking as failed.", video_id) - return self.__wrapped__._real_extract(self, url) + return self._fallback_extract(url) try: self.report_extraction(url) @@ -513,17 +509,18 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): requests = self._merge_requests(session.get_requests(), session.get_media_requests()) self.write_debug(f"Captured {len(requests)} network requests") - if self._downloader.params.get("dump_intermediate_pages"): + downloader = self._downloader + if downloader and downloader.params.get("dump_intermediate_pages"): self.to_screen(f"Browser content dump for: {url}") self.to_screen(base64.b64encode(webpage.encode("utf-8")).decode("ascii")) - if self._downloader.params.get("write_pages"): - filename = _request_dump_filename(url, video_id, None, self._downloader.params.get("trim_file_name")) + if downloader and downloader.params.get("write_pages"): + filename = _request_dump_filename(url, video_id, None, downloader.params.get("trim_file_name")) self.to_screen(f"Saving request to {filename}") with open(filename, "w", encoding="utf-8") as f: f.write(webpage) - if self._downloader.params.get("debug_printtraffic"): + if downloader and downloader.params.get("debug_printtraffic"): self.to_screen(f"[browser] {url}") self.to_screen(f"[browser] Captured '{len(requests)}' network requests") for req in requests: @@ -599,9 +596,9 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): self, requests: list[dict], video_id: str, base_info: dict[str, Any] ) -> dict[str, Any] | None: candidates = self._pick_network_candidates(requests) - formats = [] - direct_formats = [] - subtitles = {} + formats: list[dict[str, Any]] = [] + direct_formats: list[dict[str, Any]] = [] + subtitles: dict[str, Any] = {} source_counts = {} has_manifest_formats = False @@ -674,7 +671,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): self.report_warning( "Generic browser extractor found no media formats. falling back to generic extractor.", video_id ) - return self.__wrapped__._real_extract(self, self._url) + return self._fallback_extract(self._url) if not has_manifest_formats and len(direct_formats) > 1: base_title = (base_info.get("title") or "").strip() or video_id @@ -698,7 +695,7 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): ) return {"_type": "playlist", "entries": entries} - result = {"formats": formats, "direct": True} + result: dict[str, Any] = {"formats": formats, "direct": True} if subtitles: result["subtitles"] = subtitles if formats and formats[0].get("url"): @@ -733,9 +730,13 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): has_media_header_ext = header_ext and header_ext in MEDIA_EXTENSIONS resource_type = entry.get("resourceType") - if resource_type and resource_type.lower() not in MEDIA_RESOURCE_TYPES: # noqa: SIM102 - if not has_media_ext and not has_media_header_ext: - continue + if ( + resource_type + and resource_type.lower() not in MEDIA_RESOURCE_TYPES + and not has_media_ext + and not has_media_header_ext + ): + continue if not ext and not header_ext: rt = (resource_type or "").lower() @@ -779,7 +780,8 @@ class GenericBrowserIE(GenericIE, plugin_name="browser"): ) def _get_timeout_ms(self) -> int | None: - socket_timeout = self._downloader.params.get("socket_timeout") + downloader = self._downloader + socket_timeout = downloader.params.get("socket_timeout") if downloader else None if isinstance(socket_timeout, (int, float)) and socket_timeout > 0: return int(socket_timeout * 1000) return None diff --git a/app/yt_dlp_plugins/postprocessor/nfo_maker.py b/app/yt_dlp_plugins/postprocessor/nfo_maker.py index 9097e101..0950794f 100644 --- a/app/yt_dlp_plugins/postprocessor/nfo_maker.py +++ b/app/yt_dlp_plugins/postprocessor/nfo_maker.py @@ -119,7 +119,8 @@ class NFOMakerPP(PostProcessor): def pp_key(cls) -> str: return "NFOMaker" - def run(self, info: dict | None = None) -> tuple[list, dict]: + def run(self, information: Any) -> tuple[list, dict[str, Any]]: + info = information if "fail" == self.mode: msg = "Intentionally failing due to mode=fail." raise Exception(msg) diff --git a/pyproject.toml b/pyproject.toml index 2b96d829..d2756478 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,9 @@ line-ending = "auto" docstring-code-format = false docstring-code-line-length = "dynamic" +[tool.ty.src] +include = ["app"] + [tool.uv] link-mode = "copy" diff --git a/ui/app/composables/useAppSocket.ts b/ui/app/composables/useAppSocket.ts index 429e9408..87c81a27 100644 --- a/ui/app/composables/useAppSocket.ts +++ b/ui/app/composables/useAppSocket.ts @@ -255,7 +255,7 @@ const connect = () => { }); socket.value.addEventListener('message', (event: MessageEvent) => { - let payload: WebSocketEnvelope | null = null; + let payload: WebSocketEnvelope | null; try { payload = JSON.parse(event.data); } catch { diff --git a/ui/app/spa-loading-template.html b/ui/app/spa-loading-template.html index 17918b6c..3577b830 100644 --- a/ui/app/spa-loading-template.html +++ b/ui/app/spa-loading-template.html @@ -4,7 +4,7 @@ - Loading YTPTube + Loading | YTPTube