Merge pull request #618 from arabcoders/dev

chore: use ty as type checker.
This commit is contained in:
Abdulmohsen 2026-06-14 19:35:40 +03:00 committed by GitHub
commit 44f5f2eee0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
110 changed files with 1566 additions and 1357 deletions

View file

@ -15,7 +15,7 @@ on:
env:
BUN_VERSION: latest
NODE_VERSION: 20
NODE_VERSION: 22
PYTHON_VERSION: "3.13"
jobs:

View file

@ -29,7 +29,7 @@ env:
DOCKERHUB_SLUG: arabcoders/ytptube
GHCR_SLUG: ghcr.io/arabcoders/ytptube
BUN_VERSION: latest
NODE_VERSION: 20
NODE_VERSION: 22
PYTHON_VERSION: "3.13"
jobs:

View file

@ -39,7 +39,7 @@ jobs:
env:
PYTHON_VERSION: "3.13"
BUN_VERSION: latest
NODE_VERSION: 20
NODE_VERSION: 22
TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
steps:

View file

@ -6,13 +6,16 @@ from app.features.conditions.migration import Migration
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Iterable
from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
from sqlalchemy import delete, func, or_, select
from app.features.conditions.models import ConditionModel
@ -22,10 +25,26 @@ from app.library.log import get_logger
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
model = ConditionModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for ConditionModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
if isinstance(payload, ConditionModel):
return payload
return _model_from_payload(payload)
class ConditionsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session
self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -38,7 +57,7 @@ class ConditionsRepository(metaclass=Singleton):
def get_instance() -> ConditionsRepository:
return ConditionsRepository()
async def list(self) -> list[ConditionModel]:
async def all(self) -> list[ConditionModel]:
async with self.session() as session:
result: Result[tuple[ConditionModel]] = await session.execute(
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
@ -93,11 +112,11 @@ class ConditionsRepository(metaclass=Singleton):
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: ConditionModel | dict) -> ConditionModel:
async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
async with self.session() as session:
model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload
model = _coerce_model(payload)
if model.id is not None:
model.id = None
model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Condition with name '{model.name}' already exists."
@ -160,13 +179,11 @@ class ConditionsRepository(metaclass=Singleton):
await session.commit()
return model
async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]:
async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
async with self.session() as session:
try:
await session.execute(delete(ConditionModel))
models: list[ConditionModel] = [
ConditionModel(**item) if isinstance(item, dict) else item for item in items
]
models: list[ConditionModel] = [_coerce_model(item) for item in items]
session.add_all(models)
await session.commit()
except Exception:

View file

@ -131,6 +131,12 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
status=web.HTTPInternalServerError.status_code,
)
if not isinstance(data, dict):
return web.json_response(
data={"error": "Failed to extract video info."},
status=web.HTTPInternalServerError.status_code,
)
try:
from app.features.ytdlp.mini_filter import match_str

View file

@ -1,10 +1,10 @@
from collections.abc import Iterable
from numbers import Number
from aiohttp import web
from app.features.conditions.models import ConditionModel
from app.features.conditions.repository import ConditionsRepository
from app.features.conditions.schemas import Condition
from app.features.ytdlp.mini_filter import match_str
from app.library.Events import EventBus, Events
from app.library.log import get_logger
@ -13,7 +13,7 @@ from app.library.Singleton import Singleton
LOG = get_logger()
def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tuple[set[str], bool]:
def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
ignored: set[str] = set()
ignore_all = False
@ -21,7 +21,7 @@ def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tu
return ignored, ignore_all
for value in ignore_conditions:
if isinstance(value, bool) or not isinstance(value, (str, Number)):
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
continue
identifier = str(value).strip()
@ -55,7 +55,7 @@ class Conditions(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
async def get_all(self) -> list[ConditionModel]:
return await self._repo.list()
return await self._repo.all()
async def save(self, item: ConditionModel | dict) -> ConditionModel:
"""
@ -80,7 +80,7 @@ class Conditions(metaclass=Singleton):
if item.id is None or 0 == item.id:
model = await repo.create(item)
else:
model = await repo.update(item.id, item.serialize())
model = await repo.update(item.id, Condition.model_validate(item).model_dump())
except KeyError as exc:
raise ValueError(str(exc)) from exc
@ -113,13 +113,15 @@ class Conditions(metaclass=Singleton):
repo = self._repo
return await repo.get(identifier)
async def match(self, info: dict, ignore_conditions: Iterable[str | Number] | None = None) -> ConditionModel | None:
async def match(
self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
) -> ConditionModel | None:
"""
Check if any condition matches the info dict.
Args:
info (dict): The info dict to check.
ignore_conditions (Iterable[str | Number] | None): Condition ids or names to skip for this match.
ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
Returns:
Condition|None: The condition if found, None otherwise.
@ -133,7 +135,7 @@ class Conditions(metaclass=Singleton):
return None
repo = self._repo
items: list[ConditionModel] = await repo.list()
items: list[ConditionModel] = await repo.all()
if len(items) < 1:
return None
@ -187,7 +189,7 @@ class Conditions(metaclass=Singleton):
if not info or not isinstance(info, dict) or len(info) < 1:
return None
if not (item := await self.get(identifier)) or not item.enabled or not item.filter:
if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
return None
return item if match_str(item.filter, info) else None

View file

@ -6,6 +6,7 @@ import pytest
import pytest_asyncio
from types import SimpleNamespace
import pytest
from typing import Any
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
@ -39,13 +40,13 @@ async def repo():
def _json_request(path: str, payload: object) -> web.Request:
request = make_mocked_request("POST", path)
mock_request: Any = make_mocked_request("POST", path)
async def _json() -> object:
return payload
request.json = _json # type: ignore[attr-defined]
return request
mock_request.json = _json
return mock_request
class TestAllowInternalUrlsScope:
@ -208,7 +209,7 @@ class TestConditionsRepository:
await repo.create({"name": "A", "filter": "test", "priority": 1})
await repo.create({"name": "C", "filter": "test", "priority": 2})
items = await repo.list()
items = await repo.all()
assert items[0].name == "C", "Highest priority should be first"
assert items[1].name == "A", "Same priority sorted alphabetically"
@ -240,6 +241,6 @@ class TestConditionsRepository:
assert len(result) == 2, "Should create 2 new conditions"
all_items = await repo.list()
all_items = await repo.all()
assert len(all_items) == 2, "Should only have new conditions"
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"

View file

@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime as SQLADateTime
from sqlalchemy import TypeDecorator
from sqlalchemy import Dialect, TypeDecorator
from sqlalchemy.orm import DeclarativeBase
@ -20,16 +20,18 @@ class UTCDateTime(TypeDecorator):
impl = SQLADateTime
cache_ok = True
def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None:
def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
"""Convert datetime to UTC before storing."""
_ = dialect
if value is not None:
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC).replace(tzinfo=None)
return value
def process_result_value(self, value: datetime | None, _dialect) -> datetime | None:
def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
"""Ensure datetime is timezone-aware (UTC) when loading."""
_ = dialect
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value

View file

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
from sqlalchemy import delete, func, or_, select
@ -11,20 +11,39 @@ from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Iterable
from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
model = DLFieldModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
if isinstance(payload, DLFieldModel):
return payload
return _model_from_payload(payload)
class DLFieldsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session
self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -37,7 +56,7 @@ class DLFieldsRepository(metaclass=Singleton):
def get_instance() -> DLFieldsRepository:
return DLFieldsRepository()
async def list(self) -> list[DLFieldModel]:
async def all(self) -> list[DLFieldModel]:
async with self.session() as session:
result: Result[tuple[DLFieldModel]] = await session.execute(
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
@ -90,9 +109,9 @@ class DLFieldsRepository(metaclass=Singleton):
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: DLFieldModel | dict) -> DLFieldModel:
async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
async with self.session() as session:
model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload
model = _coerce_model(payload)
if await self.get_by_name(name=model.name) is not None:
msg: str = f"DL field with name '{model.name}' already exists."
@ -153,13 +172,16 @@ class DLFieldsRepository(metaclass=Singleton):
await session.commit()
return model
async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]:
async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
async with self.session() as session:
try:
await session.execute(delete(DLFieldModel))
models: list[DLFieldModel] = [
DLFieldModel(**item) if isinstance(item, dict) else item for item in items
]
models: list[DLFieldModel] = []
for item in items:
if isinstance(item, dict):
models.append(DLFieldModel(**cast("dict[str, Any]", item)))
else:
models.append(item)
session.add_all(models)
await session.commit()
except Exception:

View file

@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
async def get_all(self) -> list[DLFieldModel]:
return await self._repo.list()
return await self._repo.all()
async def get_all_serialized(self) -> list[dict[str, Any]]:
items = await self._repo.list()
items = await self._repo.all()
return [DLField.model_validate(item).model_dump() for item in items]
async def save(self, item: DLField | dict) -> DLFieldModel:

View file

@ -204,7 +204,7 @@ class TestDLFieldsRepository:
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
items = await repo.list()
items = await repo.all()
assert items[0].name == "c", "Lowest order should be first"
assert items[1].name == "a", "Same order sorted alphabetically"
@ -243,6 +243,6 @@ class TestDLFieldsRepository:
assert len(result) == 2, "Should create 2 new fields"
all_items = await repo.list()
all_items = await repo.all()
assert len(all_items) == 2, "Should only have new fields"
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"

View file

@ -11,20 +11,39 @@ from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
model = NotificationModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for NotificationModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
if isinstance(payload, NotificationModel):
return payload
return _model_from_payload(payload)
class NotificationsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session
self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -37,7 +56,7 @@ class NotificationsRepository(metaclass=Singleton):
def get_instance() -> NotificationsRepository:
return NotificationsRepository()
async def list(self) -> list[NotificationModel]:
async def all(self) -> list[NotificationModel]:
async with self.session() as session:
result: Result[tuple[NotificationModel]] = await session.execute(
select(NotificationModel).order_by(NotificationModel.name.asc())
@ -92,11 +111,11 @@ class NotificationsRepository(metaclass=Singleton):
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: NotificationModel | dict) -> NotificationModel:
async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
async with self.session() as session:
model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload
model = _coerce_model(payload)
if model.id is not None:
model.id = None
model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Notification target with name '{model.name}' already exists."

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
from app.features.notifications.models import NotificationModel
from app.features.notifications.repository import NotificationsRepository
@ -64,10 +64,10 @@ class Notifications(metaclass=Singleton):
await self._repo.run_migrations()
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
async def list(self) -> list[NotificationModel]:
return await self._repo.list()
async def all(self) -> list[NotificationModel]:
return await self._repo.all()
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
return await self._repo.list_paginated(page, per_page)
@ -92,8 +92,8 @@ class Notifications(metaclass=Singleton):
async def delete(self, identifier: int | str) -> NotificationModel:
return await self._repo.delete(identifier)
async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]:
targets = await self._repo.list()
async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
targets = await self._repo.all()
if len(targets) < 1:
return []
@ -126,7 +126,7 @@ class Notifications(metaclass=Singleton):
if wait:
return await asyncio.gather(*tasks)
return tasks
return cast("list[dict | Awaitable[dict]]", tasks)
def emit(self, e: Event, _, **__) -> None:
if not NotificationEvents.is_valid(e.event):
@ -271,7 +271,7 @@ class Notifications(metaclass=Singleton):
if not status:
msg = "Apprise failed to send notification."
raise RuntimeError(msg) # noqa: TRY301
self._raise_apprise_error(msg)
except Exception as exc:
LOG.exception(
"Failed to send Apprise notification for event '%s'.",
@ -288,6 +288,10 @@ class Notifications(metaclass=Singleton):
return {}
@staticmethod
def _raise_apprise_error(msg: str) -> None:
raise RuntimeError(msg)
async def _send(self, target: Notification, ev: Event) -> dict:
try:
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)

View file

@ -35,7 +35,7 @@ class TestNotificationsRepository:
@pytest.mark.asyncio
async def test_list_empty(self, repo):
"""List returns empty when no notifications exist."""
notifications = await repo.list()
notifications = await repo.all()
assert notifications == [], "Should return empty list when no notifications exist"
@pytest.mark.asyncio

View file

@ -46,7 +46,7 @@ class Migration(FeatureMigration):
return
inserted = 0
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()}
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
for index, item in enumerate(items):
if not (normalized := self._normalize(item, index, seen_names)):

View file

@ -30,6 +30,30 @@ if TYPE_CHECKING:
LOG = get_logger()
def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
if isinstance(payload, dict):
data: dict[str, Any] = {}
for key, value in payload.items():
if not isinstance(key, str):
msg = "Preset payload keys must be strings."
raise TypeError(msg)
data[key] = value
return data
return {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
class PresetsRepository(metaclass=Singleton):
SORT_FIELDS: dict[str, Any] = {
"id": PresetModel.id,
@ -73,15 +97,15 @@ class PresetsRepository(metaclass=Singleton):
LOG.debug("Refreshing presets cache due to configuration update.")
await self._update_cache()
Services.get_instance().add(__class__.__name__, self)
Services.get_instance().add(PresetsRepository.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
async def _update_cache(self) -> None:
from app.features.presets.service import Presets
await Presets.get_instance().refresh_cache(await self.list())
await Presets.get_instance().refresh_cache(await self.all())
@staticmethod
def get_instance() -> PresetsRepository:
@ -97,7 +121,7 @@ class PresetsRepository(metaclass=Singleton):
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
config.default_preset = "default"
async def list(self) -> list[PresetModel]:
async def all(self) -> list[PresetModel]:
async with self.session() as session:
result: Result[tuple[PresetModel]] = await session.execute(
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
@ -238,24 +262,9 @@ class PresetsRepository(metaclass=Singleton):
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: PresetModel | dict) -> PresetModel:
async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
async with self.session() as session:
data: dict[str, Any]
if isinstance(payload, dict):
data = dict(payload)
else:
data = {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
data = _payload_data(payload)
data.pop("id", None)
model = PresetModel(**data)

View file

@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
def __init__(self, repo: PresetsRepository | None = None) -> None:
self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
self._cache: list[tuple[int, str, Preset]] = []
Services.get_instance().add(__class__.__name__, self)
Services.get_instance().add(type(self).__name__, self)
@staticmethod
def get_instance() -> Presets:

View file

@ -57,7 +57,7 @@ class TestPresetsRepository:
await repo.create({"name": "A", "priority": 1})
await repo.create({"name": "C", "priority": 2})
items = await repo.list()
items = await repo.all()
assert items[0].name == "c", "Highest priority should be first"
assert items[1].name == "a", "Same priority should sort by name"

View file

@ -10,8 +10,6 @@ import os
import subprocess # qa: ignore
from pathlib import Path
import anyio
from app.features.streaming.types import FFProbeError
from app.library.log import get_logger
from app.library.Utils import timed_lru_cache
@ -39,21 +37,26 @@ class FFStream:
def __repr__(self):
if "codec_long_name" not in self.__dict__:
self.codec_long_name = self.__dict__.get("codec_name", "")
self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
index = self.__dict__.get("index", "?")
codec_type = self.__dict__.get("codec_type", "unknown")
codec_long_name = self.__dict__.get("codec_long_name", "")
if self.is_video():
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>"
return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
if self.is_audio():
return (
f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), "
"{sample_rate}Hz> "
f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
f"{self.__dict__.get('sample_rate')}Hz> "
)
if self.is_subtitle() or self.is_attachment():
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>"
return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
return f"<Stream: #{self.index} [{self.codec_type}]>"
return f"<Stream: #{index} [{codec_type}]>"
def is_audio(self):
"""
@ -252,14 +255,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
raise OSError(msg)
try:
async with await anyio.open_file(os.devnull, "w") as tempf:
await asyncio.create_subprocess_exec(
"ffprobe",
"-h",
stdout=tempf,
stderr=tempf,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
proc = await asyncio.create_subprocess_exec(
"ffprobe",
"-h",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
await proc.wait()
except FileNotFoundError as e:
msg = "ffprobe not found."
raise OSError(msg) from e

View file

@ -39,7 +39,7 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
segmentSize: float = f"{self.duration:.6f}"
segmentSize: str = f"{self.duration:.6f}"
splits: int = math.ceil(duration / self.duration)
segmentParams: dict = {}

View file

@ -14,7 +14,7 @@ class Playlist:
"The path where files are downloaded."
async def make(self, file: Path) -> str:
ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
try:
ff = await ffprobe(file)

View file

@ -1,7 +1,7 @@
# flake8: noqa: ARG002
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from functools import lru_cache
@ -16,9 +16,13 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
values are dicts with keys "full" and "lp" booleans.
"""
vainfo = shutil.which("vainfo")
if not vainfo:
return {}
try:
result: subprocess.CompletedProcess[str] = subprocess.run(
["vainfo"], # noqa: S607
[vainfo],
capture_output=True,
text=True,
check=False,
@ -87,9 +91,13 @@ def ffmpeg_encoders() -> set[str]:
"""
from app.library.config import SUPPORTED_CODECS
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
return set()
try:
result: subprocess.CompletedProcess[str] = subprocess.run(
["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607
[ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
capture_output=True,
text=True,
check=False,
@ -149,9 +157,11 @@ class _BaseBuilder:
codec_name: str
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return []
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return [*args, "-codec:v", self.codec_name]
@ -312,4 +322,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
"libx264": [],
}
return chains.get(codec, chains["libx264"])
return tuple(chains.get(codec, chains["libx264"]))

View file

@ -1,6 +1,6 @@
import asyncio
import os
import subprocess # type: ignore
import subprocess
import sys
import tempfile
from pathlib import Path
@ -178,6 +178,7 @@ class Segments:
)
try:
assert proc.stdout is not None
while True:
chunk: bytes = await proc.stdout.read(1024 * 64)
if not chunk:
@ -240,7 +241,9 @@ class Segments:
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
async def stream(self, file: Path, resp: web.StreamResponse) -> None:
codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec
codec: str = self.vcodec
if Segments._cache_initialized and Segments._cached_vcodec:
codec = Segments._cached_vcodec
if not codec or codec not in SUPPORTED_CODECS:
codec: str = select_encoder(self.vcodec or "")

View file

@ -2,6 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import anyio
import pysubs2
@ -48,7 +49,8 @@ def ms_to_timestamp(ms: int) -> str:
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
SubstationFormat.ms_to_timestamp = ms_to_timestamp
_substation_format: Any = SubstationFormat
_substation_format.ms_to_timestamp = ms_to_timestamp
class Subtitle:

View file

@ -24,6 +24,8 @@ THUMBNAIL_MISS_TTL = 3600.0
_LOCK = asyncio.Lock()
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
_SEM: asyncio.Semaphore | None = None
_SEM_LIMIT: int | None = None

View file

@ -4,6 +4,7 @@ from pathlib import Path
from aiohttp import web
from aiohttp.web import Request, Response
from aiohttp.web_response import StreamResponse
from app.features.streaming.library.m3u8 import M3u8
from app.features.streaming.library.playlist import Playlist
@ -32,7 +33,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
Response: The response object.
"""
file: str = request.match_info.get("file")
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -83,8 +84,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
Response: The response object.
"""
file: str = request.match_info.get("file")
mode: str = request.match_info.get("mode")
file: str | None = request.match_info.get("file")
mode: str | None = request.match_info.get("mode")
if mode not in ["video", "subtitle"]:
return web.json_response(
@ -94,13 +95,14 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
duration = request.query.get("duration", None)
duration: float | None = None
duration_arg = request.query.get("duration", None)
if "subtitle" in mode:
if not duration:
if not duration_arg:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
duration = float(duration)
duration = float(duration_arg)
base_path: str = config.base_path.rstrip("/")
@ -124,6 +126,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
if "subtitle" in mode:
if duration is None:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
text = await cls.make_subtitle(file=realFile, duration=duration)
else:
text = await cls.make_stream(file=realFile)
@ -149,7 +153,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
async def segments_stream(request: Request, config: Config, app: web.Application) -> Response:
async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
"""
Get the segments.
@ -162,9 +166,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
Response: The response object.
"""
file: str = request.match_info.get("file")
segment: int = request.match_info.get("segment")
sd: int = request.query.get("sd")
file: str | None = request.match_info.get("file")
segment: str | None = request.match_info.get("segment")
sd: str | None = request.query.get("sd")
vc: int = int(request.query.get("vc", 0))
ac: int = int(request.query.get("ac", 0))
@ -241,7 +245,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
Response: The response object.
"""
file: str = request.match_info.get("file")
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -299,7 +303,7 @@ async def subtitles_manifest_get(request: Request, config: Config, app: web.Appl
Response: The response object.
"""
file: str = request.match_info.get("file")
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -354,7 +358,7 @@ async def subtitles_track_get(request: Request, config: Config, app: web.Applica
Response: The response object.
"""
file: str = request.match_info.get("file")
file: str | None = request.match_info.get("file")
source_format: str | None = request.match_info.get("source_format")
if not file:

View file

@ -4,6 +4,7 @@ from pathlib import Path
from typing import Any
import pytest
from aiohttp import web
from app.features.streaming.library.segments import Segments
from app.tests.helpers import get_test_system_temp_root
@ -148,18 +149,18 @@ class _FakeProc:
self.killed = True
class _FakeResp:
def __init__(self, fail_with: Exception | None = None) -> None:
class _FakeResp(web.StreamResponse):
def __init__(self, fail_with: BaseException | None = None) -> None:
self.data: bytearray = bytearray()
self.eof = False
self._exc = fail_with
async def write(self, data: bytes) -> None:
async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
if self._exc:
raise self._exc
self.data.extend(data)
async def write_eof(self) -> None:
async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
self.eof = True

View file

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock
import pytest
@ -260,7 +261,7 @@ async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
output = temp_dir / ".jpg"
media.write_text("video")
ff_info = FFProbeResult()
ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "120.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
@ -321,7 +322,7 @@ async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
media1.write_text("video")
media2.write_text("video")
ff_info = FFProbeResult()
ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "60.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))

View file

@ -1,4 +1,3 @@
# flake8: noqa: ARG004
from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
@ -12,6 +11,7 @@ if TYPE_CHECKING:
class BaseHandler:
@staticmethod
async def can_handle(task: HandleTask) -> bool:
_ = task
return False
@staticmethod
@ -24,6 +24,7 @@ class BaseHandler:
@staticmethod
def parse(url: str) -> Any | None:
_ = url
return None
@staticmethod

View file

@ -64,7 +64,7 @@ class GenericTaskHandler(BaseHandler):
from app.features.tasks.definitions.utils import model_to_schema
repo = TaskDefinitionsRepository.get_instance()
models = await repo.list()
models = await repo.all()
cls._definitions = [model_to_schema(model) for model in models]
return cls._definitions
@ -141,7 +141,8 @@ class GenericTaskHandler(BaseHandler):
return False
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
if not definition:
return TaskFailure(message="No generic task definition matched the provided URL.")

View file

@ -9,6 +9,7 @@ from app.features.tasks.definitions.results import HandleTask, TaskFailure, Task
from app.features.ytdlp.extractor import fetch_info
from app.features.ytdlp.utils import get_archive_id
from app.library.cache import Cache
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
@ -160,12 +161,14 @@ class RssGenericHandler(BaseHandler):
return feed_url, items, real_count
@staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
"""
Extract items from an RSS/Atom feed.
Args:
task (Task): The task containing the feed URL.
config (Config | None): Optional handler configuration.
Returns:
TaskResult | TaskFailure: Extraction result with parsed items or failure information.

View file

@ -4,6 +4,7 @@ import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
@ -177,7 +178,8 @@ class TverHandler(BaseHandler):
return feed_url, items, has_items
@staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
series_id: str | None = TverHandler.parse(task.url)
if not series_id:
return TaskFailure(message="Unrecognized Tver series URL.")
@ -208,10 +210,11 @@ class TverHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str = entry.get("archive_id")
task_items.append(
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
)
archive_id: str | None = entry.get("archive_id")
metadata = entry.get("metadata", {})
if not isinstance(metadata, dict):
metadata = {}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -6,6 +6,7 @@ import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
@ -92,7 +93,8 @@ class TwitchHandler(BaseHandler):
return feed_url, items, has_items
@staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
handle_name: str | None = TwitchHandler.parse(task.url)
if not handle_name:
return TaskFailure(message="Unrecognized Twitch channel URL.")
@ -122,7 +124,7 @@ class TwitchHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str = entry.get("archive_id")
archive_id: str | None = entry.get("archive_id")
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -6,6 +6,7 @@ import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
@ -103,7 +104,8 @@ class YoutubeHandler(BaseHandler):
return feed_url, items, real_count
@staticmethod
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
if not parsed:
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
@ -133,7 +135,7 @@ class YoutubeHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str = entry.get("archive_id")
archive_id: str | None = entry.get("archive_id")
metadata: dict[str, Any] = {"published": entry.get("published")}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))

View file

@ -14,20 +14,23 @@ from app.library.Services import Services
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
class TaskDefinitionsRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False
self.session: AsyncGenerator[AsyncSession] = session or get_session
self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -51,12 +54,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
LOG.debug("Refreshing task definitions due to configuration update.")
await GenericTaskHandler.refresh_definitions(force=True)
Services.get_instance().add(__class__.__name__, self)
Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
async def list(self) -> list[TaskDefinitionModel]:
async def all(self) -> list[TaskDefinitionModel]:
async with self.session() as session:
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
@ -115,9 +118,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
async with self.session() as session:
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
model: TaskDefinitionModel = TaskDefinitionModel(**payload)
if model.id is not None:
model.id = None
model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task definition with name '{model.name}' already exists."

View file

@ -48,10 +48,13 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple):
return ret
archive_file: Path = ret.get("file")
items: set[str] = ret.get("items", set())
archive_file = ret.get("file")
items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
if len(items) < 1 or not archive_add(archive_file, list(items)):
if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
return (True, "No new items to mark as downloaded.")
return (True, f"Task '{self.name}' items marked as downloaded.")
@ -70,10 +73,13 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple):
return ret
archive_file: Path = ret.get("file")
items: set[str] = ret.get("items", set())
archive_file = ret.get("file")
items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
if len(items) < 1 or not archive_delete(archive_file, list(items)):
if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
return (True, "No items to remove from archive file.")
return (True, f"Removed '{self.name}' items from archive file.")

View file

@ -8,6 +8,7 @@ import random
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.handlers._base_handler import BaseHandler
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.tasks.models import TaskModel
from app.features.ytdlp.utils import archive_read
@ -27,7 +28,7 @@ LOG = get_logger()
class TaskHandle:
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
self._handlers: list[type] = []
self._handlers: list[type[BaseHandler]] = []
"The available handlers."
self._repo: TasksRepository = tasks
"The tasks manager."
@ -35,7 +36,7 @@ class TaskHandle:
"The scheduler."
self._config: Config = config
"The configuration."
self._task_name: str = f"{__class__.__name__}._dispatcher"
self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
"The task name for the scheduler."
self._queued: dict[str, set[str]] = {}
"Queued archive IDs per handler."
@ -45,11 +46,11 @@ class TaskHandle:
EventBus.get_instance().subscribe(
Events.ITEM_ERROR,
self._handle_item_error,
f"{__class__.__name__}.item_error",
f"{TaskHandle.__name__}.item_error",
)
def load(self) -> None:
self._handlers: list[type] = self._discover()
self._handlers: list[type[BaseHandler]] = self._discover()
timer: str = self._config.tasks_handler_timer
try:
@ -72,16 +73,16 @@ class TaskHandle:
self._scheduler.add(
timer=timer,
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
id=f"{__class__.__name__}._dispatcher",
id=f"{TaskHandle.__name__}._dispatcher",
)
async def _dispatcher(self):
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
handler_groups: dict[str, list[tuple[HandleTask, type]]] = {}
dispatches: list[tuple[HandleTask, type, asyncio.Task[TaskResult | TaskFailure | None]]] = []
handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
tasks: list[TaskModel] = await self._repo.list()
tasks: list[TaskModel] = await self._repo.all()
for task_model in tasks:
task: HandleTask = HandleTask.model_validate(task_model)
@ -100,7 +101,7 @@ class TaskHandle:
continue
try:
handler: type | None = await self._find_handler(task)
handler: type[BaseHandler] | None = await self._find_handler(task)
if handler is None:
s["u"].append(task.name)
continue
@ -181,7 +182,9 @@ class TaskHandle:
},
)
async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None:
async def _dispatch(
self, task: HandleTask, handler: type[BaseHandler], delay: float
) -> TaskResult | TaskFailure | None:
"""
Dispatch a task after a random delay to avoid rate limiting.
@ -215,7 +218,7 @@ class TaskHandle:
exc_info=(type(exc), exc, exc.__traceback__),
)
async def _find_handler(self, task: HandleTask) -> type | None:
async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
for cls in self._handlers:
try:
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
@ -239,9 +242,10 @@ class TaskHandle:
async def dispatch(
self,
task: HandleTask,
handler: type | None = None,
**kwargs, # noqa: ARG002
handler: type[BaseHandler] | None = None,
**kwargs,
) -> TaskResult | TaskFailure | None:
_ = kwargs
"""
Dispatch a task to the appropriate handler.
@ -582,11 +586,11 @@ class TaskHandle:
return TaskResult(items=list(extraction.items), metadata=combined_metadata)
def _discover(self) -> list[type]:
def _discover(self) -> list[type[BaseHandler]]:
"""Discover all available task handlers."""
import app.features.tasks.definitions.handlers as handlers_pkg
handlers: list[type] = []
handlers: list[type[BaseHandler]] = []
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
if module_name.startswith("_"):

View file

@ -8,6 +8,7 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
from app.features.tasks.definitions.schemas import (
Definition,
EngineConfig,
Parse,
RequestConfig,
ResponseConfig,
TaskDefinition,
@ -30,10 +31,12 @@ def test_build_def_payload():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
},
parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
}
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -54,15 +57,17 @@ def test_build_def_container():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
parse=Parse.model_validate(
{
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
}
},
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -82,16 +87,18 @@ def test_build_def_json():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
}
},
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -112,11 +119,13 @@ def test_parse_items_basic():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
},
parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
}
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -154,34 +163,36 @@ def test_parse_items_cards():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
parse=Parse.model_validate(
{
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
},
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
},
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
},
}
}
},
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -249,17 +260,19 @@ def test_parse_items_json():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
}
}
},
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -297,16 +310,18 @@ async def test_generic_task_handler_inspect(monkeypatch):
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
}
},
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -351,16 +366,18 @@ def test_parse_items_json_list():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse={
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
}
},
),
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),

View file

@ -69,7 +69,7 @@ class TestTaskDefinitionsRepository:
await repo.create(_sample_definition("Alpha", priority=2))
await repo.create(_sample_definition("Beta", priority=1))
items = await repo.list()
items = await repo.all()
assert len(items) == 2, "Should return two task definitions"
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"

View file

@ -1,9 +1,17 @@
from typing import Any
from typing import Any, Literal, overload
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
"""
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.

View file

@ -10,20 +10,39 @@ from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
model = TaskModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for TaskModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
if isinstance(payload, TaskModel):
return payload
return _model_from_payload(payload)
class TasksRepository(metaclass=Singleton):
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
def __init__(self, session: SessionFactory | None = None) -> None:
self._migrated = False
self.session = session or get_session
self.session: SessionFactory = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -38,7 +57,7 @@ class TasksRepository(metaclass=Singleton):
def get_instance() -> TasksRepository:
return TasksRepository()
async def list(self) -> list[TaskModel]:
async def all(self) -> list[TaskModel]:
async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
return list(result.scalars().all())
@ -90,7 +109,7 @@ class TasksRepository(metaclass=Singleton):
"""Get all enabled tasks."""
async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(
select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712
select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
)
return list(result.scalars().all())
@ -102,11 +121,11 @@ class TasksRepository(metaclass=Singleton):
)
return list(result.scalars().all())
async def create(self, payload: TaskModel | dict) -> TaskModel:
async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
async with self.session() as session:
model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload
model = _coerce_model(payload)
if model.id is not None:
model.id = None
model.id = None # ty: ignore
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task with name '{model.name}' already exists."

View file

@ -705,8 +705,9 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
if info.get("uploader"):
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
if info.get("tags", []):
for tag in info.get("tags", []):
tags = info.get("tags", [])
if isinstance(tags, list):
for tag in tags:
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
if info.get("year"):
xml_content += f" <year>{info.get('year')}</year>\n"

View file

@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
pass
async def _load_tasks(self) -> None:
tasks = await self._repo.list()
tasks = await self._repo.all()
for task in tasks:
if not task.timer or not task.enabled:
@ -134,9 +134,11 @@ class Tasks(metaclass=Singleton):
task_id = task.id
task_name = task.name
try:
if not (task := await self._repo.get(task_id)):
current_task = await self._repo.get(task_id)
if not current_task:
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
return
task = current_task
if not task.enabled:
LOG.debug(

View file

@ -260,7 +260,7 @@ class TestTasksRepository:
await repo.create({"name": "Alice", "url": "https://example.com"})
await repo.create({"name": "Bob", "url": "https://example.com"})
items = await repo.list()
items = await repo.all()
assert items[0].name == "Alice", "Should be alphabetically first"
assert items[1].name == "Bob", "Should be alphabetically second"

View file

@ -1,6 +1,7 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import pytest
import pytest_asyncio
@ -41,13 +42,13 @@ def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
request = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
async def _json() -> object:
return payload
request.json = _json # type: ignore[attr-defined]
return request
mock_request.json = _json
return mock_request
class _Notify:
@ -85,7 +86,7 @@ async def test_add_requires_timer_without_handler(repo) -> None:
assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body
assert await repo.list() == []
assert await repo.all() == []
@pytest.mark.asyncio
@ -109,7 +110,7 @@ async def test_add_all_or_nothing(repo) -> None:
assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body
assert await repo.list() == []
assert await repo.all() == []
@pytest.mark.asyncio
@ -123,7 +124,7 @@ async def test_add_allows_handler_only(repo) -> None:
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
assert response.status == web.HTTPOk.status_code
items = await repo.list()
items = await repo.all()
assert len(items) == 1
assert items[0].name == "Handler Only"

View file

@ -120,7 +120,7 @@ class ExtractorPool(metaclass=Singleton):
def attach(self, app: web.Application) -> None:
"""Attach the extractor pool to the application (no-op for now)."""
app.on_shutdown.append(self.shutdown)
Services.get_instance().add(__class__.__name__, self)
Services.get_instance().add(type(self).__name__, self)
def _ensure_initialized(self, config: ExtractorConfig) -> None:
"""

View file

@ -286,7 +286,7 @@ async def convert(request: Request) -> Response:
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
try:
response = {"opts": {}, "output_template": None, "download_path": None}
response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
data = arg_converter(args, dumps=True)
@ -384,6 +384,8 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
if cache.has(key) and not request.query.get("force", False):
data: Any | None = cache.get(key)
if data is None:
data = {}
data["_cached"] = {
"status": "hit",
"preset": preset,
@ -393,7 +395,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
"expires": data.get("_cached", {}).get("expires", time.time() + 300),
}
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
ytdlp_opts: dict = opts.get_all()
@ -450,7 +452,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
cache.set(key=key, value=data, ttl=300)
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
except Exception as e:
LOG.exception(
"Failed to get video info for '%s': %s.",
@ -486,7 +488,7 @@ async def get_options() -> Response:
"""
from app.features.ytdlp.ytdlp import ytdlp_options
return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
@ -508,7 +510,11 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
response = []
for i, url in enumerate(data):
dct = {"index": i, "url": url}
dct: dict[str, Any] = {"index": i, "url": url}
if not isinstance(url, str):
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
response.append(dct)
continue
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
dct.update(get_archive_id(url))

View file

@ -122,6 +122,7 @@ class TestExtractInfo:
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
)
assert result is not None
assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
@ -155,6 +156,7 @@ class TestExtractInfo:
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
)
assert result is not None
assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen

View file

@ -1,5 +1,6 @@
import importlib
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, Mock, patch
import pytest
@ -219,7 +220,8 @@ class TestYTDLP:
"""Test _delete_downloaded_files skips cleanup when _interrupted is True."""
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
ytdlp = YTDLP(params={})
ytdlp.to_screen = Mock()
mock_obj: Any = ytdlp
mock_obj.to_screen = Mock()
# Set interrupted flag
ytdlp._interrupted = True
@ -229,7 +231,7 @@ class TestYTDLP:
# Should not call super method
mock_super_delete.assert_not_called()
# Should show message
ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
# Should return None
assert result is None

View file

@ -776,7 +776,7 @@ class TestYTDLPCli:
from app.features.ytdlp.ytdlp_opts import YTDLPCli
with pytest.raises(ValueError, match="Expected Item instance"):
YTDLPCli(item="not an item") # type: ignore
YTDLPCli(item="not an item")
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.Config")

View file

@ -46,7 +46,8 @@ class TestLogWrapper:
def test_add_target_type_validation(self) -> None:
lw = LogWrapper()
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
lw.add_target(123) # type: ignore[arg-type]
bad: Any = 123
lw.add_target(bad)
def test_add_target_names(self) -> None:
lw = LogWrapper()
@ -128,7 +129,7 @@ class TestExtractYtdlpLogs:
def test_extract_ytdlp_logs_with_filters(self):
"""Test log extraction with filters."""
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
filters = [re.compile(r"ERROR")]
filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
result = extract_ytdlp_logs(logs, filters)
assert result == ["ERROR: Failed"]
@ -312,9 +313,12 @@ class TestGetThumbnail:
def test_non_list(self):
"""Test that None is returned for non-list input."""
assert get_thumbnail(None) is None
assert get_thumbnail("not a list") is None
assert get_thumbnail({"not": "list"}) is None
bad_none: Any = None
bad_str: Any = "not a list"
bad_dict: Any = {"not": "list"}
assert get_thumbnail(bad_none) is None
assert get_thumbnail(bad_str) is None
assert get_thumbnail(bad_dict) is None
def test_thumbnail_preference(self):
"""Test that the thumbnail with highest preference is returned."""
@ -349,6 +353,7 @@ class TestGetThumbnail:
]
result = get_thumbnail(thumbnails)
assert result is not None
assert result["url"] == "with_pref.jpg"
def test_all_equal(self):
@ -365,12 +370,15 @@ class TestGetThumbnail:
class TestGetExtras:
def test_none(self):
"""Test that empty dict is returned for None input."""
assert get_extras(None) == {}
bad: Any = None
assert get_extras(bad) == {}
def test_non_dict(self):
"""Test that empty dict is returned for non-dict input."""
assert get_extras("not a dict") == {}
assert get_extras([]) == {}
bad_str: Any = "not a dict"
bad_list: Any = []
assert get_extras(bad_str) == {}
assert get_extras(bad_list) == {}
def test_extracts_video_information(self):
"""Test extracting information from a video entry."""

View file

@ -101,12 +101,12 @@ class LogWrapper:
name (str|None): The name of the logging target. Defaults to None.
"""
if not isinstance(target, logging.Logger | Callable):
if not isinstance(target, logging.Logger) and not callable(target):
msg = "Target must be a logging.Logger instance or a callable."
raise TypeError(msg)
if name is None:
name = target.name if isinstance(target, logging.Logger) else target.__name__
name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
self.targets.append(
LogTarget(
@ -134,8 +134,9 @@ class LogWrapper:
if target.logger:
log_kwargs = {**kwargs}
log_kwargs.setdefault("stacklevel", 3)
if isinstance(target.target, logging.Logger):
target.target.log(level, msg, *args, **log_kwargs)
else:
elif callable(target.target):
target.target(level, msg, *args, **kwargs)
def debug(self, msg, *args, **kwargs):
@ -179,13 +180,17 @@ def arg_converter(
create_parser = yt_dlp.options.create_parser
def _default_opts(args: str):
def _default_opts(args: str | list[str]):
patched_parser = create_parser()
def patched_create_parser():
return patched_parser
try:
yt_dlp.options.create_parser = lambda: patched_parser
yt_dlp.options.__dict__["create_parser"] = patched_create_parser
return yt_dlp.parse_options(args)
finally:
yt_dlp.options.create_parser = create_parser
yt_dlp.options.__dict__["create_parser"] = create_parser
apply_ytdlp_patches()
@ -238,7 +243,7 @@ def arg_converter(
return diff
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
"""
Extract yt-dlp log lines matching built-in filters plus any extras.
@ -332,7 +337,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
return _DATA.YTDLP_INFO_CLS
def get_thumbnail(thumbnails: list) -> str | None:
def get_thumbnail(thumbnails: list) -> dict | None:
"""
Extract thumbnail URL from a yt-dlp entry.
@ -340,7 +345,7 @@ def get_thumbnail(thumbnails: list) -> str | None:
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
Returns:
str | None: The thumbnail URL if available, otherwise None.
dict | None: The best thumbnail dict if available, otherwise None.
"""
if not thumbnails or not isinstance(thumbnails, list):
@ -381,7 +386,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
extras[f"playlist_{property}"] = val
if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
extras["thumbnail"] = thumbnail.get("url")
extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
elif thumbnail := entry.get("thumbnail"):
extras["thumbnail"] = thumbnail
@ -439,7 +444,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
}
"""
idDict: dict[str, None] = {
idDict: dict[str, str | None] = {
"id": None,
"ie_key": None,
"archive_id": None,

View file

@ -50,12 +50,12 @@ class ARGSMerger:
"""
return str(self)
def as_dict(self) -> dict:
def as_dict(self) -> list[str]:
"""
Get all the options as a dict.
Get all the options as a shell argument list.
Returns:
dict: The options as a dict
list[str]: The options as shell arguments.
"""
return shlex.split(shlex.join(self.args))

View file

@ -29,7 +29,7 @@ class BackgroundWorker(metaclass=Singleton):
"The queue to hold the tasks."
self.running = True
"Whether the background worker is running or not."
self.thread: threading.Thread = None
self.thread: threading.Thread | None = None
"The thread that runs the background worker."
@staticmethod
@ -49,7 +49,8 @@ class BackgroundWorker(metaclass=Singleton):
try:
LOG.debug("Stopping background worker thread.")
self.queue.put((CloseThread, (), {}))
self.thread.join(timeout=5)
if self.thread:
self.thread.join(timeout=5)
LOG.debug("Background worker thread has been shut down.")
except Exception as e:
LOG.exception(

View file

@ -1,14 +1,17 @@
import base64
import hmac
import inspect
from collections.abc import Awaitable
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
import anyio
from aiohttp import web
from aiohttp.web import Request, RequestHandler, Response
from aiohttp.typedefs import Handler, Middleware
from aiohttp.web import Request, Response
from aiohttp.web_log import AccessLogger
from aiohttp.web_request import BaseRequest
from aiohttp.web_response import StreamResponse
from app.library.log import get_logger
from app.library.Services import Services
@ -24,12 +27,12 @@ LOG = get_logger("http")
class HttpAccessLogger(AccessLogger):
def log(self, request: Request, response: Response, time: float) -> None:
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
try:
fmt_info = self._format_line(request, response, time)
values: list[object] = []
extra: dict[str, object] = {"elapsed_ms": round(time * 1000.0, 2)}
extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
for key, value in fmt_info:
values.append(value)
@ -37,7 +40,7 @@ class HttpAccessLogger(AccessLogger):
extra[key] = value
else:
parent, child = key
group = extra.get(parent, {})
group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment]
if not isinstance(group, dict):
group = {}
group[child] = value
@ -177,7 +180,7 @@ class HttpAPI:
registered_options.append(route.path)
@staticmethod
def basic_auth(username: str, password: str) -> Awaitable:
def basic_auth(username: str, password: str) -> Middleware:
"""
Middleware to handle basic authentication.
@ -191,7 +194,7 @@ class HttpAPI:
"""
@web.middleware
async def auth_handler(request: Request, handler: RequestHandler) -> Response:
async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
# if OPTIONS request, skip auth
if "OPTIONS" == request.method:
return await handler(request)
@ -204,7 +207,7 @@ class HttpAPI:
auth_cookie = request.cookies.get("auth")
if auth_cookie is not None:
try:
data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key)
data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
if data is not None:
data = base64.b64encode(data.encode()).decode()
auth_header = f"Basic {data}"
@ -250,7 +253,7 @@ class HttpAPI:
},
)
response: Response = await handler(request)
response: StreamResponse = await handler(request)
contentType: str | None = response.headers.get("content-type", None)
if contentType and not contentType.startswith("text/html"):
@ -261,7 +264,7 @@ class HttpAPI:
"auth",
encrypt_data(
f"{user_name}:{user_password}",
key=Config.get_instance().secret_key,
key=cast("bytes", Config.get_instance().secret_key),
),
max_age=60 * 60 * 24 * 7,
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
@ -280,9 +283,9 @@ class HttpAPI:
return auth_handler
@staticmethod
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable:
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
@web.middleware
async def middleware_handler(request: Request, handler: RequestHandler) -> Response:
async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
static_path = str(app.router["download_static"].url_for(filename=""))
if request.path.startswith(static_path):
realFile, status = get_file(
@ -301,12 +304,13 @@ class HttpAPI:
},
)
response: Response = await handler(request)
response: StreamResponse = await handler(request)
contentType: str = str(response.headers.get("content-type", ""))
if contentType.startswith("text/html") and getattr(response, "_path", None):
rewrite_path: str = base_path.rstrip("/")
async with await anyio.open_file(response._path, "rb") as f:
response_path = cast("Any", response)._path
async with await anyio.open_file(response_path, "rb") as f:
content = await f.read()
content: str = (
content.decode("utf-8")
@ -339,7 +343,7 @@ class HttpAPI:
return middleware_handler
async def _handle(self, handler: RequestHandler, request: Request) -> Response:
async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
"""
Call the handler with the request and return the response.

View file

@ -104,19 +104,20 @@ class HttpSocket:
}
)
self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit")
self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
@staticmethod
def ws_event(func): # type: ignore
def ws_event(func):
"""
Decorator to mark a method as a socket event.
"""
@functools.wraps(func) # type: ignore
@functools.wraps(func)
async def wrapper(*args, **kwargs):
return await func(*args, **kwargs) # type: ignore
return await func(*args, **kwargs)
wrapper._ws_event = func.__name__ # type: ignore
wrapper_with_event: Any = wrapper
wrapper_with_event._ws_event = func.__name__
return wrapper
async def on_shutdown(self, _: web.Application):
@ -134,9 +135,11 @@ class HttpSocket:
if not (data and data.data):
return
await Services.get_instance().get("queue").add(item=Item.format(data.data))
queue = Services.get_instance().get("queue")
if queue is not None:
await queue.add(item=Item.format(data.data))
self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add")
self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
load_modules(self.rootPath, self.rootPath / "routes" / "socket")

View file

@ -93,7 +93,7 @@ class Item:
bool: True if the item has extras, False otherwise.
"""
return self.extras and len(self.extras) > 0
return bool(self.extras and len(self.extras) > 0)
def add_extras(self, key: str, value: Any) -> None:
"""
@ -117,7 +117,7 @@ class Item:
bool: True if the item has command options for yt, False otherwise.
"""
return self.cli and len(self.cli) > 2
return bool(self.cli and len(self.cli) > 2)
@staticmethod
def _default_preset() -> str:
@ -161,7 +161,7 @@ class Item:
Item: The formatted item.
"""
url: str = item.get("url")
url = item.get("url")
if not url or not isinstance(url, str):
msg = "url param is required."
@ -173,7 +173,7 @@ class Item:
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
url = f"https://www.youtube.com/watch?v={url}"
data: dict[str, str] = {"url": url}
data: dict[str, Any] = {"url": url}
preset: str | None = item.get("preset")
if preset and isinstance(preset, str) and preset != Item._default_preset():
@ -185,24 +185,28 @@ class Item:
data["preset"] = preset
if item.get("folder") and isinstance(item.get("folder"), str):
data["folder"] = item.get("folder")
folder = item.get("folder")
if isinstance(folder, str) and folder:
data["folder"] = folder
if item.get("cookies") and isinstance(item.get("cookies"), str):
data["cookies"] = item.get("cookies")
cookies = item.get("cookies")
if isinstance(cookies, str) and cookies:
data["cookies"] = cookies
if item.get("template") and isinstance(item.get("template"), str):
data["template"] = item.get("template")
template = item.get("template")
if isinstance(template, str) and template:
data["template"] = template
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
data["auto_start"] = bool(item.get("auto_start"))
if isinstance(item.get("auto_start"), bool):
data["auto_start"] = item["auto_start"]
extras: dict | None = item.get("extras")
if extras and isinstance(extras, dict) and len(extras) > 0:
if isinstance(extras, dict) and len(extras) > 0:
data["extras"] = extras
if item.get("requeued") and isinstance(item.get("requeued"), bool):
data["requeued"] = item.get("requeued")
requeued = item.get("requeued")
if isinstance(requeued, bool):
data["requeued"] = requeued
cli: str | None = item.get("cli")
if cli and len(cli) > 2:
@ -574,8 +578,10 @@ class ItemDTO:
self.is_archivable = bool(self._archive_file)
if not self.is_archivable:
self.is_archived = False
else:
elif self.archive_id and self._archive_file:
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
else:
self.is_archived = False
def get_archive_file(self) -> str | None:
"""

View file

@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
class Packages:
def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
from_env: list[str] = env.split() if env else []
from_file = []

View file

@ -1,4 +1,6 @@
import asyncio
from collections.abc import Callable
from typing import Any
from aiocron import Cron
from aiohttp import web
@ -66,7 +68,7 @@ class Scheduler(metaclass=Singleton):
if data and data.data:
self.add(**data.data)
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add")
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
def get_all(self) -> dict[str, Cron]:
"""Return the jobs."""
@ -99,7 +101,7 @@ class Scheduler(metaclass=Singleton):
return id in self._jobs
def add(
self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None
self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
) -> str:
"""
Add a job to the schedule.

View file

@ -1,4 +1,5 @@
import inspect
from collections.abc import Callable
from dataclasses import dataclass
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
@ -117,7 +118,7 @@ class Services(metaclass=Singleton):
return candidates[0]
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
sig: inspect.Signature = inspect.signature(handler)
try:
@ -169,10 +170,10 @@ class Services(metaclass=Singleton):
return resolved
async def handle_async(self, handler: callable, **kwargs) -> Any:
async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return await handler(**resolved)
def handle_sync(self, handler: callable, **kwargs) -> Any:
def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return handler(**resolved)

View file

@ -75,7 +75,7 @@ class TerminalSessionManager(metaclass=Singleton):
self._cleanup_job_id = Scheduler.get_instance().add(
timer=CLEANUP_SCHEDULE,
func=self.cleanup,
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
)
async def on_startup(self, _: web.Application) -> None:

View file

@ -80,7 +80,7 @@ class UpdateChecker(metaclass=Singleton):
self._notify.subscribe(
event=Events.STARTED,
callback=event_handler,
name=f"{__class__.__name__}.{__class__.attach.__name__}",
name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
)
self._schedule_check()
@ -108,7 +108,7 @@ class UpdateChecker(metaclass=Singleton):
self._job_id = self._scheduler.add(
timer=timer,
func=lambda: asyncio.create_task(self.check_for_updates()),
id=f"{__class__.__name__}.{self.check_for_updates.__name__}",
id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
)
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:

View file

@ -40,7 +40,8 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
class FileLogFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
@ -78,7 +79,8 @@ class JsonLogFormatter(logging.Formatter):
return json.dumps(data, ensure_ascii=False, default=str)
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
@staticmethod
@ -210,7 +212,7 @@ class JsonLogFormatter(logging.Formatter):
return source
def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
"""
Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
Supports both synchronous and asynchronous functions.
@ -280,8 +282,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
async_wrapper.cache_clear = cache_clear
async_wrapper.cache_info = cache_info
async_wrapper_cache: Any = async_wrapper
async_wrapper_cache.cache_clear = cache_clear
async_wrapper_cache.cache_info = cache_info
return async_wrapper
# For sync functions, use the original implementation
@ -309,8 +312,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
return result
# expose cache_clear, cache_info
sync_wrapper.cache_clear = cached.cache_clear
sync_wrapper.cache_info = cached.cache_info
sync_wrapper_cache: Any = sync_wrapper
sync_wrapper_cache.cache_clear = cached.cache_clear
sync_wrapper_cache.cache_info = cached.cache_info
return sync_wrapper
return decorator
@ -381,7 +385,12 @@ def _is_safe_key(key: Any) -> bool:
def merge_dict(
source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None
source: dict,
destination: dict,
max_depth: int = 50,
max_list_size: int = 10000,
_depth: int = 0,
_seen: set | None = None,
) -> dict:
"""
Merge data from source into destination.
@ -472,7 +481,7 @@ def merge_dict(
return destination_copy
def check_id(file: Path) -> bool | str:
def check_id(file: Path) -> bool | Path:
"""
Check if we are able to get an id from the file name.
if so check if any video file with the same id exists.
@ -491,6 +500,8 @@ def check_id(file: Path) -> bool | str:
return False
id: str | None = match.groupdict().get("id")
if id is None:
return False
try:
if not file.parent.exists():
@ -545,9 +556,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
from yarl import URL
parsed_url = URL(url)
except ValueError:
except ValueError as exc:
msg = "Invalid URL."
raise ValueError(msg) # noqa: B904
raise ValueError(msg) from exc
# Check allowed schemes
if parsed_url.scheme not in ["http", "https"]:
@ -596,7 +607,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
@timed_lru_cache(ttl_seconds=60, max_size=256)
def get_file_sidecar(file: Path | None = None) -> dict[dict]:
def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
"""
Get sidecar files for the given file.
@ -925,7 +936,7 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
download_path = Path(download_path)
try:
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
if realFile.exists():
return (realFile, 200)
except Exception as e:
@ -936,11 +947,11 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
)
return (Path(file), 404)
possibleFile: bool | str = check_id(file=realFile)
if not possibleFile:
possibleFile: bool | Path = check_id(file=realFile)
if not isinstance(possibleFile, Path):
return (realFile, 404)
return (Path(possibleFile), 302)
return (possibleFile, 302)
def encrypt_data(data: str, key: bytes) -> str:
@ -962,7 +973,7 @@ def encrypt_data(data: str, key: bytes) -> str:
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
def decrypt_data(data: str, key: bytes) -> str:
def decrypt_data(data: str, key: bytes) -> str | None:
"""
Decrypts AES-GCM encrypted data
@ -975,8 +986,8 @@ def decrypt_data(data: str, key: bytes) -> str:
"""
try:
data = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = data[:12], data[12:-16], data[-16:]
raw: bytes = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
plaintext = cipher.decrypt_and_verify(ciphertext, tag)
return plaintext.decode()
@ -987,7 +998,7 @@ def decrypt_data(data: str, key: bytes) -> str:
def get(
data: dict | list,
path: str | list | None = None,
default: any = None,
default: Any = None,
separator=".",
):
"""
@ -1111,7 +1122,7 @@ def get_files(
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
exc_info=True,
)
return []
return [], 0
try:
dir_path.relative_to(base_path)
@ -1122,7 +1133,7 @@ def get_files(
base_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return []
return [], 0
if not str(dir_path).startswith(str(base_path)):
LOG.warning(
@ -1131,7 +1142,7 @@ def get_files(
base_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return []
return [], 0
if not dir_path.is_dir():
LOG.warning(
@ -1139,7 +1150,7 @@ def get_files(
dir_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return []
return [], 0
contents: list = []
for file in dir_path.iterdir():
@ -1298,7 +1309,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
try:
from http.cookiejar import MozillaCookieJar
cookies = MozillaCookieJar(str(file), None, None)
cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
cookies.load()
return (True, cookies)

View file

@ -1,6 +1,20 @@
import sys
from typing import Any
def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
if key not in data or data[key] is None:
return False, None
return True, data[key]
def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
if key not in data:
return False
del data[key]
return True
def get_value(value: Any) -> Any:
"""
Return the result of calling `value` if it is callable, else return `value`.
@ -91,14 +105,19 @@ def ag(
return val
return get_value(default)
key = path
if isinstance(array, dict) and key in array and array[key] is not None:
return array[key]
if isinstance(array, dict):
found, value = _dict_value(array, key)
if found:
return value
if not isinstance(key, str) or separator not in key:
return get_value(default)
current = array
for segment in key.split(separator):
if isinstance(current, dict) and segment in current and current[segment] is not None:
current = current[segment]
if isinstance(current, dict):
found, value = _dict_value(current, segment)
if not found:
return get_value(default)
current = value
elif isinstance(current, list):
try:
idx = int(segment)
@ -150,7 +169,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
except TypeError:
return False
if isinstance(data, dict):
if path in data and data[path] is not None:
found, _ = _dict_value(data, path)
if found:
return True
elif isinstance(data, list) and isinstance(path, int):
return 0 <= path < len(data) and data[path] is not None
@ -158,8 +178,11 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
segments = path_str.split(separator)
current = data
for seg in segments:
if isinstance(current, dict) and seg in current and current[seg] is not None:
current = current[seg]
if isinstance(current, dict):
found, value = _dict_value(current, seg)
if not found:
return False
current = value
elif isinstance(current, list):
try:
idx = int(seg)
@ -189,17 +212,16 @@ def ag_delete(
The modified structure.
"""
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if not isinstance(data, dict | list):
try:
data = vars(data)
except TypeError:
return data # type: ignore
if isinstance(data, dict) and path in data:
del data[path]
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if isinstance(data, dict) and _dict_delete(data, path):
return data
if isinstance(data, list) and isinstance(path, int):
if 0 <= path < len(data):
@ -210,8 +232,12 @@ def ag_delete(
last = segments.pop()
current = data
for seg in segments:
if isinstance(current, dict) and seg in current:
current = current[seg]
if isinstance(current, dict):
found, value = _dict_value(current, seg)
if not found:
current = None
break
current = value
elif isinstance(current, list):
try:
idx = int(seg)
@ -228,8 +254,8 @@ def ag_delete(
break
if current is None:
return data
if isinstance(current, dict) and last in current:
del current[last]
if isinstance(current, dict):
_dict_delete(current, last)
elif isinstance(current, list):
try:
idx = int(last)
@ -246,10 +272,12 @@ def run_tests():
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
def test_ag_set_overwrite_error():
error = ""
try:
ag_set({"a": 1}, "a.b", 2)
except RuntimeError as e:
assert "Cannot set value at path" in str(e) # noqa: PT017
error = str(e)
assert "Cannot set value at path" in error
def test_ag_get_value_callable_and_value():
assert get_value(5) == 5
@ -320,9 +348,9 @@ def run_tests():
]:
try:
test()
print(f"PASS: {test.__name__}") # noqa: T201
sys.stdout.write(f"PASS: {test.__name__}\n")
except AssertionError:
print(f"FAIL {test.__name__}") # noqa: T201
sys.stdout.write(f"FAIL {test.__name__}\n")
if __name__ == "__main__":

View file

@ -38,7 +38,7 @@ class Cache(metaclass=ThreadSafe):
Scheduler.get_instance().add(
timer="* * * * *",
func=self.cleanup,
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
)
def set(self, key: str, value: Any, ttl: float | None = None) -> None:

View file

@ -3,7 +3,7 @@ from __future__ import annotations
import http.cookiejar
from abc import ABC
from collections.abc import Callable
from typing import Any, ClassVar
from typing import Any, ClassVar, cast
from yt_dlp.networking.common import (
_REQUEST_HANDLERS,
@ -223,11 +223,13 @@ class CFSolverRH(RequestHandler, ABC):
try:
response: Response = director.send(request)
except HTTPError as error:
if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers):
if error.response and is_cf_challenge(
getattr(error.response, "status", None), cast("Any", error.response.headers)
):
return self._retry_with_clearance(request, error.response, director)
raise
if is_cf_challenge(getattr(response, "status", None), response.headers):
if is_cf_challenge(getattr(response, "status", None), cast("Any", response.headers)):
return self._retry_with_clearance(request, response, director)
return response

View file

@ -1,16 +1,18 @@
# flake8: noqa: S310
from __future__ import annotations
import json
import time
import urllib.request
from typing import Any
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
from app.library.log import get_logger
from .cache import Cache
if TYPE_CHECKING:
from collections.abc import Mapping
CACHE: Cache = Cache()
LOG = get_logger()
@ -57,7 +59,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
if user_agent:
payload.setdefault("headers", {})["User-Agent"] = user_agent
req = urllib.request.Request(
req = urllib.request.Request( # noqa: S310
endpoint,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
@ -68,7 +70,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
"Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint}
)
start_time = time.time()
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: # noqa: S310
result = json.loads(resp.read().decode("utf-8"))
if "ok" != result.get("status"):
@ -98,7 +100,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
return CACHE.get(host)
def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool:
def is_cf_challenge(status: int | None, headers: Mapping[str, Any] | None) -> bool:
"""
Determine whether a response indicates a Cloudflare challenge.
"""

View file

@ -2,6 +2,7 @@ import logging
import multiprocessing
import os
import re
import shutil
import sys
import time
from logging.handlers import TimedRotatingFileHandler
@ -30,7 +31,7 @@ APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = (
if TYPE_CHECKING:
from subprocess import CompletedProcess
SUPPORTED_CODECS: tuple[str] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
SUPPORTED_CODECS: tuple[str, ...] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
"Supported encoder names in order of preference."
@ -179,7 +180,7 @@ class Config(metaclass=Singleton):
file_logging: bool = True
"Enable file logging."
secret_key: str
secret_key: str | bytes
"The secret key to use for the application."
tasks_handler_timer: str = "15 */1 * * *"
@ -377,7 +378,7 @@ class Config(metaclass=Singleton):
LOG = get_logger()
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
envFile: str = Path(self.config_path) / ".env"
envFile: Path = Path(self.config_path) / ".env"
if envFile.exists():
LOG.info("Loading environment variables from '%s'.", envFile)
@ -388,7 +389,7 @@ class Config(metaclass=Singleton):
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
Path(baseDefaultPath) / "var" / "downloads"
)
self.app_path = Path(__file__).parent.parent.absolute()
self.app_path = str(Path(__file__).parent.parent.absolute())
for k, v in self._get_attributes().items():
if k.startswith("_") or k in self._manual_vars:
@ -501,7 +502,7 @@ class Config(metaclass=Singleton):
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
key_file: str = Path(self.config_path) / "secret.key"
key_file: Path = Path(self.config_path) / "secret.key"
if key_file.exists() and key_file.stat().st_size > 2:
with open(key_file, "rb") as f:
@ -511,7 +512,7 @@ class Config(metaclass=Singleton):
with open(key_file, "wb") as f:
f.write(self.secret_key)
self.started = time.time()
self.started = int(time.time())
for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS:
logging.getLogger(_tool).setLevel(_level)
@ -545,7 +546,7 @@ class Config(metaclass=Singleton):
def _get_attributes(self) -> dict:
attrs: dict = {}
vClass: str = self.__class__
vClass: type = self.__class__
for attribute in vClass.__dict__:
if attribute.startswith("_"):
@ -601,7 +602,7 @@ class Config(metaclass=Singleton):
dict[str, str]: The replacer variables.
"""
keys: tuple[str] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
keys: tuple[str, ...] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
return {k: getattr(self, k) for k in keys}
@staticmethod
@ -619,15 +620,20 @@ class Config(metaclass=Singleton):
This is used to set the version to the latest git tag.
"""
LOG = get_logger()
git_path: str = Path(__file__).parent / ".." / ".." / ".git"
git_path: Path = Path(__file__).parent / ".." / ".." / ".git"
if not git_path.exists():
return
try:
import subprocess
git = shutil.which("git")
if not git:
LOG.warning("Git executable was not found.")
return
branch_result: CompletedProcess[str] = subprocess.run(
["git", "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607
[git, "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=os.path.dirname(git_path),
capture_output=True,
text=True,
@ -645,7 +651,7 @@ class Config(metaclass=Singleton):
return
commit_result: CompletedProcess[str] = subprocess.run(
["git", "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], # noqa: S607
[git, "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"],
cwd=os.path.dirname(git_path),
capture_output=True,
text=True,

View file

@ -39,6 +39,14 @@ if TYPE_CHECKING:
class Download:
update_task: asyncio.Task | None = None
@staticmethod
def _raise_no_formats(msg: str) -> None:
raise ValueError(msg)
@staticmethod
def _raise_interrupted() -> None:
raise SystemExit(130)
def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None):
"""
Initialize download task.
@ -51,7 +59,7 @@ class Download:
"""
config: Config = Config.get_instance()
self.download_dir: str = info.download_dir
self.download_dir: str = info.download_dir or ""
self.temp_dir: str | None = info.temp_dir
self.template: str | None = info.template
self.template_chapter: str | None = info.template_chapter
@ -98,6 +106,16 @@ class Download:
params: dict[str, Any] = {}
download_skipped = False
hook_handlers = self._hook_handlers
if hook_handlers is None:
msg = "_hook_handlers must be initialized before _download(). Call start() first."
raise RuntimeError(msg)
status_queue = self.status_queue
if status_queue is None:
msg = "status_queue must be initialized before _download(). Call start() first."
raise RuntimeError(msg)
try:
params = (
self.info.get_ytdlp_opts()
@ -125,9 +143,9 @@ class Download:
params.update(
{
"progress_hooks": [self._hook_handlers.progress_hook],
"postprocessor_hooks": [self._hook_handlers.postprocessor_hook],
"post_hooks": [self._hook_handlers.post_hook],
"progress_hooks": [hook_handlers.progress_hook],
"postprocessor_hooks": [hook_handlers.postprocessor_hook],
"post_hooks": [hook_handlers.post_hook],
}
)
@ -137,7 +155,10 @@ class Download:
if self.info.cookies:
try:
cookie_file = Path(self._temp_manager.temp_path or self.temp_dir) / f"cookie_{self.info._id}.txt"
cookie_file = (
Path(self._temp_manager.temp_path or self.temp_dir or self.download_dir)
/ f"cookie_{self.info._id}.txt"
)
self.logger.debug(
f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.",
extra={
@ -173,7 +194,9 @@ class Download:
f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info."
)
self.info_dict = None
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default:
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and (
(preset := self.info.get_preset()) is not None and preset.default
):
self.logger.debug(
"Removing download archive option for generic extractor on '%s'.",
self.info.url,
@ -267,7 +290,7 @@ class Download:
if filtered_logs := extract_ytdlp_logs(self.logs):
msg += " " + ", ".join(filtered_logs)
raise ValueError(msg) # noqa: TRY301
self._raise_no_formats(msg)
self.logger.info(
f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.",
@ -327,11 +350,11 @@ class Download:
def mark_cancelled(*_) -> None:
cls._interrupted = True
cls.to_screen("[info] Interrupt received, exiting cleanly...")
raise SystemExit(130) # noqa: TRY301
self._raise_interrupted()
signal.signal(signal.SIGUSR1, mark_cancelled)
self.status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
self.logger.debug(
@ -386,7 +409,7 @@ class Download:
)
ret = cls.download(url_list=[self.info.url])
self.status_queue.put(
status_queue.put(
{
"id": self.id,
"status": "finished" if 0 == ret else "error",
@ -407,7 +430,7 @@ class Download:
}
},
)
self.status_queue.put(
status_queue.put(
{
"id": self.id,
"status": "skip",
@ -432,7 +455,7 @@ class Download:
}
},
)
self.status_queue.put(
status_queue.put(
{
"id": self.id,
"status": "error",
@ -442,7 +465,7 @@ class Download:
}
)
finally:
self.status_queue.put(Terminator())
status_queue.put(Terminator())
if cookie_file and cookie_file.exists():
try:
cookie_file.unlink()
@ -499,24 +522,24 @@ class Download:
Process exit code or None
"""
self.status_queue = Config.get_manager().Queue()
status_queue: Any = Config.get_manager().Queue()
self.status_queue = status_queue
if temp_path := self._temp_manager.create_temp_path():
self.info.temp_path = str(temp_path)
temp_path = self._temp_manager.create_temp_path()
self._status_tracker = StatusTracker(
info=self.info,
download_id=self.id,
download_dir=self.download_dir,
temp_path=temp_path,
status_queue=self.status_queue,
status_queue=status_queue,
logger=self.logger,
debug=self.debug,
)
self._hook_handlers = HookHandlers(
download_id=self.id,
status_queue=self.status_queue,
status_queue=status_queue,
logger=self.logger,
debug=self.debug,
)
@ -529,7 +552,8 @@ class Download:
EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info)
asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}")
ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join)
proc = self._process_manager.proc
ret = await asyncio.get_running_loop().run_in_executor(None, proc.join) if proc else None
if self._status_tracker.final_update:
return ret

View file

@ -4,7 +4,7 @@ import time
import uuid
from numbers import Number
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import yt_dlp.utils
@ -67,7 +67,7 @@ async def add_item(
already=None,
logs: list | None = None,
yt_params: dict | None = None,
) -> dict[str, str]:
) -> dict[str, Any]:
"""
Route an entry to the appropriate processor based on type.
@ -102,7 +102,7 @@ async def add_item(
async def add(
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
) -> dict[str, str]:
) -> dict[str, Any]:
"""
Add an item to the download queue.
@ -204,7 +204,7 @@ async def add(
)
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
dlInfo.info.msg += f" Found in archive '{archive_file}'."
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{archive_file}'."
await queue.done.put(dlInfo)
@ -306,9 +306,12 @@ async def add(
new_archive_id: str | None = None
if entry.get("extractor_key") and entry.get("id"):
new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}"
if new_archive_id != archive_id:
extra_ids.append(new_archive_id)
_extractor_key = entry.get("extractor_key")
_entry_id = entry.get("id")
if isinstance(_extractor_key, str) and _entry_id is not None:
new_archive_id = f"{_extractor_key.lower()} {_entry_id!s}"
if new_archive_id != archive_id:
extra_ids.append(new_archive_id)
if len(extra_ids) > 0:
archive_ids: list[str] = archive_read(_archive_file, extra_ids)
@ -336,7 +339,7 @@ async def add(
)
)
dlInfo.info.msg += f" Found in archive '{_archive_file}'."
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{_archive_file}'."
await queue.done.put(dlInfo)
queue._notify.emit(
@ -374,7 +377,7 @@ async def add(
if condition.extras.get("ignore_download", False):
extra_msg: str = ""
if _archive_file and not condition.extras.get("no_archive", False):
if _archive_file and not condition.extras.get("no_archive", False) and archive_id:
archive_add(_archive_file, [archive_id])
extra_msg = f" and added to archive file '{_archive_file}'"
@ -385,7 +388,7 @@ async def add(
if not store_type:
dlInfo = Download(
info=ItemDTO(
id=entry.get("id"),
id=str(entry.get("id") or item.url),
title=_name,
url=item.url,
preset=item.preset,
@ -401,6 +404,9 @@ async def add(
LOG.info(log_message)
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
if dlInfo is None:
msg = "Download info is not available."
raise RuntimeError(msg)
queue._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},

View file

@ -121,8 +121,8 @@ async def process_playlist(
max_downloads: int = -1
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int):
max_downloads: int = ytdlp_opts.get("max_downloads")
if isinstance(ytdlp_opts.get("max_downloads"), int):
max_downloads = ytdlp_opts["max_downloads"]
results: list[dict[str, str]] = []
for i, etr in enumerate(entries, start=1):

View file

@ -99,15 +99,19 @@ class ProcessManager:
if not self.running():
return False
procId: int | None = self.proc.ident
proc = self.proc
if proc is None:
return False
procId: int | None = proc.ident
try:
self.logger.info(
"Stopping download process PID=%s.",
self.proc.pid,
proc.pid,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"process_ident": procId,
"is_live": self.is_live,
}
@ -117,25 +121,25 @@ class ProcessManager:
if self.is_live:
self.logger.debug(
"Requesting graceful live cancellation for PID=%s.",
self.proc.pid,
proc.pid,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"is_live": self.is_live,
}
},
)
self.cancel_event.set()
if wait_for_process_with_timeout(self.proc, 10):
if wait_for_process_with_timeout(proc, 10):
self.logger.debug(
"Download process PID=%s stopped gracefully.",
self.proc.pid,
proc.pid,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"is_live": self.is_live,
}
},
@ -143,43 +147,43 @@ class ProcessManager:
return True
self.logger.warning(
"Download process PID=%s did not respond to live cancellation; forcing termination.",
self.proc.pid,
proc.pid,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"is_live": self.is_live,
"force": True,
}
},
)
elif self.proc.pid and "posix" == os.name:
elif proc.pid and "posix" == os.name:
signal_name = "SIGUSR1"
try:
self.logger.debug(
"Sending %s signal to download process PID=%s.",
signal_name,
self.proc.pid,
proc.pid,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"signal": signal_name,
}
},
)
os.kill(self.proc.pid, signal.SIGUSR1)
os.kill(proc.pid, signal.SIGUSR1)
if wait_for_process_with_timeout(self.proc, 5):
if wait_for_process_with_timeout(proc, 5):
self.logger.debug(
"Download process PID=%s stopped gracefully.",
self.proc.pid,
proc.pid,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"signal": signal_name,
}
},
@ -187,12 +191,12 @@ class ProcessManager:
return True
self.logger.warning(
"Download process PID=%s did not respond to %s; forcing termination.",
self.proc.pid,
proc.pid,
signal_name,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"signal": signal_name,
"force": True,
}
@ -203,51 +207,49 @@ class ProcessManager:
self.logger.debug(
"Failed to send %s signal to download process PID=%s. %s",
signal_name,
self.proc.pid,
proc.pid,
e,
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid,
"signal": signal_name,
"exception_type": type(e).__name__,
}
},
)
if self.proc.is_alive():
if proc.is_alive():
self.logger.info(
"Terminating download process PID=%s.",
self.proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}},
proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
)
self.proc.terminate()
proc.terminate()
if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2):
if not wait_for_process_with_timeout(proc, 1 if self.is_live else 2):
self.logger.warning(
"Download process PID=%s did not terminate; killing forcefully.",
self.proc.pid,
extra={
"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}
},
proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
)
self.proc.kill()
wait_for_process_with_timeout(self.proc, 1)
proc.kill()
wait_for_process_with_timeout(proc, 1)
self.logger.info(
"Download process PID=%s stopped.",
self.proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid}},
proc.pid,
extra={"download": {"download_id": self.download_id, "process_id": proc.pid}},
)
return True
except Exception as e:
self.logger.exception(
f"Failed to stop download process PID={self.proc.pid}, ident={procId}.",
f"Failed to stop download process PID={proc.pid if proc else None}, ident={procId}.",
extra={
"download": {
"download_id": self.download_id,
"process_id": self.proc.pid,
"process_id": proc.pid if proc else None,
"process_ident": procId,
"is_live": self.is_live,
"exception_type": type(e).__name__,
@ -271,11 +273,12 @@ class ProcessManager:
return False
self.cancel_in_progress = True
procId: int | None = self.proc.ident
proc = self.proc
procId: int | None = proc.ident if proc else None
if not procId:
if self.proc:
self.proc.close()
if proc:
proc.close()
self.proc = None
self.logger.warning("Attempted to close download process, but it is not running.")
return False
@ -293,13 +296,14 @@ class ProcessManager:
loop = asyncio.get_running_loop()
if self.proc.is_alive():
current = self.proc
if current is not None and current.is_alive():
self.logger.debug(
"Waiting for download process PID='%s' to close.",
procId,
extra={"download": {"download_id": self.download_id, "process_ident": procId}},
)
await loop.run_in_executor(None, self.proc.join)
await loop.run_in_executor(None, current.join)
self.logger.debug(
"Download process PID='%s' closed.",
procId,

View file

@ -53,7 +53,7 @@ class DownloadQueue(metaclass=Singleton):
async def event_handler(_, __):
await self.initialize()
self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}")
self._notify.subscribe(Events.STARTED, event_handler, f"{DownloadQueue.__name__}.initialize")
Scheduler.get_instance().add(
timer="* * * * *",
@ -689,7 +689,7 @@ class DownloadQueue(metaclass=Singleton):
)
return {"deleted": deleted_count}
async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
async def get(self, mode: str = "all") -> dict[str, dict[str, ItemDTO]]:
"""
Get the download queue and the download history.

View file

@ -55,7 +55,7 @@ class StatusTracker:
self.final_update = False
self._terminator_sent: bool = False
self._candidate_filepath: Path | None = None
self.update_task: asyncio.Task | None = None
self.update_task: asyncio.Future[Any] | None = None
self._last_progress_time: float = 0.0
self._pending_progress: bool = False
self._progress_interval: float = 0.5
@ -256,7 +256,7 @@ class StatusTracker:
self.info.percent = status["downloaded_bytes"] / total * 100
except ZeroDivisionError:
self.info.percent = 0
self.info.total_bytes = total
self.info.total_bytes = int(total)
self.info.speed = status.get("speed")
self.info.eta = status.get("eta")
@ -280,8 +280,9 @@ class StatusTracker:
"""
while True:
try:
self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
status = await self.update_task
update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
self.update_task = asyncio.ensure_future(update_task)
status = await update_task
if status is None or isinstance(status, Terminator):
self._flush_progress()
return

View file

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import functools
import importlib.util
import threading
from dataclasses import dataclass
from typing import Any, Literal, cast, overload
@ -98,12 +99,7 @@ def _normalize_proxy(proxy: str | dict[str, str] | None) -> str | None:
@functools.lru_cache(maxsize=1)
def _curl_available() -> bool:
try:
import httpx_curl_cffi # noqa: F401
return True
except Exception:
return False
return importlib.util.find_spec("httpx_curl_cffi") is not None
def resolve_curl_transport(use_curl: bool = True) -> bool:
@ -121,7 +117,7 @@ def _build_async_curl_transport(
from httpx_curl_cffi import AsyncCurlTransport
return AsyncCurlTransport(
impersonate=curl_impersonate,
impersonate=cast("Any", curl_impersonate),
default_headers=curl_default_headers,
)
@ -308,10 +304,7 @@ def get_async_client(
curl_default_headers=curl_default_headers,
)
client = httpx.AsyncClient(
transport=cast(
"httpx.AsyncBaseTransport",
_get_transport(enable_cf, is_async=True, transport=transport),
),
transport=_get_transport(enable_cf, is_async=True, transport=transport),
proxy=cast("Any", proxy),
)
Globals.SHARED_ASYNC_CLIENTS[key] = client
@ -330,10 +323,7 @@ def get_sync_client(
return Globals.SHARED_SYNC_CLIENTS[key]
client = httpx.Client(
transport=cast(
"httpx.BaseTransport",
_get_transport(enable_cf, is_async=False, transport=None),
),
transport=_get_transport(enable_cf, is_async=False, transport=None),
proxy=cast("Any", proxy),
)
Globals.SHARED_SYNC_CLIENTS[key] = client

View file

@ -1,4 +1,3 @@
# flake8: noqa: S608
"""
This module is a fork of the Caribou library
(https://pypi.org/project/caribou/) modified to work for our specific use case.
@ -9,10 +8,11 @@ import datetime
import glob
import os.path
import traceback
from collections.abc import AsyncIterator, Iterable, Sequence
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from importlib.machinery import ModuleSpec, SourceFileLoader
from importlib.util import module_from_spec, spec_from_loader
from typing import TYPE_CHECKING
from pathlib import Path
from typing import TYPE_CHECKING, Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection
@ -44,7 +44,9 @@ class InvalidNameError(Error):
@contextlib.asynccontextmanager
async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator:
async def execute(
conn: AsyncConnection, sql: str, params: Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None
) -> AsyncIterator:
params = {} if params is None else params
result = await conn.execute(text(sql), params)
try:
@ -126,12 +128,12 @@ class Database:
self.version_table: str = version_table
self._owns_connection = bool(isinstance(db_url, str))
if self._owns_connection:
self._owns_connection = isinstance(db_url, str)
if isinstance(db_url, str):
self.conn: AsyncConnection | None = None
self.db_url: str = db_url
else:
self.conn: AsyncConnection = db_url
self.conn = db_url
async def __aenter__(self):
if self._owns_connection and self.conn is None:
@ -183,7 +185,8 @@ class Database:
async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
assert self.conn
if target_version not in (0, "0"):
_assert_migration_exists(migrations, target_version)
_assert_migration_exists(migrations, str(target_version))
target = str(target_version)
migrations.sort(key=lambda x: x.get_version(), reverse=True)
database_version: str | None = await self.get_version()
@ -192,7 +195,7 @@ class Database:
current_version: str = migration.get_version()
if database_version is not None and current_version > database_version:
continue
if current_version <= target_version:
if current_version <= target:
break
await migration.downgrade(self.conn)
next_version: str | int = 0
@ -210,14 +213,14 @@ class Database:
assert self.conn
if not await self.is_version_controlled():
return None
sql: str = f"SELECT version FROM {self.version_table}"
sql: str = f"SELECT version FROM {self.version_table}" # noqa: S608
async with execute(self.conn, sql) as result:
rows = result.fetchall()
return rows[0][0] if rows else "0"
async def update_version(self, version: str) -> None:
assert self.conn
sql: str = f"UPDATE {self.version_table} SET version = :version"
sql: str = f"UPDATE {self.version_table} SET version = :version" # noqa: S608
async with transaction(self.conn):
await self.conn.execute(text(sql), {"version": version})
@ -226,7 +229,7 @@ class Database:
sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
async with transaction(self.conn):
await self.conn.execute(text(sql))
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"})
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) # noqa: S608
def __repr__(self) -> str:
return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
@ -238,7 +241,7 @@ def _assert_migration_exists(migrations: Iterable[Migration], version: str | int
raise Error(msg)
def load_migrations(directory: str) -> list[Migration]:
def load_migrations(directory: str | Path) -> list[Migration]:
"""Return the migrations contained in the given directory."""
directory = str(directory)
if not os.path.exists(directory) or not os.path.isdir(directory):
@ -247,7 +250,7 @@ def load_migrations(directory: str) -> list[Migration]:
return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None:
async def upgrade(db_url: AsyncConnection | str, migration_dir: str | Path, version: str | None = None) -> None:
"""
Upgrade the given database with the migrations contained in the
migrations directory. If a version is not specified, upgrade
@ -260,7 +263,7 @@ async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: st
await db.upgrade(load_migrations(migration_dir), version)
async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None:
async def downgrade(db_url: str | AsyncConnection, migration_dir: str | Path, version: str) -> None:
"""
Downgrade the database to the given version with the migrations
contained in the given migration directory.

View file

@ -2,14 +2,16 @@ import asyncio
import contextlib
import json
import os
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from dataclasses import fields
from datetime import UTC, datetime
from email.utils import formatdate
from typing import Any
from urllib.parse import quote_plus
from aiohttp import web
from sqlalchemy import text
from sqlalchemy.engine.row import RowMapping
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.ext.asyncio.engine import AsyncConnection
@ -36,13 +38,13 @@ def _memory_db_url(db_path: str) -> str:
return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true"
def _decode_row(row: dict) -> ItemDTO:
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
def _decode_row(row: Mapping[str, Any] | RowMapping) -> ItemDTO:
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC)
data = json.loads(row["data"])
data.pop("_id", None)
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"]
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
item.datetime = formatdate(row_date.timestamp())
return item
@ -149,8 +151,8 @@ class SqliteStore(metaclass=ThreadSafe):
return self._sessionmaker
async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
await self.get_connection()
result = await self._conn.execute(
conn = await self.get_connection()
result = await conn.execute(
text(
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC'
),
@ -163,12 +165,12 @@ class SqliteStore(metaclass=ThreadSafe):
async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
return await self.get(type_value, key=key, url=url) is not None
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> ItemDTO | None:
if not key and not url:
msg = "key or url must be provided."
raise KeyError(msg)
await self.get_connection()
conn = await self.get_connection()
clauses: list[str] = []
params: dict[str, str] = {"type_value": type_value}
@ -186,7 +188,7 @@ class SqliteStore(metaclass=ThreadSafe):
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608
)
result = await self._conn.execute(text(query), params)
result = await conn.execute(text(query), params)
row = result.mappings().first()
if not row:
@ -195,8 +197,8 @@ class SqliteStore(metaclass=ThreadSafe):
return _decode_row(row)
async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
await self.get_connection()
result = await self._conn.execute(
conn = await self.get_connection()
result = await conn.execute(
text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'),
{"type_value": type_value, "id": id},
)
@ -212,24 +214,24 @@ class SqliteStore(metaclass=ThreadSafe):
if not ids_list:
return []
await self.get_connection()
conn = await self.get_connection()
placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list)))
params: dict[str, str] = {"type_value": type_value}
params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)})
result = await self._conn.execute(
result = await conn.execute(
text(
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608
),
params,
)
rows = result.mappings().all()
rows = list(result.mappings().all())
order = {item_id: index for index, item_id in enumerate(ids_list)}
rows.sort(key=lambda row: order.get(row["id"], len(ids_list)))
return [(row["id"], _decode_row(row)) for row in rows]
async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]:
await self.get_connection()
conn = await self.get_connection()
params: dict[str, str] = {"type_value": type_value}
where_clauses = ['"type" = :type_value']
@ -239,7 +241,7 @@ class SqliteStore(metaclass=ThreadSafe):
params.update(status_params)
query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608
result = await self._conn.execute(text(query), params)
result = await conn.execute(text(query), params)
rows = result.mappings().all()
return [(row["id"], _decode_row(row)) for row in rows]
@ -254,7 +256,7 @@ class SqliteStore(metaclass=ThreadSafe):
if not kwargs:
return None
await self.get_connection()
conn = await self.get_connection()
clauses: list[str] = []
params: dict[str, str | float | int] = {"type_value": type_value}
@ -321,7 +323,7 @@ class SqliteStore(metaclass=ThreadSafe):
where_clause: str = " OR ".join(f"({clause})" for clause in clauses)
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608
result = await self._conn.execute(text(query), params)
result = await conn.execute(text(query), params)
row = result.mappings().first()
if not row:
@ -332,7 +334,7 @@ class SqliteStore(metaclass=ThreadSafe):
return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None
async def count(self, type_value: str, status_filter: str | None = None) -> int:
await self.get_connection()
conn = await self.get_connection()
where_clauses: list[str] = ['"type" = :type_value']
params: dict[str, str] = {"type_value": type_value}
@ -344,7 +346,7 @@ class SqliteStore(metaclass=ThreadSafe):
where_clause: str = " AND ".join(where_clauses)
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
result = await self._conn.execute(text(count_query), params)
result = await conn.execute(text(count_query), params)
row = result.mappings().first()
return row["count"] if row else 0
@ -357,7 +359,7 @@ class SqliteStore(metaclass=ThreadSafe):
order: str,
status_filter: str | None = None,
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
await self.get_connection()
conn = await self.get_connection()
where_clauses: list[str] = ['"type" = :type_value']
params: dict[str, str | int] = {"type_value": type_value}
@ -369,7 +371,7 @@ class SqliteStore(metaclass=ThreadSafe):
where_clause: str = " AND ".join(where_clauses)
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
result = await self._conn.execute(text(count_query), params)
result = await conn.execute(text(count_query), params)
count_row = result.mappings().first()
total_items = count_row["count"] if count_row else 0
@ -384,7 +386,7 @@ class SqliteStore(metaclass=ThreadSafe):
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608
result = await self._conn.execute(text(query), params)
result = await conn.execute(text(query), params)
rows = result.mappings().all()
items = [(row["id"], _decode_row(row)) for row in rows]
@ -396,15 +398,15 @@ class SqliteStore(metaclass=ThreadSafe):
await self._upsert_now(type_value, item)
async def delete(self, type_value: str, key: str) -> None:
await self.get_connection()
await self._conn.execute(
conn = await self.get_connection()
await conn.execute(
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
{"type_value": type_value, "key": key},
)
await self._conn.commit()
await conn.commit()
async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
await self.get_connection()
conn = await self.get_connection()
keys_list: list[str] = list(keys)
if not keys_list:
return 0
@ -412,15 +414,15 @@ class SqliteStore(metaclass=ThreadSafe):
params: dict[str, str] = {"type_value": type_value}
params.update({f"key_{i}": key for i, key in enumerate(keys_list)})
result = await self._conn.execute(
result = await conn.execute(
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
params,
)
await self._conn.commit()
await conn.commit()
return result.rowcount if result else 0
async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int:
await self.get_connection()
conn = await self.get_connection()
params: dict[str, str] = {"type_value": type_value}
where_clauses = ['"type" = :type_value']
@ -429,11 +431,11 @@ class SqliteStore(metaclass=ThreadSafe):
where_clauses.append(status_clause)
params.update(status_params)
result = await self._conn.execute(
result = await conn.execute(
text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608
params,
)
await self._conn.commit()
await conn.commit()
return result.rowcount if result else 0
async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
@ -484,6 +486,9 @@ class SqliteStore(metaclass=ThreadSafe):
async def _enqueue(self, op: _Op) -> None:
self._ensure_worker()
if self._queue is None:
msg = "Queue was not initialized by _ensure_worker."
raise RuntimeError(msg)
await self._queue.put(op)
def _ensure_worker(self) -> None:
@ -492,10 +497,14 @@ class SqliteStore(metaclass=ThreadSafe):
self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
async def _writer(self) -> None:
queue = self._queue
if queue is None:
msg = "Writer started before queue was initialized."
raise RuntimeError(msg)
while True:
op = await self._queue.get()
op = await queue.get()
if isinstance(op.op, Terminator):
self._queue.task_done()
queue.task_done()
break
try:
async with self._lock:
@ -507,34 +516,34 @@ class SqliteStore(metaclass=ThreadSafe):
extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__},
)
finally:
self._queue.task_done()
queue.task_done()
await asyncio.sleep(self._flush_interval)
async def _apply(self, op: _Op) -> None:
await self.get_connection()
conn = await self.get_connection()
if op.op == "upsert" and op.item:
await self._upsert_now_conn(self._conn, op.type_value, op.item)
await self._conn.commit()
await self._upsert_now_conn(conn, op.type_value, op.item)
await conn.commit()
elif op.op == "delete" and op.key:
await self._conn.execute(
await conn.execute(
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
{"type_value": op.type_value, "key": op.key},
)
await self._conn.commit()
await conn.commit()
elif op.op == "bulk_delete" and op.keys:
placeholders = ",".join(f":key_{i}" for i in range(len(op.keys)))
params: dict[str, str] = {"type_value": op.type_value}
params.update({f"key_{i}": key for i, key in enumerate(op.keys)})
await self._conn.execute(
await conn.execute(
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
params,
)
await self._conn.commit()
await conn.commit()
async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
await self.get_connection()
await self._upsert_now_conn(self._conn, type_value, item)
await self._conn.commit()
conn = await self.get_connection()
await self._upsert_now_conn(conn, type_value, item)
await conn.commit()
async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None:
"""Helper to upsert using an existing connection."""
@ -560,7 +569,7 @@ class SqliteStore(metaclass=ThreadSafe):
async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None:
"""Execute raw SQL query (for testing purposes)."""
await self.get_connection()
conn = await self.get_connection()
if isinstance(params, tuple):
# Convert positional params to dict for SQLAlchemy
# Assuming queries use ? placeholders, we need to count them
@ -575,16 +584,16 @@ class SqliteStore(metaclass=ThreadSafe):
# Replace ? with :p0, :p1, etc.
for i in range(len(params)):
query = query.replace("?", f":p{i}", 1)
await self._conn.execute(text(query), param_dict)
await conn.execute(text(query), param_dict)
elif isinstance(params, dict):
await self._conn.execute(text(query), params)
await conn.execute(text(query), params)
else:
await self._conn.execute(text(query))
await self._conn.commit()
await conn.execute(text(query))
await conn.commit()
async def fetch_raw(self, query: str, params: dict | tuple | None = None):
"""Fetch results from raw SQL query (for testing purposes)."""
await self.get_connection()
conn = await self.get_connection()
if isinstance(params, tuple):
# Convert positional params to dict for SQLAlchemy
placeholders = query.count("?")
@ -596,11 +605,11 @@ class SqliteStore(metaclass=ThreadSafe):
param_dict = {f"p{i}": params[i] for i in range(len(params))}
for i in range(len(params)):
query = query.replace("?", f":p{i}", 1)
result = await self._conn.execute(text(query), param_dict)
result = await conn.execute(text(query), param_dict)
elif isinstance(params, dict):
result = await self._conn.execute(text(query), params)
result = await conn.execute(text(query), params)
else:
result = await self._conn.execute(text(query))
result = await conn.execute(text(query))
return result.mappings().all()
async def get_connection(self) -> AsyncConnection:

View file

@ -1,14 +1,14 @@
#!/usr/bin/env python3
# flake8: noqa: S310 T201
import argparse
import http.client
import os
import pathlib
import socket
import sys
import time
import urllib.request
import webbrowser
from urllib.parse import urlsplit
if __name__ == "__main__":
from multiprocessing import freeze_support
@ -26,7 +26,7 @@ APP_ROOT = str((pathlib.Path(__file__).parent / "..").resolve())
if APP_ROOT not in sys.path:
sys.path.insert(0, APP_ROOT)
import platformdirs # type: ignore
import platformdirs
def set_env():
@ -55,18 +55,27 @@ def open_browser_when_ready(url: str, timeout: float = 5.0) -> None:
import threading
def wait_then_open():
parsed = urlsplit(url)
path = parsed.path or "/"
deadline = time.time() + timeout
while time.time() < deadline:
try:
with urllib.request.urlopen(url, timeout=1) as response:
if 200 == response.status:
conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 80, timeout=1)
try:
conn.request("GET", path)
response = conn.getresponse()
if response.status == 200:
os.environ.pop("LD_LIBRARY_PATH", None)
webbrowser.open_new(url)
return
finally:
conn.close()
except Exception:
time.sleep(0.5)
print(f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.")
sys.stderr.write(
f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.\n"
)
threading.Thread(target=wait_then_open, daemon=True).start()
@ -115,7 +124,7 @@ def main():
set_env()
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH")) / ".env"
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH", "")) / ".env"
port = None
if env_file.exists():

View file

@ -199,18 +199,16 @@ class Main:
)
HTTP_LOGGER_CLASS = HttpAccessLogger
run_args = {
"host": host,
"port": port,
"loop": asyncio.get_event_loop(),
"access_log": HTTP_LOGGER,
"print": started,
"handle_signals": cb is None,
}
if HTTP_LOGGER_CLASS is not None:
run_args["access_log_class"] = HTTP_LOGGER_CLASS
web.run_app(self._app, **run_args)
web.run_app(
self._app,
host=host,
port=port,
loop=asyncio.get_event_loop(),
access_log=HTTP_LOGGER,
print=started,
handle_signals=cb is None,
access_log_class=HTTP_LOGGER_CLASS if HTTP_LOGGER_CLASS is not None else web.AccessLogger,
)
if __name__ == "__main__":

View file

@ -35,7 +35,7 @@ async def get_ffprobe(request: Request, config: Config, encoder: Encoder, app: w
Response: The response object.
"""
file: str = request.match_info.get("file")
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
@ -76,7 +76,7 @@ async def get_file_info(request: Request, config: Config, encoder: Encoder, app:
Response: The response object.
"""
file: str = request.match_info.get("file")
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
@ -137,7 +137,7 @@ async def file_browser(request: Request, config: Config, encoder: Encoder) -> Re
Response: The response object.
"""
req_path: str = request.match_info.get("path")
req_path: str | None = request.match_info.get("path")
req_path: str = "/" if not req_path else unquote_plus(req_path)
raw_req: str = (req_path or "").strip()
@ -461,7 +461,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
record(path, ok=False, error=str(e), action=action, extra={"item": params})
continue
else:
extra_info: dict[str, Path] = {"new_path": renamed}
extra_info: dict[str, Any] = {"new_path": renamed}
if sidecar_renamed:
extra_info["sidecar_count"] = len(sidecar_renamed)
record(path, ok=True, action=action, extra=extra_info)
@ -591,7 +591,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
record(path, ok=False, error=str(e), action=action, extra={"item": params})
continue
else:
extra_info: dict[str, Path] = {"new_path": moved}
extra_info: dict[str, Any] = {"new_path": moved}
if sidecar_moved:
extra_info["sidecar_count"] = len(sidecar_moved)
@ -617,7 +617,7 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
if not json or not isinstance(json, list):
return web.json_response({"error": "Invalid parameters."}, status=400)
files: list[str] = []
files: list[Path] = []
for f in json:
if not isinstance(f, str):
continue
@ -627,12 +627,10 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
continue
files.append(ref)
sc: list[dict] = get_file_sidecar(ref)
sc: dict[str, list[dict[str, Any]]] = get_file_sidecar(ref)
if sc:
for side in sc:
for scf in sc[side]:
if isinstance(scf, dict) and "file" in scf:
files.append(scf["file"]) # noqa: PERF401
for value in sc.values():
files.extend(scf["file"] for scf in value if isinstance(scf, dict) and "file" in scf)
if not files:
return web.json_response({"error": "No valid files."}, status=400)

View file

@ -95,7 +95,7 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response:
cache.set(cache_key, dct, ttl=3600)
return web.Response(**dct)
return web.Response(body=dct["body"], headers=dct["headers"])
except Exception:
LOG.exception(
"Failed to request doc '%s' from '%s'.",

View file

@ -1,5 +1,5 @@
from aiohttp import web
from aiohttp.web import Request, Response
from aiohttp.web import Request, Response, StreamResponse
from app.library.config import Config
from app.library.log import get_logger
@ -10,7 +10,7 @@ LOG = get_logger()
@route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static")
async def download_file(request: Request, config: Config, app: web.Application) -> Response:
async def download_file(request: Request, config: Config, app: web.Application) -> StreamResponse:
"""
Serve download files.

View file

@ -1,9 +1,10 @@
import asyncio
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import Any
from aiohttp import web
from aiohttp.web import Request, Response
from aiohttp.web_response import StreamResponse
from app.features.presets.schemas import Preset
from app.features.presets.service import Presets
@ -20,10 +21,6 @@ from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import calc_download_path, get_file_sidecar, rename_file
if TYPE_CHECKING:
from library.downloads import Download
LOG = get_logger()
@ -280,7 +277,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
Response: The response object.
"""
id: str = request.match_info.get("id")
id: str | None = request.match_info.get("id")
if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
@ -314,7 +311,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
@route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail")
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> Response:
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> StreamResponse:
if not (id := request.match_info.get("id")):
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
@ -458,7 +455,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
Response: The response object.
"""
id: str = request.match_info.get("id")
id: str | None = request.match_info.get("id")
if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)

View file

@ -116,8 +116,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp
safe_backend: str = _safe_url(backend)
await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24)
else:
backend = await cache.aget(CACHE_KEY_BING)
safe_backend = _safe_url(backend if isinstance(backend, str) else None)
cached_backend = await cache.aget(CACHE_KEY_BING)
backend = cached_backend if isinstance(cached_backend, str) else ""
safe_backend = _safe_url(backend)
if not isinstance(backend, str) or not backend:
return web.json_response(

View file

@ -1,6 +1,7 @@
import asyncio
import json
import os
from collections.abc import Awaitable, Callable
from pathlib import Path
from aiohttp import web
@ -110,7 +111,7 @@ async def _read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict:
return {"logs": [], "next_offset": None, "end_is_reached": True}
async def _tail_log(file: Path, emitter: callable, sleep_time: float = 0.5):
async def _tail_log(file: Path, emitter: Callable[[dict], Awaitable[None]], sleep_time: float = 0.5):
"""
Live tail.

View file

@ -2,6 +2,7 @@ import asyncio
import os
import time
from pathlib import Path
from typing import Any
from aiohttp import web
from aiohttp.web import Request, Response
@ -489,7 +490,7 @@ async def system_limits(queue: DownloadQueue, config: Config, encoder: Encoder)
for download in [*active_non_live, *queued_non_live]:
extractor = (download.info.get_extractor() or "unknown").lower()
entry = per_extractor.setdefault(
entry: dict[str, Any] = per_extractor.setdefault(
extractor,
{
"name": extractor,

View file

@ -15,7 +15,7 @@ from pathlib import Path
from typing import TYPE_CHECKING
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve())
if APP_ROOT not in sys.path:
@ -69,7 +69,7 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
async def connect_db(db_file: str) -> AsyncConnection:
async def connect_db(db_file: Path) -> tuple[AsyncConnection, AsyncEngine]:
"""Create async SQLAlchemy connection."""
engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}")
conn = await engine.connect()

View file

@ -1,3 +1,4 @@
from typing import Any
from unittest.mock import MagicMock
import pytest
@ -91,7 +92,7 @@ class TestAgSet:
def test_ag_set_error_on_non_dict_final(self):
"""Test error when final target is not a dict."""
data = "not_a_dict"
data: Any = "not_a_dict"
with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"):
ag_set(data, "key", "value")

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import http.cookiejar
from typing import Any
from unittest.mock import Mock, patch
import pytest
@ -12,10 +13,10 @@ try:
YTDLP_AVAILABLE = True
except ImportError:
YTDLP_AVAILABLE = False
Request = Mock
Response = Mock
RequestDirector = Mock
HTTPError = Exception
Request: Any = Mock
Response: Any = Mock
RequestDirector: Any = Mock
HTTPError: Any = Exception
pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")

View file

@ -1,5 +1,6 @@
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, Mock, patch
import pytest
@ -24,7 +25,7 @@ class TestConditionIgnoreMatching:
second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10)
service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"])
@ -36,7 +37,7 @@ class TestConditionIgnoreMatching:
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
matched = await service.match(info={"duration": 60}, ignore_conditions=["123"])
@ -48,7 +49,7 @@ class TestConditionIgnoreMatching:
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
matched = await service.match(info={"duration": 60}, ignore_conditions=[123])
@ -59,7 +60,7 @@ class TestConditionIgnoreMatching:
first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20)
service = Conditions.get_instance()
service._repo = Mock(list=AsyncMock(return_value=[first]))
service._repo = Mock(all=AsyncMock(return_value=[first]))
matched = await service.match(info={"duration": 60}, ignore_conditions=["*"])
@ -78,10 +79,11 @@ class TestConditionIgnorePropagation:
preset="default",
extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]},
)
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
item.get_archive_id = Mock(return_value=None)
item.get_archive_file = Mock(return_value=None)
item.is_archived = Mock(return_value=False)
mock_item: Any = item
mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
mock_item.get_archive_id = Mock(return_value=None)
mock_item.get_archive_file = Mock(return_value=None)
mock_item.is_archived = Mock(return_value=False)
matcher = Mock(match=AsyncMock(return_value=None))
entry = {"id": "video-1", "duration": 60}
@ -113,10 +115,11 @@ class TestConditionIgnorePropagation:
preset="default",
extras={"ignore_conditions": [123, "Primary", True, " * "]},
)
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
item.get_archive_id = Mock(return_value=None)
item.get_archive_file = Mock(return_value=None)
item.is_archived = Mock(return_value=False)
mock_item: Any = item
mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
mock_item.get_archive_id = Mock(return_value=None)
mock_item.get_archive_file = Mock(return_value=None)
mock_item.is_archived = Mock(return_value=False)
matcher = Mock(match=AsyncMock(return_value=None))
entry = {"id": "video-1", "duration": 60}
@ -135,7 +138,7 @@ class TestConditionIgnorePropagation:
@pytest.mark.asyncio
async def test_playlist_keeps_parent_ignore(self) -> None:
captured: dict[str, object] = {}
captured: dict[str, Any] = {}
class FakeItem:
def __init__(self) -> None:
@ -158,7 +161,8 @@ class TestConditionIgnorePropagation:
}
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
result = await process_playlist(queue=queue, entry=entry, item=FakeItem())
fake: Any = FakeItem()
result = await process_playlist(queue=queue, entry=entry, item=fake)
assert result == {"status": "ok"}
assert captured["url"] == "https://example.com/v/1"
@ -209,7 +213,8 @@ class TestConditionIgnorePropagation:
}
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
result = await process_playlist(queue=queue, entry=entry, item=FakeItem())
fake: Any = FakeItem()
result = await process_playlist(queue=queue, entry=entry, item=fake)
assert result == {"status": "ok"}
assert queue.add.await_count == 2

View file

@ -4,10 +4,13 @@ from collections import OrderedDict
from dataclasses import asdict
from datetime import UTC, datetime
from email.utils import formatdate
from typing import Any
from unittest.mock import AsyncMock, Mock
import pytest
from app.library.DataStore import DataStore, StoreType
from app.library.downloads import Download
from app.library.ItemDTO import ItemDTO
from app.library.operations import Operation
from app.library.sqlite_store import SqliteStore
@ -62,17 +65,18 @@ async def make_db(data: int = 0) -> SqliteStore:
return ins
class StubDownload:
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False):
class StubDownload(Download):
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False) -> None:
self.info = info
self._started: bool = started
self._cancelled: bool = cancelled
self.id = info._id
self._stub_started: bool = started
self._stub_cancelled: bool = cancelled
def started(self) -> bool:
return self._started
return self._stub_started
def is_cancelled(self) -> bool:
return self._cancelled
return self._stub_cancelled
async def make_store_async(store_type: StoreType) -> DataStore:
@ -357,7 +361,8 @@ class TestDataStore:
def __init__(self):
self.info = None
store._dict["broken"] = BrokenDownload()
broken: Any = BrokenDownload()
store._dict["broken"] = broken
# Should still find the valid item
result = await store.get_item(title="Video 1")
@ -960,10 +965,10 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1"
item1.filesize = 1000
setattr(item1, "filesize", 1000)
item2 = make_item(id="vid2", title="Video 2")
item2._id = "id2"
item2.filesize = 2000
setattr(item2, "filesize", 2000)
await store.put(StubDownload(info=item1))
await store.put(StubDownload(info=item2))
@ -986,10 +991,10 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1"
item1.filesize = 1000
setattr(item1, "filesize", 1000)
item2 = make_item(id="vid2", title="Video 2")
item2._id = "id2"
item2.filesize = 2000
setattr(item2, "filesize", 2000)
await store.put(StubDownload(info=item1))
await store.put(StubDownload(info=item2))
@ -1007,7 +1012,7 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1"
item1.filesize = 1000
setattr(item1, "filesize", 1000)
await store.put(StubDownload(info=item1))
@ -1033,7 +1038,7 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1"
item1.filesize = 1000
setattr(item1, "filesize", 1000)
await store.put(StubDownload(info=item1))
@ -1084,7 +1089,7 @@ class TestDataStoreOperations:
item1 = make_item(id="vid1", title="Video 1")
item1._id = "id1"
item1.description = None
setattr(item1, "description", None)
await store.put(StubDownload(info=item1))

View file

@ -307,7 +307,8 @@ class TestDownloadFlow:
postprocessor_hook=Mock(),
post_hook=Mock(),
)
download.info.get_ytdlp_opts = Mock(
download_any: Any = download.info
download_any.get_ytdlp_opts = Mock(
return_value=Mock(
add=Mock(
return_value=Mock(
@ -379,7 +380,8 @@ class TestDownloadFlow:
postprocessor_hook=Mock(),
post_hook=Mock(),
)
download.info.get_ytdlp_opts = Mock(
download_any2: Any = download.info
download_any2.get_ytdlp_opts = Mock(
return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={}))))
)
@ -481,7 +483,8 @@ class TestDownloadFlow:
)
queue.put(Terminator())
download._download = fake_download
download_mock: Any = download
download_mock._download = fake_download
class InlineProcess:
def __init__(self, target):
@ -578,7 +581,8 @@ class TestDownloadFlow:
queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)})
queue.put(Terminator())
download._download = fake_download
download_mock: Any = download
download_mock._download = fake_download
class InlineProcess:
def __init__(self, target):
@ -1028,7 +1032,7 @@ class TestStatusTracker:
"debug": False,
}
def test_init_sets_attributes(self, mock_config: dict) -> None:
def test_init_sets_attributes(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
assert st.id == "test-id", "Should set download ID"
assert st.info == mock_config["info"], "Should set info reference"
@ -1036,7 +1040,7 @@ class TestStatusTracker:
assert st.final_update is False, "Should initialize final_update as False"
@pytest.mark.asyncio
async def test_status_ignores_bad_id(self, mock_config: dict) -> None:
async def test_status_ignores_bad_id(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "wrong-id", "status": "downloading"}
@ -1044,14 +1048,14 @@ class TestStatusTracker:
assert st.info.status != "downloading", "Should not update status for wrong ID"
@pytest.mark.asyncio
async def test_status_ignores_short(self, mock_config: dict) -> None:
async def test_status_ignores_short(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id"}
await st.process_status_update(status)
@pytest.mark.asyncio
async def test_process_status_update_sets_status(self, mock_config: dict) -> None:
async def test_process_status_update_sets_status(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000}
@ -1059,7 +1063,7 @@ class TestStatusTracker:
assert st.info.status == "downloading", "Should update info status"
@pytest.mark.asyncio
async def test_status_sets_skipped(self, mock_config: dict) -> None:
async def test_status_sets_skipped(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "download_skipped": True}
@ -1067,7 +1071,7 @@ class TestStatusTracker:
assert st.info.download_skipped is True, "Should update download_skipped from status queue"
@pytest.mark.asyncio
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None:
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"}
@ -1075,7 +1079,7 @@ class TestStatusTracker:
assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename"
@pytest.mark.asyncio
async def test_status_sets_percent(self, mock_config: dict) -> None:
async def test_status_sets_percent(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {
"id": "test-id",
@ -1090,7 +1094,7 @@ class TestStatusTracker:
assert st.info.percent == 50.0, "Should calculate percent correctly"
@pytest.mark.asyncio
async def test_status_uses_estimate(self, mock_config: dict) -> None:
async def test_status_uses_estimate(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {
"id": "test-id",
@ -1104,7 +1108,7 @@ class TestStatusTracker:
assert st.info.percent == 30.0, "Should calculate percent from estimate"
@pytest.mark.asyncio
async def test_status_percent(self, mock_config: dict) -> None:
async def test_status_percent(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {
"id": "test-id",
@ -1117,7 +1121,7 @@ class TestStatusTracker:
assert st.info.percent == 50.0, "Should calculate percent correctly with valid total"
@pytest.mark.asyncio
async def test_status_sets_speed_eta(self, mock_config: dict) -> None:
async def test_status_sets_speed_eta(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60}
@ -1126,7 +1130,7 @@ class TestStatusTracker:
assert st.info.eta == 60, "Should set eta"
@pytest.mark.asyncio
async def test_process_status_update_sets_error(self, mock_config: dict) -> None:
async def test_process_status_update_sets_error(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
status = {"id": "test-id", "status": "error", "error": "Download failed"}
@ -1135,7 +1139,7 @@ class TestStatusTracker:
assert st.info.error == "Download failed", "Should set error message"
@pytest.mark.asyncio
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None:
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
test_file = tmp_path / "test.mp4"
test_file.write_text("test content")
@ -1148,20 +1152,20 @@ class TestStatusTracker:
assert st.info.filename == "test.mp4", "Should set relative filename"
@pytest.mark.asyncio
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None:
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict[str, Any]) -> None:
queue = DummyQueue()
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100})
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200})
queue.put(Terminator())
config = {**mock_config, "status_queue": queue}
config: dict[str, Any] = {**mock_config, "status_queue": queue}
st = StatusTracker(**config)
await st.drain_queue(max_iterations=10)
assert st.info.downloaded_bytes == 200, "Should process all queued updates"
@pytest.mark.asyncio
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None:
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
test_file = tmp_path / "test.mp4"
test_file.write_text("test content")
@ -1169,25 +1173,25 @@ class TestStatusTracker:
queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)})
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999})
config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
config: dict[str, Any] = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
st = StatusTracker(**config)
await st.drain_queue(max_iterations=10)
assert st.final_update is True, "Should stop draining after final update"
@pytest.mark.asyncio
async def test_drain_queue_skips_invalid(self, mock_config: dict) -> None:
async def test_drain_queue_skips_invalid(self, mock_config: dict[str, Any]) -> None:
queue = DummyQueue()
queue.put({"id": "test-id", "status": "downloading"})
queue.put(None)
config = {**mock_config, "status_queue": queue}
config: dict[str, Any] = {**mock_config, "status_queue": queue}
st = StatusTracker(**config)
await st.drain_queue(max_iterations=5)
assert st.info.status == "downloading", "valid updates should still be processed"
def test_cancel_update_task(self, mock_config: dict) -> None:
def test_cancel_update_task(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
st.update_task = Mock()
st.update_task.done = Mock(return_value=False)
@ -1196,15 +1200,15 @@ class TestStatusTracker:
st.cancel_update_task()
st.update_task.cancel.assert_called_once()
def test_cancel_update_task_noop(self, mock_config: dict) -> None:
def test_cancel_update_task_noop(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
st.cancel_update_task()
assert st.update_task is None, "missing tasks should be ignored"
def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None:
def test_put_terminator_adds_to_queue(self, mock_config: dict[str, Any]) -> None:
queue = DummyQueue()
config = {**mock_config, "status_queue": queue}
config: dict[str, Any] = {**mock_config, "status_queue": queue}
st = StatusTracker(**config)
st.put_terminator()
@ -1212,7 +1216,7 @@ class TestStatusTracker:
assert isinstance(queue.items[0], Terminator), "Should add Terminator instance"
@pytest.mark.asyncio
async def test_progress_emits_item_progress(self, mock_config: dict) -> None:
async def test_progress_emits_item_progress(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
st.info.status = "downloading"
calls: list = []
@ -1233,7 +1237,7 @@ class TestStatusTracker:
assert "options" not in payload
@pytest.mark.asyncio
async def test_status_change_emits_item_updated(self, mock_config: dict) -> None:
async def test_status_change_emits_item_updated(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
st.info.status = "started"
calls: list = []
@ -1249,7 +1253,7 @@ class TestStatusTracker:
assert updated_calls[0][1]["data"] is st.info
@pytest.mark.asyncio
async def test_progress_throttled(self, mock_config: dict) -> None:
async def test_progress_throttled(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
st.info.status = "downloading"
st._progress_interval = 0.5
@ -1272,7 +1276,7 @@ class TestStatusTracker:
assert st._pending_progress is True
@pytest.mark.asyncio
async def test_flush_on_status_change(self, mock_config: dict) -> None:
async def test_flush_on_status_change(self, mock_config: dict[str, Any]) -> None:
st = StatusTracker(**mock_config)
st.info.status = "downloading"
st._progress_interval = 10.0
@ -1345,23 +1349,29 @@ class TestQueueManager:
auto_start=True,
)
@staticmethod
def _any_video_item() -> Any:
return TestQueueManager._video_item()
def test_live_queue_caps_visible_items(self) -> None:
queue_manager = object.__new__(DownloadQueue)
items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
queue_manager.queue = self.LiveStore(items)
queue_manager.pool = Mock()
queue_manager.pool.get_active_downloads.return_value = {}
snapshot = DownloadQueue.live_queue(queue_manager, limit=2)
assert list(snapshot["queue"].keys()) == ["id0", "id1"]
queue_view = snapshot["queue"]
assert isinstance(queue_view, dict)
assert list(queue_view.keys()) == ["id0", "id1"]
assert snapshot["queue_count"] == 5
assert snapshot["queue_loaded"] == 2
assert snapshot["queue_limit"] == 2
def test_live_queue_keeps_active(self) -> None:
queue_manager = object.__new__(DownloadQueue)
items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
queue_manager.queue = self.LiveStore(items)
queue_manager.pool = Mock()
queue_manager.pool.get_active_downloads.return_value = {
@ -1371,7 +1381,9 @@ class TestQueueManager:
snapshot = DownloadQueue.live_queue(queue_manager, limit=1)
assert list(snapshot["queue"].keys()) == ["id3", "id4"]
queue_view = snapshot["queue"]
assert isinstance(queue_view, dict)
assert list(queue_view.keys()) == ["id3", "id4"]
assert snapshot["queue_count"] == 5
assert snapshot["queue_loaded"] == 2
assert snapshot["queue_limit"] == 1
@ -1388,7 +1400,7 @@ class TestQueueManager:
result = await add_video(
queue=self._video_queue(),
item=self._video_item(),
item=self._any_video_item(),
entry={
"id": "live-id",
"title": "Live stream",
@ -1420,7 +1432,7 @@ class TestQueueManager:
"formats": [{"format_id": "18"}],
}
result = await add_video(queue=self._video_queue(), item=self._video_item(), entry=entry)
result = await add_video(queue=self._video_queue(), item=self._any_video_item(), entry=entry)
assert result == {"status": "ok"}
assert seen == [entry]

View file

@ -1,6 +1,7 @@
import json
from datetime import date
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock
import pytest
@ -132,7 +133,8 @@ class TestEncoder:
return True
return original_isinstance(obj, cls)
builtins.isinstance = mock_isinstance
builtins_any: Any = builtins
builtins_any.isinstance = mock_isinstance
try:
result = self.encoder.default(mock_daterange)

View file

@ -1,5 +1,6 @@
from __future__ import annotations
from typing import Any
from unittest.mock import Mock
import pytest
@ -7,7 +8,7 @@ import pytest
from app.yt_dlp_plugins.extractor import generic_browser
def _make_ie(config: dict[str, str | None] | None = None) -> generic_browser.GenericBrowserIE:
def _make_ie(config: dict[str, str | None] | None = None) -> Any:
ie = object.__new__(generic_browser.GenericBrowserIE)
values = config or {}
ie._configuration_arg = lambda name, default: [values.get(name)]
@ -250,6 +251,7 @@ def test_extract_network_formats_playlist_entries_keep_own_urls() -> None:
{"title": "Title", "webpage_url": "https://example.com/page", "original_url": "https://example.com/page"},
)
assert result is not None
assert result["_type"] == "playlist"
assert result["entries"][0]["url"] == "https://cdn.example/1.mp3"
assert result["entries"][0]["webpage_url"] == "https://cdn.example/1.mp3"

View file

@ -36,7 +36,8 @@ def _make_download(
) -> SimpleNamespace:
base_dir = download_dir or "/downloads"
original_post_init = ItemDTO.__post_init__
ItemDTO.__post_init__ = lambda self: None
cls: Any = ItemDTO
cls.__post_init__ = lambda self: None
try:
item = ItemDTO(

View file

@ -73,5 +73,5 @@ async def test_bg_log_redact(caplog: pytest.LogCaptureFixture, monkeypatch: pyte
)
assert "apitoken=secret" not in record.getMessage()
assert "user:pass@" not in record.getMessage()
assert record.url == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted"
assert record.exception_type == "RuntimeError"
assert getattr(record, "url", None) == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted"
assert getattr(record, "exception_type", None) == "RuntimeError"

View file

@ -386,7 +386,7 @@ class TestItemAddExtras:
def test_add_extras_none(self):
"""Test adding extras when extras is None."""
item = Item(url="https://example.com")
item.extras = None
setattr(item, "extras", None)
item.add_extras("key1", "value1")

View file

@ -1,6 +1,7 @@
import asyncio
import sys
import types
from collections.abc import Callable
from typing import Any
from unittest.mock import MagicMock, patch
@ -21,7 +22,7 @@ if "aiocron" not in sys.modules:
def uuid(self) -> str:
return "stub-uuid"
aiocron_stub.Cron = _CronImportStub
setattr(aiocron_stub, "Cron", _CronImportStub)
sys.modules["aiocron"] = aiocron_stub
@ -36,7 +37,7 @@ class DummyCron:
self,
*,
spec: str,
func: callable,
func: Callable,
args: tuple = (),
kwargs: dict | None = None,
uuid: str | None = None,
@ -231,7 +232,7 @@ class TestScheduler:
self,
*_,
spec: str,
func: callable,
func: Callable,
args: tuple = (),
kwargs: dict | None = None,
uuid: str | None = None,
@ -266,7 +267,7 @@ class TestScheduler:
self,
*_,
spec: str,
func: callable,
func: Callable,
args: tuple = (),
kwargs: dict | None = None,
uuid: str | None = None,

View file

@ -1,4 +1,5 @@
from datetime import UTC, datetime, timedelta
from typing import Any
from unittest.mock import AsyncMock, patch
import pytest
@ -182,12 +183,14 @@ async def test_paginate_order_and_bounds():
encoded = itm.json()
created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S")
_list.append(itm)
assert store._conn is not None
await store._conn.execute(
text(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)'
),
{"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at},
)
assert store._conn is not None
await store._conn.commit()
items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC")
@ -208,7 +211,9 @@ async def test_get_by_id():
item = make_item(99)
await store.enqueue_upsert("queue", item)
await store.flush()
assert item._id == (await store.get_by_id("queue", item._id))._id
fetched = await store.get_by_id("queue", item._id)
assert fetched is not None
assert item._id == fetched._id
await store.close()
@ -358,7 +363,8 @@ async def test_on_shutdown_closes_connection():
await store.enqueue_upsert("queue", make_item(1))
await store.flush()
await store.on_shutdown(None)
app: Any = None
await store.on_shutdown(app)
assert store._conn is None
# Note: on_shutdown already closes the connection, so no need to call close() again

View file

@ -1,6 +1,6 @@
import asyncio
import copy
import importlib
import importlib.util
import json
import logging
import re
@ -10,6 +10,7 @@ import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
@ -861,16 +862,20 @@ class TestMergeDict:
def test_merge_dict_type_validation(self):
"""Test that non-dict parameters are properly rejected."""
# Test with non-dict source
bad_src: Any = "not_a_dict"
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
merge_dict("not_a_dict", {"key": "value"})
merge_dict(bad_src, {"key": "value"})
# Test with non-dict destination
bad_dst: Any = "not_a_dict"
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
merge_dict({"key": "value"}, "not_a_dict")
merge_dict({"key": "value"}, bad_dst)
# Test with both non-dict
bad_src2: Any = "not_a_dict"
bad_dst2: Any = ["also_not_dict"]
with pytest.raises(TypeError, match="Both source and destination must be dictionaries"):
merge_dict("not_a_dict", ["also_not_dict"])
merge_dict(bad_src2, bad_dst2)
def test_merge_dict_immutability(self):
"""Test that original dictionaries are not modified (immutability)."""
@ -1084,7 +1089,8 @@ class TestEncryptDecrypt:
assert encrypted != data
assert isinstance(encrypted, str)
decrypted: str = decrypt_data(encrypted, key)
decrypted = decrypt_data(encrypted, key)
assert decrypted is not None
assert decrypted == data
def test_encrypt_empty_string(self):
@ -1093,7 +1099,8 @@ class TestEncryptDecrypt:
data: str = ""
encrypted: str = encrypt_data(data, key)
decrypted: str = decrypt_data(encrypted, key)
decrypted = decrypt_data(encrypted, key)
assert decrypted is not None
assert decrypted == data
@ -1489,7 +1496,8 @@ class TestGet:
def test_get_list_access(self):
"""Test getting from list."""
data = ["item0", "item1", "item2"]
result = get(data, 1)
key: Any = 1
result = get(data, key)
assert result == "item1"
def test_get_empty_path(self):

Some files were not shown because too many files have changed in this diff Show more