danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

mostly linter fixes.

Browse source
This commit is contained in:
ArabCoders 2025-01-28 19:43:51 +03:00
parent 2b84aad836
commit fdec16e895
25 changed files with 442 additions and 285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.vscode/settings.json vendored
View file

@ -12,6 +12,8 @@
],
"cSpell.words": [
"aconvert",
"ahas",
"ahash",
"aiocron",
"copyts",
"dotenv",

14
app/library/AsyncPool.py
View file

@ -1,6 +1,6 @@
import asyncio
import logging
from datetime import datetime, timezone
from datetime import UTC, datetime
class Terminator:
@ -14,9 +14,9 @@ class AsyncPool:
worker_co,
name: str,
logger: logging.Logger,
loop: asyncio.AbstractEventLoop = None,
loop: asyncio.AbstractEventLoop|None = None,
load_factor: int = 1,
max_task_time: int = None,
max_task_time: int|None = None,
return_futures: bool = False,
raise_on_join: bool = False,
):
@ -150,7 +150,7 @@ class AsyncPool:
for worker_number in range(self._num_workers):
worker_id = f"worker_{worker_number+1}"
self._createWorker(worker_id)
self._create_worker(worker_id)
async def restart(self, worker_id: str, msg: str = None) -> bool:
"""Will restart the worker pool"""
@ -169,7 +169,7 @@ class AsyncPool:
if worker_id in self._workers:
self._workers.pop(worker_id)
self._createWorker(worker_id)
self._create_worker(worker_id)
return True
@ -216,7 +216,7 @@ class AsyncPool:
if self._exceptions and self._raise_on_join:
raise Exception(f"Exception occurred in {self._name} pool")
def _createWorker(self, worker_id: str) -> asyncio.Future:
def _create_worker(self, worker_id: str) -> asyncio.Future:
if worker_id in self._workers:
self._logger.debug(f"Worker {worker_id} already exists.")
return self._workers[worker_id]
@ -231,4 +231,4 @@ class AsyncPool:
return self._workers[worker_id]
def _time(self) -> datetime:
return datetime.now(tz=timezone.utc)
return datetime.now(tz=UTC)

13
app/library/DataStore.py
View file

@ -31,9 +31,10 @@ class DataStore:
for id, item in self.saved_items():
self.dict.update({id: Download(info=item)})
def exists(self, key: str = None, url: str = None) -> bool:
def exists(self, key: str | None = None, url: str | None = None) -> bool:
if not key and not url:
raise KeyError("key or url must be provided.")
msg = "key or url must be provided."
raise KeyError(msg)
if key and key in self.dict:
return True
@ -44,15 +45,17 @@ class DataStore:
return False
def get(self, key: str, url: str = None) -> Download:
def get(self, key: str, url: str|None = None) -> Download:
if not key and not url:
raise KeyError("key or url must be provided.")
msg = "key or url must be provided."
raise KeyError(msg)
for i in self.dict:
if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url):
return self.dict[i]
raise KeyError(f"{key=} or {url=} not found.")
msg = f"{key=} or {url=} not found."
raise KeyError(msg)
def getById(self, id: str) -> Download | None:
return self.dict[id] if id in self.dict else None

3
app/library/Download.py
View file

@ -235,7 +235,6 @@ class Download:
self.update_task.cancel()
except Exception as e:
LOG.error(f"Failed to close status queue: '{procId}'. {e}")
pass
self.kill()
@ -385,7 +384,7 @@ class Download:
except Exception as e:
self.info.extras["is_video"] = True
self.info.extras["is_audio"] = True
LOG.exception(f"Failed to ffprobe: {status.get}. {e}")
LOG.exception(e)
LOG.error(f"Failed to ffprobe: {status.get}. {e}")
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")

33
app/library/DownloadQueue.py
View file

@ -14,9 +14,9 @@ from .config import Config
from .DataStore import DataStore
from .Download import Download
from .Emitter import Emitter
from .EventsSubscriber import Events
from .ItemDTO import ItemDTO
from .Singleton import Singleton
from .EventsSubscriber import Events
from .Utils import ExtractInfo, calcDownloadPath, get_opts, isDownloaded, mergeConfig
LOG = logging.getLogger("DownloadQueue")
@ -77,6 +77,7 @@ class DownloadQueue(metaclass=Singleton):
Returns:
bool: True if the test is successful, False otherwise.
"""
await self.done.test()
return True
@ -96,6 +97,7 @@ class DownloadQueue(metaclass=Singleton):
Returns:
bool: True if the download is paused, False otherwise
"""
if self.paused.is_set():
self.paused.clear()
@ -110,6 +112,7 @@ class DownloadQueue(metaclass=Singleton):
Returns:
bool: True if the download is resumed, False otherwise
"""
if not self.paused.is_set():
self.paused.set()
@ -124,15 +127,16 @@ class DownloadQueue(metaclass=Singleton):
Returns:
bool: True if the download queue is paused, False otherwise
"""
return False if self.paused.is_set() else True
return not self.paused.is_set()
async def __add_entry(
self,
entry: dict,
preset: str,
folder: str,
config: dict = {},
config: dict | None = None,
cookies: str = "",
template: str = "",
already=None,
@ -151,7 +155,11 @@ class DownloadQueue(metaclass=Singleton):
Returns:
dict: The status of the operation.
"""
if not config:
config = {}
if not entry:
return {"status": "error", "msg": "Invalid/empty data was given."}
@ -304,7 +312,7 @@ class DownloadQueue(metaclass=Singleton):
url: str,
preset: str,
folder: str,
config: dict = {},
config: dict|None = None,
cookies: str = "",
template: str = "",
already=None,
@ -322,8 +330,8 @@ class DownloadQueue(metaclass=Singleton):
try:
config = json.loads(config)
except Exception as e:
LOG.error(f"Unable to load '{config=}'. {str(e)}")
return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {str(e)}"}
LOG.error(f"Unable to load '{config=}'. {e!s}")
return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {e!s}"}
already = set() if already is None else already
@ -361,13 +369,13 @@ class DownloadQueue(metaclass=Singleton):
f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'."
)
except yt_dlp.utils.ExistingVideoReached as exc:
LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{str(exc)}'.")
LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.")
return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
except yt_dlp.utils.YoutubeDLError as exc:
LOG.error(f"YoutubeDLError: Unable to extract info. '{str(exc)}'.")
LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.")
return {"status": "error", "msg": str(exc)}
except asyncio.exceptions.TimeoutError as exc:
LOG.error(f"TimeoutError: Unable to extract info. '{str(exc)}'.")
LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.")
return {
"status": "error",
"msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.",
@ -392,7 +400,7 @@ class DownloadQueue(metaclass=Singleton):
except KeyError as e:
status[id] = str(e)
status["status"] = "error"
LOG.warning(f"Requested cancel for non-existent download {id=}. {str(e)}")
LOG.warning(f"Requested cancel for non-existent download {id=}. {e!s}")
continue
itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
@ -426,7 +434,7 @@ class DownloadQueue(metaclass=Singleton):
except KeyError as e:
status[id] = str(e)
status["status"] = "error"
LOG.warning(f"Requested delete for non-existent download {id=}. {str(e)}")
LOG.warning(f"Requested delete for non-existent download {id=}. {e!s}")
continue
itemRef: str = f"{id=} {item.info.id=} {item.info.title=}"
@ -450,7 +458,7 @@ class DownloadQueue(metaclass=Singleton):
else:
LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.")
except Exception as e:
LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {str(e)}")
LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {e!s}")
self.done.delete(id)
asyncio.create_task(self.emitter.cleared(dl=item.info.serialize()), name=f"notifier-c-{id}")
@ -469,6 +477,7 @@ class DownloadQueue(metaclass=Singleton):
Returns:
dict: The download queue and the download history.
"""
items = {"queue": {}, "done": {}}

9
app/library/Emitter.py
View file

@ -2,8 +2,8 @@ import asyncio
import logging
from typing import Awaitable
from .Singleton import Singleton
from .EventsSubscriber import Events
from .Singleton import Singleton
LOG = logging.getLogger("Emitter")
@ -28,6 +28,7 @@ class Emitter(metaclass=Singleton):
Returns:
Emitter: The instance of the Emitter
"""
if not Emitter._instance:
Emitter._instance = Emitter()
@ -43,6 +44,7 @@ class Emitter(metaclass=Singleton):
Returns:
Emitter: The instance of the Emitter
"""
if not isinstance(emitter, list):
emitter = [emitter]
@ -97,6 +99,7 @@ class Emitter(metaclass=Singleton):
Returns:
None
"""
tasks = []
@ -122,8 +125,8 @@ class Emitter(metaclass=Singleton):
await asyncio.wait_for(asyncio.gather(*tasks), timeout=60)
except asyncio.CancelledError:
LOG.error(f"Cancelled sending event '{event}'.")
except asyncio.TimeoutError:
except TimeoutError:
LOG.error(f"Timed out sending event '{event}'.")
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to send event '{event}'. '{str(e)}'.")
LOG.error(f"Failed to send event '{event}'. '{e!s}'.")

17
app/library/EventsSubscriber.py
View file

@ -1,8 +1,7 @@
import asyncio
import logging
from typing import Awaitable
from dataclasses import dataclass
from typing import Awaitable
from .Singleton import Singleton
@ -32,12 +31,10 @@ class Events:
CLI_OUTPUT = "cli_output"
UPDATE = "update"
TEST = "test"
UPDATED = "updated"
ADD_URL = "add_url"
CLI_POST = "cli_post"
PAUSED = "paused"
TEST = "test"
TASKS_ADD = "task_add"
TASK_DISPATCHED = "task_dispatched"
@ -72,6 +69,7 @@ class EventsSubscriber(metaclass=Singleton):
Returns:
EventsSubscriber: The instance of the EventsSubscriber
"""
if not EventsSubscriber._instance:
EventsSubscriber._instance = EventsSubscriber()
@ -88,6 +86,7 @@ class EventsSubscriber(metaclass=Singleton):
Returns:
EventsSubscriber: The instance of the EventsSubscriber
"""
if isinstance(event, str):
event = [event]
@ -110,8 +109,8 @@ class EventsSubscriber(metaclass=Singleton):
Returns:
EventsSubscriber: The instance of the EventsSubscriber
"""
"""
if isinstance(event, str):
event = [event]
@ -148,8 +147,8 @@ class EventsSubscriber(metaclass=Singleton):
results.append(asyncio.get_event_loop().run_until_complete(callback(event, data)))
except Exception as e:
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.")
LOG.exception(e)
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{e!s}'.")
return results
@ -164,10 +163,10 @@ class EventsSubscriber(metaclass=Singleton):
Returns:
Awaitable: The task that was created to run the event.
"""
"""
if event not in self._listeners:
return
return None
tasks = []
for id, callback in self._listeners[event].items():
@ -181,7 +180,7 @@ class EventsSubscriber(metaclass=Singleton):
tasks.append(asyncio.create_task(callback(event, data)))
except Exception as e:
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.")
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{e!s}'.")
LOG.exception(e)
return asyncio.gather(*tasks)

91
app/library/HttpAPI.py
View file

@ -18,7 +18,7 @@ from aiohttp import web
from aiohttp.web import Request, RequestHandler, Response
from .cache import Cache
from .common import common
from .common import Common
from .config import Config
from .DownloadQueue import DownloadQueue
from .Emitter import Emitter
@ -37,15 +37,18 @@ LOG = logging.getLogger("http_api")
MIME = magic.Magic(mime=True)
class HttpAPI(common):
staticHolder: dict = {}
extToMime: dict = {
class HttpAPI(Common):
_static_holder: dict = {}
"""Holds loaded static assets."""
_ext_to_mime: dict = {
".html": "text/html",
".css": "text/css",
".js": "application/javascript",
".json": "application/json",
".ico": "image/x-icon",
}
"""Map ext to mimetype"""
def __init__(
self,
@ -65,6 +68,7 @@ class HttpAPI(common):
super().__init__(queue=self.queue, encoder=self.encoder, config=self.config)
@staticmethod
def route(method: str, path: str) -> Awaitable:
"""
Decorator to mark a method as an HTTP route handler.
@ -75,6 +79,7 @@ class HttpAPI(common):
Returns:
Awaitable: The decorated function.
"""
def decorator(func):
@ -97,6 +102,7 @@ class HttpAPI(common):
Returns:
HttpAPI: The instance of the HttpAPI.
"""
if self.config.auth_username and self.config.auth_password:
app.middlewares.append(HttpAPI.basic_auth(self.config.auth_username, self.config.auth_password))
@ -128,13 +134,14 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
path = req.path
if req.path not in self.staticHolder:
if req.path not in self._static_holder:
return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code)
item: dict = self.staticHolder[req.path]
item: dict = self._static_holder[req.path]
return web.Response(
body=item.get("content"),
@ -142,7 +149,7 @@ class HttpAPI(common):
"Pragma": "public",
"Cache-Control": "public, max-age=31536000",
"Content-Type": item.get("content_type"),
"X-Via": "memory" if not item.get("file", None) else "disk",
"X-Via": "memory" if not item.get("file") else "disk",
},
status=web.HTTPOk.status_code,
)
@ -156,11 +163,12 @@ class HttpAPI(common):
Returns:
HttpAPI: The instance of the HttpAPI.
"""
"""
staticDir = os.path.join(self.rootPath, "ui", "exported")
if not os.path.exists(staticDir):
raise ValueError(f"Could not find the frontend UI static assets. '{staticDir}'.")
msg = f"Could not find the frontend UI static assets. '{staticDir}'."
raise ValueError(msg)
preloaded = 0
@ -172,10 +180,12 @@ class HttpAPI(common):
file = os.path.join(root, file)
urlPath = f"/{file.replace(f'{staticDir}/', '')}"
content = open(file, "rb").read()
contentType = self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file))
with open(file, "rb") as f:
content = f.read()
self.staticHolder[urlPath] = {"content": content, "content_type": contentType}
contentType = self._ext_to_mime.get(os.path.splitext(file)[1], MIME.from_file(file))
self._static_holder[urlPath] = {"content": content, "content_type": contentType}
LOG.debug(f"Preloading '{urlPath}'.")
app.router.add_get(urlPath, self.staticFile)
preloaded += 1
@ -183,8 +193,8 @@ class HttpAPI(common):
if urlPath.endswith("/index.html") and urlPath != "/index.html":
parentSlash = urlPath.replace("/index.html", "/")
parentNoSlash = urlPath.replace("/index.html", "")
self.staticHolder[parentSlash] = {"content": content, "content_type": contentType}
self.staticHolder[parentNoSlash] = {"content": content, "content_type": contentType}
self._static_holder[parentSlash] = {"content": content, "content_type": contentType}
self._static_holder[parentNoSlash] = {"content": content, "content_type": contentType}
app.router.add_get(parentSlash, self.staticFile)
app.router.add_get(parentNoSlash, self.staticFile)
preloaded += 2
@ -210,8 +220,8 @@ class HttpAPI(common):
Returns:
HttpAPI: The instance of the HttpAPI.
"""
"""
for attr_name in dir(self):
method = getattr(self, attr_name)
if hasattr(method, "_http_method") and hasattr(method, "_http_path"):
@ -228,9 +238,11 @@ class HttpAPI(common):
app.add_routes(self.routes)
except ValueError as e:
if "ui/exported" in str(e):
raise RuntimeError(f"Could not find the frontend UI static assets. '{e}'.") from e
raise e
msg = f"Could not find the frontend UI static assets. '{e}'."
raise RuntimeError(msg) from e
raise
@staticmethod
def basic_auth(username: str, password: str) -> Awaitable:
"""
Middleware to handle basic authentication.
@ -241,6 +253,7 @@ class HttpAPI(common):
Returns:
Awaitable: The middleware handler.
"""
@web.middleware
@ -291,6 +304,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
@ -304,6 +318,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
await self.queue.test()
return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code)
@ -318,6 +333,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
post = await request.json()
args: str | None = post.get("args")
@ -345,6 +361,7 @@ class HttpAPI(common):
Returns:
Response: The response object
"""
url = request.query.get("url")
if not url:
@ -403,6 +420,7 @@ class HttpAPI(common):
Returns:
Response: The response object
"""
url: str | None = request.query.get("url")
if not url:
@ -425,6 +443,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
data = await request.json()
@ -456,6 +475,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
return web.json_response(
data=Tasks.get_instance().getTasks(), status=web.HTTPOk.status_code, dumps=self.encoder.encode
@ -471,6 +491,7 @@ class HttpAPI(common):
Returns:
Response: The response object
"""
data = await request.json()
@ -500,7 +521,7 @@ class HttpAPI(common):
item["id"] = str(uuid.uuid4())
if not item.get("timer", None) or str(item.get("timer")).strip() == "":
item["timer"] = f"{random.randint(1,59)} */1 * * *"
item["timer"] = f"{random.randint(1,59)} */1 * * *" # noqa: S311
if not item.get("cookies", None):
item["cookies"] = ""
@ -509,13 +530,13 @@ class HttpAPI(common):
item["config"] = {}
if not item.get("template", None):
item["template"] = str()
item["template"] = ""
try:
ins.validate(item)
except ValueError as e:
return web.json_response(
{"error": f"Failed to validate task '{item.get('name')}'. '{str(e)}'"},
{"error": f"Failed to validate task '{item.get('name')}'. '{e!s}'"},
status=web.HTTPBadRequest.status_code,
)
@ -542,6 +563,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
data = await request.json()
ids = data.get("ids")
@ -569,6 +591,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
id: str = request.match_info.get("id")
if not id:
@ -615,6 +638,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
data: dict = {"queue": [], "history": []}
@ -635,6 +659,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
@ -672,6 +697,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
@ -690,6 +716,7 @@ class HttpAPI(common):
Returns:
Response: The response object
"""
id: str = request.match_info.get("id")
if not id:
@ -712,6 +739,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
id: str = request.match_info.get("id")
if not id:
@ -734,6 +762,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
file: str = request.match_info.get("file")
@ -767,6 +796,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
file: str = request.match_info.get("file")
mode: str = request.match_info.get("mode")
@ -817,6 +847,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
file: str = request.match_info.get("file")
segment: int = request.match_info.get("segment")
@ -843,8 +874,8 @@ class HttpAPI(common):
segmenter = Segments(
index=int(segment),
duration=float("{:.6f}".format(float(sd if sd else M3u8.duration))),
vconvert=True if vc == 1 else False,
aconvert=True if ac == 1 else False,
vconvert=vc == 1,
aconvert=ac == 1,
)
return web.Response(
@ -875,6 +906,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
file: str = request.match_info.get("file")
file_path: str = os.path.normpath(os.path.join(self.config.download_path, file))
@ -919,14 +951,15 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
if "/index.html" not in self.staticHolder:
if "/index.html" not in self._static_holder:
LOG.error("Static frontend files not found.")
return web.json_response(
data={"error": "File not found.", "file": "/index.html"}, status=web.HTTPNotFound.status_code
)
data = self.staticHolder["/index.html"]
data = self._static_holder["/index.html"]
return web.Response(
body=data.get("content"),
content_type=data.get("content_type"),
@ -944,6 +977,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
url = request.query.get("url")
if not url:
@ -994,6 +1028,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
file: str = request.match_info.get("file")
try:
@ -1019,6 +1054,7 @@ class HttpAPI(common):
Returns:
Response: The response object
"""
cookie_file = self.config.ytdl_options.get("cookiefile", None)
if not cookie_file:
@ -1066,7 +1102,7 @@ class HttpAPI(common):
LOG.error(f"Failed to request '{url}'. '{e}'.")
LOG.exception(e)
return web.json_response(
data={"message": f"Failed to request website. {str(e)}"},
data={"message": f"Failed to request website. {e!s}"},
status=web.HTTPInternalServerError.status_code,
)
@ -1080,6 +1116,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
return web.json_response(
data={
@ -1100,6 +1137,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
post = await request.json()
if not isinstance(post, list):
@ -1125,7 +1163,7 @@ class HttpAPI(common):
Notification.validate(item)
except ValueError as e:
return web.json_response(
{"error": f"Invalid notification target settings. {str(e)}", "data": item},
{"error": f"Invalid notification target settings. {e!s}", "data": item},
status=web.HTTPBadRequest.status_code,
)
@ -1155,6 +1193,7 @@ class HttpAPI(common):
Returns:
Response: The response object.
"""
data = {"type": "test", "message": "This is a test notification."}
await self.emitter.emit(Events.TEST, data)

27
app/library/HttpSocket.py
View file

@ -7,23 +7,23 @@ import pty
import shlex
import time
from datetime import datetime
from typing import Any
import socketio
from aiohttp import web
from typing import Any
from .common import common
from .common import Common
from .config import Config
from .DownloadQueue import DownloadQueue
from .Emitter import Emitter
from .encoder import Encoder
from .Utils import isDownloaded, arg_converter
from .EventsSubscriber import EventsSubscriber, Events, Event
from .EventsSubscriber import Event, Events, EventsSubscriber
from .Utils import arg_converter, isDownloaded
LOG = logging.getLogger("socket_api")
class HttpSocket(common):
class HttpSocket(Common):
"""
This class is used to handle WebSocket events.
"""
@ -54,6 +54,7 @@ class HttpSocket(common):
super().__init__(queue=queue, encoder=encoder, config=config)
@staticmethod
def ws_event(func): # type: ignore
"""
Decorator to mark a method as a socket event.
@ -91,7 +92,7 @@ class HttpSocket(common):
try:
LOG.info(f"Cli command from client '{sid}'. '{data}'")
args = ["yt-dlp"] + shlex.split(data)
args = ["yt-dlp", *shlex.split(data)]
_env = os.environ.copy()
_env.update(
{
@ -119,7 +120,7 @@ class HttpSocket(common):
try:
os.close(slave_fd)
except Exception as e:
LOG.error(f"Error closing PTY. '{str(e)}'.")
LOG.error(f"Error closing PTY. '{e!s}'.")
async def read_pty(sid: str):
loop = asyncio.get_running_loop()
@ -130,8 +131,7 @@ class HttpSocket(common):
except OSError as e:
if e.errno == errno.EIO:
break
else:
raise
raise
if not chunk:
# No more output
@ -154,7 +154,7 @@ class HttpSocket(common):
try:
os.close(master_fd)
except Exception as e:
LOG.error(f"Error closing PTY. '{str(e)}'.")
LOG.error(f"Error closing PTY. '{e!s}'.")
# Start reading output from PTY
read_task = asyncio.create_task(read_pty(sid=sid))
@ -183,10 +183,10 @@ class HttpSocket(common):
try:
item = self.format_item(data)
except ValueError as e:
return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code)
await self.emitter.error(str(e), to=sid)
return
status = await self.add(**item)
await self.emitter.emit(event=Events.STATUS, data=status, to=sid)
@ws_event # type: ignore
@ -293,9 +293,10 @@ class HttpSocket(common):
try:
await self.emitter.emit(event=Events.YTDLP_CONVERT, data=arg_converter(args), to=sid)
return
except Exception as e:
err = str(e).strip()
err = err.split("\n")[-1] if "\n" in err else err
LOG.error(f"Failed to convert args. '{err}'.")
await self.emitter.error(f"Failed to convert options. '{err}'.", to=sid)
return

5
app/library/ItemDTO.py
View file

@ -30,7 +30,7 @@ class ItemDTO:
config: dict = field(default_factory=dict)
template: str | None = None
template_chapter: str | None = None
timestamp: float = time.time_ns()
timestamp: float = field(default_factory=lambda: time.time_ns())
is_live: bool | None = None
datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
live_in: str | None = None
@ -92,3 +92,6 @@ class ItemDTO:
def get(self, key: str, default: Any = None) -> Any:
return self.__dict__.get(key, default)
def get_id(self) -> str:
return self._id

28
app/library/M3u8.py
View file

@ -1,8 +1,9 @@
import math
import os
from urllib.parse import quote
from .Utils import calcDownloadPath, StreamingError
from .ffprobe import ffprobe
from .Utils import StreamingError, calcDownloadPath
class M3u8:
@ -19,7 +20,7 @@ class M3u8:
"mp3",
)
def __init__(self, url: str, segment_duration: float = None):
def __init__(self, url: str, segment_duration: float|None = None):
self.url = url
self.duration = float(segment_duration) if segment_duration is not None else self.duration
@ -27,7 +28,8 @@ class M3u8:
realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False)
if not os.path.exists(realFile):
raise StreamingError(f"File '{realFile}' does not exist.")
error = f"File '{realFile}' does not exist."
raise StreamingError(error)
try:
ff = await ffprobe(realFile)
@ -35,7 +37,8 @@ class M3u8:
pass
if "duration" not in ff.metadata:
raise StreamingError(f"Unable to get '{realFile}' play duration.")
error = f"Unable to get '{realFile}' play duration."
raise StreamingError(error)
duration: float = float(ff.metadata.get("duration"))
@ -47,22 +50,20 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
segmentSize: float = "{:.6f}".format(self.duration)
segmentSize: float = f"{self.duration:.6f}"
splits: int = math.ceil(duration / self.duration)
segmentParams: dict = {}
for stream in ff.streams():
if stream.is_video():
if stream.codec_name not in self.ok_vcodecs:
segmentParams["vc"] = 1
if stream.is_audio():
if stream.codec_name not in self.ok_acodecs:
segmentParams["ac"] = 1
if stream.is_video() and stream.codec_name not in self.ok_vcodecs:
segmentParams["vc"] = 1
if stream.is_audio() and stream.codec_name not in self.ok_acodecs:
segmentParams["ac"] = 1
for i in range(splits):
if (i + 1) == splits:
segmentSize = "{:.6f}".format(duration - (i * self.duration))
segmentSize = f"{duration - (i * self.duration):.6f}"
segmentParams.update({"sd": segmentSize})
m3u8.append(f"#EXTINF:{segmentSize},")
@ -81,7 +82,8 @@ class M3u8:
realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False)
if not os.path.exists(realFile):
raise StreamingError(f"File '{realFile}' does not exist.")
error = f"File '{realFile}' does not exist."
raise StreamingError(error)
m3u8 = []

69
app/library/Notifications.py
View file

@ -1,20 +1,20 @@
import asyncio
from datetime import datetime, timezone
import json
import logging
import os
from typing import List, Any
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import httpx
from .config import Config
from .ItemDTO import ItemDTO
from .Singleton import Singleton
from .Utils import validate_uuid, ag
from .version import APP_VERSION
from .encoder import Encoder
from .EventsSubscriber import Events
from .ItemDTO import ItemDTO
from .Singleton import Singleton
from .Utils import ag, validate_uuid
from .version import APP_VERSION
LOG = logging.getLogger("notifications")
@ -66,7 +66,7 @@ class Target:
id: str
name: str
on: List[str]
on: list[str]
request: targetRequest
def serialize(self) -> dict:
@ -165,16 +165,15 @@ class Notification(metaclass=Singleton):
Returns:
Notification: The Notification instance.
"""
"""
LOG.info(f"Saving notification targets to '{self._file}'.")
try:
with open(self._file, "w") as f:
json.dump([t.serialize() for t in targets], fp=f, indent=4)
except Exception as e:
LOG.exception(e)
LOG.error(f"Error saving notification targets to '{self._file}'. '{e}'")
pass
LOG.error(f"Error saving notification targets to '{self._file}'. '{e!s}'")
return self
@ -195,15 +194,14 @@ class Notification(metaclass=Singleton):
with open(self._file, "r") as f:
targets = json.load(f)
except Exception as e:
LOG.error(f"Error loading notification targets from '{self._file}'. '{e}'")
pass
LOG.error(f"Error loading notification targets from '{self._file}'. '{e!s}'")
for target in targets:
try:
try:
Notification.validate(target)
except ValueError as e:
LOG.error(f"Invalid notification target '{target}'. '{e}'")
LOG.error(f"Invalid notification target '{target}'. '{e!s}'")
continue
target = self.makeTarget(target)
@ -214,8 +212,7 @@ class Notification(metaclass=Singleton):
f"Will send '{target.on if len(target.on) > 0 else 'all'}' as {target.request.type} notification events to '{target.name}'."
)
except Exception as e:
LOG.error(f"Error loading notification target '{target}'. '{e}'")
pass
LOG.error(f"Error loading notification target '{target}'. '{e!s}'")
return self
@ -229,46 +226,57 @@ class Notification(metaclass=Singleton):
Returns:
bool: True if the target is valid, False otherwise.
"""
"""
if not isinstance(target, dict):
target = target.serialize()
if "id" not in target or validate_uuid(target["id"], version=4) is False:
raise ValueError("Invalid notification target. No ID found.")
msg = "Invalid notification target. No ID found."
raise ValueError(msg)
if "name" not in target:
raise ValueError("Invalid notification target. No name found.")
msg = "Invalid notification target. No name found."
raise ValueError(msg)
if "request" not in target:
raise ValueError("Invalid notification target. No request details found.")
msg = "Invalid notification target. No request details found."
raise ValueError(msg)
if "url" not in target["request"]:
raise ValueError("Invalid notification target. No URL found.")
msg = "Invalid notification target. No URL found."
raise ValueError(msg)
if "method" in target["request"] and target["request"]["method"].upper() not in ["POST", "PUT"]:
raise ValueError("Invalid notification target. Invalid method found.")
msg = "Invalid notification target. Invalid method found."
raise ValueError(msg)
if "type" in target["request"] and target["request"]["type"].lower() not in ["json", "form"]:
raise ValueError("Invalid notification target. Invalid type found.")
msg = "Invalid notification target. Invalid type found."
raise ValueError(msg)
if "on" in target:
if not isinstance(target["on"], list):
raise ValueError("Invalid notification target. Invalid 'on' event list found.")
msg = "Invalid notification target. Invalid 'on' event list found."
raise ValueError(msg)
for e in target["on"]:
if e not in NotificationEvents.getEvents().values():
raise ValueError(f"Invalid notification target. Invalid event '{e}' found.")
msg = f"Invalid notification target. Invalid event '{e}' found."
raise ValueError(msg)
if "headers" in target["request"]:
if not isinstance(target["request"]["headers"], list):
raise ValueError("Invalid notification target. Invalid headers list found.")
msg = "Invalid notification target. Invalid headers list found."
raise ValueError(msg)
for h in target["request"]["headers"]:
if "key" not in h:
raise ValueError("Invalid notification target. No header key found.")
msg = "Invalid notification target. No header key found."
raise ValueError(msg)
if "value" not in h:
raise ValueError("Invalid notification target. No header value found.")
msg = "Invalid notification target. No header value found."
raise ValueError(msg)
return True
@ -315,7 +323,7 @@ class Notification(metaclass=Singleton):
reqBody["json" if "json" == target.request.type.lower() else "data"] = {
"event": event,
"created_at": datetime.now(tz=timezone.utc).isoformat(),
"created_at": datetime.now(tz=UTC).isoformat(),
"payload": item.__dict__ if isinstance(item, ItemDTO) else item,
}
@ -346,6 +354,7 @@ class Notification(metaclass=Singleton):
Returns:
Target: The notification target.
"""
return Target(
id=target.get("id"),
@ -362,7 +371,7 @@ class Notification(metaclass=Singleton):
),
)
def emit(self, event, data, **kwargs):
def emit(self, event, data, **kwargs): # noqa: ARG002
if len(self._targets) < 1:
return False

13
app/library/PackageInstaller.py
View file

@ -1,8 +1,9 @@
import importlib
import logging
import os
import subprocess
import sys
import importlib
from library.config import Config
LOG = logging.getLogger("package_installer")
@ -24,8 +25,8 @@ class PackageInstaller:
return []
if os.access(file_path, os.R_OK):
with open(file_path, "r") as f:
return [pkg.strip() for pkg in f.readlines() if pkg.strip()]
with open(file_path) as f:
return [pkg.strip() for pkg in f if pkg.strip()]
else:
LOG.error(f"Could not read pip packages from '{file_path}'.")
return []
@ -37,10 +38,10 @@ class PackageInstaller:
LOG.info(f"'{pkg}' is already installed. Skipping upgrades. as requested.")
return
LOG.info(f"'{pkg}' is already installed. Checking for upgrades...")
subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", pkg], check=True)
subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", pkg], check=True) # noqa: S603
except ImportError:
LOG.info(f"'{pkg}' is not installed. Installing...")
subprocess.run([sys.executable, "-m", "pip", "install", pkg], check=True)
subprocess.run([sys.executable, "-m", "pip", "install", pkg], check=True) # noqa: S603
def check(self):
"""
@ -57,5 +58,5 @@ class PackageInstaller:
try:
self.action(package)
except Exception as e:
LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {str(e)}")
LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}")
LOG.exception(e)

16
app/library/Playlist.py
View file

@ -2,10 +2,12 @@ import glob
import re
from pathlib import Path
from urllib.parse import quote
from aiohttp.web import Response
from .ffprobe import ffprobe
from .Subtitle import Subtitle
from .Utils import calcDownloadPath, checkId, StreamingError
from .Utils import StreamingError, calcDownloadPath, checkId
class Playlist:
@ -20,7 +22,8 @@ class Playlist:
if not rFile.exists():
possibleFile = checkId(download_path, rFile)
if not possibleFile:
raise StreamingError(f"File '{rFile}' does not exist.")
msg = f"File '{rFile}' does not exist."
raise StreamingError(msg)
return Response(
status=302,
@ -35,7 +38,8 @@ class Playlist:
pass
if "duration" not in ff.metadata:
raise StreamingError(f"Unable to get '{rFile}' duration.")
msg = f"Unable to get '{rFile}' duration."
raise StreamingError(msg)
duration: float = float(ff.metadata.get("duration"))
@ -45,8 +49,8 @@ class Playlist:
subs = ""
index = 0
for item in self.getSideCarFiles(rFile):
if item.suffix not in Subtitle.allowedExtensions:
for item in self.get_sidecar_files(rFile):
if item.suffix not in Subtitle.allowed_extensions:
continue
index += 1
@ -68,7 +72,7 @@ class Playlist:
return "\n".join(playlist)
def getSideCarFiles(self, file: Path) -> list[Path]:
def get_sidecar_files(self, file: Path) -> list[Path]:
"""
Get sidecar files for the given file.

18
app/library/Segments.py
View file

@ -4,9 +4,9 @@ import logging
import os
import tempfile
from .ffprobe import ffprobe
from .config import Config
from .Utils import calcDownloadPath, StreamingError
from .ffprobe import ffprobe
from .Utils import StreamingError, calcDownloadPath
LOG = logging.getLogger("player.segments")
@ -28,7 +28,8 @@ class Segments:
realFile: str = calcDownloadPath(basePath=path, folder=file, createPath=False)
if not os.path.exists(realFile):
raise StreamingError(f"File {realFile} does not exist.")
msg = f"File {realFile} does not exist."
raise StreamingError(msg)
try:
ff = await ffprobe(realFile)
@ -36,15 +37,15 @@ class Segments:
pass
tmpDir: str = tempfile.gettempdir()
tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.md5(realFile.encode("utf-8")).hexdigest()}')
tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.md5(realFile.encode("utf-8")).hexdigest()}') # noqa: S324
if not os.path.exists(tmpFile):
os.symlink(realFile, tmpFile)
if self.index == 0:
startTime: float = "{:.6f}".format(0)
startTime: float = f"{0:.6f}"
else:
startTime: float = "{:.6f}".format((self.duration * self.index))
startTime: float = f"{self.duration * self.index:.6f}"
fargs = []
fargs.append("-xerror")
@ -55,7 +56,7 @@ class Segments:
fargs.append("-ss")
fargs.append(str(startTime if startTime else "0.00000"))
fargs.append("-t")
fargs.append(str("{:.6f}".format(self.duration)))
fargs.append(str(f"{self.duration:.6f}"))
fargs.append("-copyts")
@ -108,6 +109,7 @@ class Segments:
if 0 != proc.returncode:
LOG.error(f'Failed to stream {realFile} segment {self.index}. {err.decode("utf-8")}.')
raise StreamingError(f"Failed to stream {realFile} segment {self.index}.")
msg = f"Failed to stream {realFile} segment {self.index}."
raise StreamingError(msg)
return data

8
app/library/Singleton.py
View file

@ -1,5 +1,5 @@
from typing import Any, Dict
import threading
from typing import Any
class Singleton(type):
@ -7,7 +7,7 @@ class Singleton(type):
A metaclass that creates a Singleton base class when called.
"""
_instances: Dict[type, Any] = {}
_instances: dict[type, Any] = {}
def __call__(cls, *args: Any, **kwargs: Any) -> Any:
if cls not in cls._instances:
@ -16,12 +16,12 @@ class Singleton(type):
return cls._instances[cls]
class threadSafe(type):
class ThreadSafe(type):
"""
A metaclass that creates a Singleton base class when called.
"""
_instances: Dict[type, Any] = {}
_instances: dict[type, Any] = {}
_lock = threading.Lock()
def __call__(cls, *args: Any, **kwargs: Any) -> Any:

26
app/library/Subtitle.py
View file

@ -1,9 +1,11 @@
import logging
from .Utils import calcDownloadPath
import pathlib
import pysubs2
from pysubs2.time import ms_to_times
from pysubs2.formats.substation import SubstationFormat
from pysubs2.time import ms_to_times
from .Utils import calcDownloadPath
LOG = logging.getLogger("player.subtitle")
@ -19,7 +21,7 @@ SubstationFormat.ms_to_timestamp = ms_to_timestamp
class Subtitle:
allowedExtensions: tuple[str] = (
allowed_extensions: tuple[str] = (
".srt",
".vtt",
".ass",
@ -31,22 +33,22 @@ class Subtitle:
rFile = pathlib.Path(realFile)
if not rFile.exists():
raise Exception(f"File '{file}' does not exist.")
msg = f"File '{file}' does not exist."
raise Exception(msg)
if rFile.suffix not in self.allowedExtensions:
raise Exception(f"File '{file}' subtitle type is not supported.")
if rFile.suffix not in self.allowed_extensions:
msg = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
if rFile.suffix == ".vtt":
subData = ""
with open(realFile, "r") as f:
subData = f.read()
return subData
with open(realFile) as f:
return f.read()
subs = pysubs2.load(path=str(rFile))
if len(subs.events) < 1:
raise Exception(f"No subtitle events were found in '{rFile}'.")
msg = f"No subtitle events were found in '{rFile}'."
raise Exception(msg)
if len(subs.events) < 2:
return subs.to_string("vtt")

63
app/library/Tasks.py
View file

@ -1,11 +1,11 @@
import asyncio
from datetime import datetime
import json
import logging
import os
import time
from dataclasses import dataclass, field
from typing import Any, List
from datetime import datetime
from typing import Any
import httpx
from aiocron import Cron, crontab
@ -55,7 +55,7 @@ class Tasks(metaclass=Singleton):
This class is used to manage the tasks.
"""
_jobs: List[Job] = []
_jobs: list[Job] = []
"""The jobs for the tasks."""
_instance = None
@ -100,8 +100,8 @@ class Tasks(metaclass=Singleton):
Returns:
Tasks: The instance of the Tasks
"""
"""
if not Tasks._instance:
Tasks._instance = Tasks()
return Tasks._instance
@ -115,6 +115,7 @@ class Tasks(metaclass=Singleton):
Returns:
None
"""
self.load()
@ -124,7 +125,7 @@ class Tasks(metaclass=Singleton):
"""
self.clear()
def getTasks(self) -> List[Task]:
def getTasks(self) -> list[Task]:
"""Return the tasks."""
return [job.task for job in self._jobs]
@ -134,6 +135,7 @@ class Tasks(metaclass=Singleton):
Returns:
Tasks: The current instance.
"""
self.clear()
@ -142,7 +144,7 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Loading tasks from '{self._file}'.")
try:
with open(self._file, "r") as f:
with open(self._file) as f:
tasks = json.load(f)
except Exception as e:
LOG.error(f"Failed to parse tasks from '{self._file}'. '{e}'.")
@ -156,7 +158,7 @@ class Tasks(metaclass=Singleton):
try:
task = Task(**task)
except Exception as e:
LOG.error(f"Failed to parse task at list position '{i}'. '{str(e)}'.")
LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.")
continue
try:
@ -171,7 +173,7 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Task '{i}: {task.name}' queued to be executed every '{task.timer}'.")
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to queue task '{i}: {task['name']}'. '{str(e)}'.")
LOG.error(f"Failed to queue task '{i}: {task['name']}'. '{e!s}'.")
return self
@ -181,6 +183,7 @@ class Tasks(metaclass=Singleton):
Returns:
Tasks: The current instance.
"""
if len(self._jobs) < 1:
return self
@ -191,7 +194,7 @@ class Tasks(metaclass=Singleton):
task.job.stop()
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to stop job '{task.id}: {task.name}'. '{str(e)}'.")
LOG.error(f"Failed to stop job '{task.id}: {task.name}'. '{e!s}'.")
self._jobs.clear()
@ -202,56 +205,63 @@ class Tasks(metaclass=Singleton):
Validate the task.
Args:
tasks (Task|dict): The task to validate.
task (Task|dict): The task to validate.
Returns:
bool: True if the task is valid, False otherwise.
"""
"""
if not isinstance(task, dict):
if not isinstance(task, Task):
raise ValueError("Invalid task type.")
msg = "Invalid task type."
raise ValueError(msg) # noqa: TRY004
task = task.serialize()
if not task.get("name"):
raise ValueError("No name found.")
msg = "No name found."
raise ValueError(msg)
if not task.get("timer"):
raise ValueError("No timer found.")
msg = "No timer found."
raise ValueError(msg)
if not task.get("url"):
raise ValueError("No URL found.")
msg = "No URL found."
raise ValueError(msg)
if not isinstance(task.get("cookies"), str):
raise ValueError("Invalid cookies type.")
msg = "Invalid cookies type."
raise ValueError(msg) # noqa: TRY004
if not isinstance(task.get("config"), dict):
raise ValueError("Invalid config type.")
msg = "Invalid config type."
raise ValueError(msg) # noqa: TRY004
if not isinstance(task.get("template"), str):
raise ValueError("Invalid template type.")
msg = "Invalid template type."
raise ValueError(msg) # noqa: TRY004
return True
def save(self, tasks: List[Task | dict]) -> "Tasks":
def save(self, tasks: list[Task | dict]) -> "Tasks":
"""
Save the tasks.
Args:
tasks (List[Task]): The tasks to save.
tasks (list[Task]): The tasks to save.
Returns:
Tasks: The current instance.
"""
"""
for i, task in enumerate(tasks):
try:
if not isinstance(task, Task):
task = Task(**task)
tasks[i] = task
except Exception as e:
LOG.error(f"Failed to save task '{i}' unable to parse task. '{str(e)}'.")
LOG.error(f"Failed to save task '{i}' unable to parse task. '{e!s}'.")
continue
try:
@ -266,7 +276,7 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Tasks saved to '{self._file}'.")
except Exception as e:
LOG.error(f"Failed to save tasks to '{self._file}'. '{str(e)}'.")
LOG.error(f"Failed to save tasks to '{self._file}'. '{e!s}'.")
return self
@ -279,6 +289,7 @@ class Tasks(metaclass=Singleton):
Returns:
None
"""
try:
timeNow = datetime.now().isoformat()
@ -297,7 +308,7 @@ class Tasks(metaclass=Singleton):
try:
config = json.loads(config)
except Exception as e:
LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {str(e)}")
LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {e!s}")
return
LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.")
@ -332,5 +343,5 @@ class Tasks(metaclass=Singleton):
await self._emitter.success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds.")
except Exception as e:
timeNow = datetime.now().isoformat()
LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{str(e)}'.")
await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{str(e)}'.")
LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{e!s}'.")
await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{e!s}'.")

58
app/library/Utils.py
View file

@ -31,8 +31,6 @@ YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None
class StreamingError(Exception):
"""Raised when an error occurs during streaming."""
pass
def get_opts(preset: str, ytdl_opts: dict) -> dict:
"""
@ -44,6 +42,7 @@ def get_opts(preset: str, ytdl_opts: dict) -> dict:
Returns:
ytdl extra options
"""
opts = copy.deepcopy(ytdl_opts)
@ -90,6 +89,7 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) ->
Returns:
dict: Video information.
"""
params: dict = {
"quiet": True,
@ -110,10 +110,12 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) ->
def calcDownloadPath(basePath: str, folder: str | None = None, createPath: bool = True) -> str:
"""Calculates download path and prevents folder traversal.
"""
Calculates download path and prevents folder traversal.
Returns:
Download path with base folder factored in.
"""
if not folder:
return basePath
@ -125,7 +127,8 @@ def calcDownloadPath(basePath: str, folder: str | None = None, createPath: bool
download_path = os.path.realpath(os.path.join(basePath, folder))
if not download_path.startswith(realBasePath):
raise Exception(f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".')
msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".'
raise Exception(msg)
if not os.path.isdir(download_path) and createPath:
os.makedirs(download_path, exist_ok=True)
@ -191,6 +194,7 @@ def mergeConfig(config: dict, new_config: dict) -> dict:
Returns:
dict: Merged config
"""
for key in IGNORED_KEYS:
if key in new_config:
@ -249,31 +253,22 @@ def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, st
break
if not idDict["archive_id"]:
return (
False,
idDict,
)
return (False, idDict)
with open(archive_file, "r") as f:
for line in f.readlines():
with open(archive_file) as f:
for line in f:
if idDict["archive_id"] in line:
return (
True,
idDict,
)
return (True, idDict)
return (
False,
idDict,
)
return (False, idDict)
def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
"""Converts JSON cookies to Netscape cookies
"""
Converts JSON cookies to Netscape cookies
Returns None if no cookies are found, otherwise returns a string of cookies in Netscape format.
"""
netscapeCookies = "# Netscape HTTP Cookie File\n# https://curl.haxx.se/docs/http-cookies.html\n# This file was generated by libcurl! Edit at your own risk.\n\n"
hasCookies: bool = False
@ -327,13 +322,14 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded.
bool: True if the file was loaded successfully.
str: Error message if the file could not be loaded.
"""
try:
with open(file) as json_data:
opts = json.load(json_data)
if check_type:
assert isinstance(opts, check_type)
assert isinstance(opts, check_type) # noqa: S101
return (
opts,
@ -348,7 +344,7 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
opts = json5_load(json_data)
if check_type:
assert isinstance(opts, check_type)
assert isinstance(opts, check_type) # noqa: S101
return (
opts,
@ -379,7 +375,6 @@ def checkId(basePath: str, file: pathlib.Path) -> bool | str:
:return: False if no id found, otherwise the id.
"""
match = re.search(r"(?<=\[)(?:youtube-)?(?P<id>[a-zA-Z0-9\-_]{11})(?=\])", file.stem, re.IGNORECASE)
if not match:
return False
@ -424,7 +419,6 @@ def ag(array: dict | list, path: list[str | int] | str | int, default: Any = Non
:param separator: Separator for nested paths in strings.
:return: The found value or the default if not found.
"""
if path is None or path == "":
return array
@ -484,24 +478,29 @@ def validate_url(url: str) -> bool:
Returns:
bool: True if the URL is valid and allowed.
"""
if not url:
raise ValueError("URL is required.")
msg = "URL is required."
raise ValueError(msg)
try:
from yarl import URL
parsed_url = URL(url)
except ValueError:
raise ValueError("Invalid URL.")
msg = "Invalid URL."
raise ValueError(msg) # noqa: B904
# Check allowed schemes
if parsed_url.scheme not in ["http", "https"]:
raise ValueError("Invalid scheme usage. Only HTTP or HTTPS allowed.")
msg = "Invalid scheme usage. Only HTTP or HTTPS allowed."
raise ValueError(msg)
hostname = parsed_url.host
if not hostname or is_private_address(hostname):
raise ValueError("Access to internal urls or private networks is not allowed.")
msg = "Access to internal urls or private networks is not allowed."
raise ValueError(msg)
return True
@ -515,6 +514,7 @@ def arg_converter(args: str) -> dict:
Returns:
dict: yt-dlp options dictionary.
"""
import yt_dlp.options
@ -545,9 +545,11 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
Args:
uuid_str (str): UUID to validate.
version (int): The UUID version
Returns:
bool: True if the UUID is valid.
"""
try:
uuid.UUID(uuid_str, version=version)

16
app/library/cache.py
View file

@ -1,12 +1,12 @@
import hashlib
import threading
import time
from typing import Any, Dict, Optional, Tuple
from typing import Any
from .Singleton import threadSafe
from .Singleton import ThreadSafe
class Cache(metaclass=threadSafe):
class Cache(metaclass=ThreadSafe):
def __init__(self) -> None:
"""
Initialize the Cache.
@ -14,11 +14,11 @@ class Cache(metaclass=threadSafe):
# Prevent reinitialization in singleton context.
if hasattr(self, "_initialized") and self._initialized:
return
self._cache: Dict[str, Tuple[Any, Optional[float]]] = {}
self._cache: dict[str, tuple[Any, float | None]] = {}
self._lock = threading.Lock()
self._initialized = True
def set(self, key: str, value: Any, ttl: Optional[float] = None) -> None:
def set(self, key: str, value: Any, ttl: float | None = None) -> None:
"""
Synchronously set a value in the cache with an optional time-to-live.
If ttl is None, the entry never expires.
@ -27,7 +27,7 @@ class Cache(metaclass=threadSafe):
with self._lock:
self._cache[key] = (value, expire_at)
def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]:
def get(self, key: str, default: Any | None = None) -> Any | None:
"""
Synchronously retrieve a value from the cache if it exists and hasn't expired.
"""
@ -81,13 +81,13 @@ class Cache(metaclass=threadSafe):
return hashlib.sha256(key.encode("utf-8")).hexdigest()
# Asynchronous counterparts for non-blocking interfaces
async def aset(self, key: str, value: Any, ttl: Optional[float] = None) -> None:
async def aset(self, key: str, value: Any, ttl: float | None = None) -> None:
"""
Asynchronously set a value in the cache.
"""
self.set(key, value, ttl)
async def aget(self, key: str, default: Optional[Any] = None) -> Optional[Any]:
async def aget(self, key: str, default: Any | None = None) -> Any | None:
"""
Asynchronously retrieve a value from the cache.
"""

16
app/library/common.py
View file

@ -1,14 +1,14 @@
import json
import logging
from .config import Config
from .DownloadQueue import DownloadQueue
from .encoder import Encoder
from .config import Config
LOG = logging.getLogger("common")
class common:
class Common:
"""
This class is used to share common methods between the socket and the API gateways.
"""
@ -43,6 +43,7 @@ class common:
Returns:
dict[str, str]: The status of the download.
{ "status": "text" }
"""
if cookies and isinstance(cookies, dict):
cookies = self.encoder.encode(cookies)
@ -69,11 +70,13 @@ class common:
Returns:
dict: The formatted item
"""
url: str = item.get("url")
if not url:
raise ValueError("url param is required.")
msg = "url param is required."
raise ValueError(msg)
preset: str = str(item.get("preset", self.config.default_preset))
folder: str = str(item.get("folder")) if item.get("folder") else ""
@ -85,9 +88,10 @@ class common:
try:
config = json.loads(config)
except Exception as e:
raise ValueError(f"Failed to parse json yt-dlp config for '{url}'. {str(e)}")
msg = f"Failed to parse json yt-dlp config for '{url}'. {e!s}"
raise ValueError(msg) from e
item = {
return {
"url": url,
"preset": preset,
"folder": folder,
@ -95,5 +99,3 @@ class common:
"config": config if isinstance(config, dict) else {},
"template": template,
}
return item

42
app/library/config.py
View file

@ -113,9 +113,6 @@ class Config:
ytdl_options: dict = {}
"The options to use for yt-dlp."
tasks: list = []
"The tasks to run."
new_version_available: bool = False
"A new version of the application is available."
@ -215,9 +212,10 @@ class Config:
def __init__(self):
"""Virtually private constructor."""
if Config.__instance is not None:
raise Exception("This class is a singleton. Use Config.get_instance() instead.")
else:
Config.__instance = self
msg = "This class is a singleton. Use Config.get_instance() instead."
raise Exception(msg)
Config.__instance = self
baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute())
@ -233,7 +231,7 @@ class Config:
logging.info(f"Loading environment variables from '{envFile}'.")
load_dotenv(envFile)
for k, v in self._getAttributes().items():
for k, v in self._get_attributes().items():
if k.startswith("_") or k in self._manual_vars:
continue
@ -243,7 +241,7 @@ class Config:
continue
lookUpKey: str = f"YTP_{k}".upper()
setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v)
setattr(self, k, os.environ.get(lookUpKey, v))
for k, v in self.__dict__.items():
if k.startswith("_") or k in self._immutable or k in self._manual_vars:
@ -256,13 +254,14 @@ class Config:
logging.error(f"Config variable '{k}' had non-existing config reference '{key}'.")
sys.exit(1)
v = v.replace(key, getattr(self, localKey))
v = v.replace(key, getattr(self, localKey)) # noqa: PLW2901
setattr(self, k, v)
if k in self._boolean_vars:
if str(v).lower() not in (True, False, "true", "false", "on", "off", "1", "0"):
raise ValueError(f"Config variable '{k}' is set to a non-boolean value '{v}'.")
msg = f"Config variable '{k}' is set to a non-boolean value '{v}'."
raise ValueError(msg)
setattr(self, k, str(v).lower() in (True, "true", "on", "1"))
@ -271,7 +270,8 @@ class Config:
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f"Invalid log level '{self.log_level}' specified.")
msg = f"Invalid log level '{self.log_level}' specified."
raise TypeError(msg)
coloredlogs.install(
level=numeric_level, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S"
@ -310,20 +310,8 @@ class Config:
else:
LOG.info(f"No yt-dlp custom options found at '{optsFile}'.")
tasksFile = os.path.join(self.config_path, "tasks.json")
if os.path.exists(tasksFile) and os.path.getsize(tasksFile) > 0:
LOG.info(f"Loading tasks from '{tasksFile}'.")
try:
(tasks, status, error) = load_file(tasksFile, list)
if not status:
LOG.error(f"Could not load tasks file from '{tasksFile}'. '{error}'.")
sys.exit(1)
self.tasks.extend(tasks)
except Exception:
pass
# Load default presets.
with open(os.path.join(os.path.dirname(__file__), "presets.json"), "r") as f:
with open(os.path.join(os.path.dirname(__file__), "presets.json")) as f:
self.presets.extend(json.load(f))
# Load user defined presets.
@ -378,11 +366,11 @@ class Config:
self.started = time.time()
def _getAttributes(self) -> dict:
def _get_attributes(self) -> dict:
attrs: dict = {}
vClass: str = self.__class__
for attribute in vClass.__dict__.keys():
for attribute in vClass.__dict__:
if attribute.startswith("_"):
continue
@ -398,8 +386,8 @@ class Config:
Returns:
dict: A dictionary with the frontend configuration
"""
"""
data = {k: getattr(self, k) for k in self._frontend_vars}
hasCookies = self.ytdl_options.get("cookiefile", None)
data["has_cookies"] = hasCookies is not None and os.path.exists(hasCookies)

44
app/library/ffprobe.py
View file

@ -115,8 +115,9 @@ class FFStream:
if width and height:
try:
size = (int(width), int(height))
except ValueError:
raise FFProbeError("None integer size {}:{}".format(width, height))
except ValueError as e:
msg = f"None integer size {width}:{height}"
raise FFProbeError(msg) from e
else:
return None
@ -136,8 +137,9 @@ class FFStream:
if self.is_video() or self.is_audio():
try:
frame_count = int(self.__dict__.get("nb_frames", ""))
except ValueError:
raise FFProbeError("None integer frame count")
except ValueError as e:
msg = "None integer frame count"
raise FFProbeError(msg) from e
else:
frame_count = 0
@ -151,8 +153,9 @@ class FFStream:
if self.is_video() or self.is_audio():
try:
duration = float(self.__dict__.get("duration", ""))
except ValueError:
raise FFProbeError("None numeric duration")
except ValueError as e:
msg = "None numeric duration"
raise FFProbeError(msg) from e
else:
duration = 0.0
@ -188,8 +191,9 @@ class FFStream:
"""
try:
return int(self.__dict__.get("bit_rate", ""))
except ValueError:
raise FFProbeError("None integer bit_rate")
except ValueError as e:
msg = "None integer bit_rate"
raise FFProbeError(msg) from e
class FFProbeResult:
@ -212,19 +216,19 @@ class FFProbeResult:
return getattr(self, key) if hasattr(self, key) else default
def streams(self) -> list[FFStream]:
"List of all streams."
"""List of all streams."""
return self.video + self.audio + self.subtitle + self.attachment
def has_video(self):
"Is there a video stream?"
"""Is there a video stream?"""
return len(self.video) > 0
def has_audio(self):
"Is there an audio stream?"
"""Is there an audio stream?"""
return len(self.audio) > 0
def has_subtitle(self):
"Is there a subtitle stream?"
"""Is there a subtitle stream?"""
return len(self.subtitle) > 0
def __repr__(self):
@ -259,15 +263,18 @@ async def ffprobe(file: str) -> FFProbeResult:
Returns:
dict: A dictionary containing the parsed data.
"""
try:
with open(os.devnull, "w") as tempf:
await asyncio.create_subprocess_exec("ffprobe", "-h", stdout=tempf, stderr=tempf)
except FileNotFoundError:
raise IOError("ffprobe not found.")
except FileNotFoundError as e:
msg = "ffprobe not found."
raise OSError(msg) from e
if not os.path.isfile(file):
raise IOError(f"No such media file '{file}'.")
msg = f"No such media file '{file}'."
raise OSError(msg)
args = ["-v", "quiet", "-of", "json", "-show_streams", "-show_format", file]
@ -284,13 +291,14 @@ async def ffprobe(file: str) -> FFProbeResult:
if 0 == exitCode:
parsed: dict = json.loads(data.decode("utf-8"))
else:
raise FFProbeError(f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'")
msg = f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'"
raise FFProbeError(msg)
result = FFProbeResult()
result.metadata = parsed.get("format", {})
for stream in parsed.get("streams", []):
stream = FFStream(stream)
for raw_stream in parsed.get("streams", []):
stream = FFStream(raw_stream)
if stream.is_audio():
result.audio.append(stream)
elif stream.is_video():

24
app/main.py
View file

@ -13,7 +13,7 @@ from library.config import Config
from library.DownloadQueue import DownloadQueue
from library.Emitter import Emitter
from library.EventsSubscriber import EventsSubscriber
from library.HttpAPI import LOG as http_logger
from library.HttpAPI import LOG as http_logger # noqa: N811
from library.HttpAPI import HttpAPI
from library.HttpSocket import HttpSocket
from library.Notifications import Notification
@ -35,13 +35,13 @@ class Main:
self.rootPath = str(Path(__file__).parent.absolute())
self.app = web.Application()
self.checkFolders()
self._check_folders()
try:
PackageInstaller(self.config).check()
except Exception as e:
LOG.error(f"Failed to check for packages. Error message '{str(e)}'.")
LOG.exception(e)
LOG.error(f"Failed to check for packages. Error message '{e!s}'.")
caribou.upgrade(self.config.db_file, os.path.join(self.rootPath, "migrations"))
@ -65,7 +65,7 @@ class Main:
self.app.on_startup.append(lambda _: queue.initialize())
def checkFolders(self):
def _check_folders(self):
"""
Check if the required folders exist and create them if they do not.
"""
@ -74,27 +74,27 @@ class Main:
if not os.path.exists(self.config.download_path):
LOG.info(f"Creating download folder at '{self.config.download_path}'.")
os.makedirs(self.config.download_path, exist_ok=True)
except OSError as e:
except OSError:
LOG.error(f"Could not create download folder at '{self.config.download_path}'.")
raise e
raise
try:
LOG.debug(f"Checking temp folder at '{self.config.temp_path}'.")
if not os.path.exists(self.config.temp_path):
LOG.info(f"Creating temp folder at '{self.config.temp_path}'.")
os.makedirs(self.config.temp_path, exist_ok=True)
except OSError as e:
except OSError:
LOG.error(f"Could not create temp folder at '{self.config.temp_path}'.")
raise e
raise
try:
LOG.debug(f"Checking config folder at '{self.config.config_path}'.")
if not os.path.exists(self.config.config_path):
LOG.info(f"Creating config folder at '{self.config.config_path}'.")
os.makedirs(self.config.config_path, exist_ok=True)
except OSError as e:
except OSError:
LOG.error(f"Could not create config folder at '{self.config.config_path}'.")
raise e
raise
try:
LOG.debug(f"Checking database file at '{self.config.db_file}'.")
@ -102,9 +102,9 @@ class Main:
LOG.info(f"Creating database file at '{self.config.db_file}'.")
with open(self.config.db_file, "w") as _:
pass
except OSError as e:
except OSError:
LOG.error(f"Could not create database file at '{self.config.db_file}'.")
raise e
raise
def start(self):
"""

72
pyproject.toml
View file

@ -26,6 +26,7 @@ exclude = [
"node_modules",
"site-packages",
"venv",
".venv",
]
# Same as Black.
@ -39,8 +40,75 @@ target-version = "py311"
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
# McCabe complexity (`C901`) by default.
select = ["E4", "E7", "E9", "F", "G", "W"]
ignore = ["PTH", "G0", "G1", "G201"]
#select = ["E4", "E7", "E9", "F", "G", "W"]
#ignore = ["PTH", "G0", "G1", "G201"]
select = [
"ALL", # include all the rules, including new ones
]
ignore = [
#### modules
"ANN", # flake8-annotations
"COM", # flake8-commas
"C90", # mccabe complexity
"DJ", # django
"EXE", # flake8-executable
"T10", # debugger
"TID", # flake8-tidy-imports
#### specific rules
"D100", # ignore missing docs
"D101",
"D102",
"D103",
"D104",
"D105",
"D106",
"D107",
"D200",
"D205",
"D212",
"D400",
"D401",
"D415",
"E402", # false positives for local imports
"E501", # line too long
"TRY003", # external messages in exceptions are too verbose
"TD002",
"TD003",
"FIX002", # too verbose descriptions of todos,
"N806", # variable names should be snake_case
"PTH", # prefer using absolute imports
"PGH003",
"D404",
"PLR0913",
"INP001",
"G004",
"SIM105",
"SIM300",
"BLE001",
"TRY400",
"S104",
"PLR2004",
"S110",
"N812",
"S108",
"RUF006",
"RUF012",
"RUF013",
"TRY002",
"TRY401",
"A001",
"A002",
"SLF001",
"FBT001",
"FBT002",
"TRY300",
"PLR0911",
"PLR0912",
"PLR0915",
"PLW2901",
"ERA001"
]
# Allow fix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]