danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #187 from arabcoders/dev

Browse source 
Changed UMASK from 022 to 0002
This commit is contained in:
Abdulmohsen 2025-01-29 13:29:17 +03:00 committed by GitHub
commit d135de5088
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 910 additions and 589 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.vscode/settings.json vendored
View file

@ -12,7 +12,10 @@
], ],
"cSpell.words": [ "cSpell.words": [
"aconvert", "aconvert",
"ahas",
"ahash",
"aiocron", "aiocron",
"arrowless",
"copyts", "copyts",
"dotenv", "dotenv",
"finaldir", "finaldir",

4
Dockerfile
View file

@ -2,7 +2,7 @@ FROM node:lts-alpine AS node_builder
WORKDIR /app WORKDIR /app
COPY ui ./ COPY ui ./
RUN if [ ! -d /app/exported ]; then yarn install --production --prefer-offline --frozen-lockfile && yarn run generate; else echo "Skipping UI build, already built."; fi RUN if [ ! -f "/app/exported/index.html" ]; then yarn install --production --prefer-offline --frozen-lockfile && yarn run generate; else echo "Skipping UI build, already built."; fi
FROM python:3.11-alpine AS python_builder FROM python:3.11-alpine AS python_builder
@ -26,7 +26,7 @@ FROM python:3.11-alpine
ARG TZ=UTC ARG TZ=UTC
ARG USER_ID=1000 ARG USER_ID=1000
ENV IN_CONTAINER=1 ENV IN_CONTAINER=1
ENV UMASK=022 ENV UMASK=0002
ENV YTP_CONFIG_PATH=/config ENV YTP_CONFIG_PATH=/config
ENV YTP_TEMP_PATH=/tmp ENV YTP_TEMP_PATH=/tmp
ENV YTP_DOWNLOAD_PATH=/downloads ENV YTP_DOWNLOAD_PATH=/downloads

1
Pipfile
View file

@ -22,6 +22,7 @@ mutagen = "*"
brotli = "*" brotli = "*"
brotlicffi = "*" brotlicffi = "*"
yt-dlp = "*" yt-dlp = "*"
anyio = "*"
[dev-packages] [dev-packages]

3
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "cd9a56440fe36b901993ced28ee6ef8d301a5125344cda86519fb7c54ede1449" "sha256": "17668280c8e863033313b83f64240f6837e733957fbc2613767d297fee6c6fc2"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -128,6 +128,7 @@
"sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a", "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a",
"sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a" "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"
], ],
"index": "pypi",
"markers": "python_version >= '3.9'", "markers": "python_version >= '3.9'",
"version": "==4.8.0" "version": "==4.8.0"
}, },

52
app/library/AsyncPool.py
View file

@ -1,6 +1,6 @@
import asyncio import asyncio
import logging import logging
from datetime import datetime, timezone from datetime import UTC, datetime
class Terminator: class Terminator:
@ -14,9 +14,9 @@ class AsyncPool:
worker_co, worker_co,
name: str, name: str,
logger: logging.Logger, logger: logging.Logger,
loop: asyncio.AbstractEventLoop = None, loop: asyncio.AbstractEventLoop | None = None,
load_factor: int = 1, load_factor: int = 1,
max_task_time: int = None, max_task_time: int | None = None,
return_futures: bool = False, return_futures: bool = False,
raise_on_join: bool = False, raise_on_join: bool = False,
): ):
@ -54,6 +54,12 @@ class AsyncPool:
self._status: dict[str, dict | None] = {} self._status: dict[str, dict | None] = {}
async def _worker_loop(self, worker_id: str): async def _worker_loop(self, worker_id: str):
"""
This is the main loop for each worker. It will pull from the queue and call the worker_co
coroutine with the arguments passed to `push`.
:param worker_id: id of the worker
:return: None
"""
while True: while True:
got_obj = False got_obj = False
future = None future = None
@ -84,7 +90,7 @@ class AsyncPool:
raise raise
except BaseException as e: except BaseException as e:
if isinstance(e, asyncio.exceptions.CancelledError): if isinstance(e, asyncio.exceptions.CancelledError):
raise e raise
self._exceptions = True self._exceptions = True
@ -92,7 +98,7 @@ class AsyncPool:
# don't log the failure when the client is receiving the future # don't log the failure when the client is receiving the future
future.set_exception(e) future.set_exception(e)
else: else:
self._logger.exception(f"Worker call failed. {str(e)}") self._logger.exception(f"Worker call failed. {e!s}")
finally: finally:
self._status[worker_id] = None self._status[worker_id] = None
@ -150,10 +156,15 @@ class AsyncPool:
for worker_number in range(self._num_workers): for worker_number in range(self._num_workers):
worker_id = f"worker_{worker_number+1}" worker_id = f"worker_{worker_number+1}"
self._createWorker(worker_id) self._create_worker(worker_id)
async def restart(self, worker_id: str, msg: str = None) -> bool: async def restart(self, worker_id: str, msg: str = None) -> bool:
"""Will restart the worker pool""" """
Will restart the worker pool
:param worker_id: worker id to restart
:param msg: message to send to the worker
:return: True if worker was restarted
"""
if worker_id not in self._workers: if worker_id not in self._workers:
self._logger.warning(f"Worker {worker_id} does not exist.") self._logger.warning(f"Worker {worker_id} does not exist.")
return False return False
@ -162,19 +173,31 @@ class AsyncPool:
self._workers[worker_id].cancel(msg) self._workers[worker_id].cancel(msg)
await self._workers[worker_id] await self._workers[worker_id]
except asyncio.exceptions.CancelledError as e: except asyncio.exceptions.CancelledError as e:
self._logger.warning(f"Worker {worker_id} restarted. {str(e)}") self._logger.warning(f"Worker {worker_id} restarted. {e!s}")
if worker_id in self._status: if worker_id in self._status:
self._status.pop(worker_id) self._status.pop(worker_id)
if worker_id in self._workers: if worker_id in self._workers:
self._workers.pop(worker_id) self._workers.pop(worker_id)
self._createWorker(worker_id) self._create_worker(worker_id)
return True return True
async def on_shutdown(self, _) -> None:
"""Will shutdown the worker pool"""
try:
await asyncio.wait_for(asyncio.gather(*[self.stop(worker_id) for worker_id in self._workers]), timeout=10)
except Exception:
self._logger.error(f"Exception shutting down {self._name}")
async def stop(self, worker_id: str, msg: str = None) -> bool: async def stop(self, worker_id: str, msg: str = None) -> bool:
"""Will stop the worker""" """
Will stop the worker
:param worker_id: worker id to stop
:param msg: message to send to the worker
:return: True if worker was stopped
"""
if worker_id not in self._workers: if worker_id not in self._workers:
self._logger.warning(f"Worker {worker_id} does not exist.") self._logger.warning(f"Worker {worker_id} does not exist.")
return False return False
@ -183,7 +206,7 @@ class AsyncPool:
try: try:
await self._workers[worker_id] await self._workers[worker_id]
except asyncio.exceptions.CancelledError as e: except asyncio.exceptions.CancelledError as e:
self._logger.warning(f"Worker {worker_id} stopped. {str(e)}") self._logger.warning(f"Worker {worker_id} stopped. {e!s}")
if worker_id in self._status: if worker_id in self._status:
self._status.pop(worker_id) self._status.pop(worker_id)
@ -214,9 +237,10 @@ class AsyncPool:
self._logger.info(f"Completed {self._name}") self._logger.info(f"Completed {self._name}")
if self._exceptions and self._raise_on_join: if self._exceptions and self._raise_on_join:
raise Exception(f"Exception occurred in {self._name} pool") msg = f"Exception occurred in {self._name} pool"
raise Exception(msg)
def _createWorker(self, worker_id: str) -> asyncio.Future: def _create_worker(self, worker_id: str) -> asyncio.Future:
if worker_id in self._workers: if worker_id in self._workers:
self._logger.debug(f"Worker {worker_id} already exists.") self._logger.debug(f"Worker {worker_id} already exists.")
return self._workers[worker_id] return self._workers[worker_id]
@ -231,4 +255,4 @@ class AsyncPool:
return self._workers[worker_id] return self._workers[worker_id]
def _time(self) -> datetime: def _time(self) -> datetime:
return datetime.now(tz=timezone.utc) return datetime.now(tz=UTC)

49
app/library/DataStore.py
View file

@ -1,7 +1,7 @@
import copy import copy
import json import json
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime, timezone from datetime import UTC, datetime
from email.utils import formatdate from email.utils import formatdate
from sqlite3 import Connection from sqlite3 import Connection
@ -31,31 +31,30 @@ class DataStore:
for id, item in self.saved_items(): for id, item in self.saved_items():
self.dict.update({id: Download(info=item)}) self.dict.update({id: Download(info=item)})
def exists(self, key: str = None, url: str = None) -> bool: def exists(self, key: str | None = None, url: str | None = None) -> bool:
if not key and not url: if not key and not url:
raise KeyError("key or url must be provided.") msg = "key or url must be provided."
raise KeyError(msg)
if key and key in self.dict: if key and key in self.dict:
return True return True
for i in self.dict: return any((key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url) for i in self.dict)
if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url):
return True
return False def get(self, key: str, url: str | None = None) -> Download:
def get(self, key: str, url: str = None) -> Download:
if not key and not url: if not key and not url:
raise KeyError("key or url must be provided.") msg = "key or url must be provided."
raise KeyError(msg)
for i in self.dict: for i in self.dict:
if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url): if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url):
return self.dict[i] return self.dict[i]
raise KeyError(f"{key=} or {url=} not found.") msg = f"{key=} or {url=} not found."
raise KeyError(msg)
def getById(self, id: str) -> Download | None: def get_by_id(self, id: str) -> Download | None:
return self.dict[id] if id in self.dict else None return self.dict.get(id, None)
def items(self) -> list[tuple[str, Download]]: def items(self) -> list[tuple[str, Download]]:
return self.dict.items() return self.dict.items()
@ -68,25 +67,25 @@ class DataStore:
) )
for row in cursor: for row in cursor:
rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data: dict = json.loads(row["data"]) data: dict = json.loads(row["data"])
key: str = data.pop("_id") key: str = data.pop("_id")
item: ItemDTO = ItemDTO(**data) item: ItemDTO = ItemDTO(**data)
item._id = key item._id = key
item.datetime = formatdate(rowDate.replace(tzinfo=timezone.utc).timestamp()) item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
items.append((row["id"], item)) items.append((row["id"], item))
return items return items
def put(self, value: Download) -> Download: def put(self, value: Download) -> Download:
self.dict.update({value.info._id: value}) self.dict.update({value.info._id: value})
self._updateStoreItem(self.type, value.info) self._update_store_item(self.type, value.info)
return self.dict[value.info._id] return self.dict[value.info._id]
def delete(self, key: str) -> None: def delete(self, key: str) -> None:
self.dict.pop(key, None) self.dict.pop(key, None)
self._deleteStoreItem(key) self._delete_store_item(key)
def next(self) -> tuple[str, Download]: def next(self) -> tuple[str, Download]:
return next(iter(self.dict.items())) return next(iter(self.dict.items()))
@ -94,17 +93,13 @@ class DataStore:
def empty(self): def empty(self):
return not bool(self.dict) return not bool(self.dict)
def hasDownloads(self): def has_downloads(self):
if 0 == len(self.dict): if 0 == len(self.dict):
return False return False
for key in self.dict: return any(self.dict[key].started() is False for key in self.dict)
if self.dict[key].started() is False:
return True
return False def get_next_download(self) -> Download:
def getNextDownload(self) -> Download:
for key in self.dict: for key in self.dict:
if self.dict[key].started() is False and self.dict[key].is_cancelled() is False: if self.dict[key].started() is False and self.dict[key].is_cancelled() is False:
return self.dict[key] return self.dict[key]
@ -115,7 +110,7 @@ class DataStore:
self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone()
return True return True
def _updateStoreItem(self, type: str, item: ItemDTO) -> None: def _update_store_item(self, type: str, item: ItemDTO) -> None:
sqlStatement = """ sqlStatement = """
INSERT INTO "history" ("id", "type", "url", "data") INSERT INTO "history" ("id", "type", "url", "data")
VALUES (?, ?, ?, ?) VALUES (?, ?, ?, ?)
@ -146,11 +141,11 @@ class DataStore:
type, type,
stored.url, stored.url,
stored.json(), stored.json(),
datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
), ),
) )
def _deleteStoreItem(self, key: str) -> None: def _delete_store_item(self, key: str) -> None:
self.connection.execute( self.connection.execute(
'DELETE FROM "history" WHERE "type" = ? AND "id" = ?', 'DELETE FROM "history" WHERE "type" = ? AND "id" = ?',
( (

44
app/library/Download.py
View file

@ -15,7 +15,7 @@ from .config import Config
from .Emitter import Emitter from .Emitter import Emitter
from .ffprobe import ffprobe from .ffprobe import ffprobe
from .ItemDTO import ItemDTO from .ItemDTO import ItemDTO
from .Utils import get_opts, jsonCookie, mergeConfig from .Utils import get_opts, json_cookie, merge_config
LOG = logging.getLogger("download") LOG = logging.getLogger("download")
@ -34,7 +34,7 @@ class Download:
info: ItemDTO = None info: ItemDTO = None
default_ytdl_opts: dict = None default_ytdl_opts: dict = None
debug: bool = False debug: bool = False
tempPath: str = None temp_path: str = None
emitter: Emitter = None emitter: Emitter = None
cancelled: bool = False cancelled: bool = False
is_live: bool = False is_live: bool = False
@ -64,7 +64,7 @@ class Download:
) )
"Fields to be extracted from yt-dlp progress hook." "Fields to be extracted from yt-dlp progress hook."
tempKeep: bool = False temp_keep: bool = False
"Keep temp folder after download." "Keep temp folder after download."
def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False): def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False):
@ -86,7 +86,7 @@ class Download:
self.proc = None self.proc = None
self.emitter = None self.emitter = None
self.max_workers = int(config.max_workers) self.max_workers = int(config.max_workers)
self.tempKeep = bool(config.temp_keep) self.temp_keep = bool(config.temp_keep)
self.is_live = bool(info.is_live) or info.live_in is not None self.is_live = bool(info.is_live) or info.live_in is not None
self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
self.info_dict = info_dict self.info_dict = info_dict
@ -94,7 +94,7 @@ class Download:
def _progress_hook(self, data: dict): def _progress_hook(self, data: dict):
dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
if "finished" == data.get("status", None) and data.get("info_dict", {}).get("filename", None): if "finished" == data.get("status") and data.get("info_dict", {}).get("filename", None):
dataDict["filename"] = data["info_dict"]["filename"] dataDict["filename"] = data["info_dict"]["filename"]
self.status_queue.put({"id": self.id, **dataDict}) self.status_queue.put({"id": self.id, **dataDict})
@ -115,11 +115,11 @@ class Download:
def _download(self): def _download(self):
try: try:
params: dict = get_opts(self.preset, mergeConfig(self.default_ytdl_opts, self.ytdl_opts)) params: dict = get_opts(self.preset, merge_config(self.default_ytdl_opts, self.ytdl_opts))
params.update( params.update(
{ {
"color": "no_color", "color": "no_color",
"paths": {"home": self.download_dir, "temp": self.tempPath}, "paths": {"home": self.download_dir, "temp": self.temp_path},
"outtmpl": {"default": self.template, "chapter": self.template_chapter}, "outtmpl": {"default": self.template, "chapter": self.template_chapter},
"noprogress": True, "noprogress": True,
"break_on_existing": True, "break_on_existing": True,
@ -138,17 +138,17 @@ class Download:
if self.info.cookies: if self.info.cookies:
try: try:
data = jsonCookie(json.loads(self.info.cookies)) data = json_cookie(json.loads(self.info.cookies))
if not data: if not data:
LOG.warning( LOG.warning(
f"The cookie string that was provided for {self.info.title} is empty or not in expected spec." f"The cookie string that was provided for {self.info.title} is empty or not in expected spec."
) )
with open(os.path.join(self.tempPath, f"cookie_{self.info._id}.txt"), "w") as f: with open(os.path.join(self.temp_path, f"cookie_{self.info._id}.txt"), "w") as f:
f.write(data) f.write(data)
params["cookiefile"] = f.name params["cookiefile"] = f.name
except ValueError as e: except ValueError as e:
LOG.error(f"Invalid cookies: was provided for '{self.info.title}'. '{str(e)}'.") LOG.error(f"Invalid cookies: was provided for '{self.info.title}'. '{e!s}'.")
if self.is_live or self.is_manifestless: if self.is_live or self.is_manifestless:
hasDeletedOptions = False hasDeletedOptions = False
@ -191,10 +191,10 @@ class Download:
self.status_queue = Config.get_manager().Queue() self.status_queue = Config.get_manager().Queue()
# Create temp dir for each download. # Create temp dir for each download.
self.tempPath = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode("utf-8")).hexdigest(5)) self.temp_path = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5))
if not os.path.exists(self.tempPath): if not os.path.exists(self.temp_path):
os.makedirs(self.tempPath, exist_ok=True) os.makedirs(self.temp_path, exist_ok=True)
self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download) self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download)
self.proc.start() self.proc.start()
@ -235,7 +235,6 @@ class Download:
self.update_task.cancel() self.update_task.cancel()
except Exception as e: except Exception as e:
LOG.error(f"Failed to close status queue: '{procId}'. {e}") LOG.error(f"Failed to close status queue: '{procId}'. {e}")
pass
self.kill() self.kill()
@ -288,26 +287,26 @@ class Download:
return False return False
def delete_temp(self): def delete_temp(self):
if self.tempKeep is True or not self.tempPath: if self.temp_keep is True or not self.temp_path:
return return
if "finished" != self.info.status and self.is_live: if "finished" != self.info.status and self.is_live:
LOG.warning( LOG.warning(
f"Keeping live temp folder '{self.tempPath}', as the reported status is not finished '{self.info.status}'." f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'."
) )
return return
if not os.path.exists(self.tempPath): if not os.path.exists(self.temp_path):
return return
if self.tempPath == self.temp_dir: if self.temp_path == self.temp_dir:
LOG.warning( LOG.warning(
f"Attempted to delete video temp folder: {self.tempPath}, but it is the same as main temp folder." f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder."
) )
return return
LOG.info(f"Deleting Temp folder '{self.tempPath}'.") LOG.info(f"Deleting Temp folder '{self.temp_path}'.")
shutil.rmtree(self.tempPath, ignore_errors=True) shutil.rmtree(self.temp_path, ignore_errors=True)
async def progress_update(self): async def progress_update(self):
""" """
@ -343,7 +342,6 @@ class Download:
self.info.file_size = os.path.getsize(status.get("filename")) self.info.file_size = os.path.getsize(status.get("filename"))
except FileNotFoundError: except FileNotFoundError:
self.info.file_size = 0 self.info.file_size = 0
pass
self.info.status = status.get("status", self.info.status) self.info.status = status.get("status", self.info.status)
self.info.msg = status.get("msg") self.info.msg = status.get("msg")
@ -385,7 +383,7 @@ class Download:
except Exception as e: except Exception as e:
self.info.extras["is_video"] = True self.info.extras["is_video"] = True
self.info.extras["is_audio"] = True self.info.extras["is_audio"] = True
LOG.exception(f"Failed to ffprobe: {status.get}. {e}")
LOG.exception(e) LOG.exception(e)
LOG.error(f"Failed to ffprobe: {status.get}. {e}")
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")

199
app/library/DownloadQueue.py
View file

@ -1,23 +1,24 @@
import asyncio import asyncio
import datetime
import json import json
import logging import logging
import os import os
import time import time
from datetime import UTC, datetime
from email.utils import formatdate from email.utils import formatdate
from sqlite3 import Connection from sqlite3 import Connection
import yt_dlp import yt_dlp
from aiohttp import web
from .AsyncPool import AsyncPool from .AsyncPool import AsyncPool
from .config import Config from .config import Config
from .DataStore import DataStore from .DataStore import DataStore
from .Download import Download from .Download import Download
from .Emitter import Emitter from .Emitter import Emitter
from .EventsSubscriber import Events
from .ItemDTO import ItemDTO from .ItemDTO import ItemDTO
from .Singleton import Singleton from .Singleton import Singleton
from .EventsSubscriber import Events from .Utils import calc_download_path, extract_info, get_opts, is_downloaded, merge_config
from .Utils import ExtractInfo, calcDownloadPath, get_opts, isDownloaded, mergeConfig
LOG = logging.getLogger("DownloadQueue") LOG = logging.getLogger("DownloadQueue")
@ -42,6 +43,8 @@ class DownloadQueue(metaclass=Singleton):
pool: AsyncPool | None = None pool: AsyncPool | None = None
"""Pool of workers to download the files.""" """Pool of workers to download the files."""
_active_downloads: dict[str, Download] = {}
_instance = None _instance = None
"""Instance of the DownloadQueue.""" """Instance of the DownloadQueue."""
@ -65,18 +68,39 @@ class DownloadQueue(metaclass=Singleton):
Returns: Returns:
DownloadQueue: The instance of the DownloadQueue DownloadQueue: The instance of the DownloadQueue
""" """
if not DownloadQueue._instance: if not DownloadQueue._instance:
DownloadQueue._instance = DownloadQueue() DownloadQueue._instance = DownloadQueue()
return DownloadQueue._instance return DownloadQueue._instance
def attach(self, app: web.Application):
"""
Attach the download queue to the application.
Args:
app (web.Application): The application to attach the download queue to.
"""
app.on_startup.append(lambda _: self.initialize())
# app.on_shutdown.append(self.on_shutdown)
# async def close_pool(_: web.Application):
# try:
# await self.pool.on_shutdown(_)
# except Exception as e:
# LOG.error(f"Failed to cleanup download pool. {e!s}")
# app.on_cleanup.append(close_pool)
async def test(self) -> bool: async def test(self) -> bool:
""" """
Test the datastore connection to the database. Test the datastore connection to the database.
Returns: Returns:
bool: True if the test is successful, False otherwise. bool: True if the test is successful, False otherwise.
""" """
await self.done.test() await self.done.test()
return True return True
@ -88,18 +112,20 @@ class DownloadQueue(metaclass=Singleton):
LOG.info( LOG.info(
f"Using '{self.config.max_workers}' worker/s for downloading. Can be configured via `YTP_MAX_WORKERS` environment variable." f"Using '{self.config.max_workers}' worker/s for downloading. Can be configured via `YTP_MAX_WORKERS` environment variable."
) )
asyncio.create_task(self.__download_pool(), name="download_pool") asyncio.create_task(self._download_pool(), name="download_pool")
def pause(self) -> bool: def pause(self, shutdown: bool = False) -> bool:
""" """
Pause the download queue. Pause the download queue.
Returns: Returns:
bool: True if the download is paused, False otherwise bool: True if the download is paused, False otherwise
""" """
if self.paused.is_set(): if self.paused.is_set():
self.paused.clear() self.paused.clear()
LOG.warning(f"Download paused at. {datetime.datetime.now().isoformat()}") if not shutdown:
LOG.warning(f"Download paused at. {datetime.now(tz=UTC).isoformat()}")
return True return True
return False return False
@ -110,29 +136,40 @@ class DownloadQueue(metaclass=Singleton):
Returns: Returns:
bool: True if the download is resumed, False otherwise bool: True if the download is resumed, False otherwise
""" """
if not self.paused.is_set(): if not self.paused.is_set():
self.paused.set() self.paused.set()
LOG.warning(f"Downloading resumed at. {datetime.datetime.now().isoformat()}") LOG.warning(f"Downloading resumed at. {datetime.now(tz=UTC).isoformat()}")
return True return True
return False return False
def isPaused(self) -> bool: def is_paused(self) -> bool:
""" """
Check if the download queue is paused. Check if the download queue is paused.
Returns: Returns:
bool: True if the download queue is paused, False otherwise bool: True if the download queue is paused, False otherwise
""" """
return False if self.paused.is_set() else True return not self.paused.is_set()
async def on_shutdown(self, _: web.Application):
LOG.debug("Canceling all active downloads.")
if self._active_downloads:
self.pause()
try:
await self.cancel(list(self._active_downloads.keys()))
except Exception as e:
LOG.error(f"Failed to cancel downloads. {e!s}")
async def __add_entry( async def __add_entry(
self, self,
entry: dict, entry: dict,
preset: str, preset: str,
folder: str, folder: str,
config: dict = {}, config: dict | None = None,
cookies: str = "", cookies: str = "",
template: str = "", template: str = "",
already=None, already=None,
@ -151,7 +188,11 @@ class DownloadQueue(metaclass=Singleton):
Returns: Returns:
dict: The status of the operation. dict: The status of the operation.
""" """
if not config:
config = {}
if not entry: if not entry:
return {"status": "error", "msg": "Invalid/empty data was given."} return {"status": "error", "msg": "Invalid/empty data was given."}
@ -161,13 +202,14 @@ class DownloadQueue(metaclass=Singleton):
live_in: str | None = None live_in: str | None = None
eventType = entry.get("_type") or "video" eventType = entry.get("_type") or "video"
if "playlist" == eventType: if "playlist" == eventType:
entries = entry.get("entries", []) entries = entry.get("entries", [])
playlist_index_digits = len(str(len(entries))) playlist_index_digits = len(str(len(entries)))
results = [] results = []
for i, etr in enumerate(entries, start=1): for i, etr in enumerate(entries, start=1):
etr["playlist"] = entry.get("id") etr["playlist"] = entry.get("id")
etr["playlist_index"] = "{{0:0{0:d}d}}".format(playlist_index_digits).format(i) etr["playlist_index"] = f"{{0:0{playlist_index_digits:d}d}}".format(i)
for property in ("id", "title", "uploader", "uploader_id"): for property in ("id", "title", "uploader", "uploader_id"):
if property in entry: if property in entry:
etr[f"playlist_{property}"] = entry.get(property) etr[f"playlist_{property}"] = entry.get(property)
@ -194,7 +236,8 @@ class DownloadQueue(metaclass=Singleton):
} }
return {"status": "ok"} return {"status": "ok"}
elif ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
# check if the video is live stream. # check if the video is live stream.
if "live_status" in entry and "is_upcoming" == entry.get("live_status"): if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
if "release_timestamp" in entry and entry.get("release_timestamp"): if "release_timestamp" in entry and entry.get("release_timestamp"):
@ -202,11 +245,9 @@ class DownloadQueue(metaclass=Singleton):
else: else:
error = "Live stream not yet started. And no date is set." error = "Live stream not yet started. And no date is set."
else: else:
error = entry.get("msg", None) error = entry.get("msg")
LOG.debug( LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
f"Entry id '{entry.get('id', None)}' url '{entry.get('webpage_url', None)} - {entry.get('url', None)}'."
)
if self.done.exists(key=entry["id"], url=str(entry.get("webpage_url") or entry.get("url"))): if self.done.exists(key=entry["id"], url=str(entry.get("webpage_url") or entry.get("url"))):
item = self.done.get(key=entry["id"], url=entry.get("webpage_url") or entry["url"]) item = self.done.get(key=entry["id"], url=entry.get("webpage_url") or entry["url"])
@ -226,10 +267,10 @@ class DownloadQueue(metaclass=Singleton):
options.update({"is_manifestless": is_manifestless}) options.update({"is_manifestless": is_manifestless})
live_status: list = ["is_live", "is_upcoming"] live_status: list = ["is_live", "is_upcoming"]
is_live = bool(entry.get("is_live", None) or live_in or entry.get("live_status", None) in live_status) is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
try: try:
download_dir = calcDownloadPath(basePath=self.config.download_path, folder=folder) download_dir = calc_download_path(base_path=self.config.download_path, folder=folder)
except Exception as e: except Exception as e:
LOG.exception(e) LOG.exception(e)
return {"status": "error", "msg": str(e)} return {"status": "error", "msg": str(e)}
@ -237,7 +278,7 @@ class DownloadQueue(metaclass=Singleton):
extras: dict = {} extras: dict = {}
fields: tuple = ("uploader", "channel", "thumbnail") fields: tuple = ("uploader", "channel", "thumbnail")
for field in fields: for field in fields:
if entry.get(field, None): if entry.get(field):
extras[field] = entry.get(field) extras[field] = entry.get(field)
dl = ItemDTO( dl = ItemDTO(
@ -245,7 +286,6 @@ class DownloadQueue(metaclass=Singleton):
title=str(entry.get("title")), title=str(entry.get("title")),
url=str(entry.get("webpage_url") or entry.get("url")), url=str(entry.get("webpage_url") or entry.get("url")),
preset=preset, preset=preset,
thumbnail=entry.get("thumbnail", None),
folder=folder, folder=folder,
download_dir=download_dir, download_dir=download_dir,
temp_dir=self.config.temp_path, temp_dir=self.config.temp_path,
@ -267,7 +307,7 @@ class DownloadQueue(metaclass=Singleton):
dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug)) dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug))
if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status", None): if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"):
dlInfo.info.status = "not_live" dlInfo.info.status = "not_live"
itemDownload = self.done.put(dlInfo) itemDownload = self.done.put(dlInfo)
NotifyEvent = Events.COMPLETED NotifyEvent = Events.COMPLETED
@ -286,7 +326,8 @@ class DownloadQueue(metaclass=Singleton):
) )
return {"status": "ok"} return {"status": "ok"}
elif eventType.startswith("url"):
if eventType.startswith("url"):
return await self.add( return await self.add(
url=str(entry.get("url")), url=str(entry.get("url")),
preset=preset, preset=preset,
@ -304,7 +345,7 @@ class DownloadQueue(metaclass=Singleton):
url: str, url: str,
preset: str, preset: str,
folder: str, folder: str,
config: dict = {}, config: dict | None = None,
cookies: str = "", cookies: str = "",
template: str = "", template: str = "",
already=None, already=None,
@ -312,7 +353,7 @@ class DownloadQueue(metaclass=Singleton):
config = config if config else {} config = config if config else {}
folder = str(folder) if folder else "" folder = str(folder) if folder else ""
filePath = calcDownloadPath(basePath=self.config.download_path, folder=folder) filePath = calc_download_path(base_path=self.config.download_path, folder=folder)
LOG.info( LOG.info(
f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {cookies}' 'YTConfig: {config}'." f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {cookies}' 'YTConfig: {config}'."
@ -322,8 +363,8 @@ class DownloadQueue(metaclass=Singleton):
try: try:
config = json.loads(config) config = json.loads(config)
except Exception as e: except Exception as e:
LOG.error(f"Unable to load '{config=}'. {str(e)}") LOG.error(f"Unable to load '{config=}'. {e!s}")
return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {str(e)}"} return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {e!s}"}
already = set() if already is None else already already = set() if already is None else already
@ -334,7 +375,7 @@ class DownloadQueue(metaclass=Singleton):
already.add(url) already.add(url)
try: try:
downloaded, id_dict = self.isDownloaded(url) downloaded, id_dict = self._is_downloaded(url)
if downloaded is True and id_dict: if downloaded is True and id_dict:
message = f"This url with ID '{id_dict.get('id')}' has been downloaded already and recorded in archive." message = f"This url with ID '{id_dict.get('id')}' has been downloaded already and recorded in archive."
LOG.info(message) LOG.info(message)
@ -346,8 +387,8 @@ class DownloadQueue(metaclass=Singleton):
entry = await asyncio.wait_for( entry = await asyncio.wait_for(
fut=asyncio.get_running_loop().run_in_executor( fut=asyncio.get_running_loop().run_in_executor(
None, None,
ExtractInfo, extract_info,
get_opts(preset, mergeConfig(self.config.ytdl_options, config)), get_opts(preset, merge_config(self.config.ytdl_options, config)),
url, url,
bool(self.config.ytdl_debug), bool(self.config.ytdl_debug),
), ),
@ -361,13 +402,13 @@ class DownloadQueue(metaclass=Singleton):
f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'." f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'."
) )
except yt_dlp.utils.ExistingVideoReached as exc: except yt_dlp.utils.ExistingVideoReached as exc:
LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{str(exc)}'.") LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.")
return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
except yt_dlp.utils.YoutubeDLError as exc: except yt_dlp.utils.YoutubeDLError as exc:
LOG.error(f"YoutubeDLError: Unable to extract info. '{str(exc)}'.") LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.")
return {"status": "error", "msg": str(exc)} return {"status": "error", "msg": str(exc)}
except asyncio.exceptions.TimeoutError as exc: except asyncio.exceptions.TimeoutError as exc:
LOG.error(f"TimeoutError: Unable to extract info. '{str(exc)}'.") LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.")
return { return {
"status": "error", "status": "error",
"msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.", "msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.",
@ -384,6 +425,16 @@ class DownloadQueue(metaclass=Singleton):
) )
async def cancel(self, ids: list[str]) -> dict[str, str]: async def cancel(self, ids: list[str]) -> dict[str, str]:
"""
Cancel the download.
Args:
ids (list): The list of ids to cancel.
Returns:
dict: The status of the operation.
"""
status: dict[str, str] = {"status": "ok"} status: dict[str, str] = {"status": "ok"}
for id in ids: for id in ids:
@ -392,32 +443,43 @@ class DownloadQueue(metaclass=Singleton):
except KeyError as e: except KeyError as e:
status[id] = str(e) status[id] = str(e)
status["status"] = "error" status["status"] = "error"
LOG.warning(f"Requested cancel for non-existent download {id=}. {str(e)}") LOG.warning(f"Requested cancel for non-existent download {id=}. {e!s}")
continue continue
itemMessage = f"{id=} {item.info.id=} {item.info.title=}" item_ref = f"{id=} {item.info.id=} {item.info.title=}"
if item.running(): if item.running():
LOG.debug(f"Canceling {itemMessage}") LOG.debug(f"Canceling {item_ref}")
item.cancel() item.cancel()
LOG.info(f"Cancelled {itemMessage}") LOG.info(f"Cancelled {item_ref}")
await item.close() await item.close()
else: else:
await item.close() await item.close()
LOG.debug(f"Deleting from queue {itemMessage}") LOG.debug(f"Deleting from queue {item_ref}")
self.queue.delete(id) self.queue.delete(id)
asyncio.create_task(self.emitter.cancelled(dl=item.info.serialize()), name=f"notifier-c-{id}") asyncio.create_task(self.emitter.cancelled(dl=item.info.serialize()), name=f"notifier-c-{id}")
item.info.status = "cancelled" item.info.status = "cancelled"
item.info.error = "Cancelled by user." item.info.error = "Cancelled by user."
self.done.put(item) self.done.put(item)
asyncio.create_task(self.emitter.completed(dl=item.info.serialize()), name=f"notifier-d-{id}") asyncio.create_task(self.emitter.completed(dl=item.info.serialize()), name=f"notifier-d-{id}")
LOG.info(f"Deleted from queue {itemMessage}") LOG.info(f"Deleted from queue {item_ref}")
status[id] = "ok" status[id] = "ok"
return status return status
async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]: async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]:
"""
Clear the download history.
Args:
ids (list): The list of ids to clear.
remove_file (bool): True to remove the file, False otherwise. Default is False.
Returns:
dict: The status of the operation.
"""
status: dict[str, str] = {"status": "ok"} status: dict[str, str] = {"status": "ok"}
for id in ids: for id in ids:
@ -426,7 +488,7 @@ class DownloadQueue(metaclass=Singleton):
except KeyError as e: except KeyError as e:
status[id] = str(e) status[id] = str(e)
status["status"] = "error" status["status"] = "error"
LOG.warning(f"Requested delete for non-existent download {id=}. {str(e)}") LOG.warning(f"Requested delete for non-existent download {id=}. {e!s}")
continue continue
itemRef: str = f"{id=} {item.info.id=} {item.info.title=}" itemRef: str = f"{id=} {item.info.id=} {item.info.title=}"
@ -439,10 +501,10 @@ class DownloadQueue(metaclass=Singleton):
filename = f"{item.info.folder}/{item.info.filename}" filename = f"{item.info.folder}/{item.info.filename}"
try: try:
realFile: str = calcDownloadPath( realFile: str = calc_download_path(
basePath=self.config.download_path, base_path=self.config.download_path,
folder=filename, folder=filename,
createPath=False, create_path=False,
) )
if realFile and os.path.exists(realFile): if realFile and os.path.exists(realFile):
os.remove(realFile) os.remove(realFile)
@ -450,7 +512,7 @@ class DownloadQueue(metaclass=Singleton):
else: else:
LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.") LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.")
except Exception as e: except Exception as e:
LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {str(e)}") LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {e!s}")
self.done.delete(id) self.done.delete(id)
asyncio.create_task(self.emitter.cleared(dl=item.info.serialize()), name=f"notifier-c-{id}") asyncio.create_task(self.emitter.cleared(dl=item.info.serialize()), name=f"notifier-c-{id}")
@ -469,6 +531,7 @@ class DownloadQueue(metaclass=Singleton):
Returns: Returns:
dict: The download queue and the download history. dict: The download queue and the download history.
""" """
items = {"queue": {}, "done": {}} items = {"queue": {}, "done": {}}
@ -488,11 +551,18 @@ class DownloadQueue(metaclass=Singleton):
return items return items
async def __download_pool(self): async def _download_pool(self) -> None:
"""
Create a pool of workers to download the files.
Returns:
None
"""
self.pool = AsyncPool( self.pool = AsyncPool(
loop=asyncio.get_running_loop(), loop=asyncio.get_running_loop(),
num_workers=self.config.max_workers, num_workers=self.config.max_workers,
worker_co=self.__downloadFile, worker_co=self._download_file,
name="download_pool", name="download_pool",
logger=logging.getLogger("WorkerPool"), logger=logging.getLogger("WorkerPool"),
) )
@ -510,19 +580,19 @@ class DownloadQueue(metaclass=Singleton):
LOG.info(f"Waiting for worker to be free. {self.pool.get_workers_status()}") LOG.info(f"Waiting for worker to be free. {self.pool.get_workers_status()}")
await asyncio.sleep(1) await asyncio.sleep(1)
while not self.queue.hasDownloads(): while not self.queue.has_downloads():
LOG.info(f"Waiting for item to download. '{self.pool.get_available_workers()}' free workers.") LOG.info(f"Waiting for item to download. '{self.pool.get_available_workers()}' free workers.")
if self.event: if self.event:
await self.event.wait() await self.event.wait()
self.event.clear() self.event.clear()
LOG.debug("Cleared wait event.") LOG.debug("Cleared wait event.")
if self.paused and isinstance(self.paused, asyncio.Event) and self.isPaused(): if self.paused and isinstance(self.paused, asyncio.Event) and self.is_paused():
LOG.info("Download pool is paused.") LOG.info("Download pool is paused.")
await self.paused.wait() await self.paused.wait()
LOG.info("Download pool resumed downloading.") LOG.info("Download pool resumed downloading.")
entry = self.queue.getNextDownload() entry = self.queue.get_next_download()
await asyncio.sleep(0.2) await asyncio.sleep(0.2)
if entry is None: if entry is None:
@ -535,13 +605,25 @@ class DownloadQueue(metaclass=Singleton):
LOG.debug(f"Pushed {entry=} to executor.") LOG.debug(f"Pushed {entry=} to executor.")
await asyncio.sleep(1) await asyncio.sleep(1)
async def __downloadFile(self, id: str, entry: Download): async def _download_file(self, id: str, entry: Download) -> None:
filePath = calcDownloadPath(basePath=self.config.download_path, folder=entry.info.folder) """
Download the file.
Args:
id (str): The id of the download.
entry (Download): The download entry.
Returns:
None
"""
filePath = calc_download_path(base_path=self.config.download_path, folder=entry.info.folder)
LOG.info( LOG.info(
f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {filePath}'." f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {filePath}'."
) )
try: try:
self._active_downloads[entry.info._id] = entry
await entry.start(self.emitter) await entry.start(self.emitter)
if "finished" != entry.info.status: if "finished" != entry.info.status:
@ -554,6 +636,9 @@ class DownloadQueue(metaclass=Singleton):
entry.info.status = "error" entry.info.status = "error"
finally: finally:
if entry.info._id in self._active_downloads:
self._active_downloads.pop(entry.info._id, None)
await entry.close() await entry.close()
if self.queue.exists(key=id): if self.queue.exists(key=id):
@ -573,8 +658,18 @@ class DownloadQueue(metaclass=Singleton):
if self.event: if self.event:
self.event.set() self.event.set()
def isDownloaded(self, url: str) -> tuple[bool, dict | None]: def _is_downloaded(self, url: str) -> tuple[bool, dict | None]:
"""
Check if the url has been downloaded already.
Args:
url (str): The url to check.
Returns:
tuple: A tuple with the status of the operation and the id of the downloaded item.
"""
if not url or not self.config.keep_archive: if not url or not self.config.keep_archive:
return False, None return False, None
return isDownloaded(self.config.ytdl_options.get("download_archive", None), url) return is_downloaded(self.config.ytdl_options.get("download_archive", None), url)

17
app/library/Emitter.py
View file

@ -1,9 +1,9 @@
import asyncio import asyncio
import logging import logging
from typing import Awaitable from collections.abc import Awaitable
from .Singleton import Singleton
from .EventsSubscriber import Events from .EventsSubscriber import Events
from .Singleton import Singleton
LOG = logging.getLogger("Emitter") LOG = logging.getLogger("Emitter")
@ -28,6 +28,7 @@ class Emitter(metaclass=Singleton):
Returns: Returns:
Emitter: The instance of the Emitter Emitter: The instance of the Emitter
""" """
if not Emitter._instance: if not Emitter._instance:
Emitter._instance = Emitter() Emitter._instance = Emitter()
@ -43,6 +44,7 @@ class Emitter(metaclass=Singleton):
Returns: Returns:
Emitter: The instance of the Emitter Emitter: The instance of the Emitter
""" """
if not isinstance(emitter, list): if not isinstance(emitter, list):
emitter = [emitter] emitter = [emitter]
@ -67,19 +69,19 @@ class Emitter(metaclass=Singleton):
async def cleared(self, dl: dict | None = None, local: bool = False, **kwargs): async def cleared(self, dl: dict | None = None, local: bool = False, **kwargs):
await self.emit(Events.CLEARED, data=dl, local=local, **kwargs) await self.emit(Events.CLEARED, data=dl, local=local, **kwargs)
async def error(self, message: str, data: dict = {}, local: bool = False, **kwargs): async def error(self, message: str, data: dict|None = None, local: bool = False, **kwargs):
msg = {"type": "error", "message": message, "data": {}} msg = {"type": "error", "message": message, "data": {}}
if data: if data:
msg.update({"data": data}) msg.update({"data": data})
await self.emit(Events.ERROR, data=msg, local=local, **kwargs) await self.emit(Events.ERROR, data=msg, local=local, **kwargs)
async def info(self, message: str, data: dict = {}, local: bool = False, **kwargs): async def info(self, message: str, data: dict|None = None, local: bool = False, **kwargs):
msg = {"type": "info", "message": message, "data": {}} msg = {"type": "info", "message": message, "data": {}}
if data: if data:
msg.update({"data": data}) msg.update({"data": data})
await self.emit(Events.LOG_INFO, data=msg, local=local, **kwargs) await self.emit(Events.LOG_INFO, data=msg, local=local, **kwargs)
async def success(self, message: str, data: dict = {}, local: bool = False, **kwargs): async def success(self, message: str, data: dict|None = None, local: bool = False, **kwargs):
msg = {"type": "success", "message": message, "data": {}} msg = {"type": "success", "message": message, "data": {}}
if data: if data:
msg.update({"data": data}) msg.update({"data": data})
@ -97,6 +99,7 @@ class Emitter(metaclass=Singleton):
Returns: Returns:
None None
""" """
tasks = [] tasks = []
@ -122,8 +125,8 @@ class Emitter(metaclass=Singleton):
await asyncio.wait_for(asyncio.gather(*tasks), timeout=60) await asyncio.wait_for(asyncio.gather(*tasks), timeout=60)
except asyncio.CancelledError: except asyncio.CancelledError:
LOG.error(f"Cancelled sending event '{event}'.") LOG.error(f"Cancelled sending event '{event}'.")
except asyncio.TimeoutError: except TimeoutError:
LOG.error(f"Timed out sending event '{event}'.") LOG.error(f"Timed out sending event '{event}'.")
except Exception as e: except Exception as e:
LOG.exception(e) LOG.exception(e)
LOG.error(f"Failed to send event '{event}'. '{str(e)}'.") LOG.error(f"Failed to send event '{event}'. '{e!s}'.")

17
app/library/EventsSubscriber.py
View file

@ -1,7 +1,6 @@
import asyncio import asyncio
import logging import logging
from typing import Awaitable from collections.abc import Awaitable
from dataclasses import dataclass from dataclasses import dataclass
from .Singleton import Singleton from .Singleton import Singleton
@ -32,12 +31,10 @@ class Events:
CLI_OUTPUT = "cli_output" CLI_OUTPUT = "cli_output"
UPDATE = "update" UPDATE = "update"
TEST = "test" TEST = "test"
UPDATED = "updated"
ADD_URL = "add_url" ADD_URL = "add_url"
CLI_POST = "cli_post" CLI_POST = "cli_post"
PAUSED = "paused" PAUSED = "paused"
TEST = "test"
TASKS_ADD = "task_add" TASKS_ADD = "task_add"
TASK_DISPATCHED = "task_dispatched" TASK_DISPATCHED = "task_dispatched"
@ -72,6 +69,7 @@ class EventsSubscriber(metaclass=Singleton):
Returns: Returns:
EventsSubscriber: The instance of the EventsSubscriber EventsSubscriber: The instance of the EventsSubscriber
""" """
if not EventsSubscriber._instance: if not EventsSubscriber._instance:
EventsSubscriber._instance = EventsSubscriber() EventsSubscriber._instance = EventsSubscriber()
@ -88,6 +86,7 @@ class EventsSubscriber(metaclass=Singleton):
Returns: Returns:
EventsSubscriber: The instance of the EventsSubscriber EventsSubscriber: The instance of the EventsSubscriber
""" """
if isinstance(event, str): if isinstance(event, str):
event = [event] event = [event]
@ -110,8 +109,8 @@ class EventsSubscriber(metaclass=Singleton):
Returns: Returns:
EventsSubscriber: The instance of the EventsSubscriber EventsSubscriber: The instance of the EventsSubscriber
"""
"""
if isinstance(event, str): if isinstance(event, str):
event = [event] event = [event]
@ -148,8 +147,8 @@ class EventsSubscriber(metaclass=Singleton):
results.append(asyncio.get_event_loop().run_until_complete(callback(event, data))) results.append(asyncio.get_event_loop().run_until_complete(callback(event, data)))
except Exception as e: except Exception as e:
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.")
LOG.exception(e) LOG.exception(e)
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{e!s}'.")
return results return results
@ -164,10 +163,10 @@ class EventsSubscriber(metaclass=Singleton):
Returns: Returns:
Awaitable: The task that was created to run the event. Awaitable: The task that was created to run the event.
"""
"""
if event not in self._listeners: if event not in self._listeners:
return return None
tasks = [] tasks = []
for id, callback in self._listeners[event].items(): for id, callback in self._listeners[event].items():
@ -181,7 +180,7 @@ class EventsSubscriber(metaclass=Singleton):
tasks.append(asyncio.create_task(callback(event, data))) tasks.append(asyncio.create_task(callback(event, data)))
except Exception as e: except Exception as e:
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{str(e)}'.")
LOG.exception(e) LOG.exception(e)
LOG.error(f"Failed to emit event '{event}' to '{id}'. Error message '{e!s}'.")
return asyncio.gather(*tasks) return asyncio.gather(*tasks)

153
app/library/HttpAPI.py
View file

@ -8,9 +8,9 @@ import os
import random import random
import time import time
import uuid import uuid
from datetime import datetime from collections.abc import Awaitable
from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Awaitable
import httpx import httpx
import magic import magic
@ -18,7 +18,7 @@ from aiohttp import web
from aiohttp.web import Request, RequestHandler, Response from aiohttp.web import Request, RequestHandler, Response
from .cache import Cache from .cache import Cache
from .common import common from .common import Common
from .config import Config from .config import Config
from .DownloadQueue import DownloadQueue from .DownloadQueue import DownloadQueue
from .Emitter import Emitter from .Emitter import Emitter
@ -31,21 +31,24 @@ from .Playlist import Playlist
from .Segments import Segments from .Segments import Segments
from .Subtitle import Subtitle from .Subtitle import Subtitle
from .Tasks import Task, Tasks from .Tasks import Task, Tasks
from .Utils import StreamingError, arg_converter, calcDownloadPath, getVideoInfo, validate_url, validate_uuid from .Utils import StreamingError, arg_converter, calc_download_path, get_video_info, validate_url, validate_uuid
LOG = logging.getLogger("http_api") LOG = logging.getLogger("http_api")
MIME = magic.Magic(mime=True) MIME = magic.Magic(mime=True)
class HttpAPI(common): class HttpAPI(Common):
staticHolder: dict = {} _static_holder: dict = {}
extToMime: dict = { """Holds loaded static assets."""
_ext_to_mime: dict = {
".html": "text/html", ".html": "text/html",
".css": "text/css", ".css": "text/css",
".js": "application/javascript", ".js": "application/javascript",
".json": "application/json", ".json": "application/json",
".ico": "image/x-icon", ".ico": "image/x-icon",
} }
"""Map ext to mimetype"""
def __init__( def __init__(
self, self,
@ -65,6 +68,7 @@ class HttpAPI(common):
super().__init__(queue=self.queue, encoder=self.encoder, config=self.config) super().__init__(queue=self.queue, encoder=self.encoder, config=self.config)
@staticmethod
def route(method: str, path: str) -> Awaitable: def route(method: str, path: str) -> Awaitable:
""" """
Decorator to mark a method as an HTTP route handler. Decorator to mark a method as an HTTP route handler.
@ -75,6 +79,7 @@ class HttpAPI(common):
Returns: Returns:
Awaitable: The decorated function. Awaitable: The decorated function.
""" """
def decorator(func): def decorator(func):
@ -88,6 +93,9 @@ class HttpAPI(common):
return decorator return decorator
async def on_shutdown(self, _: web.Application):
LOG.debug("Shutting down http API server.")
def attach(self, app: web.Application) -> "HttpAPI": def attach(self, app: web.Application) -> "HttpAPI":
""" """
Attach the routes to the application. Attach the routes to the application.
@ -97,6 +105,7 @@ class HttpAPI(common):
Returns: Returns:
HttpAPI: The instance of the HttpAPI. HttpAPI: The instance of the HttpAPI.
""" """
if self.config.auth_username and self.config.auth_password: if self.config.auth_username and self.config.auth_password:
app.middlewares.append(HttpAPI.basic_auth(self.config.auth_username, self.config.auth_password)) app.middlewares.append(HttpAPI.basic_auth(self.config.auth_username, self.config.auth_password))
@ -117,9 +126,10 @@ class HttpAPI(common):
except Exception as e: except Exception as e:
LOG.exception(e) LOG.exception(e)
app.on_shutdown.append(self.on_shutdown)
return self return self
async def staticFile(self, req: Request) -> Response: async def _static_file(self, req: Request) -> Response:
""" """
Preload static files from the ui/exported folder. Preload static files from the ui/exported folder.
@ -128,13 +138,14 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
path = req.path path = req.path
if req.path not in self.staticHolder: if req.path not in self._static_holder:
return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code) return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code)
item: dict = self.staticHolder[req.path] item: dict = self._static_holder[req.path]
return web.Response( return web.Response(
body=item.get("content"), body=item.get("content"),
@ -142,12 +153,12 @@ class HttpAPI(common):
"Pragma": "public", "Pragma": "public",
"Cache-Control": "public, max-age=31536000", "Cache-Control": "public, max-age=31536000",
"Content-Type": item.get("content_type"), "Content-Type": item.get("content_type"),
"X-Via": "memory" if not item.get("file", None) else "disk", "X-Via": "memory" if not item.get("file") else "disk",
}, },
status=web.HTTPOk.status_code, status=web.HTTPOk.status_code,
) )
def preloadStatic(self, app: web.Application) -> "HttpAPI": def _preload_static(self, app: web.Application) -> "HttpAPI":
""" """
Preload static files from the ui/exported folder. Preload static files from the ui/exported folder.
@ -156,11 +167,12 @@ class HttpAPI(common):
Returns: Returns:
HttpAPI: The instance of the HttpAPI. HttpAPI: The instance of the HttpAPI.
"""
"""
staticDir = os.path.join(self.rootPath, "ui", "exported") staticDir = os.path.join(self.rootPath, "ui", "exported")
if not os.path.exists(staticDir): if not os.path.exists(staticDir):
raise ValueError(f"Could not find the frontend UI static assets. '{staticDir}'.") msg = f"Could not find the frontend UI static assets. '{staticDir}'."
raise ValueError(msg)
preloaded = 0 preloaded = 0
@ -172,21 +184,23 @@ class HttpAPI(common):
file = os.path.join(root, file) file = os.path.join(root, file)
urlPath = f"/{file.replace(f'{staticDir}/', '')}" urlPath = f"/{file.replace(f'{staticDir}/', '')}"
content = open(file, "rb").read() with open(file, "rb") as f:
contentType = self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)) content = f.read()
self.staticHolder[urlPath] = {"content": content, "content_type": contentType} contentType = self._ext_to_mime.get(os.path.splitext(file)[1], MIME.from_file(file))
self._static_holder[urlPath] = {"content": content, "content_type": contentType}
LOG.debug(f"Preloading '{urlPath}'.") LOG.debug(f"Preloading '{urlPath}'.")
app.router.add_get(urlPath, self.staticFile) app.router.add_get(urlPath, self._static_file)
preloaded += 1 preloaded += 1
if urlPath.endswith("/index.html") and urlPath != "/index.html": if urlPath.endswith("/index.html") and urlPath != "/index.html":
parentSlash = urlPath.replace("/index.html", "/") parentSlash = urlPath.replace("/index.html", "/")
parentNoSlash = urlPath.replace("/index.html", "") parentNoSlash = urlPath.replace("/index.html", "")
self.staticHolder[parentSlash] = {"content": content, "content_type": contentType} self._static_holder[parentSlash] = {"content": content, "content_type": contentType}
self.staticHolder[parentNoSlash] = {"content": content, "content_type": contentType} self._static_holder[parentNoSlash] = {"content": content, "content_type": contentType}
app.router.add_get(parentSlash, self.staticFile) app.router.add_get(parentSlash, self._static_file)
app.router.add_get(parentNoSlash, self.staticFile) app.router.add_get(parentNoSlash, self._static_file)
preloaded += 2 preloaded += 2
if preloaded < 1: if preloaded < 1:
@ -210,8 +224,8 @@ class HttpAPI(common):
Returns: Returns:
HttpAPI: The instance of the HttpAPI. HttpAPI: The instance of the HttpAPI.
"""
"""
for attr_name in dir(self): for attr_name in dir(self):
method = getattr(self, attr_name) method = getattr(self, attr_name)
if hasattr(method, "_http_method") and hasattr(method, "_http_path"): if hasattr(method, "_http_method") and hasattr(method, "_http_path"):
@ -222,15 +236,17 @@ class HttpAPI(common):
self.routes.route(method._http_method, f"/{http_path}")(method) self.routes.route(method._http_method, f"/{http_path}")(method)
self.routes.static("/api/download/", self.config.download_path) self.routes.static("/api/download/", self.config.download_path)
self.preloadStatic(app) self._preload_static(app)
try: try:
app.add_routes(self.routes) app.add_routes(self.routes)
except ValueError as e: except ValueError as e:
if "ui/exported" in str(e): if "ui/exported" in str(e):
raise RuntimeError(f"Could not find the frontend UI static assets. '{e}'.") from e msg = f"Could not find the frontend UI static assets. '{e}'."
raise e raise RuntimeError(msg) from e
raise
@staticmethod
def basic_auth(username: str, password: str) -> Awaitable: def basic_auth(username: str, password: str) -> Awaitable:
""" """
Middleware to handle basic authentication. Middleware to handle basic authentication.
@ -241,6 +257,7 @@ class HttpAPI(common):
Returns: Returns:
Awaitable: The middleware handler. Awaitable: The middleware handler.
""" """
@web.middleware @web.middleware
@ -291,6 +308,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code) return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
@ -304,6 +322,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
await self.queue.test() await self.queue.test()
return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code) return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code)
@ -318,6 +337,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
post = await request.json() post = await request.json()
args: str | None = post.get("args") args: str | None = post.get("args")
@ -345,6 +365,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object Response: The response object
""" """
url = request.query.get("url") url = request.query.get("url")
if not url: if not url:
@ -372,7 +393,7 @@ class HttpAPI(common):
"proxy": self.config.ytdl_options.get("proxy", None), "proxy": self.config.ytdl_options.get("proxy", None),
} }
data = getVideoInfo(url=url, ytdlp_opts=opts, no_archive=True) data = get_video_info(url=url, ytdlp_opts=opts, no_archive=True)
self.cache.set(key=self.cache.hash(url), value=data, ttl=300) self.cache.set(key=self.cache.hash(url), value=data, ttl=300)
data["_cached"] = { data["_cached"] = {
"key": self.cache.hash(url), "key": self.cache.hash(url),
@ -403,6 +424,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object Response: The response object
""" """
url: str | None = request.query.get("url") url: str | None = request.query.get("url")
if not url: if not url:
@ -425,6 +447,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
data = await request.json() data = await request.json()
@ -456,9 +479,10 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
return web.json_response( return web.json_response(
data=Tasks.get_instance().getTasks(), status=web.HTTPOk.status_code, dumps=self.encoder.encode data=Tasks.get_instance().get_tasks(), status=web.HTTPOk.status_code, dumps=self.encoder.encode
) )
@route("PUT", "api/tasks") @route("PUT", "api/tasks")
@ -471,6 +495,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object Response: The response object
""" """
data = await request.json() data = await request.json()
@ -500,7 +525,7 @@ class HttpAPI(common):
item["id"] = str(uuid.uuid4()) item["id"] = str(uuid.uuid4())
if not item.get("timer", None) or str(item.get("timer")).strip() == "": if not item.get("timer", None) or str(item.get("timer")).strip() == "":
item["timer"] = f"{random.randint(1,59)} */1 * * *" item["timer"] = f"{random.randint(1,59)} */1 * * *" # noqa: S311
if not item.get("cookies", None): if not item.get("cookies", None):
item["cookies"] = "" item["cookies"] = ""
@ -509,20 +534,20 @@ class HttpAPI(common):
item["config"] = {} item["config"] = {}
if not item.get("template", None): if not item.get("template", None):
item["template"] = str() item["template"] = ""
try: try:
ins.validate(item) ins.validate(item)
except ValueError as e: except ValueError as e:
return web.json_response( return web.json_response(
{"error": f"Failed to validate task '{item.get('name')}'. '{str(e)}'"}, {"error": f"Failed to validate task '{item.get('name')}'. '{e!s}'"},
status=web.HTTPBadRequest.status_code, status=web.HTTPBadRequest.status_code,
) )
tasks.append(Task(**item)) tasks.append(Task(**item))
try: try:
tasks = ins.save(tasks=tasks).load().getTasks() tasks = ins.save(tasks=tasks).load().get_tasks()
except Exception as e: except Exception as e:
LOG.exception(e) LOG.exception(e)
return web.json_response( return web.json_response(
@ -542,6 +567,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
data = await request.json() data = await request.json()
ids = data.get("ids") ids = data.get("ids")
@ -569,12 +595,13 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
id: str = request.match_info.get("id") id: str = request.match_info.get("id")
if not id: if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
item = self.queue.done.getById(id) item = self.queue.done.get_by_id(id)
if not item: if not item:
return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code)
@ -615,6 +642,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
data: dict = {"queue": [], "history": []} data: dict = {"queue": [], "history": []}
@ -635,6 +663,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
if self.queue.pool is None: if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
@ -672,6 +701,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
if self.queue.pool is None: if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code) return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
@ -690,6 +720,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object Response: The response object
""" """
id: str = request.match_info.get("id") id: str = request.match_info.get("id")
if not id: if not id:
@ -712,6 +743,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
id: str = request.match_info.get("id") id: str = request.match_info.get("id")
if not id: if not id:
@ -734,6 +766,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str = request.match_info.get("file")
@ -767,6 +800,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str = request.match_info.get("file")
mode: str = request.match_info.get("mode") mode: str = request.match_info.get("mode")
@ -817,6 +851,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str = request.match_info.get("file")
segment: int = request.match_info.get("segment") segment: int = request.match_info.get("segment")
@ -829,7 +864,7 @@ class HttpAPI(common):
if request.if_modified_since: if request.if_modified_since:
lastMod = time.strftime( lastMod = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple()
) )
if os.path.exists(file_path) and request.if_modified_since.timestamp() == os.path.getmtime(file_path): if os.path.exists(file_path) and request.if_modified_since.timestamp() == os.path.getmtime(file_path):
return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
@ -842,9 +877,9 @@ class HttpAPI(common):
segmenter = Segments( segmenter = Segments(
index=int(segment), index=int(segment),
duration=float("{:.6f}".format(float(sd if sd else M3u8.duration))), duration=float(f"{float(sd if sd else M3u8.duration):.6f}"),
vconvert=True if vc == 1 else False, vconvert=vc == 1,
aconvert=True if ac == 1 else False, aconvert=ac == 1,
) )
return web.Response( return web.Response(
@ -856,10 +891,10 @@ class HttpAPI(common):
"Pragma": "public", "Pragma": "public",
"Cache-Control": f"public, max-age={time.time() + 31536000}", "Cache-Control": f"public, max-age={time.time() + 31536000}",
"Last-Modified": time.strftime( "Last-Modified": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple()
), ),
"Expires": time.strftime( "Expires": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000).timetuple() "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
), ),
}, },
status=web.HTTPOk.status_code, status=web.HTTPOk.status_code,
@ -875,6 +910,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str = request.match_info.get("file")
file_path: str = os.path.normpath(os.path.join(self.config.download_path, file)) file_path: str = os.path.normpath(os.path.join(self.config.download_path, file))
@ -883,7 +919,7 @@ class HttpAPI(common):
if request.if_modified_since: if request.if_modified_since:
lastMod = time.strftime( lastMod = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple()
) )
if os.path.exists(file_path) and request.if_modified_since.timestamp() == os.path.getmtime(file_path): if os.path.exists(file_path) and request.if_modified_since.timestamp() == os.path.getmtime(file_path):
return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod}) return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
@ -900,10 +936,10 @@ class HttpAPI(common):
"Pragma": "public", "Pragma": "public",
"Cache-Control": f"public, max-age={time.time() + 31536000}", "Cache-Control": f"public, max-age={time.time() + 31536000}",
"Last-Modified": time.strftime( "Last-Modified": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path)).timetuple() "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC).timetuple()
), ),
"Expires": time.strftime( "Expires": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000).timetuple() "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
), ),
}, },
status=web.HTTPOk.status_code, status=web.HTTPOk.status_code,
@ -919,14 +955,15 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
if "/index.html" not in self.staticHolder: if "/index.html" not in self._static_holder:
LOG.error("Static frontend files not found.") LOG.error("Static frontend files not found.")
return web.json_response( return web.json_response(
data={"error": "File not found.", "file": "/index.html"}, status=web.HTTPNotFound.status_code data={"error": "File not found.", "file": "/index.html"}, status=web.HTTPNotFound.status_code
) )
data = self.staticHolder["/index.html"] data = self._static_holder["/index.html"]
return web.Response( return web.Response(
body=data.get("content"), body=data.get("content"),
content_type=data.get("content_type"), content_type=data.get("content_type"),
@ -944,6 +981,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
url = request.query.get("url") url = request.query.get("url")
if not url: if not url:
@ -974,7 +1012,8 @@ class HttpAPI(common):
"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Origin": "*",
"Cache-Control": f"public, max-age={time.time() + 31536000}", "Cache-Control": f"public, max-age={time.time() + 31536000}",
"Expires": time.strftime( "Expires": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000).timetuple() "%a, %d %b %Y %H:%M:%S GMT",
datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple(),
), ),
}, },
) )
@ -994,11 +1033,15 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
file: str = request.match_info.get("file") file: str = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
try: try:
realFile: str = calcDownloadPath(basePath=self.config.download_path, folder=file, createPath=False) realFile: str = calc_download_path(base_path=self.config.download_path, folder=file, create_path=False)
if not os.path.exists(realFile): if not os.path.exists(realFile) or not os.path.isfile(realFile):
return web.json_response( return web.json_response(
data={"error": f"File '{file}' does not exist."}, status=web.HTTPNotFound.status_code data={"error": f"File '{file}' does not exist."}, status=web.HTTPNotFound.status_code
) )
@ -1019,6 +1062,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object Response: The response object
""" """
cookie_file = self.config.ytdl_options.get("cookiefile", None) cookie_file = self.config.ytdl_options.get("cookiefile", None)
if not cookie_file: if not cookie_file:
@ -1066,7 +1110,7 @@ class HttpAPI(common):
LOG.error(f"Failed to request '{url}'. '{e}'.") LOG.error(f"Failed to request '{url}'. '{e}'.")
LOG.exception(e) LOG.exception(e)
return web.json_response( return web.json_response(
data={"message": f"Failed to request website. {str(e)}"}, data={"message": f"Failed to request website. {e!s}"},
status=web.HTTPInternalServerError.status_code, status=web.HTTPInternalServerError.status_code,
) )
@ -1080,11 +1124,12 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
return web.json_response( return web.json_response(
data={ data={
"notifications": Notification.get_instance().getTargets(), "notifications": Notification.get_instance().get_targets(),
"allowedTypes": list(NotificationEvents.getEvents().values()), "allowedTypes": list(NotificationEvents.get_events().values()),
}, },
status=web.HTTPOk.status_code, status=web.HTTPOk.status_code,
dumps=self.encoder.encode, dumps=self.encoder.encode,
@ -1100,6 +1145,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
post = await request.json() post = await request.json()
if not isinstance(post, list): if not isinstance(post, list):
@ -1125,11 +1171,11 @@ class HttpAPI(common):
Notification.validate(item) Notification.validate(item)
except ValueError as e: except ValueError as e:
return web.json_response( return web.json_response(
{"error": f"Invalid notification target settings. {str(e)}", "data": item}, {"error": f"Invalid notification target settings. {e!s}", "data": item},
status=web.HTTPBadRequest.status_code, status=web.HTTPBadRequest.status_code,
) )
targets.append(ins.makeTarget(item)) targets.append(ins.make_target(item))
try: try:
if len(targets) < 1: if len(targets) < 1:
@ -1141,7 +1187,7 @@ class HttpAPI(common):
LOG.exception(e) LOG.exception(e)
return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code) return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code)
data = {"notifications": targets, "allowedTypes": list(NotificationEvents.getEvents().values())} data = {"notifications": targets, "allowedTypes": list(NotificationEvents.get_events().values())}
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode) return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
@ -1155,6 +1201,7 @@ class HttpAPI(common):
Returns: Returns:
Response: The response object. Response: The response object.
""" """
data = {"type": "test", "message": "This is a test notification."} data = {"type": "test", "message": "This is a test notification."}
await self.emitter.emit(Events.TEST, data) await self.emitter.emit(Events.TEST, data)

78
app/library/HttpSocket.py
View file

@ -6,24 +6,25 @@ import os
import pty import pty
import shlex import shlex
import time import time
from datetime import datetime from datetime import UTC, datetime
import socketio
from aiohttp import web
from typing import Any from typing import Any
from .common import common import anyio
import socketio
from aiohttp import web
from .common import Common
from .config import Config from .config import Config
from .DownloadQueue import DownloadQueue from .DownloadQueue import DownloadQueue
from .Emitter import Emitter from .Emitter import Emitter
from .encoder import Encoder from .encoder import Encoder
from .Utils import isDownloaded, arg_converter from .EventsSubscriber import Event, Events, EventsSubscriber
from .EventsSubscriber import EventsSubscriber, Events, Event from .Utils import arg_converter, is_downloaded
LOG = logging.getLogger("socket_api") LOG = logging.getLogger("socket_api")
class HttpSocket(common): class HttpSocket(Common):
""" """
This class is used to handle WebSocket events. This class is used to handle WebSocket events.
""" """
@ -54,6 +55,7 @@ class HttpSocket(common):
super().__init__(queue=queue, encoder=encoder, config=config) super().__init__(queue=queue, encoder=encoder, config=config)
@staticmethod
def ws_event(func): # type: ignore def ws_event(func): # type: ignore
""" """
Decorator to mark a method as a socket event. Decorator to mark a method as a socket event.
@ -66,6 +68,9 @@ class HttpSocket(common):
wrapper._ws_event = func.__name__ # type: ignore wrapper._ws_event = func.__name__ # type: ignore
return wrapper return wrapper
async def on_shutdown(self, _: web.Application):
LOG.debug("Shutting down socket server.")
def attach(self, app: web.Application): def attach(self, app: web.Application):
self.sio.attach(app, socketio_path=self.config.url_socketio) self.sio.attach(app, socketio_path=self.config.url_socketio)
@ -82,7 +87,10 @@ class HttpSocket(common):
EventsSubscriber.get_instance().subscribe(Events.ADD_URL, "socket_add_url", handle_event) EventsSubscriber.get_instance().subscribe(Events.ADD_URL, "socket_add_url", handle_event)
@ws_event # type: ignore # register the shutdown event.
app.on_shutdown.append(self.on_shutdown)
@ws_event
async def cli_post(self, sid: str, data): async def cli_post(self, sid: str, data):
if not data: if not data:
await self.emitter.emit(Events.CLI_CLOSE, {"exitcode": 0}, to=sid) await self.emitter.emit(Events.CLI_CLOSE, {"exitcode": 0}, to=sid)
@ -91,7 +99,7 @@ class HttpSocket(common):
try: try:
LOG.info(f"Cli command from client '{sid}'. '{data}'") LOG.info(f"Cli command from client '{sid}'. '{data}'")
args = ["yt-dlp"] + shlex.split(data) args = ["yt-dlp", *shlex.split(data)]
_env = os.environ.copy() _env = os.environ.copy()
_env.update( _env.update(
{ {
@ -119,7 +127,7 @@ class HttpSocket(common):
try: try:
os.close(slave_fd) os.close(slave_fd)
except Exception as e: except Exception as e:
LOG.error(f"Error closing PTY. '{str(e)}'.") LOG.error(f"Error closing PTY. '{e!s}'.")
async def read_pty(sid: str): async def read_pty(sid: str):
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
@ -130,8 +138,7 @@ class HttpSocket(common):
except OSError as e: except OSError as e:
if e.errno == errno.EIO: if e.errno == errno.EIO:
break break
else: raise
raise
if not chunk: if not chunk:
# No more output # No more output
@ -154,7 +161,7 @@ class HttpSocket(common):
try: try:
os.close(master_fd) os.close(master_fd)
except Exception as e: except Exception as e:
LOG.error(f"Error closing PTY. '{str(e)}'.") LOG.error(f"Error closing PTY. '{e!s}'.")
# Start reading output from PTY # Start reading output from PTY
read_task = asyncio.create_task(read_pty(sid=sid)) read_task = asyncio.create_task(read_pty(sid=sid))
@ -172,7 +179,7 @@ class HttpSocket(common):
await self.emitter.emit(Events.CLI_OUTPUT, {"type": "stderr", "line": str(e)}, to=sid) await self.emitter.emit(Events.CLI_OUTPUT, {"type": "stderr", "line": str(e)}, to=sid)
await self.emitter.emit(Events.CLI_CLOSE, {"exitcode": -1}, to=sid) await self.emitter.emit(Events.CLI_CLOSE, {"exitcode": -1}, to=sid)
@ws_event # type: ignore @ws_event
async def add_url(self, sid: str, data: dict): async def add_url(self, sid: str, data: dict):
url: str | None = data.get("url") url: str | None = data.get("url")
@ -183,13 +190,13 @@ class HttpSocket(common):
try: try:
item = self.format_item(data) item = self.format_item(data)
except ValueError as e: except ValueError as e:
return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) await self.emitter.error(str(e), to=sid)
return
status = await self.add(**item) status = await self.add(**item)
await self.emitter.emit(event=Events.STATUS, data=status, to=sid) await self.emitter.emit(event=Events.STATUS, data=status, to=sid)
@ws_event # type: ignore @ws_event
async def item_cancel(self, sid: str, id: str): async def item_cancel(self, sid: str, id: str):
if not id: if not id:
await self.emitter.error("Invalid request.", to=sid) await self.emitter.error("Invalid request.", to=sid)
@ -201,7 +208,7 @@ class HttpSocket(common):
await self.emitter.emit(event=Events.ITEM_CANCEL, data=status) await self.emitter.emit(event=Events.ITEM_CANCEL, data=status)
@ws_event # type: ignore @ws_event
async def item_delete(self, sid: str, data: dict): async def item_delete(self, sid: str, data: dict):
if not data: if not data:
await self.emitter.error("Invalid request.", to=sid) await self.emitter.error("Invalid request.", to=sid)
@ -218,8 +225,8 @@ class HttpSocket(common):
await self.emitter.emit(event=Events.ITEM_DELETE, data=status) await self.emitter.emit(event=Events.ITEM_DELETE, data=status)
@ws_event # type: ignore @ws_event
async def archive_item(self, sid: str, data: dict): async def archive_item(self, _: str, data: dict):
if not isinstance(data, dict) or "url" not in data or not self.config.keep_archive: if not isinstance(data, dict) or "url" not in data or not self.config.keep_archive:
return return
@ -231,36 +238,36 @@ class HttpSocket(common):
if not file: if not file:
return return
exists, idDict = isDownloaded(file, data["url"]) exists, idDict = is_downloaded(file, data["url"])
if exists or "archive_id" not in idDict or idDict["archive_id"] is None: if exists or "archive_id" not in idDict or idDict["archive_id"] is None:
return return
with open(file, "a") as f: async with await anyio.open_file(file, "a") as f:
f.write(f"{idDict['archive_id']}\n") await f.write(f"{idDict['archive_id']}\n")
manual_archive = self.config.manual_archive manual_archive = self.config.manual_archive
if manual_archive: if manual_archive:
previouslyArchived = False previouslyArchived = False
if os.path.exists(manual_archive): if os.path.exists(manual_archive):
with open(manual_archive, "r") as f: async with await anyio.open_file(manual_archive) as f:
for line in f.readlines(): async for line in f:
if idDict["archive_id"] in line: if idDict["archive_id"] in line:
previouslyArchived = True previouslyArchived = True
break break
if not previouslyArchived: if not previouslyArchived:
with open(manual_archive, "a") as f: async with await anyio.open_file(manual_archive, "a") as f:
f.write(f"{idDict['archive_id']} - at: {datetime.now().isoformat()}\n") await f.write(f"{idDict['archive_id']} - at: {datetime.now(UTC).isoformat()}\n")
LOG.info(f"Archiving url '{data['url']}' with id '{idDict['archive_id']}'.") LOG.info(f"Archiving url '{data['url']}' with id '{idDict['archive_id']}'.")
@ws_event # type: ignore @ws_event
async def connect(self, sid: str, _=None): async def connect(self, sid: str, _=None):
data = { data = {
**self.queue.get(), **self.queue.get(),
"config": self.config.frontend(), "config": self.config.frontend(),
"presets": self.config.presets, "presets": self.config.presets,
"paused": self.queue.isPaused(), "paused": self.queue.is_paused(),
} }
# get download folder listing. # get download folder listing.
@ -269,13 +276,13 @@ class HttpSocket(common):
await self.emitter.emit(event=Events.INITIAL_DATA, data=data, to=sid) await self.emitter.emit(event=Events.INITIAL_DATA, data=data, to=sid)
@ws_event # type: ignore @ws_event
async def pause(self, sid: str, _=None): async def pause(self, *_, **__):
self.queue.pause() self.queue.pause()
await self.emitter.emit(event=Events.PAUSED, data={"paused": True, "at": time.time()}) await self.emitter.emit(event=Events.PAUSED, data={"paused": True, "at": time.time()})
@ws_event # type: ignore @ws_event
async def resume(self, sid: str, _=None): async def resume(self, *_, **__):
self.queue.resume() self.queue.resume()
await self.emitter.emit(event=Events.PAUSED, data={"paused": False, "at": time.time()}) await self.emitter.emit(event=Events.PAUSED, data={"paused": False, "at": time.time()})
@ -293,9 +300,10 @@ class HttpSocket(common):
try: try:
await self.emitter.emit(event=Events.YTDLP_CONVERT, data=arg_converter(args), to=sid) await self.emitter.emit(event=Events.YTDLP_CONVERT, data=arg_converter(args), to=sid)
return
except Exception as e: except Exception as e:
err = str(e).strip() err = str(e).strip()
err = err.split("\n")[-1] if "\n" in err else err err = err.split("\n")[-1] if "\n" in err else err
LOG.error(f"Failed to convert args. '{err}'.") LOG.error(f"Failed to convert args. '{err}'.")
await self.emitter.error(f"Failed to convert options. '{err}'.", to=sid) await self.emitter.error(f"Failed to convert options. '{err}'.", to=sid)
return

5
app/library/ItemDTO.py
View file

@ -30,7 +30,7 @@ class ItemDTO:
config: dict = field(default_factory=dict) config: dict = field(default_factory=dict)
template: str | None = None template: str | None = None
template_chapter: str | None = None template_chapter: str | None = None
timestamp: float = time.time_ns() timestamp: float = field(default_factory=lambda: time.time_ns())
is_live: bool | None = None is_live: bool | None = None
datetime: str = field(default_factory=lambda: str(formatdate(time.time()))) datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
live_in: str | None = None live_in: str | None = None
@ -92,3 +92,6 @@ class ItemDTO:
def get(self, key: str, default: Any = None) -> Any: def get(self, key: str, default: Any = None) -> Any:
return self.__dict__.get(key, default) return self.__dict__.get(key, default)
def get_id(self) -> str:
return self._id

44
app/library/M3u8.py
View file

@ -1,33 +1,27 @@
import math import math
import os import os
from urllib.parse import quote from urllib.parse import quote
from .Utils import calcDownloadPath, StreamingError
from .ffprobe import ffprobe from .ffprobe import ffprobe
from .Utils import StreamingError, calc_download_path
class M3u8: class M3u8:
duration: float = 6.000000 duration: float = 6.000000
ok_vcodecs: tuple = ( ok_vcodecs: tuple = ("h264", "x264", "avc")
"h264", ok_acodecs: tuple = ("aac", "m4a", "mp3")
"x264",
"avc",
)
ok_acodecs: tuple = (
"aac",
"m4a",
"mp3",
)
def __init__(self, url: str, segment_duration: float = None): def __init__(self, url: str, segment_duration: float | None = None):
self.url = url self.url = url
self.duration = float(segment_duration) if segment_duration is not None else self.duration self.duration = float(segment_duration) if segment_duration is not None else self.duration
async def make_stream(self, download_path: str, file: str) -> str: async def make_stream(self, download_path: str, file: str) -> str:
realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) realFile: str = calc_download_path(base_path=download_path, folder=file, create_path=False)
if not os.path.exists(realFile): if not os.path.exists(realFile):
raise StreamingError(f"File '{realFile}' does not exist.") error = f"File '{realFile}' does not exist."
raise StreamingError(error)
try: try:
ff = await ffprobe(realFile) ff = await ffprobe(realFile)
@ -35,7 +29,8 @@ class M3u8:
pass pass
if "duration" not in ff.metadata: if "duration" not in ff.metadata:
raise StreamingError(f"Unable to get '{realFile}' play duration.") error = f"Unable to get '{realFile}' play duration."
raise StreamingError(error)
duration: float = float(ff.metadata.get("duration")) duration: float = float(ff.metadata.get("duration"))
@ -47,22 +42,20 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0") m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD") m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
segmentSize: float = "{:.6f}".format(self.duration) segmentSize: float = f"{self.duration:.6f}"
splits: int = math.ceil(duration / self.duration) splits: int = math.ceil(duration / self.duration)
segmentParams: dict = {} segmentParams: dict = {}
for stream in ff.streams(): for stream in ff.streams():
if stream.is_video(): if stream.is_video() and stream.codec_name not in self.ok_vcodecs:
if stream.codec_name not in self.ok_vcodecs: segmentParams["vc"] = 1
segmentParams["vc"] = 1 if stream.is_audio() and stream.codec_name not in self.ok_acodecs:
if stream.is_audio(): segmentParams["ac"] = 1
if stream.codec_name not in self.ok_acodecs:
segmentParams["ac"] = 1
for i in range(splits): for i in range(splits):
if (i + 1) == splits: if (i + 1) == splits:
segmentSize = "{:.6f}".format(duration - (i * self.duration)) segmentSize = f"{duration - (i * self.duration):.6f}"
segmentParams.update({"sd": segmentSize}) segmentParams.update({"sd": segmentSize})
m3u8.append(f"#EXTINF:{segmentSize},") m3u8.append(f"#EXTINF:{segmentSize},")
@ -78,10 +71,11 @@ class M3u8:
return "\n".join(m3u8) return "\n".join(m3u8)
async def make_subtitle(self, download_path: str, file: str, duration: float) -> str: async def make_subtitle(self, download_path: str, file: str, duration: float) -> str:
realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) realFile: str = calc_download_path(base_path=download_path, folder=file, create_path=False)
if not os.path.exists(realFile): if not os.path.exists(realFile):
raise StreamingError(f"File '{realFile}' does not exist.") error = f"File '{realFile}' does not exist."
raise StreamingError(error)
m3u8 = [] m3u8 = []

106
app/library/Notifications.py
View file

@ -1,26 +1,26 @@
import asyncio import asyncio
from datetime import datetime, timezone
import json import json
import logging import logging
import os import os
from typing import List, Any
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import httpx import httpx
from .config import Config from .config import Config
from .ItemDTO import ItemDTO
from .Singleton import Singleton
from .Utils import validate_uuid, ag
from .version import APP_VERSION
from .encoder import Encoder from .encoder import Encoder
from .EventsSubscriber import Events from .EventsSubscriber import Events
from .ItemDTO import ItemDTO
from .Singleton import Singleton
from .Utils import ag, validate_uuid
from .version import APP_VERSION
LOG = logging.getLogger("notifications") LOG = logging.getLogger("notifications")
@dataclass(kw_only=True) @dataclass(kw_only=True)
class targetRequestHeader: class TargetRequestHeader:
"""Request header details for a notification target.""" """Request header details for a notification target."""
key: str key: str
@ -37,13 +37,13 @@ class targetRequestHeader:
@dataclass(kw_only=True) @dataclass(kw_only=True)
class targetRequest: class TargetRequest:
"""Request details for a notification target.""" """Request details for a notification target."""
type: str type: str
method: str method: str
url: str url: str
headers: list[targetRequestHeader] = field(default_factory=list) headers: list[TargetRequestHeader] = field(default_factory=list)
def serialize(self) -> dict: def serialize(self) -> dict:
return { return {
@ -66,8 +66,8 @@ class Target:
id: str id: str
name: str name: str
on: List[str] on: list[str]
request: targetRequest request: TargetRequest
def serialize(self) -> dict: def serialize(self) -> dict:
return { return {
@ -95,17 +95,12 @@ class NotificationEvents:
TEST = Events.TEST TEST = Events.TEST
@staticmethod @staticmethod
def getEvents() -> dict[str, str]: def get_events() -> dict[str, str]:
data: dict[str, str] = {} return {k: v for k, v in vars(NotificationEvents).items() if not k.startswith("__") and not callable(v)}
for k, v in vars(NotificationEvents).items():
if not k.startswith("__") and not callable(v):
data[k] = v
return data
@staticmethod @staticmethod
def isValid(event: str) -> bool: def is_valid(event: str) -> bool:
return event in NotificationEvents.getEvents().values() return event in NotificationEvents.get_events().values()
class Notification(metaclass=Singleton): class Notification(metaclass=Singleton):
@ -147,7 +142,7 @@ class Notification(metaclass=Singleton):
return Notification._instance return Notification._instance
def getTargets(self) -> list[Target]: def get_targets(self) -> list[Target]:
"""Get the list of notification targets.""" """Get the list of notification targets."""
return self._targets return self._targets
@ -165,16 +160,15 @@ class Notification(metaclass=Singleton):
Returns: Returns:
Notification: The Notification instance. Notification: The Notification instance.
"""
"""
LOG.info(f"Saving notification targets to '{self._file}'.") LOG.info(f"Saving notification targets to '{self._file}'.")
try: try:
with open(self._file, "w") as f: with open(self._file, "w") as f:
json.dump([t.serialize() for t in targets], fp=f, indent=4) json.dump([t.serialize() for t in targets], fp=f, indent=4)
except Exception as e: except Exception as e:
LOG.exception(e) LOG.exception(e)
LOG.error(f"Error saving notification targets to '{self._file}'. '{e}'") LOG.error(f"Error saving notification targets to '{self._file}'. '{e!s}'")
pass
return self return self
@ -192,21 +186,20 @@ class Notification(metaclass=Singleton):
LOG.info(f"Loading notification targets from '{self._file}'.") LOG.info(f"Loading notification targets from '{self._file}'.")
try: try:
with open(self._file, "r") as f: with open(self._file) as f:
targets = json.load(f) targets = json.load(f)
except Exception as e: except Exception as e:
LOG.error(f"Error loading notification targets from '{self._file}'. '{e}'") LOG.error(f"Error loading notification targets from '{self._file}'. '{e!s}'")
pass
for target in targets: for target in targets:
try: try:
try: try:
Notification.validate(target) Notification.validate(target)
except ValueError as e: except ValueError as e:
LOG.error(f"Invalid notification target '{target}'. '{e}'") LOG.error(f"Invalid notification target '{target}'. '{e!s}'")
continue continue
target = self.makeTarget(target) target = self.make_target(target)
self._targets.append(target) self._targets.append(target)
@ -214,8 +207,7 @@ class Notification(metaclass=Singleton):
f"Will send '{target.on if len(target.on) > 0 else 'all'}' as {target.request.type} notification events to '{target.name}'." f"Will send '{target.on if len(target.on) > 0 else 'all'}' as {target.request.type} notification events to '{target.name}'."
) )
except Exception as e: except Exception as e:
LOG.error(f"Error loading notification target '{target}'. '{e}'") LOG.error(f"Error loading notification target '{target}'. '{e!s}'")
pass
return self return self
@ -229,46 +221,57 @@ class Notification(metaclass=Singleton):
Returns: Returns:
bool: True if the target is valid, False otherwise. bool: True if the target is valid, False otherwise.
"""
"""
if not isinstance(target, dict): if not isinstance(target, dict):
target = target.serialize() target = target.serialize()
if "id" not in target or validate_uuid(target["id"], version=4) is False: if "id" not in target or validate_uuid(target["id"], version=4) is False:
raise ValueError("Invalid notification target. No ID found.") msg = "Invalid notification target. No ID found."
raise ValueError(msg)
if "name" not in target: if "name" not in target:
raise ValueError("Invalid notification target. No name found.") msg = "Invalid notification target. No name found."
raise ValueError(msg)
if "request" not in target: if "request" not in target:
raise ValueError("Invalid notification target. No request details found.") msg = "Invalid notification target. No request details found."
raise ValueError(msg)
if "url" not in target["request"]: if "url" not in target["request"]:
raise ValueError("Invalid notification target. No URL found.") msg = "Invalid notification target. No URL found."
raise ValueError(msg)
if "method" in target["request"] and target["request"]["method"].upper() not in ["POST", "PUT"]: if "method" in target["request"] and target["request"]["method"].upper() not in ["POST", "PUT"]:
raise ValueError("Invalid notification target. Invalid method found.") msg = "Invalid notification target. Invalid method found."
raise ValueError(msg)
if "type" in target["request"] and target["request"]["type"].lower() not in ["json", "form"]: if "type" in target["request"] and target["request"]["type"].lower() not in ["json", "form"]:
raise ValueError("Invalid notification target. Invalid type found.") msg = "Invalid notification target. Invalid type found."
raise ValueError(msg)
if "on" in target: if "on" in target:
if not isinstance(target["on"], list): if not isinstance(target["on"], list):
raise ValueError("Invalid notification target. Invalid 'on' event list found.") msg = "Invalid notification target. Invalid 'on' event list found."
raise ValueError(msg)
for e in target["on"]: for e in target["on"]:
if e not in NotificationEvents.getEvents().values(): if e not in NotificationEvents.get_events().values():
raise ValueError(f"Invalid notification target. Invalid event '{e}' found.") msg = f"Invalid notification target. Invalid event '{e}' found."
raise ValueError(msg)
if "headers" in target["request"]: if "headers" in target["request"]:
if not isinstance(target["request"]["headers"], list): if not isinstance(target["request"]["headers"], list):
raise ValueError("Invalid notification target. Invalid headers list found.") msg = "Invalid notification target. Invalid headers list found."
raise ValueError(msg)
for h in target["request"]["headers"]: for h in target["request"]["headers"]:
if "key" not in h: if "key" not in h:
raise ValueError("Invalid notification target. No header key found.") msg = "Invalid notification target. No header key found."
raise ValueError(msg)
if "value" not in h: if "value" not in h:
raise ValueError("Invalid notification target. No header value found.") msg = "Invalid notification target. No header value found."
raise ValueError(msg)
return True return True
@ -315,7 +318,7 @@ class Notification(metaclass=Singleton):
reqBody["json" if "json" == target.request.type.lower() else "data"] = { reqBody["json" if "json" == target.request.type.lower() else "data"] = {
"event": event, "event": event,
"created_at": datetime.now(tz=timezone.utc).isoformat(), "created_at": datetime.now(tz=UTC).isoformat(),
"payload": item.__dict__ if isinstance(item, ItemDTO) else item, "payload": item.__dict__ if isinstance(item, ItemDTO) else item,
} }
@ -337,7 +340,7 @@ class Notification(metaclass=Singleton):
LOG.error(f"Error sending Notification event '{event}' id '{itemId}' to '{target.name}'. '{e}'.") LOG.error(f"Error sending Notification event '{event}' id '{itemId}' to '{target.name}'. '{e}'.")
return {"url": target.request.url, "status": 500, "text": str(e)} return {"url": target.request.url, "status": 500, "text": str(e)}
def makeTarget(self, target: dict) -> Target: def make_target(self, target: dict) -> Target:
""" """
Make a notification target from a dictionary. Make a notification target from a dictionary.
@ -346,27 +349,28 @@ class Notification(metaclass=Singleton):
Returns: Returns:
Target: The notification target. Target: The notification target.
""" """
return Target( return Target(
id=target.get("id"), id=target.get("id"),
name=target.get("name"), name=target.get("name"),
on=target.get("on", []), on=target.get("on", []),
request=targetRequest( request=TargetRequest(
type=target.get("request", {}).get("type", "json"), type=target.get("request", {}).get("type", "json"),
method=target.get("request", {}).get("method", "POST"), method=target.get("request", {}).get("method", "POST"),
url=target.get("request", {}).get("url"), url=target.get("request", {}).get("url"),
headers=[ headers=[
targetRequestHeader(key=h.get("key"), value=h.get("value")) TargetRequestHeader(key=h.get("key"), value=h.get("value"))
for h in target.get("request", {}).get("headers", []) for h in target.get("request", {}).get("headers", [])
], ],
), ),
) )
def emit(self, event, data, **kwargs): def emit(self, event, data, **kwargs): # noqa: ARG002
if len(self._targets) < 1: if len(self._targets) < 1:
return False return False
if not NotificationEvents.isValid(event): if not NotificationEvents.is_valid(event):
return False return False
return self.send(event, data) return self.send(event, data)

13
app/library/PackageInstaller.py
View file

@ -1,8 +1,9 @@
import importlib
import logging import logging
import os import os
import subprocess import subprocess
import sys import sys
import importlib
from library.config import Config from library.config import Config
LOG = logging.getLogger("package_installer") LOG = logging.getLogger("package_installer")
@ -24,8 +25,8 @@ class PackageInstaller:
return [] return []
if os.access(file_path, os.R_OK): if os.access(file_path, os.R_OK):
with open(file_path, "r") as f: with open(file_path) as f:
return [pkg.strip() for pkg in f.readlines() if pkg.strip()] return [pkg.strip() for pkg in f if pkg.strip()]
else: else:
LOG.error(f"Could not read pip packages from '{file_path}'.") LOG.error(f"Could not read pip packages from '{file_path}'.")
return [] return []
@ -37,10 +38,10 @@ class PackageInstaller:
LOG.info(f"'{pkg}' is already installed. Skipping upgrades. as requested.") LOG.info(f"'{pkg}' is already installed. Skipping upgrades. as requested.")
return return
LOG.info(f"'{pkg}' is already installed. Checking for upgrades...") LOG.info(f"'{pkg}' is already installed. Checking for upgrades...")
subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", pkg], check=True) subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade", pkg], check=True) # noqa: S603
except ImportError: except ImportError:
LOG.info(f"'{pkg}' is not installed. Installing...") LOG.info(f"'{pkg}' is not installed. Installing...")
subprocess.run([sys.executable, "-m", "pip", "install", pkg], check=True) subprocess.run([sys.executable, "-m", "pip", "install", pkg], check=True) # noqa: S603
def check(self): def check(self):
""" """
@ -57,5 +58,5 @@ class PackageInstaller:
try: try:
self.action(package) self.action(package)
except Exception as e: except Exception as e:
LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {str(e)}") LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}")
LOG.exception(e) LOG.exception(e)

24
app/library/Playlist.py
View file

@ -2,10 +2,12 @@ import glob
import re import re
from pathlib import Path from pathlib import Path
from urllib.parse import quote from urllib.parse import quote
from aiohttp.web import Response
from aiohttp.web import HTTPFound, Response
from .ffprobe import ffprobe from .ffprobe import ffprobe
from .Subtitle import Subtitle from .Subtitle import Subtitle
from .Utils import calcDownloadPath, checkId, StreamingError from .Utils import StreamingError, calc_download_path, check_id
class Playlist: class Playlist:
@ -15,15 +17,16 @@ class Playlist:
self.url = url self.url = url
async def make(self, download_path: str, file: str) -> str | Response: async def make(self, download_path: str, file: str) -> str | Response:
rFile = Path(calcDownloadPath(basePath=download_path, folder=file, createPath=False)) rFile = Path(calc_download_path(base_path=download_path, folder=file, create_path=False))
if not rFile.exists(): if not rFile.exists():
possibleFile = checkId(download_path, rFile) possibleFile = check_id(file=rFile)
if not possibleFile: if not possibleFile:
raise StreamingError(f"File '{rFile}' does not exist.") msg = f"File '{rFile}' does not exist."
raise StreamingError(msg)
return Response( return Response(
status=302, status=HTTPFound.status_code,
headers={ headers={
"Location": f"{self.url}api/player/playlist/{quote(str(possibleFile).replace(download_path, '').strip('/'))}.m3u8" "Location": f"{self.url}api/player/playlist/{quote(str(possibleFile).replace(download_path, '').strip('/'))}.m3u8"
}, },
@ -35,7 +38,8 @@ class Playlist:
pass pass
if "duration" not in ff.metadata: if "duration" not in ff.metadata:
raise StreamingError(f"Unable to get '{rFile}' duration.") msg = f"Unable to get '{rFile}' duration."
raise StreamingError(msg)
duration: float = float(ff.metadata.get("duration")) duration: float = float(ff.metadata.get("duration"))
@ -45,8 +49,8 @@ class Playlist:
subs = "" subs = ""
index = 0 index = 0
for item in self.getSideCarFiles(rFile): for item in self.get_sidecar_files(rFile):
if item.suffix not in Subtitle.allowedExtensions: if item.suffix not in Subtitle.allowed_extensions:
continue continue
index += 1 index += 1
@ -68,7 +72,7 @@ class Playlist:
return "\n".join(playlist) return "\n".join(playlist)
def getSideCarFiles(self, file: Path) -> list[Path]: def get_sidecar_files(self, file: Path) -> list[Path]:
""" """
Get sidecar files for the given file. Get sidecar files for the given file.

22
app/library/Segments.py
View file

@ -4,9 +4,9 @@ import logging
import os import os
import tempfile import tempfile
from .ffprobe import ffprobe
from .config import Config from .config import Config
from .Utils import calcDownloadPath, StreamingError from .ffprobe import ffprobe
from .Utils import StreamingError, calc_download_path
LOG = logging.getLogger("player.segments") LOG = logging.getLogger("player.segments")
@ -25,10 +25,11 @@ class Segments:
self.aconvert = True self.aconvert = True
async def stream(self, path: str, file: str) -> bytes: async def stream(self, path: str, file: str) -> bytes:
realFile: str = calcDownloadPath(basePath=path, folder=file, createPath=False) realFile: str = calc_download_path(base_path=path, folder=file, create_path=False)
if not os.path.exists(realFile): if not os.path.exists(realFile):
raise StreamingError(f"File {realFile} does not exist.") msg = f"File {realFile} does not exist."
raise StreamingError(msg)
try: try:
ff = await ffprobe(realFile) ff = await ffprobe(realFile)
@ -36,15 +37,15 @@ class Segments:
pass pass
tmpDir: str = tempfile.gettempdir() tmpDir: str = tempfile.gettempdir()
tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.md5(realFile.encode("utf-8")).hexdigest()}') tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.sha256(realFile.encode("utf-8")).hexdigest()}')
if not os.path.exists(tmpFile): if not os.path.exists(tmpFile):
os.symlink(realFile, tmpFile) os.symlink(realFile, tmpFile)
if self.index == 0: if self.index == 0:
startTime: float = "{:.6f}".format(0) startTime: float = f"{0:.6f}"
else: else:
startTime: float = "{:.6f}".format((self.duration * self.index)) startTime: float = f"{self.duration * self.index:.6f}"
fargs = [] fargs = []
fargs.append("-xerror") fargs.append("-xerror")
@ -55,7 +56,7 @@ class Segments:
fargs.append("-ss") fargs.append("-ss")
fargs.append(str(startTime if startTime else "0.00000")) fargs.append(str(startTime if startTime else "0.00000"))
fargs.append("-t") fargs.append("-t")
fargs.append(str("{:.6f}".format(self.duration))) fargs.append(str(f"{self.duration:.6f}"))
fargs.append("-copyts") fargs.append("-copyts")
@ -107,7 +108,8 @@ class Segments:
data, err = await proc.communicate() data, err = await proc.communicate()
if 0 != proc.returncode: if 0 != proc.returncode:
LOG.error(f'Failed to stream {realFile} segment {self.index}. {err.decode("utf-8")}.') LOG.error(f"Failed to stream '{realFile}' segment '{self.index}'. {err.decode('utf-8')}.")
raise StreamingError(f"Failed to stream {realFile} segment {self.index}.") msg = f"Failed to stream '{realFile}' segment '{self.index}'."
raise StreamingError(msg)
return data return data

8
app/library/Singleton.py
View file

@ -1,5 +1,5 @@
from typing import Any, Dict
import threading import threading
from typing import Any
class Singleton(type): class Singleton(type):
@ -7,7 +7,7 @@ class Singleton(type):
A metaclass that creates a Singleton base class when called. A metaclass that creates a Singleton base class when called.
""" """
_instances: Dict[type, Any] = {} _instances: dict[type, Any] = {}
def __call__(cls, *args: Any, **kwargs: Any) -> Any: def __call__(cls, *args: Any, **kwargs: Any) -> Any:
if cls not in cls._instances: if cls not in cls._instances:
@ -16,12 +16,12 @@ class Singleton(type):
return cls._instances[cls] return cls._instances[cls]
class threadSafe(type): class ThreadSafe(type):
""" """
A metaclass that creates a Singleton base class when called. A metaclass that creates a Singleton base class when called.
""" """
_instances: Dict[type, Any] = {} _instances: dict[type, Any] = {}
_lock = threading.Lock() _lock = threading.Lock()
def __call__(cls, *args: Any, **kwargs: Any) -> Any: def __call__(cls, *args: Any, **kwargs: Any) -> Any:

29
app/library/Subtitle.py
View file

@ -1,9 +1,12 @@
import logging import logging
from .Utils import calcDownloadPath
import pathlib import pathlib
import anyio
import pysubs2 import pysubs2
from pysubs2.time import ms_to_times
from pysubs2.formats.substation import SubstationFormat from pysubs2.formats.substation import SubstationFormat
from pysubs2.time import ms_to_times
from .Utils import calc_download_path
LOG = logging.getLogger("player.subtitle") LOG = logging.getLogger("player.subtitle")
@ -19,34 +22,34 @@ SubstationFormat.ms_to_timestamp = ms_to_timestamp
class Subtitle: class Subtitle:
allowedExtensions: tuple[str] = ( allowed_extensions: tuple[str] = (
".srt", ".srt",
".vtt", ".vtt",
".ass", ".ass",
) )
async def make(self, path: str, file: str) -> str: async def make(self, path: str, file: str) -> str:
realFile: str = calcDownloadPath(basePath=path, folder=file, createPath=False) realFile: str = calc_download_path(base_path=path, folder=file, create_path=False)
rFile = pathlib.Path(realFile) rFile = pathlib.Path(realFile)
if not rFile.exists(): if not rFile.exists():
raise Exception(f"File '{file}' does not exist.") msg = f"File '{file}' does not exist."
raise Exception(msg)
if rFile.suffix not in self.allowedExtensions: if rFile.suffix not in self.allowed_extensions:
raise Exception(f"File '{file}' subtitle type is not supported.") msg = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
if rFile.suffix == ".vtt": if rFile.suffix == ".vtt":
subData = "" async with await anyio.open_file(realFile) as f:
with open(realFile, "r") as f: return await f.read()
subData = f.read()
return subData
subs = pysubs2.load(path=str(rFile)) subs = pysubs2.load(path=str(rFile))
if len(subs.events) < 1: if len(subs.events) < 1:
raise Exception(f"No subtitle events were found in '{rFile}'.") msg = f"No subtitle events were found in '{rFile}'."
raise Exception(msg)
if len(subs.events) < 2: if len(subs.events) < 2:
return subs.to_string("vtt") return subs.to_string("vtt")

92
app/library/Tasks.py
View file

@ -1,11 +1,11 @@
import asyncio import asyncio
from datetime import datetime
import json import json
import logging import logging
import os import os
import time import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Any, List from datetime import UTC, datetime
from typing import Any
import httpx import httpx
from aiocron import Cron, crontab from aiocron import Cron, crontab
@ -55,7 +55,7 @@ class Tasks(metaclass=Singleton):
This class is used to manage the tasks. This class is used to manage the tasks.
""" """
_jobs: List[Job] = [] _jobs: list[Job] = []
"""The jobs for the tasks.""" """The jobs for the tasks."""
_instance = None _instance = None
@ -100,12 +100,24 @@ class Tasks(metaclass=Singleton):
Returns: Returns:
Tasks: The instance of the Tasks Tasks: The instance of the Tasks
"""
"""
if not Tasks._instance: if not Tasks._instance:
Tasks._instance = Tasks() Tasks._instance = Tasks()
return Tasks._instance return Tasks._instance
async def on_shutdown(self, _: web.Application):
"""
Shutdown the socket server.
Args:
_: The aiohttp application.
"""
LOG.debug("Shutting down tasks runner.")
self.clear(shutdown=True)
LOG.debug("Tasks runner has been shut down.")
def attach(self, _: web.Application): def attach(self, _: web.Application):
""" """
Attach the tasks to the aiohttp application. Attach the tasks to the aiohttp application.
@ -115,16 +127,11 @@ class Tasks(metaclass=Singleton):
Returns: Returns:
None None
""" """
self.load() self.load()
def shutdown(self): def get_tasks(self) -> list[Task]:
"""
Shutdown the tasks service.
"""
self.clear()
def getTasks(self) -> List[Task]:
"""Return the tasks.""" """Return the tasks."""
return [job.task for job in self._jobs] return [job.task for job in self._jobs]
@ -134,6 +141,7 @@ class Tasks(metaclass=Singleton):
Returns: Returns:
Tasks: The current instance. Tasks: The current instance.
""" """
self.clear() self.clear()
@ -142,7 +150,7 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Loading tasks from '{self._file}'.") LOG.info(f"Loading tasks from '{self._file}'.")
try: try:
with open(self._file, "r") as f: with open(self._file) as f:
tasks = json.load(f) tasks = json.load(f)
except Exception as e: except Exception as e:
LOG.error(f"Failed to parse tasks from '{self._file}'. '{e}'.") LOG.error(f"Failed to parse tasks from '{self._file}'. '{e}'.")
@ -156,7 +164,7 @@ class Tasks(metaclass=Singleton):
try: try:
task = Task(**task) task = Task(**task)
except Exception as e: except Exception as e:
LOG.error(f"Failed to parse task at list position '{i}'. '{str(e)}'.") LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.")
continue continue
try: try:
@ -171,16 +179,17 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Task '{i}: {task.name}' queued to be executed every '{task.timer}'.") LOG.info(f"Task '{i}: {task.name}' queued to be executed every '{task.timer}'.")
except Exception as e: except Exception as e:
LOG.exception(e) LOG.exception(e)
LOG.error(f"Failed to queue task '{i}: {task['name']}'. '{str(e)}'.") LOG.error(f"Failed to queue task '{i}: {task['name']}'. '{e!s}'.")
return self return self
def clear(self) -> "Tasks": def clear(self, shutdown: bool = False) -> "Tasks":
""" """
Clear all tasks. Clear all tasks.
Returns: Returns:
Tasks: The current instance. Tasks: The current instance.
""" """
if len(self._jobs) < 1: if len(self._jobs) < 1:
return self return self
@ -190,8 +199,9 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Stopping job '{task.id}: {task.name}'.") LOG.info(f"Stopping job '{task.id}: {task.name}'.")
task.job.stop() task.job.stop()
except Exception as e: except Exception as e:
LOG.exception(e) if not shutdown:
LOG.error(f"Failed to stop job '{task.id}: {task.name}'. '{str(e)}'.") LOG.exception(e)
LOG.error(f"Failed to stop job '{task.id}: {task.name}'. '{e!s}'.")
self._jobs.clear() self._jobs.clear()
@ -202,56 +212,63 @@ class Tasks(metaclass=Singleton):
Validate the task. Validate the task.
Args: Args:
tasks (Task|dict): The task to validate. task (Task|dict): The task to validate.
Returns: Returns:
bool: True if the task is valid, False otherwise. bool: True if the task is valid, False otherwise.
"""
"""
if not isinstance(task, dict): if not isinstance(task, dict):
if not isinstance(task, Task): if not isinstance(task, Task):
raise ValueError("Invalid task type.") msg = "Invalid task type."
raise ValueError(msg) # noqa: TRY004
task = task.serialize() task = task.serialize()
if not task.get("name"): if not task.get("name"):
raise ValueError("No name found.") msg = "No name found."
raise ValueError(msg)
if not task.get("timer"): if not task.get("timer"):
raise ValueError("No timer found.") msg = "No timer found."
raise ValueError(msg)
if not task.get("url"): if not task.get("url"):
raise ValueError("No URL found.") msg = "No URL found."
raise ValueError(msg)
if not isinstance(task.get("cookies"), str): if not isinstance(task.get("cookies"), str):
raise ValueError("Invalid cookies type.") msg = "Invalid cookies type."
raise ValueError(msg) # noqa: TRY004
if not isinstance(task.get("config"), dict): if not isinstance(task.get("config"), dict):
raise ValueError("Invalid config type.") msg = "Invalid config type."
raise ValueError(msg) # noqa: TRY004
if not isinstance(task.get("template"), str): if not isinstance(task.get("template"), str):
raise ValueError("Invalid template type.") msg = "Invalid template type."
raise ValueError(msg) # noqa: TRY004
return True return True
def save(self, tasks: List[Task | dict]) -> "Tasks": def save(self, tasks: list[Task | dict]) -> "Tasks":
""" """
Save the tasks. Save the tasks.
Args: Args:
tasks (List[Task]): The tasks to save. tasks (list[Task]): The tasks to save.
Returns: Returns:
Tasks: The current instance. Tasks: The current instance.
"""
"""
for i, task in enumerate(tasks): for i, task in enumerate(tasks):
try: try:
if not isinstance(task, Task): if not isinstance(task, Task):
task = Task(**task) task = Task(**task)
tasks[i] = task tasks[i] = task
except Exception as e: except Exception as e:
LOG.error(f"Failed to save task '{i}' unable to parse task. '{str(e)}'.") LOG.error(f"Failed to save task '{i}' unable to parse task. '{e!s}'.")
continue continue
try: try:
@ -266,7 +283,7 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Tasks saved to '{self._file}'.") LOG.info(f"Tasks saved to '{self._file}'.")
except Exception as e: except Exception as e:
LOG.error(f"Failed to save tasks to '{self._file}'. '{str(e)}'.") LOG.error(f"Failed to save tasks to '{self._file}'. '{e!s}'.")
return self return self
@ -279,9 +296,10 @@ class Tasks(metaclass=Singleton):
Returns: Returns:
None None
""" """
try: try:
timeNow = datetime.now().isoformat() timeNow = datetime.now(UTC).isoformat()
started = time.time() started = time.time()
if not task.url: if not task.url:
LOG.error(f"Failed to dispatch task '{task.id}: {task.name}'. No URL found.") LOG.error(f"Failed to dispatch task '{task.id}: {task.name}'. No URL found.")
@ -297,7 +315,7 @@ class Tasks(metaclass=Singleton):
try: try:
config = json.loads(config) config = json.loads(config)
except Exception as e: except Exception as e:
LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {str(e)}") LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {e!s}")
return return
LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.") LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.")
@ -324,13 +342,13 @@ class Tasks(metaclass=Singleton):
await asyncio.wait_for(asyncio.gather(*tasks), timeout=None) await asyncio.wait_for(asyncio.gather(*tasks), timeout=None)
timeNow = datetime.now().isoformat() timeNow = datetime.now(UTC).isoformat()
ended = time.time() ended = time.time()
LOG.info(f"Task '{task.id}: {task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") LOG.info(f"Task '{task.id}: {task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.")
await self._emitter.success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds.") await self._emitter.success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds.")
except Exception as e: except Exception as e:
timeNow = datetime.now().isoformat() timeNow = datetime.now(UTC).isoformat()
LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{str(e)}'.") LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{e!s}'.")
await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{str(e)}'.") await self._emitter.error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{e!s}'.")

115
app/library/Utils.py
View file

@ -8,7 +8,7 @@ import re
import shlex import shlex
import socket import socket
import uuid import uuid
from datetime import datetime, timezone from datetime import UTC, datetime
from functools import lru_cache from functools import lru_cache
from typing import Any from typing import Any
@ -26,14 +26,11 @@ IGNORED_KEYS: tuple[str] = (
"download_archive", "download_archive",
) )
YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None
OS_ALT_SEP: list[str] = list(sep for sep in [os.sep, os.path.altsep] if sep is not None and sep != "/")
class StreamingError(Exception): class StreamingError(Exception):
"""Raised when an error occurs during streaming.""" """Raised when an error occurs during streaming."""
pass
def get_opts(preset: str, ytdl_opts: dict) -> dict: def get_opts(preset: str, ytdl_opts: dict) -> dict:
""" """
@ -45,6 +42,7 @@ def get_opts(preset: str, ytdl_opts: dict) -> dict:
Returns: Returns:
ytdl extra options ytdl extra options
""" """
opts = copy.deepcopy(ytdl_opts) opts = copy.deepcopy(ytdl_opts)
@ -80,7 +78,7 @@ def get_opts(preset: str, ytdl_opts: dict) -> dict:
return opts return opts
def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> Any | dict[str, Any] | None: def get_video_info(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> Any | dict[str, Any] | None:
""" """
Extracts video information from the given URL. Extracts video information from the given URL.
@ -91,6 +89,7 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) ->
Returns: Returns:
dict: Video information. dict: Video information.
""" """
params: dict = { params: dict = {
"quiet": True, "quiet": True,
@ -110,28 +109,34 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None, no_archive: bool = True) ->
return yt_dlp.YoutubeDL().extract_info(url, download=False) return yt_dlp.YoutubeDL().extract_info(url, download=False)
def calcDownloadPath(basePath: str, folder: str | None = None, createPath: bool = True) -> str: def calc_download_path(base_path: str, folder: str | None = None, create_path: bool = True) -> str:
"""Calculates download path and prevents folder traversal. """
Calculates download path and prevents folder traversal.
Returns: Returns:
Download path with base folder factored in. Download path with base folder factored in.
""" """
if not folder: if not folder:
return basePath return base_path
realBasePath = os.path.realpath(basePath) if folder.startswith("/"):
download_path = os.path.realpath(os.path.join(basePath, folder)) folder = folder[1:]
realBasePath = os.path.realpath(base_path)
download_path = os.path.realpath(os.path.join(base_path, folder))
if not download_path.startswith(realBasePath): if not download_path.startswith(realBasePath):
raise Exception(f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".') msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".'
raise Exception(msg)
if not os.path.isdir(download_path) and createPath: if not os.path.isdir(download_path) and create_path:
os.makedirs(download_path, exist_ok=True) os.makedirs(download_path, exist_ok=True)
return download_path return download_path
def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict: def extract_info(config: dict, url: str, debug: bool = False) -> dict:
params: dict = { params: dict = {
"color": "no_color", "color": "no_color",
"extract_flat": True, "extract_flat": True,
@ -163,14 +168,14 @@ def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict:
return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)
def mergeDict(source: dict, destination: dict) -> dict: def merge_dict(source: dict, destination: dict) -> dict:
"""Merge data from source into destination""" """Merge data from source into destination"""
destination_copy = copy.deepcopy(destination) destination_copy = copy.deepcopy(destination)
for key, value in source.items(): for key, value in source.items():
destination_key_value = destination_copy.get(key) destination_key_value = destination_copy.get(key)
if isinstance(value, dict) and isinstance(destination_key_value, dict): if isinstance(value, dict) and isinstance(destination_key_value, dict):
destination_copy[key] = mergeDict(source=value, destination=destination_copy.setdefault(key, {})) destination_copy[key] = merge_dict(source=value, destination=destination_copy.setdefault(key, {}))
elif isinstance(value, list) and isinstance(destination_key_value, list): elif isinstance(value, list) and isinstance(destination_key_value, list):
destination_copy[key] = destination_key_value + value destination_copy[key] = destination_key_value + value
else: else:
@ -179,7 +184,7 @@ def mergeDict(source: dict, destination: dict) -> dict:
return destination_copy return destination_copy
def mergeConfig(config: dict, new_config: dict) -> dict: def merge_config(config: dict, new_config: dict) -> dict:
""" """
Merge user provided config into default config Merge user provided config into default config
@ -189,13 +194,14 @@ def mergeConfig(config: dict, new_config: dict) -> dict:
Returns: Returns:
dict: Merged config dict: Merged config
""" """
for key in IGNORED_KEYS: for key in IGNORED_KEYS:
if key in new_config: if key in new_config:
LOG.error(f"Key '{key}' is not allowed to be manually set.") LOG.error(f"Key '{key}' is not allowed to be manually set.")
del new_config[key] del new_config[key]
conf = mergeDict(new_config, config) conf = merge_dict(new_config, config)
if "impersonate" in conf: if "impersonate" in conf:
conf["impersonate"] = ImpersonateTarget.from_str(conf["impersonate"]) conf["impersonate"] = ImpersonateTarget.from_str(conf["impersonate"])
@ -203,8 +209,8 @@ def mergeConfig(config: dict, new_config: dict) -> dict:
return conf return conf
def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]: def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]:
global YTDLP_INFO_CLS global YTDLP_INFO_CLS # noqa: PLW0603
idDict = { idDict = {
"id": None, "id": None,
@ -247,35 +253,26 @@ def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, st
break break
if not idDict["archive_id"]: if not idDict["archive_id"]:
return ( return (False, idDict)
False,
idDict,
)
with open(archive_file, "r") as f: with open(archive_file) as f:
for line in f.readlines(): for line in f:
if idDict["archive_id"] in line: if idDict["archive_id"] in line:
return ( return (True, idDict)
True,
idDict,
)
return ( return (False, idDict)
False,
idDict,
)
def jsonCookie(cookies: dict[dict[str, any]]) -> str | None: def json_cookie(cookies: dict[dict[str, any]]) -> str | None:
"""Converts JSON cookies to Netscape cookies """
Converts JSON cookies to Netscape cookies
Returns None if no cookies are found, otherwise returns a string of cookies in Netscape format. Returns None if no cookies are found, otherwise returns a string of cookies in Netscape format.
""" """
netscapeCookies = "# Netscape HTTP Cookie File\n# https://curl.haxx.se/docs/http-cookies.html\n# This file was generated by libcurl! Edit at your own risk.\n\n" netscapeCookies = "# Netscape HTTP Cookie File\n# https://curl.haxx.se/docs/http-cookies.html\n# This file was generated by libcurl! Edit at your own risk.\n\n"
hasCookies: bool = False hasCookies: bool = False
for domain in cookies: for domain in cookies: # noqa: PLC0206
if not isinstance(cookies[domain], dict): if not isinstance(cookies[domain], dict):
continue continue
@ -290,7 +287,7 @@ def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
cookieDict = cookies[domain][subDomain][cookie] cookieDict = cookies[domain][subDomain][cookie]
if 0 == int(cookieDict["expirationDate"]): if 0 == int(cookieDict["expirationDate"]):
cookieDict["expirationDate"] = datetime.now(timezone.utc).timestamp() + (86400 * 1000) cookieDict["expirationDate"] = datetime.now(UTC).timestamp() + (86400 * 1000)
hasCookies = True hasCookies = True
netscapeCookies += ( netscapeCookies += (
@ -325,13 +322,14 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded. dict|list: Dictionary or list of the file contents. Empty dict if the file could not be loaded.
bool: True if the file was loaded successfully. bool: True if the file was loaded successfully.
str: Error message if the file could not be loaded. str: Error message if the file could not be loaded.
""" """
try: try:
with open(file) as json_data: with open(file) as json_data:
opts = json.load(json_data) opts = json.load(json_data)
if check_type: if check_type:
assert isinstance(opts, check_type) assert isinstance(opts, check_type) # noqa: S101
return ( return (
opts, opts,
@ -346,7 +344,7 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
opts = json5_load(json_data) opts = json5_load(json_data)
if check_type: if check_type:
assert isinstance(opts, check_type) assert isinstance(opts, check_type) # noqa: S101
return ( return (
opts, opts,
@ -367,7 +365,7 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
) )
def checkId(basePath: str, file: pathlib.Path) -> bool | str: def check_id(file: pathlib.Path) -> bool | str:
""" """
Check if we are able to get an id from the file name. Check if we are able to get an id from the file name.
if so check if any video file with the same id exists. if so check if any video file with the same id exists.
@ -377,7 +375,6 @@ def checkId(basePath: str, file: pathlib.Path) -> bool | str:
:return: False if no id found, otherwise the id. :return: False if no id found, otherwise the id.
""" """
match = re.search(r"(?<=\[)(?:youtube-)?(?P<id>[a-zA-Z0-9\-_]{11})(?=\])", file.stem, re.IGNORECASE) match = re.search(r"(?<=\[)(?:youtube-)?(?P<id>[a-zA-Z0-9\-_]{11})(?=\])", file.stem, re.IGNORECASE)
if not match: if not match:
return False return False
@ -388,16 +385,7 @@ def checkId(basePath: str, file: pathlib.Path) -> bool | str:
if id not in f.stem: if id not in f.stem:
continue continue
if f.suffix not in ( if f.suffix not in (".mp4", ".mkv", ".webm", ".m4v", ".m4a", ".mp3", ".aac", ".ogg"):
".mp4",
".mkv",
".webm",
".m4v",
".m4a",
".mp3",
".aac",
".ogg",
):
continue continue
return f.absolute() return f.absolute()
@ -422,7 +410,6 @@ def ag(array: dict | list, path: list[str | int] | str | int, default: Any = Non
:param separator: Separator for nested paths in strings. :param separator: Separator for nested paths in strings.
:return: The found value or the default if not found. :return: The found value or the default if not found.
""" """
if path is None or path == "": if path is None or path == "":
return array return array
@ -482,24 +469,29 @@ def validate_url(url: str) -> bool:
Returns: Returns:
bool: True if the URL is valid and allowed. bool: True if the URL is valid and allowed.
""" """
if not url: if not url:
raise ValueError("URL is required.") msg = "URL is required."
raise ValueError(msg)
try: try:
from yarl import URL from yarl import URL
parsed_url = URL(url) parsed_url = URL(url)
except ValueError: except ValueError:
raise ValueError("Invalid URL.") msg = "Invalid URL."
raise ValueError(msg) # noqa: B904
# Check allowed schemes # Check allowed schemes
if parsed_url.scheme not in ["http", "https"]: if parsed_url.scheme not in ["http", "https"]:
raise ValueError("Invalid scheme usage. Only HTTP or HTTPS allowed.") msg = "Invalid scheme usage. Only HTTP or HTTPS allowed."
raise ValueError(msg)
hostname = parsed_url.host hostname = parsed_url.host
if not hostname or is_private_address(hostname): if not hostname or is_private_address(hostname):
raise ValueError("Access to internal urls or private networks is not allowed.") msg = "Access to internal urls or private networks is not allowed."
raise ValueError(msg)
return True return True
@ -509,16 +501,17 @@ def arg_converter(args: str) -> dict:
Convert yt-dlp options to a dictionary. Convert yt-dlp options to a dictionary.
Args: Args:
opts (str): yt-dlp options string. args (str): yt-dlp options string.
Returns: Returns:
dict: yt-dlp options dictionary. dict: yt-dlp options dictionary.
""" """
import yt_dlp.options import yt_dlp.options
create_parser = yt_dlp.options.create_parser create_parser = yt_dlp.options.create_parser
def defaultOpts(args: str): def _default_opts(args: str):
patched_parser = create_parser() patched_parser = create_parser()
try: try:
yt_dlp.options.create_parser = lambda: patched_parser yt_dlp.options.create_parser = lambda: patched_parser
@ -526,7 +519,7 @@ def arg_converter(args: str) -> dict:
finally: finally:
yt_dlp.options.create_parser = create_parser yt_dlp.options.create_parser = create_parser
default_opts = defaultOpts([]).ydl_opts default_opts = _default_opts([]).ydl_opts
opts = yt_dlp.parse_options(shlex.split(args)).ydl_opts opts = yt_dlp.parse_options(shlex.split(args)).ydl_opts
@ -543,9 +536,11 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
Args: Args:
uuid_str (str): UUID to validate. uuid_str (str): UUID to validate.
version (int): The UUID version
Returns: Returns:
bool: True if the UUID is valid. bool: True if the UUID is valid.
""" """
try: try:
uuid.UUID(uuid_str, version=version) uuid.UUID(uuid_str, version=version)

16
app/library/cache.py
View file

@ -1,12 +1,12 @@
import hashlib import hashlib
import threading import threading
import time import time
from typing import Any, Dict, Optional, Tuple from typing import Any
from .Singleton import threadSafe from .Singleton import ThreadSafe
class Cache(metaclass=threadSafe): class Cache(metaclass=ThreadSafe):
def __init__(self) -> None: def __init__(self) -> None:
""" """
Initialize the Cache. Initialize the Cache.
@ -14,11 +14,11 @@ class Cache(metaclass=threadSafe):
# Prevent reinitialization in singleton context. # Prevent reinitialization in singleton context.
if hasattr(self, "_initialized") and self._initialized: if hasattr(self, "_initialized") and self._initialized:
return return
self._cache: Dict[str, Tuple[Any, Optional[float]]] = {} self._cache: dict[str, tuple[Any, float | None]] = {}
self._lock = threading.Lock() self._lock = threading.Lock()
self._initialized = True self._initialized = True
def set(self, key: str, value: Any, ttl: Optional[float] = None) -> None: def set(self, key: str, value: Any, ttl: float | None = None) -> None:
""" """
Synchronously set a value in the cache with an optional time-to-live. Synchronously set a value in the cache with an optional time-to-live.
If ttl is None, the entry never expires. If ttl is None, the entry never expires.
@ -27,7 +27,7 @@ class Cache(metaclass=threadSafe):
with self._lock: with self._lock:
self._cache[key] = (value, expire_at) self._cache[key] = (value, expire_at)
def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]: def get(self, key: str, default: Any | None = None) -> Any | None:
""" """
Synchronously retrieve a value from the cache if it exists and hasn't expired. Synchronously retrieve a value from the cache if it exists and hasn't expired.
""" """
@ -81,13 +81,13 @@ class Cache(metaclass=threadSafe):
return hashlib.sha256(key.encode("utf-8")).hexdigest() return hashlib.sha256(key.encode("utf-8")).hexdigest()
# Asynchronous counterparts for non-blocking interfaces # Asynchronous counterparts for non-blocking interfaces
async def aset(self, key: str, value: Any, ttl: Optional[float] = None) -> None: async def aset(self, key: str, value: Any, ttl: float | None = None) -> None:
""" """
Asynchronously set a value in the cache. Asynchronously set a value in the cache.
""" """
self.set(key, value, ttl) self.set(key, value, ttl)
async def aget(self, key: str, default: Optional[Any] = None) -> Optional[Any]: async def aget(self, key: str, default: Any | None = None) -> Any | None:
""" """
Asynchronously retrieve a value from the cache. Asynchronously retrieve a value from the cache.
""" """

16
app/library/common.py
View file

@ -1,14 +1,14 @@
import json import json
import logging import logging
from .config import Config
from .DownloadQueue import DownloadQueue from .DownloadQueue import DownloadQueue
from .encoder import Encoder from .encoder import Encoder
from .config import Config
LOG = logging.getLogger("common") LOG = logging.getLogger("common")
class common: class Common:
""" """
This class is used to share common methods between the socket and the API gateways. This class is used to share common methods between the socket and the API gateways.
""" """
@ -43,6 +43,7 @@ class common:
Returns: Returns:
dict[str, str]: The status of the download. dict[str, str]: The status of the download.
{ "status": "text" } { "status": "text" }
""" """
if cookies and isinstance(cookies, dict): if cookies and isinstance(cookies, dict):
cookies = self.encoder.encode(cookies) cookies = self.encoder.encode(cookies)
@ -69,11 +70,13 @@ class common:
Returns: Returns:
dict: The formatted item dict: The formatted item
""" """
url: str = item.get("url") url: str = item.get("url")
if not url: if not url:
raise ValueError("url param is required.") msg = "url param is required."
raise ValueError(msg)
preset: str = str(item.get("preset", self.config.default_preset)) preset: str = str(item.get("preset", self.config.default_preset))
folder: str = str(item.get("folder")) if item.get("folder") else "" folder: str = str(item.get("folder")) if item.get("folder") else ""
@ -85,9 +88,10 @@ class common:
try: try:
config = json.loads(config) config = json.loads(config)
except Exception as e: except Exception as e:
raise ValueError(f"Failed to parse json yt-dlp config for '{url}'. {str(e)}") msg = f"Failed to parse json yt-dlp config for '{url}'. {e!s}"
raise ValueError(msg) from e
item = { return {
"url": url, "url": url,
"preset": preset, "preset": preset,
"folder": folder, "folder": folder,
@ -95,5 +99,3 @@ class common:
"config": config if isinstance(config, dict) else {}, "config": config if isinstance(config, dict) else {},
"template": template, "template": template,
} }
return item

44
app/library/config.py
View file

@ -12,7 +12,7 @@ import coloredlogs
from dotenv import load_dotenv from dotenv import load_dotenv
from yt_dlp.version import __version__ as YTDLP_VERSION from yt_dlp.version import __version__ as YTDLP_VERSION
from .Utils import IGNORED_KEYS, load_file, mergeDict from .Utils import IGNORED_KEYS, load_file, merge_dict
from .version import APP_VERSION from .version import APP_VERSION
@ -113,9 +113,6 @@ class Config:
ytdl_options: dict = {} ytdl_options: dict = {}
"The options to use for yt-dlp." "The options to use for yt-dlp."
tasks: list = []
"The tasks to run."
new_version_available: bool = False new_version_available: bool = False
"A new version of the application is available." "A new version of the application is available."
@ -215,9 +212,10 @@ class Config:
def __init__(self): def __init__(self):
"""Virtually private constructor.""" """Virtually private constructor."""
if Config.__instance is not None: if Config.__instance is not None:
raise Exception("This class is a singleton. Use Config.get_instance() instead.") msg = "This class is a singleton. Use Config.get_instance() instead."
else: raise Exception(msg)
Config.__instance = self
Config.__instance = self
baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute()) baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute())
@ -233,7 +231,7 @@ class Config:
logging.info(f"Loading environment variables from '{envFile}'.") logging.info(f"Loading environment variables from '{envFile}'.")
load_dotenv(envFile) load_dotenv(envFile)
for k, v in self._getAttributes().items(): for k, v in self._get_attributes().items():
if k.startswith("_") or k in self._manual_vars: if k.startswith("_") or k in self._manual_vars:
continue continue
@ -243,7 +241,7 @@ class Config:
continue continue
lookUpKey: str = f"YTP_{k}".upper() lookUpKey: str = f"YTP_{k}".upper()
setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v) setattr(self, k, os.environ.get(lookUpKey, v))
for k, v in self.__dict__.items(): for k, v in self.__dict__.items():
if k.startswith("_") or k in self._immutable or k in self._manual_vars: if k.startswith("_") or k in self._immutable or k in self._manual_vars:
@ -262,7 +260,8 @@ class Config:
if k in self._boolean_vars: if k in self._boolean_vars:
if str(v).lower() not in (True, False, "true", "false", "on", "off", "1", "0"): if str(v).lower() not in (True, False, "true", "false", "on", "off", "1", "0"):
raise ValueError(f"Config variable '{k}' is set to a non-boolean value '{v}'.") msg = f"Config variable '{k}' is set to a non-boolean value '{v}'."
raise ValueError(msg)
setattr(self, k, str(v).lower() in (True, "true", "on", "1")) setattr(self, k, str(v).lower() in (True, "true", "on", "1"))
@ -271,7 +270,8 @@ class Config:
numeric_level = getattr(logging, self.log_level.upper(), None) numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int): if not isinstance(numeric_level, int):
raise ValueError(f"Invalid log level '{self.log_level}' specified.") msg = f"Invalid log level '{self.log_level}' specified."
raise TypeError(msg)
coloredlogs.install( coloredlogs.install(
level=numeric_level, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S" level=numeric_level, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S"
@ -304,26 +304,14 @@ class Config:
LOG.error(f"Key '{key}' is not allowed to be loaded via 'ytdlp.json' file.") LOG.error(f"Key '{key}' is not allowed to be loaded via 'ytdlp.json' file.")
del opts[key] del opts[key]
self.ytdl_options = mergeDict(self.ytdl_options, opts) self.ytdl_options = merge_dict(self.ytdl_options, opts)
else: else:
LOG.error(f"Invalid yt-dlp custom options file '{optsFile}'.") LOG.error(f"Invalid yt-dlp custom options file '{optsFile}'.")
else: else:
LOG.info(f"No yt-dlp custom options found at '{optsFile}'.") LOG.info(f"No yt-dlp custom options found at '{optsFile}'.")
tasksFile = os.path.join(self.config_path, "tasks.json")
if os.path.exists(tasksFile) and os.path.getsize(tasksFile) > 0:
LOG.info(f"Loading tasks from '{tasksFile}'.")
try:
(tasks, status, error) = load_file(tasksFile, list)
if not status:
LOG.error(f"Could not load tasks file from '{tasksFile}'. '{error}'.")
sys.exit(1)
self.tasks.extend(tasks)
except Exception:
pass
# Load default presets. # Load default presets.
with open(os.path.join(os.path.dirname(__file__), "presets.json"), "r") as f: with open(os.path.join(os.path.dirname(__file__), "presets.json")) as f:
self.presets.extend(json.load(f)) self.presets.extend(json.load(f))
# Load user defined presets. # Load user defined presets.
@ -378,11 +366,11 @@ class Config:
self.started = time.time() self.started = time.time()
def _getAttributes(self) -> dict: def _get_attributes(self) -> dict:
attrs: dict = {} attrs: dict = {}
vClass: str = self.__class__ vClass: str = self.__class__
for attribute in vClass.__dict__.keys(): for attribute in vClass.__dict__:
if attribute.startswith("_"): if attribute.startswith("_"):
continue continue
@ -398,8 +386,8 @@ class Config:
Returns: Returns:
dict: A dictionary with the frontend configuration dict: A dictionary with the frontend configuration
"""
"""
data = {k: getattr(self, k) for k in self._frontend_vars} data = {k: getattr(self, k) for k in self._frontend_vars}
hasCookies = self.ytdl_options.get("cookiefile", None) hasCookies = self.ytdl_options.get("cookiefile", None)
data["has_cookies"] = hasCookies is not None and os.path.exists(hasCookies) data["has_cookies"] = hasCookies is not None and os.path.exists(hasCookies)

53
app/library/ffprobe.py
View file

@ -8,6 +8,8 @@ import json
import operator import operator
import os import os
import anyio
# parameter-less decorator # parameter-less decorator
def async_lru_cache_decorator(async_function): def async_lru_cache_decorator(async_function):
@ -85,10 +87,7 @@ class FFStream:
if self.__dict__.get("codec_type", None) != "video": if self.__dict__.get("codec_type", None) != "video":
return False return False
if self.__dict__.get("codec_name", None) in ["png", "mjpeg", "gif", "bmp", "tiff", "webp"]: return self.__dict__.get("codec_name", None) not in ["png", "mjpeg", "gif", "bmp", "tiff", "webp"]
return False
return True
def is_subtitle(self): def is_subtitle(self):
""" """
@ -115,8 +114,9 @@ class FFStream:
if width and height: if width and height:
try: try:
size = (int(width), int(height)) size = (int(width), int(height))
except ValueError: except ValueError as e:
raise FFProbeError("None integer size {}:{}".format(width, height)) msg = f"None integer size {width}:{height}"
raise FFProbeError(msg) from e
else: else:
return None return None
@ -136,8 +136,9 @@ class FFStream:
if self.is_video() or self.is_audio(): if self.is_video() or self.is_audio():
try: try:
frame_count = int(self.__dict__.get("nb_frames", "")) frame_count = int(self.__dict__.get("nb_frames", ""))
except ValueError: except ValueError as e:
raise FFProbeError("None integer frame count") msg = "None integer frame count"
raise FFProbeError(msg) from e
else: else:
frame_count = 0 frame_count = 0
@ -151,8 +152,9 @@ class FFStream:
if self.is_video() or self.is_audio(): if self.is_video() or self.is_audio():
try: try:
duration = float(self.__dict__.get("duration", "")) duration = float(self.__dict__.get("duration", ""))
except ValueError: except ValueError as e:
raise FFProbeError("None numeric duration") msg = "None numeric duration"
raise FFProbeError(msg) from e
else: else:
duration = 0.0 duration = 0.0
@ -188,8 +190,9 @@ class FFStream:
""" """
try: try:
return int(self.__dict__.get("bit_rate", "")) return int(self.__dict__.get("bit_rate", ""))
except ValueError: except ValueError as e:
raise FFProbeError("None integer bit_rate") msg = "None integer bit_rate"
raise FFProbeError(msg) from e
class FFProbeResult: class FFProbeResult:
@ -212,19 +215,19 @@ class FFProbeResult:
return getattr(self, key) if hasattr(self, key) else default return getattr(self, key) if hasattr(self, key) else default
def streams(self) -> list[FFStream]: def streams(self) -> list[FFStream]:
"List of all streams." """List of all streams."""
return self.video + self.audio + self.subtitle + self.attachment return self.video + self.audio + self.subtitle + self.attachment
def has_video(self): def has_video(self):
"Is there a video stream?" """Is there a video stream?"""
return len(self.video) > 0 return len(self.video) > 0
def has_audio(self): def has_audio(self):
"Is there an audio stream?" """Is there an audio stream?"""
return len(self.audio) > 0 return len(self.audio) > 0
def has_subtitle(self): def has_subtitle(self):
"Is there a subtitle stream?" """Is there a subtitle stream?"""
return len(self.subtitle) > 0 return len(self.subtitle) > 0
def __repr__(self): def __repr__(self):
@ -259,15 +262,18 @@ async def ffprobe(file: str) -> FFProbeResult:
Returns: Returns:
dict: A dictionary containing the parsed data. dict: A dictionary containing the parsed data.
""" """
try: try:
with open(os.devnull, "w") as tempf: async with await anyio.open_file(os.devnull, "w") as tempf:
await asyncio.create_subprocess_exec("ffprobe", "-h", stdout=tempf, stderr=tempf) await asyncio.create_subprocess_exec("ffprobe", "-h", stdout=tempf, stderr=tempf)
except FileNotFoundError: except FileNotFoundError as e:
raise IOError("ffprobe not found.") msg = "ffprobe not found."
raise OSError(msg) from e
if not os.path.isfile(file): if not os.path.isfile(file):
raise IOError(f"No such media file '{file}'.") msg = f"No such media file '{file}'."
raise OSError(msg)
args = ["-v", "quiet", "-of", "json", "-show_streams", "-show_format", file] args = ["-v", "quiet", "-of", "json", "-show_streams", "-show_format", file]
@ -284,13 +290,14 @@ async def ffprobe(file: str) -> FFProbeResult:
if 0 == exitCode: if 0 == exitCode:
parsed: dict = json.loads(data.decode("utf-8")) parsed: dict = json.loads(data.decode("utf-8"))
else: else:
raise FFProbeError(f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'") msg = f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'"
raise FFProbeError(msg)
result = FFProbeResult() result = FFProbeResult()
result.metadata = parsed.get("format", {}) result.metadata = parsed.get("format", {})
for stream in parsed.get("streams", []): for raw_stream in parsed.get("streams", []):
stream = FFStream(stream) stream = FFStream(raw_stream)
if stream.is_audio(): if stream.is_audio():
result.audio.append(stream) result.audio.append(stream)
elif stream.is_video(): elif stream.is_video():

120
app/main.py
View file

@ -13,7 +13,6 @@ from library.config import Config
from library.DownloadQueue import DownloadQueue from library.DownloadQueue import DownloadQueue
from library.Emitter import Emitter from library.Emitter import Emitter
from library.EventsSubscriber import EventsSubscriber from library.EventsSubscriber import EventsSubscriber
from library.HttpAPI import LOG as http_logger
from library.HttpAPI import HttpAPI from library.HttpAPI import HttpAPI
from library.HttpSocket import HttpSocket from library.HttpSocket import HttpSocket
from library.Notifications import Notification from library.Notifications import Notification
@ -25,110 +24,97 @@ MIME = magic.Magic(mime=True)
class Main: class Main:
config: Config
app: web.Application
http: HttpAPI
socket: HttpSocket
def __init__(self): def __init__(self):
self.config = Config.get_instance() self._config = Config.get_instance()
self.rootPath = str(Path(__file__).parent.absolute()) self.rootPath = str(Path(__file__).parent.absolute())
self.app = web.Application() self._app = web.Application()
self.checkFolders() self._check_folders()
try: try:
PackageInstaller(self.config).check() PackageInstaller(self._config).check()
except Exception as e: except Exception as e:
LOG.error(f"Failed to check for packages. Error message '{str(e)}'.")
LOG.exception(e) LOG.exception(e)
LOG.error(f"Failed to check for packages. Error message '{e!s}'.")
caribou.upgrade(self.config.db_file, os.path.join(self.rootPath, "migrations")) caribou.upgrade(self._config.db_file, os.path.join(self.rootPath, "migrations"))
connection = sqlite3.connect(database=self.config.db_file, isolation_level=None) connection = sqlite3.connect(database=self._config.db_file, isolation_level=None)
connection.row_factory = sqlite3.Row connection.row_factory = sqlite3.Row
connection.execute("PRAGMA journal_mode=wal") connection.execute("PRAGMA journal_mode=wal")
async def _close_connection(_):
LOG.debug("Closing database connection.")
connection.close()
LOG.debug("Database connection closed.")
try: try:
if "600" != oct(os.stat(self.config.db_file).st_mode)[-3:]: if "600" != oct(os.stat(self._config.db_file).st_mode)[-3:]:
os.chmod(self.config.db_file, 0o600) os.chmod(self._config.db_file, 0o600)
except Exception: except Exception:
pass pass
queue = DownloadQueue(connection=connection) self._queue = DownloadQueue(connection=connection)
self._http = HttpAPI(queue=self._queue)
self.http = HttpAPI(queue=queue) self._socket = HttpSocket(queue=self._queue)
self.socket = HttpSocket(queue=queue)
Emitter.get_instance().add_emitter([Notification().emit], local=False).add_emitter( Emitter.get_instance().add_emitter([Notification().emit], local=False).add_emitter(
[EventsSubscriber().emit], local=True [EventsSubscriber().emit], local=True
) )
self.app.on_startup.append(lambda _: queue.initialize()) self._app.on_cleanup.append(_close_connection)
def checkFolders(self): def _check_folders(self):
""" """Check if the required folders exist and create them if they do not."""
Check if the required folders exist and create them if they do not. folders = (self._config.download_path, self._config.temp_path, self._config.config_path)
"""
try: for folder in folders:
LOG.debug(f"Checking download folder at '{self.config.download_path}'.") try:
if not os.path.exists(self.config.download_path): LOG.debug(f"Checking folder at '{folder}'.")
LOG.info(f"Creating download folder at '{self.config.download_path}'.") if not os.path.exists(folder):
os.makedirs(self.config.download_path, exist_ok=True) LOG.info(f"Creating folder at '{folder}'.")
except OSError as e: os.makedirs(folder, exist_ok=True)
LOG.error(f"Could not create download folder at '{self.config.download_path}'.") except OSError:
raise e LOG.error(f"Could not create folder at '{folder}'.")
raise
try: try:
LOG.debug(f"Checking temp folder at '{self.config.temp_path}'.") LOG.debug(f"Checking database file at '{self._config.db_file}'.")
if not os.path.exists(self.config.temp_path): if not os.path.exists(self._config.db_file):
LOG.info(f"Creating temp folder at '{self.config.temp_path}'.") LOG.info(f"Creating database file at '{self._config.db_file}'.")
os.makedirs(self.config.temp_path, exist_ok=True) with open(self._config.db_file, "w") as _:
except OSError as e:
LOG.error(f"Could not create temp folder at '{self.config.temp_path}'.")
raise e
try:
LOG.debug(f"Checking config folder at '{self.config.config_path}'.")
if not os.path.exists(self.config.config_path):
LOG.info(f"Creating config folder at '{self.config.config_path}'.")
os.makedirs(self.config.config_path, exist_ok=True)
except OSError as e:
LOG.error(f"Could not create config folder at '{self.config.config_path}'.")
raise e
try:
LOG.debug(f"Checking database file at '{self.config.db_file}'.")
if not os.path.exists(self.config.db_file):
LOG.info(f"Creating database file at '{self.config.db_file}'.")
with open(self.config.db_file, "w") as _:
pass pass
except OSError as e: except OSError:
LOG.error(f"Could not create database file at '{self.config.db_file}'.") LOG.error(f"Could not create database file at '{self._config.db_file}'.")
raise e raise
def start(self): def start(self):
""" """
Start the application. Start the application.
""" """
self.socket.attach(self.app) self._socket.attach(self._app)
self.http.attach(self.app) self._http.attach(self._app)
Tasks.get_instance().attach(self.app) self._queue.attach(self._app)
Tasks.get_instance().attach(self._app)
def started(_): def started(_):
LOG.info("=" * 40) LOG.info("=" * 40)
LOG.info(f"YTPTube v{self.config.version} - started on http://{self.config.host}:{self.config.port}") LOG.info(f"YTPTube v{self._config.version} - started on http://{self._config.host}:{self._config.port}")
LOG.info("=" * 40) LOG.info("=" * 40)
if self.config.access_log: HTTP_LOGGER = None
http_logger.addFilter(lambda record: "GET /api/ping" not in record.getMessage()) if self._config.access_log:
from library.HttpAPI import LOG as HTTP_LOGGER
HTTP_LOGGER.addFilter(lambda record: "GET /api/ping" not in record.getMessage())
web.run_app( web.run_app(
self.app, self._app,
host=self.config.host, host=self._config.host,
port=self.config.port, port=self._config.port,
reuse_port=True, reuse_port=True,
loop=asyncio.get_event_loop(), loop=asyncio.get_event_loop(),
access_log=http_logger if self.config.access_log else None, access_log=HTTP_LOGGER,
print=started, print=started,
) )

13
entrypoint.sh
View file

@ -19,4 +19,17 @@ if [ ! -w "${YTP_CONFIG_PATH}" ]; then
exit 1 exit 1
fi fi
if [ ! -w "${YTP_DOWNLOAD_PATH}" ]; then
CH_USER=$(stat -c "%u" "${YTP_DOWNLOAD_PATH}")
CH_GRP=$(stat -c "%g" "${YTP_DOWNLOAD_PATH}")
echo_err "ERROR: Unable to write to '${YTP_DOWNLOAD_PATH}' downloads directory. Current user id '${UID}' while directory owner is '${CH_USER}'."
echo_err "[Running under docker]"
echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
echo_err "Run the following command to change the directory ownership"
echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config"
echo_err "[Running under podman]"
echo_err "change docker-compose.yaml user: to user:\"0:0\""
exit 1
fi
exec "${@}" exec "${@}"

72
pyproject.toml
View file

@ -26,6 +26,7 @@ exclude = [
"node_modules", "node_modules",
"site-packages", "site-packages",
"venv", "venv",
".venv",
] ]
# Same as Black. # Same as Black.
@ -39,8 +40,75 @@ target-version = "py311"
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
# McCabe complexity (`C901`) by default. # McCabe complexity (`C901`) by default.
select = ["E4", "E7", "E9", "F", "G", "W"] #select = ["E4", "E7", "E9", "F", "G", "W"]
ignore = ["PTH", "G0", "G1", "G201"] #ignore = ["PTH", "G0", "G1", "G201"]
select = [
"ALL", # include all the rules, including new ones
]
ignore = [
#### modules
"ANN", # flake8-annotations
"COM", # flake8-commas
"C90", # mccabe complexity
"DJ", # django
"EXE", # flake8-executable
"T10", # debugger
"TID", # flake8-tidy-imports
#### specific rules
"D100", # ignore missing docs
"D101",
"D102",
"D103",
"D104",
"D105",
"D106",
"D107",
"D200",
"D205",
"D212",
"D400",
"D401",
"D415",
"E402", # false positives for local imports
"E501", # line too long
"TRY003", # external messages in exceptions are too verbose
"TD002",
"TD003",
"FIX002", # too verbose descriptions of todos,
"N806", # variable names should be snake_case
"PTH", # prefer using absolute imports
"PGH003",
"D404",
"PLR0913",
"INP001",
"G004",
"SIM105",
"SIM300",
"BLE001",
"TRY400",
"S104",
"PLR2004",
"S110",
"N812",
"S108",
"RUF006",
"RUF012",
"RUF013",
"TRY002",
"TRY401",
"A001",
"A002",
"SLF001",
"FBT001",
"FBT002",
"TRY300",
"PLR0911",
"PLR0912",
"PLR0915",
"PLW2901",
"ERA001"
]
# Allow fix for all enabled rules (when `--fix`) is provided. # Allow fix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"] fixable = ["ALL"]

68
ui/layouts/default.vue
View file

@ -13,11 +13,40 @@
<div class="navbar-end is-flex" style="flex-flow:wrap"> <div class="navbar-end is-flex" style="flex-flow:wrap">
<div class="navbar-item">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/" v-tooltip.bottom="'Downloads'">
<span class="icon"><i class="fa-solid fa-download" /></span>
</NuxtLink>
</div>
<div class="navbar-item has-dropdown is-unselectable" :class="{ 'is-active': theme_open }"
@click.prevent="openCloseTheme" v-tooltip.bottom="'Change Theme'">
<a class="navbar-link is-arrowless">
<span class="icon"><i class="fas fa-repeat" /></span>
</a>
<div class="navbar-dropdown">
<a class="navbar-item" :class="{ 'is-selected': 'auto' === selectedTheme }" @click="selectTheme('auto')">
<span class="icon"><i class="fas fa-sync" /></span>
<span>Auto</span>
</a>
<hr class="navbar-divider">
<a class="navbar-item" :class="{ 'is-selected': 'light' === selectedTheme }" @click="selectTheme('light')">
<span class="icon"><i class="fas fa-sun" /></span>
<span>Light</span>
</a>
<a class="navbar-item" :class="{ 'is-selected': 'dark' === selectedTheme }" @click="selectTheme('dark')">
<span class="icon"><i class="fas fa-moon" /></span>
<span>Dark</span>
</a>
</div>
</div>
<div class="navbar-item" v-if="!config.app.basic_mode"> <div class="navbar-item" v-if="!config.app.basic_mode">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/console" v-tooltip.bottom="'Terminal'"> <NuxtLink class="button is-dark has-tooltip-bottom" to="/console" v-tooltip.bottom="'Terminal'">
<span class="icon"><i class="fa-solid fa-terminal" /></span> <span class="icon"><i class="fa-solid fa-terminal" /></span>
</NuxtLink> </NuxtLink>
</div> </div>
<div class="navbar-item" v-if="!config.app.basic_mode" v-tooltip.bottom="'Tasks'"> <div class="navbar-item" v-if="!config.app.basic_mode" v-tooltip.bottom="'Tasks'">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/tasks"> <NuxtLink class="button is-dark has-tooltip-bottom" to="/tasks">
<span class="icon"><i class="fa-solid fa-tasks" /></span> <span class="icon"><i class="fa-solid fa-tasks" /></span>
@ -30,17 +59,6 @@
</NuxtLink> </NuxtLink>
</div> </div>
<div class="navbar-item">
<button class="button is-dark" @click="selectedTheme = 'light'" v-if="'dark' === selectedTheme"
v-tooltip="'Switch to light theme'">
<span class="icon has-text-warning"><i class="fas fa-sun"></i></span>
</button>
<button class="button is-dark" @click="selectedTheme = 'dark'" v-if="'light' === selectedTheme"
v-tooltip="'Switch to dark theme'">
<span class="icon"><i class="fas fa-moon"></i></span>
</button>
</div>
<div class="navbar-item"> <div class="navbar-item">
<button class="button is-dark" @click="reloadPage"> <button class="button is-dark" @click="reloadPage">
<span class="icon"><i class="fas fa-refresh"></i></span> <span class="icon"><i class="fas fa-refresh"></i></span>
@ -80,11 +98,17 @@ import { useStorage } from '@vueuse/core'
import moment from 'moment' import moment from 'moment'
const Year = new Date().getFullYear() const Year = new Date().getFullYear()
const selectedTheme = useStorage('theme', (() => window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light')()) const selectedTheme = useStorage('theme', 'auto')
const socket = useSocketStore() const socket = useSocketStore()
const config = useConfigStore() const config = useConfigStore()
const theme_open = ref(false)
const theme_inner_click = ref(false)
const applyPreferredColorScheme = scheme => { const applyPreferredColorScheme = scheme => {
if (!scheme || 'auto' === scheme) {
return
}
for (let s = 0; s < document.styleSheets.length; s++) { for (let s = 0; s < document.styleSheets.length; s++) {
for (let i = 0; i < document.styleSheets[s].cssRules.length; i++) { for (let i = 0; i < document.styleSheets[s].cssRules.length; i++) {
try { try {
@ -140,4 +164,24 @@ watch(selectedTheme, value => {
const reloadPage = () => window.location.reload() const reloadPage = () => window.location.reload()
const selectTheme = theme => {
theme_inner_click.value = true
theme_open.value = false
console.log('selectedTheme', selectedTheme.value, theme_open.value)
if (theme === selectedTheme.value) {
return
}
selectedTheme.value = theme
if ('auto' === theme) {
return reloadPage()
}
}
const openCloseTheme = () => {
if (theme_inner_click.value) {
theme_inner_click.value = false
return
}
theme_open.value = !theme_open.value
}
</script> </script>

19
ui/pages/index.vue
View file

@ -1,3 +1,18 @@
<style>
.navbar-dropdown {
display: none;
}
.navbar-item.is-active .navbar-dropdown,
.navbar-item.is-hoverable:focus .navbar-dropdown,
.navbar-item.is-hoverable:focus-within .navbar-dropdown,
.navbar-item.is-hoverable:hover .navbar-dropdown {
display: block;
}
.navbar-item.has-dropdown{
padding: 0.5rem 0.75rem;
}
</style>
<template> <template>
<div> <div>
<div class="mt-1 columns is-multiline"> <div class="mt-1 columns is-multiline">
@ -48,9 +63,9 @@
</div> </div>
<NewDownload v-if="config.showForm || config.app.basic_mode" @getInfo="url => get_info = url" /> <NewDownload v-if="config.showForm || config.app.basic_mode" @getInfo="url => get_info = url" />
<Queue @getInfo="url => get_info = url" />
<History @getInfo="url => get_info = url" />
<GetInfo v-if="get_info" :link="get_info" @closeModel="get_info = ''" /> <GetInfo v-if="get_info" :link="get_info" @closeModel="get_info = ''" />
<Queue @getInfo="url => emitter('getInfo', url)" />
<History @getInfo="url => emitter('getInfo', url)" />
</div> </div>
</template> </template>