diff --git a/app/library/DataStore.py b/app/library/DataStore.py index eb92e4a5..cce7766a 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -1,5 +1,6 @@ import copy import json +import logging from collections import OrderedDict from datetime import UTC, datetime from email.utils import formatdate @@ -8,7 +9,9 @@ from sqlite3 import Connection from .config import Config from .Download import Download from .ItemDTO import ItemDTO -from .Utils import clean_item +from .Utils import init_class + +LOG = logging.getLogger("datastore") class DataStore: @@ -69,9 +72,9 @@ class DataStore: for row in cursor: rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data, _ = clean_item(json.loads(row["data"]), keys=ItemDTO.removed_fields()) + data: dict = json.loads(row["data"]) data.pop("_id", None) - item: ItemDTO = ItemDTO(**data) + item: ItemDTO = init_class(ItemDTO, data) item._id = row["id"] item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp()) items.append((row["id"], item)) @@ -147,10 +150,4 @@ class DataStore: ) def _delete_store_item(self, key: str) -> None: - self.connection.execute( - 'DELETE FROM "history" WHERE "type" = ? AND "id" = ?', - ( - self.type, - key, - ), - ) + self.connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (self.type, key)) diff --git a/app/library/Download.py b/app/library/Download.py index 7bfaf4b4..71da56df 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -4,10 +4,10 @@ import logging import multiprocessing import os import re -import shutil import time from datetime import UTC, datetime from email.utils import formatdate +from pathlib import Path import yt_dlp @@ -16,7 +16,7 @@ from .config import Config from .Events import EventBus, Events from .ffprobe import ffprobe from .ItemDTO import ItemDTO -from .Utils import extract_info, extract_ytdlp_logs, load_cookies +from .Utils import delete_dir, extract_info, extract_ytdlp_logs, load_cookies from .YTDLPOpts import YTDLPOpts @@ -137,7 +137,7 @@ class Download: return if "__finaldir" in data["info_dict"]: - filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"])) + filename = str(Path(data["info_dict"]["__finaldir"]) / Path(data["info_dict"]["filepath"]).name) else: filename = data["info_dict"]["filepath"] @@ -160,8 +160,8 @@ class Download: config={ "color": "no_color", "paths": { - "home": self.download_dir, - "temp": self.temp_path, + "home": str(self.download_dir), + "temp": str(self.temp_path), }, "outtmpl": { "default": self.template, @@ -189,13 +189,12 @@ class Download: if self.info.cookies: try: - cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt") + cookie_file = Path(self.temp_path) / f"cookie_{self.info._id}.txt" self.logger.debug( f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'." ) - with open(cookie_file, "w") as f: - f.write(self.info.cookies) - params["cookiefile"] = f.name + cookie_file.write_text(self.info.cookies) + params["cookiefile"] = str(cookie_file.as_posix()) load_cookies(cookie_file) except Exception as e: @@ -280,10 +279,12 @@ class Download: self.status_queue = Config.get_manager().Queue() # Create temp dir for each download. - self.temp_path = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5)) + self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5) - if not os.path.exists(self.temp_path): - os.makedirs(self.temp_path, exist_ok=True) + if not self.temp_path.exists(): + self.temp_path.mkdir(parents=True, exist_ok=True) + + self.info.temp_path = str(self.temp_path) self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download) self.proc.start() @@ -385,17 +386,19 @@ class Download: ) return - if not os.path.exists(self.temp_path): + tmp_dir = Path(self.temp_path) + + if not tmp_dir.exists(): return - if self.temp_path == self.temp_dir: + if str(tmp_dir) == str(self.temp_dir): self.logger.warning( - f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder." + f"Attempted to delete video temp folder '{self.temp_path}', but it is the same as main temp folder." ) return - self.logger.info(f"Deleting Temp folder '{self.temp_path}'.") - shutil.rmtree(self.temp_path, ignore_errors=True) + status = delete_dir(tmp_dir) + self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.") async def progress_update(self): """ @@ -424,11 +427,15 @@ class Download: self.tmpfilename = status.get("tmpfilename") if "filename" in status: - self.info.filename = os.path.relpath(status.get("filename"), self.download_dir) + fl = Path(status.get("filename")) + try: + self.info.filename = str(Path(status.get("filename")).relative_to(Path(self.download_dir))) + except ValueError: + self.info.filename = str(Path(status.get("filename")).relative_to(Path(self.temp_path))) - if os.path.exists(status.get("filename")): + if fl.is_file() and fl.exists(): try: - self.info.file_size = os.path.getsize(status.get("filename")) + self.info.file_size = fl.stat().st_size except FileNotFoundError: self.info.file_size = 0 @@ -439,7 +446,8 @@ class Download: self.info.error = status.get("error") await self._notify.emit(Events.ERROR, data={"message": self.info.error, "data": self.info}) - if "downloaded_bytes" in status: + if "downloaded_bytes" in status and status.get("downloaded_bytes") > 0: + self.info.downloaded_bytes = status.get("downloaded_bytes") total = status.get("total_bytes") or status.get("total_bytes_estimate") if total: try: @@ -451,17 +459,11 @@ class Download: self.info.speed = status.get("speed") self.info.eta = status.get("eta") - if ( - "finished" == self.info.status - and "filename" in status - and os.path.isfile(status.get("filename")) - and os.path.exists(status.get("filename")) - ): - try: - self.info.file_size = os.path.getsize(status.get("filename")) - self.info.datetime = str(formatdate(time.time())) - except FileNotFoundError: - pass + fl = Path(status.get("filename")) if status and "filename" in status else None + + if "finished" == self.info.status and fl and fl.is_file() and fl.exists(): + self.info.file_size = fl.stat().st_size + self.info.datetime = str(formatdate(time.time())) try: ff = await ffprobe(status.get("filename")) @@ -471,7 +473,7 @@ class Download: self.info.extras["is_video"] = True self.info.extras["is_audio"] = True self.logger.exception(e) - self.logger.error(f"Failed to ffprobe: {status.get}. {e}") + self.logger.error(f"Failed to run ffprobe. {status.get}. {e}") await self._notify.emit(Events.UPDATED, data=self.info) diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index 19436886..68c0e804 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -247,4 +247,5 @@ class ItemDTO: "output_template", "output_template_chapter", "config", + "temp_path" ) diff --git a/app/library/Utils.py b/app/library/Utils.py index d1d29296..8c7e4a55 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -1131,3 +1131,47 @@ def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) matched.extend(line for line in logs if line and any(p.search(line) for p in compiled)) return list(dict.fromkeys(matched)) + + +def delete_dir(dir: pathlib.Path) -> bool: + """ + Delete a directory and all its contents. + + Args: + dir (Path): The directory to delete. + + Returns: + bool: True if the directory was deleted successfully, False otherwise. + + """ + if not dir or not dir.exists() or not dir.is_dir(): + return False + + try: + for item in dir.iterdir(): + if item.is_dir(): + delete_dir(item) + else: + item.unlink() + dir.rmdir() + return True + except Exception as e: + LOG.error(f"Failed to delete directory '{dir}': {e}") + return False + + +def init_class(cls: type, data: dict): + """ + Initialize a class instance with data from a dictionary, filtering out keys not present in the class fields. + + Args: + cls (type): The class to initialize. + data (dict): The data to use for initialization. + + Returns: + object: An instance of the class initialized with the provided data. + + """ + from dataclasses import fields + + return cls(**{k: v for k, v in data.items() if k in {f.name for f in fields(cls)}}) diff --git a/app/library/conditions.py b/app/library/conditions.py index eff44d6f..7b3a2909 100644 --- a/app/library/conditions.py +++ b/app/library/conditions.py @@ -1,8 +1,8 @@ import json import logging -import os import uuid from dataclasses import dataclass, field +from pathlib import Path from typing import Any from aiohttp import web @@ -13,7 +13,7 @@ from .Events import EventBus, Events from .Singleton import Singleton from .Utils import arg_converter -LOG = logging.getLogger("presets") +LOG = logging.getLogger("conditions") @dataclass(kw_only=True) @@ -51,16 +51,16 @@ class Conditions(metaclass=Singleton): _instance = None """The instance of the class.""" - def __init__(self, file: str | None = None, config: Config | None = None): + def __init__(self, file: Path | str | None = None, config: Config | None = None): Conditions._instance = self config = config or Config.get_instance() - self._file: str = file or os.path.join(config.config_path, "conditions.json") + self._file: Path = Path(file) if file else Path(config.config_path) / "conditions.json" - if os.path.exists(self._file) and "600" != oct(os.stat(self._file).st_mode)[-3:]: + if self._file.exists() and "600" != self._file.stat().st_mode: try: - os.chmod(self._file, 0o600) + self._file.chmod(0o600) except Exception: pass @@ -68,7 +68,7 @@ class Conditions(metaclass=Singleton): msg = "Not implemented" raise Exception(msg) - EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.add") + EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.save") @staticmethod def get_instance() -> "Conditions": @@ -114,7 +114,7 @@ class Conditions(metaclass=Singleton): """ self.clear() - if not os.path.exists(self._file) or os.path.getsize(self._file) < 10: + if not self._file.exists() or self._file.stat().st_size < 1: return self LOG.info(f"Loading '{self._file}'.") diff --git a/app/main.py b/app/main.py index 6253cfc9..f184e6cc 100644 --- a/app/main.py +++ b/app/main.py @@ -2,7 +2,6 @@ import asyncio import logging -import os import sqlite3 import sys from pathlib import Path @@ -34,7 +33,7 @@ class Main: self._check_folders() - caribou.upgrade(self._config.db_file, os.path.join(ROOT_PATH, "migrations")) + caribou.upgrade(self._config.db_file, ROOT_PATH / "migrations") connection = sqlite3.connect(database=self._config.db_file, isolation_level=None) connection.row_factory = sqlite3.Row @@ -46,8 +45,9 @@ class Main: LOG.debug("Database connection closed.") try: - if "600" != oct(os.stat(self._config.db_file).st_mode)[-3:]: - os.chmod(self._config.db_file, 0o600) + db_file = Path(self._config.db_file) + if "600" != oct(db_file.stat().st_mode)[-3:]: + db_file.chmod(0o600) except Exception: pass @@ -62,23 +62,24 @@ class Main: folders = (self._config.download_path, self._config.temp_path, self._config.config_path) for folder in folders: + folder = Path(folder) try: LOG.debug(f"Checking folder at '{folder}'.") - if not os.path.exists(folder): + if not folder.exists(): LOG.info(f"Creating folder at '{folder}'.") - os.makedirs(folder, exist_ok=True) + folder.mkdir(parents=True, exist_ok=True) except OSError: LOG.error(f"Could not create folder at '{folder}'.") raise try: - LOG.debug(f"Checking database file at '{self._config.db_file}'.") - if not os.path.exists(self._config.db_file): - LOG.info(f"Creating database file at '{self._config.db_file}'.") - with open(self._config.db_file, "w") as _: - pass - except OSError: - LOG.error(f"Could not create database file at '{self._config.db_file}'.") + db_file = Path(self._config.db_file) + LOG.debug(f"Checking database file at '{db_file}'.") + if not db_file.exists(): + LOG.info(f"Creating database file at '{db_file}'.") + db_file.touch(exist_ok=True) + except OSError as e: + LOG.error(f"Could not create database file at '{self._config.db_file}'. {e!s}") raise def start(self, host: str | None = None, port: int | None = None, cb=None): diff --git a/entrypoint.sh b/entrypoint.sh index 9a341cf9..85246d8d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,11 +11,11 @@ if [ ! -w "${YTP_CONFIG_PATH}" ]; then CH_GRP=$(stat -c "%g" "${YTP_CONFIG_PATH}") echo_err "ERROR: Unable to write to '${YTP_CONFIG_PATH}' data directory. Current user id '${UID}' while directory owner is '${CH_USER}'." echo_err "[Running under docker]" - echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\"" + echo_err "change compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\"" echo_err "Run the following command to change the directory ownership" echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config" echo_err "[Running under podman]" - echo_err "change docker-compose.yaml user: to user:\"0:0\"" + echo_err "change compose.yaml user: to user:\"0:0\"" exit 1 fi @@ -24,11 +24,11 @@ if [ ! -w "${YTP_DOWNLOAD_PATH}" ]; then CH_GRP=$(stat -c "%g" "${YTP_DOWNLOAD_PATH}") echo_err "ERROR: Unable to write to '${YTP_DOWNLOAD_PATH}' downloads directory. Current user id '${UID}' while directory owner is '${CH_USER}'." echo_err "[Running under docker]" - echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\"" + echo_err "change compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\"" echo_err "Run the following command to change the directory ownership" echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config" echo_err "[Running under podman]" - echo_err "change docker-compose.yaml user: to user:\"0:0\"" + echo_err "change compose.yaml user: to user:\"0:0\"" exit 1 fi diff --git a/ui/components/Queue.vue b/ui/components/Queue.vue index eff156c8..9baaa87b 100644 --- a/ui/components/Queue.vue +++ b/ui/components/Queue.vue @@ -66,6 +66,9 @@