WIP: more changes to support webview in the future.
This commit is contained in:
arabcoders
parent
fee8939c80
commit
1f1fbbac01
8 changed files with 120 additions and 68 deletions
|
|
@ -1,5 +1,6 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import formatdate
|
||||
|
|
@ -8,7 +9,9 @@ from sqlite3 import Connection
|
|||
from .config import Config
|
||||
from .Download import Download
|
||||
from .ItemDTO import ItemDTO
|
||||
from .Utils import clean_item
|
||||
from .Utils import init_class
|
||||
|
||||
LOG = logging.getLogger("datastore")
|
||||
|
||||
|
||||
class DataStore:
|
||||
|
|
@ -69,9 +72,9 @@ class DataStore:
|
|||
|
||||
for row in cursor:
|
||||
rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
|
||||
data, _ = clean_item(json.loads(row["data"]), keys=ItemDTO.removed_fields())
|
||||
data: dict = json.loads(row["data"])
|
||||
data.pop("_id", None)
|
||||
item: ItemDTO = ItemDTO(**data)
|
||||
item: ItemDTO = init_class(ItemDTO, data)
|
||||
item._id = row["id"]
|
||||
item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
|
||||
items.append((row["id"], item))
|
||||
|
|
@ -147,10 +150,4 @@ class DataStore:
|
|||
)
|
||||
|
||||
def _delete_store_item(self, key: str) -> None:
|
||||
self.connection.execute(
|
||||
'DELETE FROM "history" WHERE "type" = ? AND "id" = ?',
|
||||
(
|
||||
self.type,
|
||||
key,
|
||||
),
|
||||
)
|
||||
self.connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (self.type, key))
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ import logging
|
|||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import formatdate
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
|
||||
|
|
@ -16,7 +16,7 @@ from .config import Config
|
|||
from .Events import EventBus, Events
|
||||
from .ffprobe import ffprobe
|
||||
from .ItemDTO import ItemDTO
|
||||
from .Utils import extract_info, extract_ytdlp_logs, load_cookies
|
||||
from .Utils import delete_dir, extract_info, extract_ytdlp_logs, load_cookies
|
||||
from .YTDLPOpts import YTDLPOpts
|
||||
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ class Download:
|
|||
return
|
||||
|
||||
if "__finaldir" in data["info_dict"]:
|
||||
filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"]))
|
||||
filename = str(Path(data["info_dict"]["__finaldir"]) / Path(data["info_dict"]["filepath"]).name)
|
||||
else:
|
||||
filename = data["info_dict"]["filepath"]
|
||||
|
||||
|
|
@ -160,8 +160,8 @@ class Download:
|
|||
config={
|
||||
"color": "no_color",
|
||||
"paths": {
|
||||
"home": self.download_dir,
|
||||
"temp": self.temp_path,
|
||||
"home": str(self.download_dir),
|
||||
"temp": str(self.temp_path),
|
||||
},
|
||||
"outtmpl": {
|
||||
"default": self.template,
|
||||
|
|
@ -189,13 +189,12 @@ class Download:
|
|||
|
||||
if self.info.cookies:
|
||||
try:
|
||||
cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt")
|
||||
cookie_file = Path(self.temp_path) / f"cookie_{self.info._id}.txt"
|
||||
self.logger.debug(
|
||||
f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
|
||||
)
|
||||
with open(cookie_file, "w") as f:
|
||||
f.write(self.info.cookies)
|
||||
params["cookiefile"] = f.name
|
||||
cookie_file.write_text(self.info.cookies)
|
||||
params["cookiefile"] = str(cookie_file.as_posix())
|
||||
|
||||
load_cookies(cookie_file)
|
||||
except Exception as e:
|
||||
|
|
@ -280,10 +279,12 @@ class Download:
|
|||
self.status_queue = Config.get_manager().Queue()
|
||||
|
||||
# Create temp dir for each download.
|
||||
self.temp_path = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5))
|
||||
self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5)
|
||||
|
||||
if not os.path.exists(self.temp_path):
|
||||
os.makedirs(self.temp_path, exist_ok=True)
|
||||
if not self.temp_path.exists():
|
||||
self.temp_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.info.temp_path = str(self.temp_path)
|
||||
|
||||
self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download)
|
||||
self.proc.start()
|
||||
|
|
@ -385,17 +386,19 @@ class Download:
|
|||
)
|
||||
return
|
||||
|
||||
if not os.path.exists(self.temp_path):
|
||||
tmp_dir = Path(self.temp_path)
|
||||
|
||||
if not tmp_dir.exists():
|
||||
return
|
||||
|
||||
if self.temp_path == self.temp_dir:
|
||||
if str(tmp_dir) == str(self.temp_dir):
|
||||
self.logger.warning(
|
||||
f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder."
|
||||
f"Attempted to delete video temp folder '{self.temp_path}', but it is the same as main temp folder."
|
||||
)
|
||||
return
|
||||
|
||||
self.logger.info(f"Deleting Temp folder '{self.temp_path}'.")
|
||||
shutil.rmtree(self.temp_path, ignore_errors=True)
|
||||
status = delete_dir(tmp_dir)
|
||||
self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.")
|
||||
|
||||
async def progress_update(self):
|
||||
"""
|
||||
|
|
@ -424,11 +427,15 @@ class Download:
|
|||
self.tmpfilename = status.get("tmpfilename")
|
||||
|
||||
if "filename" in status:
|
||||
self.info.filename = os.path.relpath(status.get("filename"), self.download_dir)
|
||||
fl = Path(status.get("filename"))
|
||||
try:
|
||||
self.info.filename = str(Path(status.get("filename")).relative_to(Path(self.download_dir)))
|
||||
except ValueError:
|
||||
self.info.filename = str(Path(status.get("filename")).relative_to(Path(self.temp_path)))
|
||||
|
||||
if os.path.exists(status.get("filename")):
|
||||
if fl.is_file() and fl.exists():
|
||||
try:
|
||||
self.info.file_size = os.path.getsize(status.get("filename"))
|
||||
self.info.file_size = fl.stat().st_size
|
||||
except FileNotFoundError:
|
||||
self.info.file_size = 0
|
||||
|
||||
|
|
@ -439,7 +446,8 @@ class Download:
|
|||
self.info.error = status.get("error")
|
||||
await self._notify.emit(Events.ERROR, data={"message": self.info.error, "data": self.info})
|
||||
|
||||
if "downloaded_bytes" in status:
|
||||
if "downloaded_bytes" in status and status.get("downloaded_bytes") > 0:
|
||||
self.info.downloaded_bytes = status.get("downloaded_bytes")
|
||||
total = status.get("total_bytes") or status.get("total_bytes_estimate")
|
||||
if total:
|
||||
try:
|
||||
|
|
@ -451,17 +459,11 @@ class Download:
|
|||
self.info.speed = status.get("speed")
|
||||
self.info.eta = status.get("eta")
|
||||
|
||||
if (
|
||||
"finished" == self.info.status
|
||||
and "filename" in status
|
||||
and os.path.isfile(status.get("filename"))
|
||||
and os.path.exists(status.get("filename"))
|
||||
):
|
||||
try:
|
||||
self.info.file_size = os.path.getsize(status.get("filename"))
|
||||
self.info.datetime = str(formatdate(time.time()))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
fl = Path(status.get("filename")) if status and "filename" in status else None
|
||||
|
||||
if "finished" == self.info.status and fl and fl.is_file() and fl.exists():
|
||||
self.info.file_size = fl.stat().st_size
|
||||
self.info.datetime = str(formatdate(time.time()))
|
||||
|
||||
try:
|
||||
ff = await ffprobe(status.get("filename"))
|
||||
|
|
@ -471,7 +473,7 @@ class Download:
|
|||
self.info.extras["is_video"] = True
|
||||
self.info.extras["is_audio"] = True
|
||||
self.logger.exception(e)
|
||||
self.logger.error(f"Failed to ffprobe: {status.get}. {e}")
|
||||
self.logger.error(f"Failed to run ffprobe. {status.get}. {e}")
|
||||
|
||||
await self._notify.emit(Events.UPDATED, data=self.info)
|
||||
|
||||
|
|
|
|||
|
|
@ -247,4 +247,5 @@ class ItemDTO:
|
|||
"output_template",
|
||||
"output_template_chapter",
|
||||
"config",
|
||||
"temp_path"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1131,3 +1131,47 @@ def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None)
|
|||
matched.extend(line for line in logs if line and any(p.search(line) for p in compiled))
|
||||
|
||||
return list(dict.fromkeys(matched))
|
||||
|
||||
|
||||
def delete_dir(dir: pathlib.Path) -> bool:
|
||||
"""
|
||||
Delete a directory and all its contents.
|
||||
|
||||
Args:
|
||||
dir (Path): The directory to delete.
|
||||
|
||||
Returns:
|
||||
bool: True if the directory was deleted successfully, False otherwise.
|
||||
|
||||
"""
|
||||
if not dir or not dir.exists() or not dir.is_dir():
|
||||
return False
|
||||
|
||||
try:
|
||||
for item in dir.iterdir():
|
||||
if item.is_dir():
|
||||
delete_dir(item)
|
||||
else:
|
||||
item.unlink()
|
||||
dir.rmdir()
|
||||
return True
|
||||
except Exception as e:
|
||||
LOG.error(f"Failed to delete directory '{dir}': {e}")
|
||||
return False
|
||||
|
||||
|
||||
def init_class(cls: type, data: dict):
|
||||
"""
|
||||
Initialize a class instance with data from a dictionary, filtering out keys not present in the class fields.
|
||||
|
||||
Args:
|
||||
cls (type): The class to initialize.
|
||||
data (dict): The data to use for initialization.
|
||||
|
||||
Returns:
|
||||
object: An instance of the class initialized with the provided data.
|
||||
|
||||
"""
|
||||
from dataclasses import fields
|
||||
|
||||
return cls(**{k: v for k, v in data.items() if k in {f.name for f in fields(cls)}})
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from aiohttp import web
|
||||
|
|
@ -13,7 +13,7 @@ from .Events import EventBus, Events
|
|||
from .Singleton import Singleton
|
||||
from .Utils import arg_converter
|
||||
|
||||
LOG = logging.getLogger("presets")
|
||||
LOG = logging.getLogger("conditions")
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
|
|
@ -51,16 +51,16 @@ class Conditions(metaclass=Singleton):
|
|||
_instance = None
|
||||
"""The instance of the class."""
|
||||
|
||||
def __init__(self, file: str | None = None, config: Config | None = None):
|
||||
def __init__(self, file: Path | str | None = None, config: Config | None = None):
|
||||
Conditions._instance = self
|
||||
|
||||
config = config or Config.get_instance()
|
||||
|
||||
self._file: str = file or os.path.join(config.config_path, "conditions.json")
|
||||
self._file: Path = Path(file) if file else Path(config.config_path) / "conditions.json"
|
||||
|
||||
if os.path.exists(self._file) and "600" != oct(os.stat(self._file).st_mode)[-3:]:
|
||||
if self._file.exists() and "600" != self._file.stat().st_mode:
|
||||
try:
|
||||
os.chmod(self._file, 0o600)
|
||||
self._file.chmod(0o600)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ class Conditions(metaclass=Singleton):
|
|||
msg = "Not implemented"
|
||||
raise Exception(msg)
|
||||
|
||||
EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.add")
|
||||
EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.save")
|
||||
|
||||
@staticmethod
|
||||
def get_instance() -> "Conditions":
|
||||
|
|
@ -114,7 +114,7 @@ class Conditions(metaclass=Singleton):
|
|||
"""
|
||||
self.clear()
|
||||
|
||||
if not os.path.exists(self._file) or os.path.getsize(self._file) < 10:
|
||||
if not self._file.exists() or self._file.stat().st_size < 1:
|
||||
return self
|
||||
|
||||
LOG.info(f"Loading '{self._file}'.")
|
||||
|
|
|
|||
27
app/main.py
27
app/main.py
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
|
@ -34,7 +33,7 @@ class Main:
|
|||
|
||||
self._check_folders()
|
||||
|
||||
caribou.upgrade(self._config.db_file, os.path.join(ROOT_PATH, "migrations"))
|
||||
caribou.upgrade(self._config.db_file, ROOT_PATH / "migrations")
|
||||
|
||||
connection = sqlite3.connect(database=self._config.db_file, isolation_level=None)
|
||||
connection.row_factory = sqlite3.Row
|
||||
|
|
@ -46,8 +45,9 @@ class Main:
|
|||
LOG.debug("Database connection closed.")
|
||||
|
||||
try:
|
||||
if "600" != oct(os.stat(self._config.db_file).st_mode)[-3:]:
|
||||
os.chmod(self._config.db_file, 0o600)
|
||||
db_file = Path(self._config.db_file)
|
||||
if "600" != oct(db_file.stat().st_mode)[-3:]:
|
||||
db_file.chmod(0o600)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -62,23 +62,24 @@ class Main:
|
|||
folders = (self._config.download_path, self._config.temp_path, self._config.config_path)
|
||||
|
||||
for folder in folders:
|
||||
folder = Path(folder)
|
||||
try:
|
||||
LOG.debug(f"Checking folder at '{folder}'.")
|
||||
if not os.path.exists(folder):
|
||||
if not folder.exists():
|
||||
LOG.info(f"Creating folder at '{folder}'.")
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
except OSError:
|
||||
LOG.error(f"Could not create folder at '{folder}'.")
|
||||
raise
|
||||
|
||||
try:
|
||||
LOG.debug(f"Checking database file at '{self._config.db_file}'.")
|
||||
if not os.path.exists(self._config.db_file):
|
||||
LOG.info(f"Creating database file at '{self._config.db_file}'.")
|
||||
with open(self._config.db_file, "w") as _:
|
||||
pass
|
||||
except OSError:
|
||||
LOG.error(f"Could not create database file at '{self._config.db_file}'.")
|
||||
db_file = Path(self._config.db_file)
|
||||
LOG.debug(f"Checking database file at '{db_file}'.")
|
||||
if not db_file.exists():
|
||||
LOG.info(f"Creating database file at '{db_file}'.")
|
||||
db_file.touch(exist_ok=True)
|
||||
except OSError as e:
|
||||
LOG.error(f"Could not create database file at '{self._config.db_file}'. {e!s}")
|
||||
raise
|
||||
|
||||
def start(self, host: str | None = None, port: int | None = None, cb=None):
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ if [ ! -w "${YTP_CONFIG_PATH}" ]; then
|
|||
CH_GRP=$(stat -c "%g" "${YTP_CONFIG_PATH}")
|
||||
echo_err "ERROR: Unable to write to '${YTP_CONFIG_PATH}' data directory. Current user id '${UID}' while directory owner is '${CH_USER}'."
|
||||
echo_err "[Running under docker]"
|
||||
echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
|
||||
echo_err "change compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
|
||||
echo_err "Run the following command to change the directory ownership"
|
||||
echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config"
|
||||
echo_err "[Running under podman]"
|
||||
echo_err "change docker-compose.yaml user: to user:\"0:0\""
|
||||
echo_err "change compose.yaml user: to user:\"0:0\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -24,11 +24,11 @@ if [ ! -w "${YTP_DOWNLOAD_PATH}" ]; then
|
|||
CH_GRP=$(stat -c "%g" "${YTP_DOWNLOAD_PATH}")
|
||||
echo_err "ERROR: Unable to write to '${YTP_DOWNLOAD_PATH}' downloads directory. Current user id '${UID}' while directory owner is '${CH_USER}'."
|
||||
echo_err "[Running under docker]"
|
||||
echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
|
||||
echo_err "change compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
|
||||
echo_err "Run the following command to change the directory ownership"
|
||||
echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config"
|
||||
echo_err "[Running under podman]"
|
||||
echo_err "change docker-compose.yaml user: to user:\"0:0\""
|
||||
echo_err "change compose.yaml user: to user:\"0:0\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,9 @@
|
|||
</label>
|
||||
</td>
|
||||
<td class="is-text-overflow is-vcentered">
|
||||
<div class="is-inline is-pulled-right" v-if="item.downloaded_bytes">
|
||||
<span class="tag">{{ formatBytes(item.downloaded_bytes) }}</span>
|
||||
</div>
|
||||
<div v-if="showThumbnails && item.extras?.thumbnail">
|
||||
<FloatingImage :image="uri('/api/thumbnail?url=' + encodePath(item.extras.thumbnail))"
|
||||
:title="item.title">
|
||||
|
|
@ -180,6 +183,10 @@
|
|||
<span v-tooltip="moment(item.datetime).format('MMMM Do YYYY, h:mm:ss a')" :data-datetime="item.datetime"
|
||||
v-rtime="item.datetime" />
|
||||
</div>
|
||||
<div class="column is-half-mobile has-text-centered is-text-overflow is-unselectable"
|
||||
v-if="item.downloaded_bytes">
|
||||
{{ formatBytes(item.downloaded_bytes) }}
|
||||
</div>
|
||||
<div class="column is-half-mobile has-text-centered is-text-overflow is-unselectable">
|
||||
<label class="checkbox is-block">
|
||||
<input class="completed-checkbox" type="checkbox" v-model="selectedElms" :id="'checkbox-' + item._id"
|
||||
|
|
|
|||
Loading…
Reference in a new issue