danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #297 from arabcoders/dev

Browse source 
Support running on windows
This commit is contained in:
Abdulmohsen 2025-06-10 13:44:17 +03:00 committed by GitHub
commit 670c99510e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 2784 additions and 2240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
.github/workflows/update-yt-dlp.yml vendored
View file

@ -23,26 +23,38 @@ jobs:
uses: actions/checkout@v4
with:
ref: "dev"
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Update yt-dlp
- name: Install uv
run: pip install uv
- name: Sync dependencies
run: uv venv && . .venv/bin/activate && uv sync
- name: Check and update yt-dlp
id: ytdlp_update
run: |
pip install pipenv
pipenv sync
VER=`pipenv run pip list -o | awk '$1 == "yt-dlp" {print $3}'`
. .venv/bin/activate
VER=$(uv pip list --outdated | awk '$1 == "yt-dlp" {print $3}')
if [ -n "$VER" ]; then
echo "YTLDLP_VER=${VER}" >> "$GITHUB_OUTPUT"
pipenv update yt-dlp
uv pip install --upgrade yt-dlp
uv sync --upgrade
UPDATED=true
else
UPDATED=false
fi
echo "UPDATED=${UPDATED}" >> "$GITHUB_OUTPUT"
- name: Create Pull Request
if: steps.ytdlp_update.outputs.UPDATED == 'true'
uses: peter-evans/create-pull-request@v5
with:
title: "[yt-dlp] automated update to ${{ steps.ytdlp_update.outputs.YTLDLP_VER }}"
commit-message: "Update yt-dlp to ${{ steps.ytdlp_update.outputs.YTLDLP_VER }}"
body: "This is an automated request to update yt-dlp dependency to ${{ steps.ytdlp_update.outputs.YTLDLP_VER }}"
delete-branch: true

2
.gitignore vendored
View file

@ -33,3 +33,5 @@ var/*
__pycache__
.venv
.idea
dist
build

19
.vscode/settings.json vendored
View file

@ -15,15 +15,19 @@
"ahas",
"ahash",
"aiocron",
"anyio",
"arrowless",
"attl",
"autonumber",
"brotlicffi",
"consoletitle",
"cookiesfrombrowser",
"copyts",
"cronsim",
"datas",
"daterange",
"dotenv",
"edgechromium",
"euuo",
"finaldir",
"flac",
@ -32,12 +36,17 @@
"fribidi",
"getpid",
"gpac",
"hiddenimports",
"hookspath",
"httpx",
"levelno",
"libcurl",
"libjavascriptcoregtk",
"libwebkit",
"libx",
"matroska",
"mbed",
"mccabe",
"Microformat",
"microformats",
"mkvtoolsnix",
@ -46,14 +55,24 @@
"muxdelay",
"nodesc",
"noprogress",
"onefile",
"pathex",
"platformdirs",
"plexmatch",
"postprocessor",
"preferredcodec",
"preferredquality",
"printtraffic",
"pycryptodome",
"pyinstaller",
"pypi",
"pyside",
"pywebview",
"qtpy",
"quicktime",
"rtime",
"smhd",
"socketio",
"timespec",
"tmpfilename",
"upgrader",

84
API.md
View file

@ -26,10 +26,6 @@ This document describes the available endpoints and their usage. All endpoints r
- [GET /api/history](#get-apihistory)
- [GET /api/tasks](#get-apitasks)
- [PUT /api/tasks](#put-apitasks)
- [GET /api/workers](#get-apiworkers)
- [POST /api/workers](#post-apiworkers)
- [PATCH /api/workers/{id}](#patch-apiworkersid)
- [DELETE /api/workers/{id}](#delete-apiworkersid)
- [GET /api/player/playlist/{file:.\*}.m3u8](#get-apiplayerplaylistfilem3u8)
- [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8)
- [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets)
@ -391,86 +387,6 @@ or on error
---
### GET /api/workers
**Purpose**: Returns the status of the worker pool and all workers.
**Response**:
```json
{
"open": true|false,
"count": 4,
"workers": [
{
"id": "worker-1",
"data": { "status": "downloading", ... }
},
{
"id": "worker-2",
"data": { "status": "Waiting for download." }
},
...
]
}
```
- `open`: Indicates if there are any available workers.
- `count`: Total number of available workers.
---
### POST /api/workers
**Purpose**: Restart the entire worker pool.
**Response**:
```json
{
"message": "Workers pool being restarted."
}
```
---
### PATCH /api/workers/{id}
**Purpose**: Restart a single worker by ID.
**Path Parameter**:
- `id` = The worker ID.
**Response**:
```json
{
"status": "restarted"
}
```
or
```json
{
"status": "in_error_state"
}
```
---
### DELETE /api/workers/{id}
**Purpose**: Stop a single worker by ID.
**Path Parameter**:
- `id` = The worker ID.
**Response**:
```json
{
"status": "stopped"
}
```
or
```json
{
"status": "in_error_state"
}
```
---
### GET /api/player/playlist/{file:.*}.m3u8
**Purpose**: Generate a playlist for a given local media file.

19
Dockerfile
View file

@ -13,16 +13,16 @@ ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONFAULTHANDLER=1
ENV PIP_NO_CACHE_DIR=off
ENV PIP_CACHE_DIR=/root/.cache/pip
ENV UV_CACHE_DIR=/root/.cache/pip
# Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows)
# Install dependencies
RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev curl make && pip install pipenv
RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev curl make && pip install uv
WORKDIR /app
WORKDIR /opt/
ARG PIPENV_FLAGS="--deploy"
COPY ./Pipfile* .
RUN --mount=type=cache,target=/root/.cache/pip PIPENV_VENV_IN_PROJECT=1 pipenv install ${PIPENV_FLAGS}
COPY ./pyproject.toml ./uv.lock ./
RUN --mount=type=cache,target=/root/.cache/pip uv venv --system-site-packages --relocatable ./python && \
VIRTUAL_ENV=/opt/python uv sync --link-mode=copy --active
FROM python:3.13-alpine
@ -51,16 +51,13 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
COPY --chown=app:app ./app /app/app
COPY --chown=app:app --from=node_builder /app/exported /app/ui/exported
COPY --chown=app:app --from=python_builder /app/.venv /opt/python
COPY --chown=app:app --from=python_builder /opt/python /opt/python
COPY --chown=app:app --from=ghcr.io/arabcoders/alpine-mp4box /usr/bin/mp4box /usr/bin/mp4box
COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck
ENV PATH="/opt/python/bin:$PATH"
RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck && \
sed -i 's$#!\/app\/\.venv\/bin\/python$#!/opt/python/bin/python$' /opt/python/bin/* && \
sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate* && \
chmod +x /usr/bin/mp4box
RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck /usr/bin/mp4box
VOLUME /config
VOLUME /downloads

31
Pipfile
View file

@ -1,31 +0,0 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
python-socketio = ">=5.11.1"
aiohttp = ">=3.9.3"
caribou = ">=0.3.0"
coloredlogs = ">=15.0.1"
aiocron = ">=1.8"
python-dotenv = ">=1.0.1"
python-magic = ">=0.4.27"
debugpy = ">=1.8.1"
httpx = "*"
async-timeout = "*"
pyjson5 = "*"
curl_cffi = "0.7.1"
pysubs2 = "*"
regex = "*"
mutagen = "*"
brotli = "*"
brotlicffi = "*"
anyio = "*"
pycryptodome = "*"
yt-dlp = "*"
[dev-packages]
[requires]
python_version = "*"

1374
Pipfile.lock generated
View file

File diff suppressed because it is too large Load diff

68
app.spec Normal file
View file

@ -0,0 +1,68 @@
import os
import platform
import sys
import tomllib
block_cipher = None
machine = platform.machine().lower()
binaries = []
if sys.platform.startswith("linux"):
if machine in ("x86_64", "amd64"):
libdir = "/usr/lib/x86_64-linux-gnu"
elif machine in ("aarch64", "arm64"):
libdir = "/usr/lib/aarch64-linux-gnu"
else:
libdir = "/usr/lib"
binaries = []
elif sys.platform == "darwin":
binaries = []
elif sys.platform.startswith("win"):
# Windows DLLs
binaries = []
with open("./uv.lock", "rb") as f:
lock = tomllib.load(f)
hidden = [
*lock.get("dependencies", {}).keys(),
"aiohttp",
"socketio",
"engineio",
"engineio.async_drivers.aiohttp",
"socketio.async_drivers.aiohttp",
]
hidden = [f.replace("-", "_") for f in hidden]
a = Analysis( # noqa: F821 # type: ignore
["app/native.py"],
pathex=[os.getcwd()],
binaries=binaries,
datas=[
("ui/exported", "ui/exported"),
("app/migrations", "migrations"),
("app/library/presets.json", "library"),
],
hiddenimports=hidden,
hookspath=[],
runtime_hooks=[],
excludes=[],
cipher=block_cipher,
)
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) # type: ignore # noqa: F821
exe = EXE( # type: ignore # noqa: F821
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name="YTPTube",
debug=False,
strip=False,
upx=True,
console=False,
onefile=True,
)

17
app/library/DataStore.py
View file

@ -1,5 +1,6 @@
import copy
import json
import logging
from collections import OrderedDict
from datetime import UTC, datetime
from email.utils import formatdate
@ -8,7 +9,9 @@ from sqlite3 import Connection
from .config import Config
from .Download import Download
from .ItemDTO import ItemDTO
from .Utils import clean_item
from .Utils import init_class
LOG = logging.getLogger("datastore")
class DataStore:
@ -69,9 +72,9 @@ class DataStore:
for row in cursor:
rowDate = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data, _ = clean_item(json.loads(row["data"]), keys=ItemDTO.removed_fields())
data: dict = json.loads(row["data"])
data.pop("_id", None)
item: ItemDTO = ItemDTO(**data)
item: ItemDTO = init_class(ItemDTO, data)
item._id = row["id"]
item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
items.append((row["id"], item))
@ -147,10 +150,4 @@ class DataStore:
)
def _delete_store_item(self, key: str) -> None:
self.connection.execute(
'DELETE FROM "history" WHERE "type" = ? AND "id" = ?',
(
self.type,
key,
),
)
self.connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (self.type, key))

72
app/library/Download.py
View file

@ -4,10 +4,10 @@ import logging
import multiprocessing
import os
import re
import shutil
import time
from datetime import UTC, datetime
from email.utils import formatdate
from pathlib import Path
import yt_dlp
@ -16,7 +16,7 @@ from .config import Config
from .Events import EventBus, Events
from .ffprobe import ffprobe
from .ItemDTO import ItemDTO
from .Utils import extract_info, extract_ytdlp_logs, load_cookies
from .Utils import delete_dir, extract_info, extract_ytdlp_logs, load_cookies
from .YTDLPOpts import YTDLPOpts
@ -137,7 +137,7 @@ class Download:
return
if "__finaldir" in data["info_dict"]:
filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"]))
filename = str(Path(data["info_dict"]["__finaldir"]) / Path(data["info_dict"]["filepath"]).name)
else:
filename = data["info_dict"]["filepath"]
@ -151,17 +151,17 @@ class Download:
def _download(self):
try:
params = (
params: dict = (
YTDLPOpts.get_instance()
.preset(self.preset, with_cookies=not self.info.cookies)
.preset(self.preset)
.add({"break_on_existing": True})
.add_cli(args=self.info.cli, from_user=True)
.add(
config={
"color": "no_color",
"paths": {
"home": self.download_dir,
"temp": self.temp_path,
"home": str(self.download_dir),
"temp": str(self.temp_path),
},
"outtmpl": {
"default": self.template,
@ -189,13 +189,12 @@ class Download:
if self.info.cookies:
try:
cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt")
cookie_file = Path(self.temp_path) / f"cookie_{self.info._id}.txt"
self.logger.debug(
f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
)
with open(cookie_file, "w") as f:
f.write(self.info.cookies)
params["cookiefile"] = f.name
cookie_file.write_text(self.info.cookies)
params["cookiefile"] = str(cookie_file.as_posix())
load_cookies(cookie_file)
except Exception as e:
@ -280,10 +279,12 @@ class Download:
self.status_queue = Config.get_manager().Queue()
# Create temp dir for each download.
self.temp_path = os.path.join(self.temp_dir, hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5))
self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5)
if not os.path.exists(self.temp_path):
os.makedirs(self.temp_path, exist_ok=True)
if not self.temp_path.exists():
self.temp_path.mkdir(parents=True, exist_ok=True)
self.info.temp_path = str(self.temp_path)
self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download)
self.proc.start()
@ -385,17 +386,19 @@ class Download:
)
return
if not os.path.exists(self.temp_path):
tmp_dir = Path(self.temp_path)
if not tmp_dir.exists():
return
if self.temp_path == self.temp_dir:
if str(tmp_dir) == str(self.temp_dir):
self.logger.warning(
f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder."
f"Attempted to delete video temp folder '{self.temp_path}', but it is the same as main temp folder."
)
return
self.logger.info(f"Deleting Temp folder '{self.temp_path}'.")
shutil.rmtree(self.temp_path, ignore_errors=True)
status = delete_dir(tmp_dir)
self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.")
async def progress_update(self):
"""
@ -424,11 +427,15 @@ class Download:
self.tmpfilename = status.get("tmpfilename")
if "filename" in status:
self.info.filename = os.path.relpath(status.get("filename"), self.download_dir)
fl = Path(status.get("filename"))
try:
self.info.filename = str(Path(status.get("filename")).relative_to(Path(self.download_dir)))
except ValueError:
self.info.filename = str(Path(status.get("filename")).relative_to(Path(self.temp_path)))
if os.path.exists(status.get("filename")):
if fl.is_file() and fl.exists():
try:
self.info.file_size = os.path.getsize(status.get("filename"))
self.info.file_size = fl.stat().st_size
except FileNotFoundError:
self.info.file_size = 0
@ -439,7 +446,8 @@ class Download:
self.info.error = status.get("error")
await self._notify.emit(Events.ERROR, data={"message": self.info.error, "data": self.info})
if "downloaded_bytes" in status:
if "downloaded_bytes" in status and status.get("downloaded_bytes") > 0:
self.info.downloaded_bytes = status.get("downloaded_bytes")
total = status.get("total_bytes") or status.get("total_bytes_estimate")
if total:
try:
@ -451,17 +459,11 @@ class Download:
self.info.speed = status.get("speed")
self.info.eta = status.get("eta")
if (
"finished" == self.info.status
and "filename" in status
and os.path.isfile(status.get("filename"))
and os.path.exists(status.get("filename"))
):
try:
self.info.file_size = os.path.getsize(status.get("filename"))
self.info.datetime = str(formatdate(time.time()))
except FileNotFoundError:
pass
fl = Path(status.get("filename")) if status and "filename" in status else None
if "finished" == self.info.status and fl and fl.is_file() and fl.exists():
self.info.file_size = fl.stat().st_size
self.info.datetime = str(formatdate(time.time()))
try:
ff = await ffprobe(status.get("filename"))
@ -471,7 +473,7 @@ class Download:
self.info.extras["is_video"] = True
self.info.extras["is_audio"] = True
self.logger.exception(e)
self.logger.error(f"Failed to ffprobe: {status.get}. {e}")
self.logger.error(f"Failed to run ffprobe. {status.get}. {e}")
await self._notify.emit(Events.UPDATED, data=self.info)

64
app/library/DownloadQueue.py
View file

@ -2,7 +2,6 @@ import asyncio
import functools
import glob
import logging
import os
import time
import uuid
from datetime import UTC, datetime, timedelta
@ -10,7 +9,6 @@ from email.utils import formatdate, parsedate_to_datetime
from pathlib import Path
from sqlite3 import Connection
import anyio
import yt_dlp
from aiohttp import web
@ -399,7 +397,7 @@ class DownloadQueue(metaclass=Singleton):
item.template = _preset.template
yt_conf = {}
cookie_file = os.path.join(self.config.temp_path, f"c_{uuid.uuid4().hex}.txt")
cookie_file = Path(self.config.temp_path) / f"c_{uuid.uuid4().hex}.txt"
LOG.info(f"Adding '{item.__repr__()}'.")
@ -420,10 +418,7 @@ class DownloadQueue(metaclass=Singleton):
"level": logging.WARNING,
"name": "callback-logger",
},
**YTDLPOpts.get_instance()
.preset(name=item.preset, with_cookies=not item.cookies)
.add_cli(args=item.cli, from_user=True)
.get_all(),
**YTDLPOpts.get_instance().preset(name=item.preset).add_cli(args=item.cli, from_user=True).get_all(),
}
if yt_conf.get("external_downloader"):
@ -441,10 +436,8 @@ class DownloadQueue(metaclass=Singleton):
if item.cookies:
try:
async with await anyio.open_file(cookie_file, "w") as f:
await f.write(item.cookies)
yt_conf["cookiefile"] = f.name
cookie_file.write_text(item.cookies)
yt_conf["cookiefile"] = str(cookie_file.as_posix())
load_cookies(cookie_file)
except Exception as e:
msg = f"Failed to create cookie file for '{item.url}'. '{e!s}'."
@ -493,10 +486,10 @@ class DownloadQueue(metaclass=Singleton):
"msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.",
}
finally:
if cookie_file and os.path.exists(cookie_file):
if cookie_file and cookie_file.exists():
try:
os.remove(cookie_file)
del yt_conf["cookiefile"]
cookie_file.unlink(missing_ok=True)
yt_conf.pop("cookiefile", None)
except Exception as e:
LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
@ -562,7 +555,7 @@ class DownloadQueue(metaclass=Singleton):
for id in ids:
try:
item = self.done.get(key=id)
item: Download = self.done.get(key=id)
except KeyError as e:
status[id] = str(e)
status["status"] = "error"
@ -573,24 +566,34 @@ class DownloadQueue(metaclass=Singleton):
removed_files = 0
filename: str = ""
LOG.info(
f"{remove_file=} {itemRef} - Removing local files: {self.config.remove_files}, {item.info.status=}"
)
if remove_file and self.config.remove_files and "finished" == item.info.status:
filename = str(item.info.filename)
if item.info.folder:
filename = f"{item.info.folder}/{item.info.filename}"
try:
realFile: str = calc_download_path(
base_path=self.config.download_path,
folder=filename,
create_path=False,
rf = Path(
calc_download_path(
base_path=Path(self.config.download_path),
folder=filename,
create_path=False,
)
)
rf = Path(realFile)
if rf.is_file() and rf.exists():
for f in rf.parent.glob(f"{glob.escape(rf.stem)}.*"):
if f.is_file() and f.exists() and not f.name.startswith("."):
removed_files += 1
LOG.debug(f"Removing '{itemRef}' local file '{f.name}'.")
os.remove(f)
if rf.stem and rf.suffix:
for f in rf.parent.glob(f"{glob.escape(rf.stem)}.*"):
if f.is_file() and f.exists() and not f.name.startswith("."):
removed_files += 1
LOG.debug(f"Removing '{itemRef}' local file '{f.name}'.")
f.unlink(missing_ok=True)
else:
LOG.debug(f"Removing '{itemRef}' local file '{rf.name}'.")
rf.unlink(missing_ok=True)
removed_files += 1
else:
LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.")
except Exception as e:
@ -701,22 +704,13 @@ class DownloadQueue(metaclass=Singleton):
"""
filePath = calc_download_path(base_path=self.config.download_path, folder=entry.info.folder)
LOG.info(
f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'Folder: {filePath}'."
)
LOG.info(f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' To '{filePath}'.")
try:
self._active[entry.info._id] = entry
await entry.start()
if "finished" != entry.info.status:
if entry.tmpfilename and os.path.isfile(entry.tmpfilename):
try:
os.remove(entry.tmpfilename)
entry.tmpfilename = None
except Exception:
pass
entry.info.status = "error"
finally:
if entry.info._id in self._active:

427
app/library/HttpAPI.py
View file

@ -4,7 +4,6 @@ import functools
import hmac
import json
import logging
import os
import random
import time
import uuid
@ -18,6 +17,7 @@ import httpx
import magic
from aiohttp import web
from aiohttp.web import Request, RequestHandler, Response
from library.ag_utils import ag
from yt_dlp.cookies import LenientSimpleCookie
from .cache import Cache
@ -47,6 +47,7 @@ from .Utils import (
get_file_sidecar,
get_files,
get_mime_type,
init_class,
read_logfile,
validate_url,
validate_uuid,
@ -85,6 +86,7 @@ class HttpAPI(Common):
def __init__(
self,
root_path: str,
queue: DownloadQueue | None = None,
encoder: Encoder | None = None,
config: Config | None = None,
@ -94,10 +96,11 @@ class HttpAPI(Common):
self.config = config or Config.get_instance()
self._notify = EventBus.get_instance()
self.rootPath = str(Path(__file__).parent.parent.parent)
self.rootPath = root_path
self.routes = web.RouteTableDef()
self.cache = Cache()
self.app: web.Application | None = None
self.isRequestingBackground = False
super().__init__(queue=self.queue, encoder=self.encoder, config=self.config)
@ -129,7 +132,7 @@ class HttpAPI(Common):
return decorator
async def on_shutdown(self, _: web.Application):
LOG.debug("Shutting down http API server.")
pass
def attach(self, app: web.Application) -> "HttpAPI":
"""
@ -225,43 +228,48 @@ class HttpAPI(Common):
HttpAPI: The instance of the HttpAPI.
"""
staticDir = os.path.join(self.rootPath, "ui", "exported")
if not os.path.exists(staticDir):
msg = f"Could not find the frontend UI static assets. '{staticDir}'."
raise ValueError(msg)
staticDir = (Path(self.rootPath) / "ui" / "exported").absolute()
if not staticDir.exists():
staticDir = (Path(self.rootPath).parent / "ui" / "exported").absolute()
if not staticDir.exists():
msg = f"Could not find the frontend UI static assets. '{staticDir}'."
raise ValueError(msg)
preloaded = 0
base_path: str = self.config.base_path.rstrip("/")
base_path: str = str(Path(self.config.base_path).as_posix()).rstrip("/")
for root, _, files in os.walk(staticDir):
for file in files:
if file.endswith(".map"):
continue
for file in staticDir.rglob("*.*"):
if ".map" == file.suffix:
continue
file = os.path.join(root, file)
urlPath: str = f"{base_path}/{file.replace(f'{staticDir}/', '')}"
urlPath: str = f"{base_path}/{str(file.as_posix()).replace(f'{staticDir.as_posix()!s}/', '')}"
with open(file, "rb") as f:
content = f.read()
with open(file, "rb") as f:
content = f.read()
contentType = self._ext_to_mime.get(os.path.splitext(file)[1], MIME.from_file(file))
contentType = self._ext_to_mime.get(file.suffix, MIME.from_file(file))
self._static_holder[urlPath] = {"content": content, "content_type": contentType}
LOG.debug(f"Preloading '{urlPath}'.")
app.router.add_get(urlPath, self._static_file)
self._static_holder[urlPath] = {"content": content, "content_type": contentType}
LOG.debug(f"Preloading static '{urlPath}'.")
app.router.add_get(urlPath, self._static_file)
preloaded += 1
if "/index.html" in self._static_holder:
for path in self._frontend_routes:
path: str = f"{base_path}/{path.lstrip('/')}"
self._static_holder[path] = self._static_holder["/index.html"]
app.router.add_get(path, self._static_file)
if "{" not in path:
self._static_holder[path + "/"] = self._static_holder["/index.html"]
app.router.add_get(path + "/", self._static_file)
LOG.debug(f"Preloading static route '{path}'.")
preloaded += 1
if urlPath.endswith("/index.html"):
for path in self._frontend_routes:
path: str = f"{base_path}/{path.lstrip('/')}"
self._static_holder[path] = {"content": content, "content_type": contentType}
app.router.add_get(path, self._static_file)
if "{" not in path:
self._static_holder[path + "/"] = {"content": content, "content_type": contentType}
app.router.add_get(path + "/", self._static_file)
LOG.debug(f"Preloading '{path}'.")
preloaded += 1
if "/" != self.config.base_path:
LOG.debug(f"adding base_path folder '{base_path}' to routes.")
app.router.add_get(base_path, self._static_file, name="_base_path")
app.router.add_get(f"{base_path}/", self._static_file, name="_base_path_slash")
if preloaded < 1:
message = f"Failed to find any static files in '{staticDir}'."
@ -273,13 +281,6 @@ class HttpAPI(Common):
LOG.info(f"Preloaded '{preloaded}' static files.")
if "/" != self.config.base_path:
LOG.debug(f"adding base_path folder '{self.config.base_path}' to routes.")
app.router.add_get(self.config.base_path.rstrip("/"), self._static_file, name="base_path_static")
app.router.add_get(
f"{self.config.base_path.rstrip('/')}/", self._static_file, name="base_path_static_slash"
)
return self
def add_routes(self, app: web.Application) -> "HttpAPI":
@ -308,7 +309,7 @@ class HttpAPI(Common):
if hasattr(method, "_http_name") and method._http_name:
opts["name"] = method._http_name
LOG.debug(f"Registering route {method._http_method} {base_path}/{http_path}' {opts}.")
LOG.debug(f"Adding API route {method._http_method} {base_path}/{http_path}' {opts}.")
self.routes.route(method._http_method, f"{base_path}/{http_path}", **opts)(method)
if http_path in registered_options:
@ -597,7 +598,7 @@ class HttpAPI(Common):
if ytdlp_proxy := self.config.get_ytdlp_args().get("proxy", None):
opts["proxy"] = ytdlp_proxy
ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all()
ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset).add(opts).get_all()
data = extract_info(
config=ytdlp_opts,
@ -657,7 +658,8 @@ class HttpAPI(Common):
data={"error": "Manual archive is not enabled."}, status=web.HTTPNotFound.status_code
)
if not os.path.exists(manual_archive):
manual_archive = Path(manual_archive)
if not manual_archive.exists():
return web.json_response(
data={"error": "Manual archive file not found.", "file": manual_archive},
status=web.HTTPNotFound.status_code,
@ -705,7 +707,7 @@ class HttpAPI(Common):
continue
tasks.append(
asyncio.get_event_loop().run_in_executor(None, lambda id=id, url=url: info_wrapper(id=id, url=url))
asyncio.get_event_loop().run_in_executor(None, lambda i=id, url=url: info_wrapper(id=i, url=url))
)
if len(tasks) > 0:
@ -818,7 +820,7 @@ class HttpAPI(Common):
limit = 50
logs_data = await read_logfile(
file=os.path.join(self.config.config_path, "logs", "app.log"),
file=Path(self.config.config_path) / "logs" / "app.log",
offset=offset,
limit=limit,
)
@ -910,7 +912,7 @@ class HttpAPI(Common):
status=web.HTTPBadRequest.status_code,
)
items.append(Condition(**item))
items.append(init_class(Condition, item))
try:
items = cls.save(items=items).load().get_all()
except Exception as e:
@ -958,7 +960,7 @@ class HttpAPI(Common):
opts = {}
if ytdlp_proxy := self.config.get_ytdlp_args().get("proxy", None):
opts["proxy"] = ytdlp_proxy
ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset, with_cookies=True).add(opts).get_all()
ytdlp_opts = YTDLPOpts.get_instance().preset(name=preset).add(opts).get_all()
data = extract_info(
config=ytdlp_opts,
@ -1143,7 +1145,7 @@ class HttpAPI(Common):
item["cli"] = ""
try:
ins.validate(item)
Tasks.validate(item)
except ValueError as e:
return web.json_response(
{"error": f"Failed to validate task '{item.get('name')}'. '{e!s}'"},
@ -1258,109 +1260,6 @@ class HttpAPI(Common):
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
@route("GET", "api/workers", "pool_list")
async def pool_list(self, _) -> Response:
"""
Get the workers status.
Args:
_: The request object.
Returns:
Response: The response object.
"""
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
status = self.queue.pool.get_workers_status()
data = []
for worker in status:
worker_status = status.get(worker)
data.append(
{
"id": worker,
"data": {"status": "Waiting for download."} if worker_status is None else worker_status,
}
)
return web.json_response(
data={
"open": self.queue.pool.has_open_workers(),
"count": self.queue.pool.get_available_workers(),
"workers": data,
},
status=web.HTTPOk.status_code,
dumps=lambda obj: json.dumps(obj, default=lambda o: f"<<non-serializable: {type(o).__qualname__}>>"),
)
@route("POST", "api/workers", "pool_start")
async def pool_restart(self, _) -> Response:
"""
Restart the workers pool.
Args:
_: The request object.
Returns:
Response: The response object.
"""
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
self.queue.pool.start()
return web.json_response({"message": "Workers pool being restarted."}, status=web.HTTPOk.status_code)
@route("PATCH", "api/workers/{id}", "worker_restart")
async def worker_restart(self, request: Request) -> Response:
"""
Restart a worker.
Args:
request (Request): The request object.
Returns:
Response: The response object
"""
id: str = request.match_info.get("id")
if not id:
return web.json_response({"error": "worker id is required."}, status=web.HTTPBadRequest.status_code)
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
status = await self.queue.pool.restart(id, "requested by user.")
return web.json_response({"status": "restarted" if status else "in_error_state"}, status=web.HTTPOk.status_code)
@route("DELETE", "api/workers/{id}", "worker_stop")
async def worker_stop(self, request: Request) -> Response:
"""
Stop a worker.
Args:
request (Request): The request object.
Returns:
Response: The response object.
"""
id: str = request.match_info.get("id")
if not id:
raise web.HTTPBadRequest(text="worker id is required.")
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
status = await self.queue.pool.stop(id, "requested by user.")
return web.json_response({"status": "stopped" if status else "in_error_state"}, status=web.HTTPOk.status_code)
@route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist")
async def playlist(self, request: Request) -> Response:
"""
@ -1398,7 +1297,9 @@ class HttpAPI(Common):
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
return web.Response(
text=await Playlist(download_path=self.config.download_path, url=f"{base_path}/").make(file=realFile),
text=await Playlist(download_path=Path(self.config.download_path), url=f"{base_path}/").make(
file=realFile
),
headers={
"Content-Type": "application/x-mpegURL",
"Cache-Control": "no-cache",
@ -1443,7 +1344,7 @@ class HttpAPI(Common):
base_path: str = self.config.base_path.rstrip("/")
try:
cls = M3u8(download_path=self.config.download_path, url=f"{base_path}/")
cls = M3u8(download_path=Path(self.config.download_path), url=f"{base_path}/")
realFile, status = get_file(download_path=self.config.download_path, file=file)
if web.HTTPFound.status_code == status:
@ -1596,7 +1497,7 @@ class HttpAPI(Common):
return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
return web.Response(
body=await Subtitle(download_path=self.config.download_path).make(file=realFile),
body=await Subtitle().make(file=realFile),
headers={
"Content-Type": "text/vtt; charset=UTF-8",
"X-Accel-Buffering": "no",
@ -1707,8 +1608,13 @@ class HttpAPI(Common):
"""
backend = None
if self.isRequestingBackground:
return web.Response(status=web.HTTPTooManyRequests.status_code)
try:
self.isRequestingBackground = True
backend = random.choice(self.config.pictures_backends) # noqa: S311
CACHE_KEY_BING = "random_background_bing"
CACHE_KEY = "random_background"
if self.cache.has(CACHE_KEY) and not request.query.get("force", False):
@ -1732,6 +1638,29 @@ class HttpAPI(Common):
}
async with httpx.AsyncClient(**opts) as client:
if backend.startswith("https://www.bing.com/HPImageArchive.aspx"):
if not self.cache.has(CACHE_KEY_BING):
response = await client.request(method="GET", url=backend)
if response.status_code != web.HTTPOk.status_code:
return web.json_response(
data={"error": "failed to retrieve the random background image."},
status=web.HTTPInternalServerError.status_code,
)
img_url: str | None = ag(response.json(), "images.0.url")
if not img_url:
return web.json_response(
data={"error": "failed to retrieve the random background image."},
status=web.HTTPInternalServerError.status_code,
)
backend = f"https://www.bing.com{img_url}"
await self.cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24)
else:
backend: str = await self.cache.aget(CACHE_KEY_BING)
LOG.debug(f"Requesting random picture from '{backend!s}'.")
response = await client.request(method="GET", url=backend, follow_redirects=True)
if response.status_code != web.HTTPOk.status_code:
@ -1751,6 +1680,8 @@ class HttpAPI(Common):
await self.cache.aset(key=CACHE_KEY, value=data, ttl=3600)
LOG.debug(f"Random background image from '{backend!s}' cached.")
return web.Response(
body=data.get("content"),
headers={
@ -1766,6 +1697,8 @@ class HttpAPI(Common):
data={"error": "failed to retrieve the random background image."},
status=web.HTTPInternalServerError.status_code,
)
finally:
self.isRequestingBackground = False
@route("GET", "api/file/ffprobe/{file:.*}", "ffprobe")
async def get_ffprobe(self, request: Request) -> Response:
@ -1791,7 +1724,7 @@ class HttpAPI(Common):
headers={
"Location": str(
self.app.router["ffprobe"].url_for(
file=str(realFile).replace(self.config.download_path, "").strip("/")
file=str(realFile.relative_to(self.config.download_path).as_posix()).strip("/")
)
),
},
@ -1830,7 +1763,7 @@ class HttpAPI(Common):
headers={
"Location": str(
self.app.router["file_info"].url_for(
file=str(realFile).replace(self.config.download_path, "").strip("/")
file=str(realFile.relative_to(self.config.download_path).as_posix()).strip("/")
)
),
},
@ -1842,7 +1775,7 @@ class HttpAPI(Common):
ff_info = await ffprobe(realFile)
response = {
"title": str(Path(realFile).stem),
"title": realFile.stem,
"ffprobe": ff_info,
"mimetype": get_mime_type(ff_info.get("metadata", {}), realFile),
"sidecar": get_file_sidecar(realFile),
@ -1850,11 +1783,9 @@ class HttpAPI(Common):
for key in response["sidecar"]:
for i, f in enumerate(response["sidecar"][key]):
response["sidecar"][key][i]["file"] = (
str(Path(realFile).with_name(Path(f["file"]).name))
.replace(self.config.download_path, "")
.strip("/")
)
response["sidecar"][key][i]["file"] = str(
realFile.with_name(f["file"].name).relative_to(self.config.download_path)
).strip("/")
return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
except Exception as e:
@ -1925,9 +1856,6 @@ class HttpAPI(Common):
targets.append(ins.make_target(item))
try:
if len(targets) < 1:
ins.clear()
ins.save(targets=targets)
ins.load()
except Exception as e:
@ -1971,20 +1899,25 @@ class HttpAPI(Common):
if not self.config.browser_enabled:
return web.json_response(data={"error": "File browser is disabled."}, status=web.HTTPForbidden.status_code)
path = request.match_info.get("path")
path = "/" if not path else unquote_plus(path)
req_path: str = request.match_info.get("path")
req_path: str = "/" if not req_path else unquote_plus(req_path)
test = os.path.realpath(os.path.join(self.config.download_path, path))
if not os.path.exists(test):
test: Path = Path(self.config.download_path).joinpath(req_path)
if not test.exists():
return web.json_response(
data={"error": f"path '{path}' does not exist."}, status=web.HTTPNotFound.status_code
data={"error": f"path '{req_path}' does not exist."}, status=web.HTTPNotFound.status_code
)
if not test.is_dir() and not test.is_symlink():
return web.json_response(
data={"error": f"path '{req_path}' is not a directory."}, status=web.HTTPBadRequest.status_code
)
try:
return web.json_response(
data={
"path": path,
"contents": get_files(base_path=self.config.download_path, dir=path),
"path": req_path,
"contents": get_files(base_path=Path(self.config.download_path), dir=req_path),
},
status=web.HTTPOk.status_code,
dumps=self.encoder.encode,
@ -1992,3 +1925,159 @@ class HttpAPI(Common):
except OSError as e:
LOG.exception(e)
return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code)
@route("GET", "api/dev/loop")
async def debug_asyncio(self, _: Request) -> Response:
if not self.config.is_dev():
return web.json_response(
data={"error": "This endpoint is only available in development mode."},
status=web.HTTPForbidden.status_code,
)
import traceback
tasks = []
for task in asyncio.all_tasks():
task_info = {"task": str(task), "stack": []}
for frame in task.get_stack():
formatted = traceback.format_stack(f=frame)
task_info["stack"].extend(formatted)
tasks.append(task_info)
return web.json_response(
data={
"total_tasks": len(tasks),
"loop": str(asyncio.get_event_loop()),
"tasks": tasks,
},
status=web.HTTPOk.status_code,
dumps=self.encoder.encode,
)
@route("GET", "api/dev/workers", "pool_list")
async def pool_list(self, _) -> Response:
"""
Get the workers status.
Args:
_: The request object.
Returns:
Response: The response object.
"""
if not self.config.is_dev():
return web.json_response(
{"error": "This endpoint is only available in development mode."},
status=web.HTTPNotFound.status_code,
)
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
status = self.queue.pool.get_workers_status()
data = []
for worker in status:
worker_status = status.get(worker)
data.append(
{
"id": worker,
"data": {"status": "Waiting for download."} if worker_status is None else worker_status,
}
)
return web.json_response(
data={
"open": self.queue.pool.has_open_workers(),
"count": self.queue.pool.get_available_workers(),
"workers": data,
},
status=web.HTTPOk.status_code,
dumps=lambda obj: json.dumps(obj, default=lambda o: f"<<non-serializable: {type(o).__qualname__}>>"),
)
@route("POST", "api/dev/workers", "pool_start")
async def pool_restart(self, _) -> Response:
"""
Restart the workers pool.
Args:
_: The request object.
Returns:
Response: The response object.
"""
if not self.config.is_dev():
return web.json_response(
{"error": "This endpoint is only available in development mode."},
status=web.HTTPNotFound.status_code,
)
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
self.queue.pool.start()
return web.json_response({"message": "Workers pool being restarted."}, status=web.HTTPOk.status_code)
@route("PATCH", "api/dev/workers/{id}", "worker_restart")
async def worker_restart(self, request: Request) -> Response:
"""
Restart a worker.
Args:
request (Request): The request object.
Returns:
Response: The response object
"""
if not self.config.is_dev():
return web.json_response(
{"error": "This endpoint is only available in development mode."},
status=web.HTTPNotFound.status_code,
)
worker_id: str = request.match_info.get("id")
if not worker_id:
return web.json_response({"error": "worker id is required."}, status=web.HTTPBadRequest.status_code)
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
status = await self.queue.pool.restart(worker_id, "requested by user.")
return web.json_response({"status": "restarted" if status else "in_error_state"}, status=web.HTTPOk.status_code)
@route("DELETE", "api/dev/workers/{id}", "worker_stop")
async def worker_stop(self, request: Request) -> Response:
"""
Stop a worker.
Args:
request (Request): The request object.
Returns:
Response: The response object.
"""
if not self.config.is_dev():
return web.json_response(
{"error": "This endpoint is only available in development mode."},
status=web.HTTPNotFound.status_code,
)
worker_id: str = request.match_info.get("id")
if not worker_id:
raise web.HTTPBadRequest(text="worker id is required.")
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
status = await self.queue.pool.stop(worker_id, "requested by user.")
return web.json_response({"status": "stopped" if status else "in_error_state"}, status=web.HTTPOk.status_code)

90
app/library/HttpSocket.py
View file

@ -3,10 +3,10 @@ import errno
import functools
import logging
import os
import pty
import shlex
import time
from datetime import UTC, datetime
from pathlib import Path
import anyio
import socketio
@ -50,8 +50,17 @@ class HttpSocket(Common):
self.queue = queue or DownloadQueue.get_instance()
self._notify = EventBus.get_instance()
#logger=True, engineio_logger=True,
self.sio = sio or socketio.AsyncServer(cors_allowed_origins="*")
# logger=True, engineio_logger=True,
self.sio = sio or socketio.AsyncServer(
async_handlers=True,
async_mode="aiohttp",
cors_allowed_origins=[],
transports=["websocket"],
logger=self.config.debug,
engineio_logger=self.config.debug,
ping_interval=10,
ping_timeout=5,
)
encoder = encoder or Encoder()
def emit(e: Event, _, **kwargs):
@ -77,6 +86,12 @@ class HttpSocket(Common):
async def on_shutdown(self, _: web.Application):
LOG.debug("Shutting down socket server.")
for sid in self.sio.manager.get_participants("/", None):
LOG.debug(f"Disconnecting client '{sid}'.")
await self.sio.disconnect(sid[0], namespace="/")
LOG.debug("Socket server shutdown complete.")
def attach(self, app: web.Application):
self.sio.attach(app, socketio_path=f"{self.config.base_path.rstrip('/')}/socket.io")
@ -121,23 +136,47 @@ class HttpSocket(Common):
}
)
master_fd, slave_fd = pty.openpty()
try:
import pty
master_fd, slave_fd = pty.openpty()
stdin_arg = asyncio.subprocess.DEVNULL
stdout_arg = stderr_arg = slave_fd
use_pty = True
except ImportError:
use_pty = False
master_fd = slave_fd = None
stdin_arg = asyncio.subprocess.DEVNULL
stdout_arg = asyncio.subprocess.PIPE
stderr_arg = asyncio.subprocess.STDOUT
proc = await asyncio.create_subprocess_exec(
*args,
cwd=self.config.download_path,
stdin=asyncio.subprocess.DEVNULL,
stdout=slave_fd,
stderr=slave_fd,
stdin=stdin_arg,
stdout=stdout_arg,
stderr=stderr_arg,
env=_env,
)
try:
os.close(slave_fd)
except Exception as e:
LOG.error(f"Error closing PTY. '{e!s}'.")
if use_pty:
try:
os.close(slave_fd)
except Exception as e:
LOG.error(f"Error closing PTY. '{e!s}'.")
async def reader(sid: str):
if use_pty is False:
assert proc.stdout is not None
async for raw_line in proc.stdout:
line = raw_line.rstrip(b"\n")
await self._notify.emit(
Events.CLI_OUTPUT,
data={"type": "stdout", "line": line.decode("utf-8", errors="replace")},
to=sid,
)
return
async def read_pty(sid: str):
loop = asyncio.get_running_loop()
buffer = b""
while True:
@ -172,7 +211,7 @@ class HttpSocket(Common):
LOG.error(f"Error closing PTY. '{e!s}'.")
# Start reading output from PTY
read_task = asyncio.create_task(read_pty(sid=sid))
read_task = asyncio.create_task(reader(sid=sid))
# Wait until process finishes
returncode = await proc.wait()
@ -266,13 +305,17 @@ class HttpSocket(Common):
manual_archive = self.config.manual_archive
if manual_archive:
manual_archive = Path(manual_archive)
if not manual_archive.exists():
manual_archive.touch(exist_ok=True)
previouslyArchived = False
if os.path.exists(manual_archive):
async with await anyio.open_file(manual_archive) as f:
async for line in f:
if idDict["archive_id"] in line:
previouslyArchived = True
break
async with await anyio.open_file(manual_archive) as f:
async for line in f:
if idDict["archive_id"] in line:
previouslyArchived = True
break
if not previouslyArchived:
async with await anyio.open_file(manual_archive, "a") as f:
@ -290,9 +333,7 @@ class HttpSocket(Common):
"paused": self.queue.is_paused(),
}
# get download folder listing.
downloadPath: str = self.config.download_path
data["folders"] = [name for name in os.listdir(downloadPath) if os.path.isdir(os.path.join(downloadPath, name))]
data["folders"] = [folder.name for folder in Path(self.config.download_path).iterdir() if folder.is_dir()]
await self._notify.emit(Events.INITIAL_DATA, data=data, to=sid)
@ -332,10 +373,11 @@ class HttpSocket(Common):
await self.subscribe_emit(event=event, data=data)
if "log_lines" == event and self.log_task is None:
LOG.debug("Starting log tailing task.")
log_file = Path(self.config.config_path) / "logs" / "app.log"
LOG.debug(f"Starting tailing '{log_file!s}'.")
self.log_task = asyncio.create_task(
tail_log(
file=os.path.join(self.config.config_path, "logs", "app.log"),
file=log_file,
emitter=emit_logs,
),
name="tail_log",

1
app/library/ItemDTO.py
View file

@ -247,4 +247,5 @@ class ItemDTO:
"output_template",
"output_template_chapter",
"config",
"temp_path"
)

14
app/library/M3u8.py
View file

@ -12,9 +12,9 @@ class M3u8:
ok_vcodecs: tuple = ("h264", "x264", "avc")
ok_acodecs: tuple = ("aac", "m4a", "mp3")
def __init__(self, download_path: str, url: str, segment_duration: float | None = None):
def __init__(self, download_path: Path, url: str, segment_duration: float | None = None):
self.url = url
self.download_path = download_path
self.download_path: Path = download_path
self.duration = float(segment_duration) if segment_duration is not None else self.duration
async def make_stream(self, file: Path) -> str:
@ -23,7 +23,7 @@ class M3u8:
except UnicodeDecodeError:
pass
file = str(file).replace(self.download_path, "").strip("/")
urlPath: str = str(file.relative_to(self.download_path).as_posix()).strip("/")
if "duration" not in ff.metadata:
error = f"Unable to get '{file}' play duration."
@ -31,7 +31,7 @@ class M3u8:
duration: float = float(ff.metadata.get("duration"))
m3u8 = []
m3u8: list = []
m3u8.append("#EXTM3U")
m3u8.append("#EXT-X-VERSION:3")
@ -57,7 +57,7 @@ class M3u8:
m3u8.append(f"#EXTINF:{segmentSize},")
url = f"{self.url}api/player/segments/{i}/{quote(file)}.ts"
url = f"{self.url}api/player/segments/{i}/{quote(urlPath)}.ts"
if len(segmentParams) > 0:
url += "?" + "&".join([f"{key}={value}" for key, value in segmentParams.items()])
@ -70,13 +70,15 @@ class M3u8:
async def make_subtitle(self, file: Path, duration: float) -> str:
m3u8 = []
urlPath: str = str(file.relative_to(self.download_path).as_posix()).strip("/")
m3u8.append("#EXTM3U")
m3u8.append("#EXT-X-VERSION:3")
m3u8.append(f"#EXT-X-TARGETDURATION:{int(self.duration)}")
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
m3u8.append(f"#EXTINF:{duration},")
m3u8.append(f"{self.url}api/player/subtitle/{quote(str(file).replace(self.download_path, '').strip('/'))}.vtt")
m3u8.append(f"{self.url}api/player/subtitle/{quote(urlPath)}.vtt")
m3u8.append("#EXT-X-ENDLIST")
return "\n".join(m3u8)

36
app/library/Notifications.py
View file

@ -1,10 +1,10 @@
import asyncio
import json
import logging
import os
from collections.abc import Awaitable
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any
import httpx
@ -17,7 +17,6 @@ from .Events import Event, EventBus, Events
from .ItemDTO import ItemDTO
from .Singleton import Singleton
from .Utils import validate_uuid
from .version import APP_VERSION
LOG = logging.getLogger("notifications")
@ -135,14 +134,14 @@ class Notification(metaclass=Singleton):
config: Config = config or Config.get_instance()
self._debug = config.debug
self._file: str = file or os.path.join(config.config_path, "notifications.json")
self._file: Path = Path(file) if file else Path(config.config_path).joinpath("notifications.json")
self._client: httpx.AsyncClient = client or httpx.AsyncClient()
self._encoder: Encoder = encoder or Encoder()
self._version = config.version
if os.path.exists(self._file):
if self._file.exists() and "600" != self._file.stat().st_mode:
try:
if "600" != oct(os.stat(self._file).st_mode)[-3:]:
os.chmod(self._file, 0o600)
self._file.chmod(0o600)
except Exception:
pass
@ -184,50 +183,45 @@ class Notification(metaclass=Singleton):
Notification: The Notification instance.
"""
LOG.info(f"Saving notification targets to '{self._file}'.")
try:
with open(self._file, "w") as f:
json.dump([t.serialize() for t in targets], fp=f, indent=4)
self._file.write_text(json.dumps([t.serialize() for t in targets], indent=4))
LOG.info(f"Updated '{self._file}'.")
except Exception as e:
LOG.exception(e)
LOG.error(f"Error saving notification targets to '{self._file}'. '{e!s}'")
LOG.error(f"Error saving '{self._file}'. '{e!s}'")
return self
def load(self) -> "Notification":
"""Load or reload notification targets from the file."""
if len(self._targets) > 0:
LOG.info("Clearing existing notification targets.")
self.clear()
if not os.path.exists(self._file) or os.path.getsize(self._file) < 10:
if not self._file.exists() or self._file.stat().st_size < 1:
return self
targets = []
LOG.info(f"Loading notification targets from '{self._file}'.")
try:
with open(self._file) as f:
targets = json.load(f)
LOG.info(f"Loading '{self._file}'.")
targets = json.loads(self._file.read_text())
except Exception as e:
LOG.error(f"Error loading notification targets from '{self._file}'. '{e!s}'")
LOG.error(f"Error loading '{self._file}'. '{e!s}'")
for target in targets:
try:
try:
Notification.validate(target)
target: Target = self.make_target(target)
except ValueError as e:
name = target.get("name") or target.get("id") or target.get("request", {}).get("url") or "unknown"
LOG.error(f"Invalid notification target '{name}'. '{e!s}'")
continue
target = self.make_target(target)
self._targets.append(target)
LOG.info(
f"Will send {target.request.type} request on '{', '.join(target.on) if len(target.on) > 0 else 'all events'}' to '{target.name}'."
f"Send '{target.request.type}' request on '{', '.join(target.on) if len(target.on) > 0 else 'all events'}' to '{target.name}'."
)
except Exception as e:
LOG.error(f"Error loading notification target '{target}'. '{e!s}'")
@ -360,7 +354,7 @@ class Notification(metaclass=Singleton):
"method": target.request.method.upper(),
"url": target.request.url,
"headers": {
"User-Agent": f"YTPTube/{APP_VERSION}",
"User-Agent": f"YTPTube/{self._version}",
"X-Event-Id": ev.id,
"X-Event": ev.event,
"Content-Type": "application/json"

13
app/library/PackageInstaller.py
View file

@ -3,6 +3,7 @@ import logging
import os
import subprocess
import sys
from pathlib import Path
LOG = logging.getLogger("package_installer")
@ -12,11 +13,15 @@ class Packages:
from_env = env.split() if env else []
from_file = []
if os.path.exists(file) and os.access(file, os.R_OK):
with open(file) as f:
from_file = [pkg.strip() for pkg in f if pkg.strip()]
if file:
file = Path(file)
if file.exists() and os.access(str(file), os.R_OK):
with open(file) as f:
from_file: list[str] = [pkg.strip() for pkg in f if pkg.strip()]
else:
LOG.error(f"pip packages file '{file}' doesn't exist or is not readable.")
self.packages: list = list(set(from_env + from_file))
self.packages: list[str] = list(set(from_env + from_file))
self.upgrade = bool(upgrade)
def has_packages(self) -> bool:

20
app/library/Playlist.py
View file

@ -8,12 +8,12 @@ from .Utils import StreamingError, get_file_sidecar
class Playlist:
_url: str = None
def __init__(self, download_path: str, url: str):
self.url = url
self.download_path = download_path
def __init__(self, download_path: Path, url: str):
self.url: str = url
self.download_path: Path = download_path
async def make(self, file: Path) -> str:
ref = str(file).replace(self.download_path, "").strip("/")
ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
try:
ff = await ffprobe(file)
@ -24,19 +24,19 @@ class Playlist:
msg = f"Unable to get '{ref}' duration."
raise StreamingError(msg)
playlist = []
playlist: list[str] = []
playlist.append("#EXTM3U")
subs = ""
subs: str = ""
duration: float = float(ff.metadata.get("duration"))
for sub_file in get_file_sidecar(file).get("subtitle", []):
lang = sub_file["lang"]
item = Path(sub_file["file"])
name = sub_file["name"]
lang: str = sub_file["lang"]
item: Path = sub_file["file"]
name: str = sub_file["name"]
subs = ',SUBTITLES="subs"'
url = f"{self.url}api/player/m3u8/subtitle/{quote(str(Path(ref).with_name(item.name)))}.m3u8?duration={duration}"
url = f"{self.url}api/player/m3u8/subtitle/{quote(str(ref.with_name(item.name)))}.m3u8?duration={duration}"
playlist.append(
f'#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="{name}",DEFAULT=NO,AUTOSELECT=NO,FORCED=NO,LANGUAGE="{lang}",URI="{url}"'
)

91
app/library/Presets.py
View file

@ -1,8 +1,8 @@
import json
import logging
import os
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from aiohttp import web
@ -11,10 +11,51 @@ from .config import Config
from .encoder import Encoder
from .Events import EventBus, Events
from .Singleton import Singleton
from .Utils import arg_converter, clean_item
from .Utils import arg_converter, init_class
LOG = logging.getLogger("presets")
DEFAULT_PRESETS: list[dict[int, dict[str, str | bool]]] = [
{
"id": "3e163c6c-64eb-4448-924f-814b629b3810",
"name": "default",
"default": True,
},
{
"id": "5bf9c42b-8852-468a-99f5-915622dfba25",
"name": "Best video and audio",
"cli": "--format 'bv+ba/b'",
"default": True,
},
{
"id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48b",
"name": "1080p H264/m4a or best available",
"cli": "-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
"default": True,
},
{
"id": "9719fcc3-4cf2-4d88-b1e4-74dff3dba00e",
"name": "720p h264/m4a or best available",
"cli": "-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
"default": True,
},
{
"id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330",
"name": "Audio only",
"cli": "--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'",
"default": True,
},
{
"id": "2ade2c28-cad4-4a06-b7eb-2439fdf46f60",
"name": "yt-dlp info reader plugin",
"description": 'This preset generate specific filename format and metadata to work with yt-dlp info reader plugins for jellyfin/emby and plex and to make play state sync work for WatchState.\n\nThere is one more step you need to do via Other > Terminal if you have it enabled or directly from container shell\n\nyt-dlp -I0 --write-info-json --write-thumbnail --convert-thumbnails jpg --paths /downloads/youtube -o "%(channel|Unknown_title)s [%(channel_id|Unknown_id)s]/%(title).180B [%(channel_id|Unknown_id)s].%(ext)s" -- https://www.youtube.com/channel/UCClfFsWcT3N2I7VTXXyt84A\n\nChange the url to the channel you want to download.\n\nFor more information please visit \nhttps://github.com/arabcoders/watchstate/blob/master/FAQ.md#how-to-get-watchstate-working-with-youtube-contentlibrary',
"folder": "youtube",
"template": "%(channel)s %(channel_id|Unknown_id)s/Season %(release_date>%Y,upload_date>%Y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s",
"cli": "--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata",
"default": True,
},
]
@dataclass(kw_only=True)
class Preset:
@ -65,28 +106,26 @@ class Presets(metaclass=Singleton):
_default: list[Preset] = []
def __init__(self, file: str | None = None, config: Config | None = None):
def __init__(self, file: str | Path | None = None, config: Config | None = None):
Presets._instance = self
config = config or Config.get_instance()
self._file: str = file or os.path.join(config.config_path, "presets.json")
self._file: Path = Path(file) if file else Path(config.config_path).joinpath("presets.json")
if os.path.exists(self._file) and "600" != oct(os.stat(self._file).st_mode)[-3:]:
if self._file.exists() and "600" != self._file.stat().st_mode:
try:
os.chmod(self._file, 0o600)
self._file.chmod(0o600)
except Exception:
pass
default_file = os.path.join(os.path.dirname(__file__), "presets.json")
with open(default_file) as f:
for i, preset in enumerate(json.load(f)):
try:
self.validate(preset)
self._default.append(Preset(**preset))
except Exception as e:
LOG.error(f"Failed to parse '{default_file}:{i}'. '{e!s}'.")
continue
for i, preset in enumerate(DEFAULT_PRESETS):
try:
self.validate(preset)
self._default.append(init_class(Preset, preset))
except Exception as e:
LOG.error(f"Failed to parse default preset ':{i}'. '{e!s}'.")
continue
def event_handler(_, __):
msg = "Not implemented"
@ -138,13 +177,12 @@ class Presets(metaclass=Singleton):
"""
self.clear()
if not os.path.exists(self._file) or os.path.getsize(self._file) < 10:
if not self._file.exists() or self._file.stat().st_size < 10:
return self
LOG.info(f"Loading '{self._file}'.")
try:
with open(self._file) as f:
presets = json.load(f)
LOG.info(f"Loading '{self._file}'.")
presets: dict = json.loads(self._file.read_text())
except Exception as e:
LOG.error(f"Failed to parse '{self._file}'. '{e}'.")
return self
@ -160,7 +198,6 @@ class Presets(metaclass=Singleton):
preset["id"] = str(uuid.uuid4())
need_save = True
preset, preset_status = clean_item(preset, keys=("args", "postprocessors"))
if preset.get("format"):
if not preset.get("cli"):
preset.update({"cli": f"--format {preset['format']}"})
@ -172,10 +209,7 @@ class Presets(metaclass=Singleton):
preset.pop("format")
need_save = True
preset = Preset(**preset)
if preset_status:
need_save = True
preset: Preset = init_class(Preset, preset)
self._items.append(preset)
except Exception as e:
@ -183,7 +217,7 @@ class Presets(metaclass=Singleton):
continue
if need_save:
LOG.info(f"Saving '{self._file}' due to changes.")
LOG.info(f"Saving '{self._file}'.")
self.save(self._items)
return self
@ -255,7 +289,7 @@ class Presets(metaclass=Singleton):
for i, preset in enumerate(items):
try:
if not isinstance(preset, Preset):
preset = Preset(**preset)
preset: Preset = init_class(Preset, preset)
items[i] = preset
except Exception as e:
LOG.error(f"Failed to save item '{i}' due to parsing error. '{e!s}'.")
@ -268,8 +302,9 @@ class Presets(metaclass=Singleton):
continue
try:
with open(self._file, "w") as f:
json.dump(obj=[preset.serialize() for preset in items if preset.default is False], fp=f, indent=4)
self._file.write_text(
json.dumps(obj=[preset.serialize() for preset in items if preset.default is False], indent=4)
)
LOG.info(f"Saved '{self._file}'.")
except Exception as e:

16
app/library/Segments.py
View file

@ -1,7 +1,6 @@
import asyncio
import hashlib
import logging
import os
import tempfile
from pathlib import Path
@ -33,13 +32,14 @@ class Segments:
except UnicodeDecodeError:
pass
tmpDir = tempfile.gettempdir()
tmpFile = os.path.join(tmpDir, f"ytptube_stream.{hashlib.sha256(str(file).encode()).hexdigest()}")
tmpFile = Path(tempfile.gettempdir()).joinpath(
f"ytptube_stream.{hashlib.sha256(str(file).encode()).hexdigest()}"
)
if not os.path.exists(tmpFile):
os.symlink(file, tmpFile)
if not tmpFile.exists():
tmpFile.symlink_to(file, target_is_directory=False)
startTime = f"{0:.6f}" if self.index == 0 else f"{self.duration * self.index:.6f}"
startTime: str = f"{0:.6f}" if self.index == 0 else f"{self.duration * self.index:.6f}"
fargs = [
"-xerror",
@ -78,7 +78,7 @@ class Segments:
return fargs
async def stream(self, file: Path, resp: web.StreamResponse):
ffmpeg_args = await self.build_ffmpeg_args(file)
ffmpeg_args: list[str] = await self.build_ffmpeg_args(file)
proc = await asyncio.create_subprocess_exec(
"ffmpeg",
@ -94,7 +94,7 @@ class Segments:
try:
while True:
chunk = await proc.stdout.read(1024 * 64)
chunk: bytes = await proc.stdout.read(1024 * 64)
if not chunk:
break
try:

12
app/library/Subtitle.py
View file

@ -22,9 +22,6 @@ SubstationFormat.ms_to_timestamp = ms_to_timestamp
class Subtitle:
def __init__(self, download_path: str):
self.download_path = download_path
async def make(self, file: Path) -> str:
if file.suffix not in ALLOWED_SUBS_EXTENSIONS:
msg = f"File '{file}' subtitle type is not supported."
@ -34,7 +31,7 @@ class Subtitle:
async with await anyio.open_file(file) as f:
return await f.read()
subs = pysubs2.load(path=str(file))
subs: pysubs2.SSAFile = pysubs2.load(path=str(file))
if len(subs.events) < 1:
msg = f"No subtitle events were found in '{file}'."
@ -43,7 +40,10 @@ class Subtitle:
if len(subs.events) < 2:
return subs.to_string("vtt")
if subs.events[0].end == subs.events[len(subs.events) - 1].end:
subs.events.pop(0)
try:
if subs.events[0].end == subs.events[len(subs.events) - 1].end:
subs.events.pop(0)
except Exception:
pass
return subs.to_string("vtt")

110
app/library/Tasks.py
View file

@ -1,10 +1,10 @@
import asyncio
import json
import logging
import os
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import httpx
@ -15,7 +15,7 @@ from .encoder import Encoder
from .Events import EventBus, Events, error, info, success
from .Scheduler import Scheduler
from .Singleton import Singleton
from .Utils import clean_item
from .Utils import init_class
LOG = logging.getLogger("tasks")
@ -65,18 +65,18 @@ class Tasks(metaclass=Singleton):
config = config or Config.get_instance()
self._debug = config.debug
self._default_preset = config.default_preset
self._file = file or os.path.join(config.config_path, "tasks.json")
self._client = client or httpx.AsyncClient()
self._encoder = encoder or Encoder()
self._loop = loop or asyncio.get_event_loop()
self._scheduler = scheduler or Scheduler.get_instance()
self._notify = EventBus.get_instance()
self._debug: bool = config.debug
self._default_preset: str = config.default_preset
self._file: Path = Path(file) if file else Path(config.config_path).joinpath("tasks.json")
self._client: httpx.AsyncClient = client or httpx.AsyncClient()
self._encoder: Encoder = encoder or Encoder()
self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_event_loop()
self._scheduler: Scheduler = scheduler or Scheduler.get_instance()
self._notify: EventBus = EventBus.get_instance()
if os.path.exists(self._file) and "600" != oct(os.stat(self._file).st_mode)[-3:]:
if self._file.exists() and "600" != self._file.stat().st_mode:
try:
os.chmod(self._file, 0o600)
self._file.chmod(0o600)
except Exception:
pass
@ -126,29 +126,23 @@ class Tasks(metaclass=Singleton):
"""
self.clear()
if not os.path.exists(self._file) or os.path.getsize(self._file) < 10:
if not self._file.exists() or self._file.stat().st_size < 1:
return self
LOG.info(f"Loading tasks from '{self._file}'.")
try:
with open(self._file) as f:
tasks = json.load(f)
LOG.info(f"Loading '{self._file}'.")
tasks = json.loads(self._file.read_text())
except Exception as e:
LOG.error(f"Failed to parse tasks from '{self._file}'. '{e!s}'.")
LOG.error(f"Error loading '{self._file}'. '{e!s}'.")
return self
if not tasks or len(tasks) < 1:
LOG.info(f"No tasks were defined in '{self._file}'.")
return self
need_save = False
for i, task in enumerate(tasks):
try:
task, task_status = clean_item(task, keys=("cookies", "config"))
self.validate(task)
task = Task(**task)
if task_status:
need_save = True
Tasks.validate(task)
task: Task = init_class(Task, task)
except Exception as e:
LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.")
continue
@ -172,11 +166,7 @@ class Tasks(metaclass=Singleton):
LOG.info(f"Task '{i}: {task.name}' queued to be executed '{schedule_time}'.")
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to queue task '{i}: {task.name}'. '{e!s}'.")
if need_save:
LOG.info("Updating tasks file to remove old keys.")
self.save(self.get_all())
LOG.error(f"Failed to queue '{i}: {task.name}'. '{e!s}'.")
return self
@ -193,18 +183,19 @@ class Tasks(metaclass=Singleton):
for task in self._tasks:
try:
LOG.info(f"Stopping task '{task.id}: {task.name}'.")
LOG.info(f"Stopping '{task.id}: {task.name}'.")
self._scheduler.remove(task.id)
except Exception as e:
if not shutdown:
LOG.exception(e)
LOG.error(f"Failed to stop task '{task.id}: {task.name}'. '{e!s}'.")
LOG.error(f"Failed to stop '{task.id}: {task.name}'. '{e!s}'.")
self._tasks.clear()
return self
def validate(self, task: Task | dict) -> bool:
@staticmethod
def validate(task: Task | dict) -> bool:
"""
Validate the task.
@ -262,31 +253,28 @@ class Tasks(metaclass=Singleton):
"""
for i, task in enumerate(tasks):
try:
if not isinstance(task, Task):
task = Task(**task)
tasks[i] = task
except Exception as e:
LOG.error(f"Failed to save task '{i}' unable to parse task. '{e!s}'.")
continue
try:
self.validate(task)
if not isinstance(task, Task):
task: Task = init_class(Task, task)
tasks[i] = task
except ValueError as e:
LOG.error(f"Failed to add task '{i}: {task.name}'. '{e}'.")
LOG.error(f"Failed to validate item '{i}: {task.name}'. '{e}'.")
continue
except Exception as e:
LOG.error(f"Failed to save task '{i}'. '{e!s}'.")
continue
try:
with open(self._file, "w") as f:
json.dump(obj=[task.serialize() for task in tasks], fp=f, indent=4)
LOG.info(f"Tasks saved to '{self._file}'.")
self._file.write_text(json.dumps([i.serialize() for i in tasks], indent=4))
LOG.info(f"Updated '{self._file}'.")
except Exception as e:
LOG.error(f"Failed to save tasks to '{self._file}'. '{e!s}'.")
LOG.error(f"Error saving '{self._file}'. '{e!s}'.")
return self
async def _runner(self, task: Task):
async def _runner(self, task: Task) -> None:
"""
Run the task.
@ -297,11 +285,11 @@ class Tasks(metaclass=Singleton):
None
"""
timeNow: str = datetime.now(UTC).isoformat()
try:
timeNow = datetime.now(UTC).isoformat()
started = time.time()
started: float = time.time()
if not task.url:
LOG.error(f"Failed to dispatch task '{task.id}: {task.name}'. No URL found.")
LOG.error(f"Failed to dispatch '{task.id}: {task.name}'. No URL found.")
return
preset: str = str(task.preset or self._default_preset)
@ -309,13 +297,10 @@ class Tasks(metaclass=Singleton):
template: str = task.template if task.template else ""
cli: str = task.cli if task.cli else ""
LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.")
LOG.info(f"Dispatched '{task.id}: {task.name}' at '{timeNow}'.")
tasks = []
tasks.append(
self._notify.emit(Events.LOG_INFO, data=info(f"Task '{task.name}' dispatched at '{timeNow}'."))
)
tasks.append(
tasks: list = [
self._notify.emit(Events.LOG_INFO, data=info(f"Dispatched '{task.name}' at '{timeNow}'.")),
self._notify.emit(
Events.ADD_URL,
data={
@ -327,22 +312,21 @@ class Tasks(metaclass=Singleton):
},
id=task.id,
),
)
]
await asyncio.wait_for(asyncio.gather(*tasks), timeout=None)
timeNow = datetime.now(UTC).isoformat()
ended = time.time()
LOG.info(f"Task '{task.id}: {task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.")
ended: float = time.time()
LOG.info(f"Completed '{task.id}: {task.name}' at '{timeNow}' took '{ended - started:.2f}' seconds.")
await self._notify.emit(
Events.LOG_SUCCESS,
data=success(f"Task '{task.name}' completed in '{ended - started:.2f}' seconds."),
data=success(f"Completed '{task.name}' in '{ended - started:.2f}' seconds."),
)
except Exception as e:
timeNow = datetime.now(UTC).isoformat()
LOG.error(f"Task '{task.id}: {task.name}' has failed to execute at '{timeNow}'. '{e!s}'.")
LOG.error(f"Failed to execute '{task.id}: {task.name}' at '{timeNow}'. '{e!s}'.")
await self._notify.emit(
Events.ERROR, data=error(f"Task '{task.name}' failed to execute at '{timeNow}'. '{e!s}'.")
Events.ERROR, data=error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.")
)

187
app/library/Utils.py
View file

@ -5,7 +5,6 @@ import ipaddress
import json
import logging
import os
import pathlib
import re
import shlex
import socket
@ -13,6 +12,8 @@ import uuid
from datetime import UTC, datetime, timedelta
from functools import lru_cache
from http.cookiejar import MozillaCookieJar
from pathlib import Path
from typing import TypeVar
import yt_dlp
from Crypto.Cipher import AES
@ -60,6 +61,8 @@ FILES_TYPE: list = [
DATETIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?")
T = TypeVar("T")
class StreamingError(Exception):
"""Raised when an error occurs during streaming."""
@ -70,7 +73,7 @@ class FileLogFormatter(logging.Formatter):
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
def calc_download_path(base_path: str, folder: str | None = None, create_path: bool = True) -> str:
def calc_download_path(base_path: str | Path, folder: str | None = None, create_path: bool = True) -> str:
"""
Calculates download path and prevents folder traversal.
@ -83,23 +86,32 @@ def calc_download_path(base_path: str, folder: str | None = None, create_path: b
Download path with base folder factored in.
"""
if not isinstance(base_path, Path):
base_path = Path(base_path)
if not folder:
return base_path
return str(base_path)
if folder.startswith("/"):
folder = folder[1:]
realBasePath = os.path.realpath(base_path)
download_path = os.path.realpath(os.path.join(base_path, folder))
realBasePath = base_path.resolve()
download_path = Path(realBasePath).joinpath(folder).resolve(strict=False)
if not download_path.startswith(realBasePath):
if not str(download_path).startswith(str(realBasePath)):
msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".'
raise Exception(msg)
if not os.path.isdir(download_path) and create_path:
os.makedirs(download_path, exist_ok=True)
try:
download_path.relative_to(realBasePath)
except ValueError as e:
msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".'
raise Exception(msg) from e
return download_path
if not download_path.is_dir() and create_path:
download_path.mkdir(parents=True, exist_ok=True)
return str(download_path)
def extract_info(
@ -248,7 +260,10 @@ def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, s
"""
idDict = {"id": None, "ie_key": None, "archive_id": None}
if not url or not archive_file or not os.path.exists(archive_file):
if not url or not archive_file:
return (False, idDict)
if not Path(archive_file).exists():
return (False, idDict)
idDict = get_archive_id(url=url)
@ -301,13 +316,13 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]:
return ({}, False, f"{e}")
def check_id(file: pathlib.Path) -> bool | str:
def check_id(file: Path) -> bool | str:
"""
Check if we are able to get an id from the file name.
if so check if any video file with the same id exists.
Args:
file (pathlib.Path): File to check.
file (Path): File to check.
Returns:
bool|str: False if no file found, else the file path.
@ -477,12 +492,12 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
return False
def get_file_sidecar(file: pathlib.Path) -> list[dict]:
def get_file_sidecar(file: Path) -> list[dict]:
"""
Get sidecar files for the given file.
Args:
file (pathlib.Path): The video file.
file (Path): The video file.
Returns:
list: List of sidecar files.
@ -531,7 +546,7 @@ def get_file_sidecar(file: pathlib.Path) -> list[dict]:
def get_possible_images(dir: str) -> list[dict]:
images = []
path_loc = pathlib.Path(dir, "test.jpg")
path_loc = Path(dir, "test.jpg")
for filename in ["poster", "thumbnail", "artwork", "cover", "fanart"]:
for ext in [".jpg", ".jpeg", ".png", ".webp"]:
@ -542,7 +557,7 @@ def get_possible_images(dir: str) -> list[dict]:
return images
def get_mime_type(metadata: dict, file_path: pathlib.Path) -> str:
def get_mime_type(metadata: dict, file_path: Path) -> str:
"""
Determine the correct MIME type for a video file based on ffprobe metadata.
@ -587,32 +602,35 @@ def get_mime_type(metadata: dict, file_path: pathlib.Path) -> str:
return "application/octet-stream"
def get_file(download_path: str, file: str | pathlib.Path) -> tuple[pathlib.Path, int]:
def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
"""
Get the real file path.
Args:
download_path (str): Base download path.
file (str|pathlib.Path): File path.
download_path (str|Path): Base download path.
file (str|Path): File path.
Returns:
pathlib.Path: Real file path.
Path: Real file path.
int: http status code.
"""
file_path: str = os.path.normpath(os.path.join(download_path, str(file)))
if not file_path.startswith(download_path):
return (pathlib.Path(file_path), 404)
if not isinstance(download_path, Path):
download_path = Path(download_path)
realFile: str = pathlib.Path(calc_download_path(base_path=str(download_path), folder=str(file), create_path=False))
if realFile.exists():
return (realFile, 200)
try:
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
if realFile.exists():
return (realFile, 200)
except Exception as e:
LOG.error(f"Error calculating download path. {e!s}")
return (Path(file), 404)
possibleFile = check_id(file=realFile)
if not possibleFile:
return (realFile, 404)
return (pathlib.Path(possibleFile), 302)
return (Path(possibleFile), 302)
def encrypt_data(data: str, key: bytes) -> str:
@ -736,12 +754,12 @@ def get(
return data
def get_files(base_path: str, dir: str | None = None):
def get_files(base_path: Path | str, dir: str | None = None):
"""
Get directory contents.
Args:
base_path (str): Base download path.
base_path (Path|str): Base download path.
dir (str): Directory to check.
Returns:
@ -751,31 +769,33 @@ def get_files(base_path: str, dir: str | None = None):
OSError: If the directory is invalid or not a directory.
"""
if not isinstance(base_path, Path):
base_path = Path(base_path)
base_path = base_path.resolve()
dir_path = base_path
if dir and dir != "/":
path = os.path.normpath(os.path.join(base_path, str(dir)))
if not path.startswith(base_path):
msg = f"Invalid path: '{dir}' - '{path}' - must be inside '{base_path}'."
raise OSError(msg)
dir_path = os.path.realpath(path)
else:
dir_path = base_path
dir_path: Path = base_path.joinpath(dir)
dir_path = pathlib.Path(dir_path)
if dir_path.is_symlink():
try:
dir_path = dir_path.resolve()
except OSError as e:
LOG.warning(f"Skipping broken symlink: {dir_path} - {e}")
return []
try:
dir_path = dir_path.resolve()
except OSError as e:
LOG.warning(f"Failed to resolve '{dir}' - {e}")
return []
if not str(dir_path).startswith(base_path):
msg = f"Invalid path: '{dir_path}' - must be inside '{base_path}'."
LOG.warning(msg)
try:
dir_path.relative_to(base_path)
except ValueError:
LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
return []
if not str(dir_path).startswith(str(base_path)):
LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
return []
if not dir_path.is_dir():
msg = f"Invalid path: '{dir_path}' - must be a directory."
LOG.warning(msg)
LOG.warning(f"Invalid path: '{dir_path}' - must be a directory.")
return []
contents: list = []
@ -785,11 +805,14 @@ def get_files(base_path: str, dir: str | None = None):
if file.is_symlink():
try:
test: pathlib.Path = file.resolve()
if not str(test).startswith(base_path):
msg = f"Invalid symlink: '{file}' - must resolve inside '{base_path}'."
LOG.warning(msg)
test: Path = file.resolve()
test.relative_to(base_path)
if not str(test).startswith(str(base_path)):
LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
continue
except ValueError:
LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
continue
except OSError:
LOG.warning(f"Skipping broken symlink: {file}")
continue
@ -813,7 +836,7 @@ def get_files(base_path: str, dir: str | None = None):
"type": "file" if file.is_file() else "dir",
"content_type": content_type,
"name": file.name,
"path": str(file).replace(base_path, "").strip("/"),
"path": str(file.relative_to(base_path)).strip("/"),
"size": stat.st_size,
"mime": get_mime_type({}, file) if file.is_file() else "directory",
"mtime": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
@ -883,12 +906,12 @@ def strip_newline(string: str) -> str:
return res.strip() if res else ""
async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict:
async def read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict:
"""
Read a log file and return a set of log lines along with pagination metadata.
Args:
file (str): The log file path.
file (Path): The log file path.
offset (int): Number of lines to skip from the end (newer entries).
limit (int): Number of lines to return.
@ -903,7 +926,7 @@ async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict:
from anyio import open_file
if not pathlib.Path(file).exists():
if not file.exists():
return {"logs": [], "next_offset": None, "end_is_reached": True}
result = []
@ -951,7 +974,7 @@ async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict:
return {"logs": [], "next_offset": None, "end_is_reached": True}
async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
async def tail_log(file: Path, emitter: callable, sleep_time: float = 0.5):
"""
Continuously read a log file and emit new lines.
@ -966,7 +989,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
from anyio import open_file
if not pathlib.Path(file).exists():
if not file.exists():
return
try:
@ -993,7 +1016,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
return
def load_cookies(file: str) -> tuple[bool, MozillaCookieJar]:
def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
"""
Validate and load a cookie file.
@ -1007,7 +1030,7 @@ def load_cookies(file: str) -> tuple[bool, MozillaCookieJar]:
try:
from http.cookiejar import MozillaCookieJar
cookies = MozillaCookieJar(file, None, None)
cookies = MozillaCookieJar(str(file), None, None)
cookies.load()
return (True, cookies)
@ -1131,3 +1154,47 @@ def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None)
matched.extend(line for line in logs if line and any(p.search(line) for p in compiled))
return list(dict.fromkeys(matched))
def delete_dir(dir: Path) -> bool:
"""
Delete a directory and all its contents.
Args:
dir (Path): The directory to delete.
Returns:
bool: True if the directory was deleted successfully, False otherwise.
"""
if not dir or not dir.exists() or not dir.is_dir():
return False
try:
for item in dir.iterdir():
if item.is_dir():
delete_dir(item)
else:
item.unlink()
dir.rmdir()
return True
except Exception as e:
LOG.error(f"Failed to delete directory '{dir}': {e}")
return False
def init_class(cls: type[T], data: dict) -> T:
"""
Initialize a class instance with data from a dictionary, filtering out keys not present in the class fields.
Args:
cls (type): The class to initialize.
data (dict): The data to use for initialization.
Returns:
T: An instance of the class initialized with the provided data.
"""
from dataclasses import fields
return cls(**{k: v for k, v in data.items() if k in {f.name for f in fields(cls)}})

48
app/library/YTDLPOpts.py
View file

@ -2,11 +2,11 @@ import logging
from pathlib import Path
from .config import Config
from .Presets import Presets
from .Presets import Presets, Preset
from .Singleton import Singleton
from .Utils import REMOVE_KEYS, arg_converter, calc_download_path, load_cookies, merge_dict
LOG = logging.getLogger("YTDLPOpts")
LOG: logging.Logger = logging.getLogger("YTDLPOpts")
class YTDLPOpts(metaclass=Singleton):
@ -79,11 +79,11 @@ class YTDLPOpts(metaclass=Singleton):
YTDLPOpts: The instance of the class
"""
bad_options = {}
bad_options: dict = {}
if from_user:
bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()}
bad_options: dict[str, str] = {k: v for d in REMOVE_KEYS for k, v in d.items()}
removed_options = []
removed_options: list = []
for key, value in config.items():
if from_user and key in bad_options:
@ -97,19 +97,18 @@ class YTDLPOpts(metaclass=Singleton):
return self
def preset(self, name: str, with_cookies: bool = False) -> "YTDLPOpts":
def preset(self, name: str) -> "YTDLPOpts":
"""
Add the preset options to the item options.
Args:
name (str): The name of the preset
with_cookies (bool): If the cookies should be added
Returns:
YTDLPOpts: The instance of the class
"""
preset = Presets.get_instance().get(name)
preset: Preset | None = Presets.get_instance().get(name)
if not preset or "default" == name:
return self
@ -122,25 +121,26 @@ class YTDLPOpts(metaclass=Singleton):
msg = f"Invalid preset '{preset.name}' command options for yt-dlp. '{e!s}'."
raise ValueError(msg) from e
if preset.cookies and with_cookies:
file = Path(self._config.config_path, "cookies", f"{preset.id}.txt")
if preset.cookies:
file: Path = Path(self._config.config_path) / "cookies" / f"{preset.id}.txt"
if not file.parent.exists():
file.parent.mkdir(parents=True)
file.parent.mkdir(parents=True, exist_ok=True)
with open(file, "w") as f:
f.write(preset.cookies)
file.write_text(preset.cookies)
load_cookies(str(file))
self._preset_opts["cookiefile"] = str(file)
try:
load_cookies(file)
self._preset_opts["cookiefile"] = str(file)
except ValueError as e:
LOG.error(str(e))
if preset.template:
self._preset_opts["outtmpl"] = {"default": preset.template, "chapter": self._config.output_template_chapter}
if preset.folder:
self._preset_opts["paths"] = {
"home": calc_download_path(base_path=self._config.download_path, folder=preset.folder),
"home": calc_download_path(base_path=Path(self._config.download_path), folder=preset.folder),
"temp": self._config.temp_path,
}
@ -157,7 +157,7 @@ class YTDLPOpts(metaclass=Singleton):
dict: The options
"""
default_opts = {}
default_opts: dict = {}
default_opts["paths"] = {"home": self._config.download_path, "temp": self._config.temp_path}
default_opts["outtmpl"] = {
"default": self._config.output_template,
@ -167,7 +167,7 @@ class YTDLPOpts(metaclass=Singleton):
if not isinstance(self._item_cli, list):
self._item_cli = []
merge = []
merge: list[str] = []
if self._config._ytdlp_cli_mutable and len(self._config._ytdlp_cli_mutable) > 1:
merge.append(self._config._ytdlp_cli_mutable)
@ -178,12 +178,16 @@ class YTDLPOpts(metaclass=Singleton):
# prepend the yt-dlp command options to the list
self._item_cli = merge + self._item_cli
user_cli = {}
user_cli: dict = {}
if len(self._item_cli) > 0:
try:
removed_options = []
user_cli = arg_converter(args="\n".join(self._item_cli), level=True, removed_options=removed_options)
removed_options: list = []
user_cli: dict = arg_converter(
args="\n".join(self._item_cli),
level=True,
removed_options=removed_options,
)
if len(removed_options) > 0:
LOG.warning("Removed the following options: '%s'.", ", ".join(removed_options))

40
app/library/conditions.py
View file

@ -1,8 +1,8 @@
import json
import logging
import os
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from aiohttp import web
@ -11,9 +11,9 @@ from .config import Config
from .encoder import Encoder
from .Events import EventBus, Events
from .Singleton import Singleton
from .Utils import arg_converter
from .Utils import arg_converter, init_class
LOG = logging.getLogger("presets")
LOG = logging.getLogger("conditions")
@dataclass(kw_only=True)
@ -51,16 +51,16 @@ class Conditions(metaclass=Singleton):
_instance = None
"""The instance of the class."""
def __init__(self, file: str | None = None, config: Config | None = None):
def __init__(self, file: Path | str | None = None, config: Config | None = None):
Conditions._instance = self
config = config or Config.get_instance()
self._file: str = file or os.path.join(config.config_path, "conditions.json")
self._file: Path = Path(file) if file else Path(config.config_path) / "conditions.json"
if os.path.exists(self._file) and "600" != oct(os.stat(self._file).st_mode)[-3:]:
if self._file.exists() and "600" != self._file.stat().st_mode:
try:
os.chmod(self._file, 0o600)
self._file.chmod(0o600)
except Exception:
pass
@ -68,7 +68,7 @@ class Conditions(metaclass=Singleton):
msg = "Not implemented"
raise Exception(msg)
EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.add")
EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.save")
@staticmethod
def get_instance() -> "Conditions":
@ -114,15 +114,15 @@ class Conditions(metaclass=Singleton):
"""
self.clear()
if not os.path.exists(self._file) or os.path.getsize(self._file) < 10:
if not self._file.exists() or self._file.stat().st_size < 1:
return self
LOG.info(f"Loading '{self._file}'.")
try:
with open(self._file) as f:
items = json.load(f)
LOG.info(f"Loading '{self._file}'.")
items = json.loads(self._file.read_text())
except Exception as e:
LOG.error(f"Failed to parse '{self._file}'. '{e}'.")
LOG.exception(e)
LOG.error(f"Error loading '{self._file}'. '{e}'.")
return self
if not items or len(items) < 1:
@ -137,15 +137,15 @@ class Conditions(metaclass=Singleton):
item["id"] = str(uuid.uuid4())
need_save = True
item = Condition(**item)
item: Condition = init_class(Condition, item)
self._items.append(item)
except Exception as e:
LOG.error(f"Failed to parse failure condition at list position '{i}'. '{e!s}'.")
LOG.error(f"Failed to parse condition at list position '{i}'. '{e!s}'.")
continue
if need_save:
LOG.info("Saving failure conditions due to format, or id change.")
LOG.info("Saving conditions due to format, or id change.")
self.save(self._items)
return self
@ -229,7 +229,7 @@ class Conditions(metaclass=Singleton):
for i, item in enumerate(items):
try:
if not isinstance(item, Condition):
item = Condition(**item)
item: Condition = init_class(Condition, item)
items[i] = item
except Exception as e:
LOG.error(f"Failed to save '{i}' due to parsing error. '{e!s}'.")
@ -242,12 +242,10 @@ class Conditions(metaclass=Singleton):
continue
try:
with open(self._file, "w") as f:
json.dump(obj=[item.serialize() for item in items], fp=f, indent=4)
self._file.write_text(json.dumps(obj=[item.serialize() for item in items], indent=4))
LOG.info(f"Updated '{self._file}'.")
except Exception as e:
LOG.error(f"Failed to save '{self._file}'. '{e!s}'.")
LOG.error(f"Error saving '{self._file}'. '{e!s}'.")
return self

162
app/library/config.py
View file

@ -11,11 +11,14 @@ from pathlib import Path
import coloredlogs
from dotenv import load_dotenv
from .Utils import FileLogFormatter, arg_converter, load_cookies
from .Utils import FileLogFormatter, arg_converter
from .version import APP_VERSION
class Config:
app_env: str = "production"
"""The application environment, can be 'production' or 'development'."""
config_path: str = "."
"""The path to the configuration directory."""
@ -152,10 +155,13 @@ class Config:
_ytdlp_cli_mutable: str = ""
"""The command line options to use for yt-dlp."""
is_native: bool = False
"Is the application running in webview."
pictures_backends: list[str] = [
"https://unsplash.it/1920/1080?random",
"https://picsum.photos/1920/1080",
"https://placedog.net/1920/1080",
"https://www.bing.com/HPImageArchive.aspx?format=js&idx=0&n=1&mkt=en-US",
]
"The list of picture backends to use for the background."
@ -175,6 +181,7 @@ class Config:
"started",
"ytdlp_cli",
"_ytdlp_cli_mutable",
"is_native",
)
"The variables that are immutable."
@ -223,6 +230,8 @@ class Config:
"ytdlp_cli",
"file_logging",
"base_path",
"is_native",
"app_env",
)
"The variables that are relevant to the frontend."
@ -230,9 +239,9 @@ class Config:
"The manager instance."
@staticmethod
def get_instance():
def get_instance(is_native: bool = False) -> "Config":
"""Static access method."""
return Config() if not Config.__instance else Config.__instance
return Config(is_native) if not Config.__instance else Config.__instance
@staticmethod
def get_manager() -> SyncManager:
@ -241,7 +250,7 @@ class Config:
return Config._manager
def __init__(self):
def __init__(self, is_native: bool = False):
"""Virtually private constructor."""
if Config.__instance is not None:
msg = "This class is a singleton. Use Config.get_instance() instead."
@ -251,15 +260,16 @@ class Config:
baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute())
self.temp_path = os.environ.get("YTP_TEMP_PATH", None) or os.path.join(baseDefaultPath, "var", "tmp")
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or os.path.join(baseDefaultPath, "var", "config")
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or os.path.join(
baseDefaultPath, "var", "downloads"
self.is_native = is_native
self.temp_path = os.environ.get("YTP_TEMP_PATH", None) or str(Path(baseDefaultPath) / "var" / "tmp")
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
Path(baseDefaultPath) / "var" / "downloads"
)
envFile: str = os.path.join(self.config_path, ".env")
envFile: str = Path(self.config_path) / ".env"
if os.path.exists(envFile):
if envFile.exists():
logging.info(f"Loading environment variables from '{envFile}'.")
load_dotenv(envFile)
@ -326,8 +336,8 @@ class Config:
LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}")
ytdl_options = {}
opts_file: str = os.path.join(self.config_path, "ytdlp.cli")
if os.path.exists(opts_file) and os.path.getsize(opts_file) > 2:
opts_file: Path = Path(self.config_path) / "ytdlp.cli"
if opts_file.exists() and opts_file.stat().st_size > 2:
LOG.info(f"Loading yt-dlp custom options from '{opts_file}'.")
with open(opts_file) as f:
self.ytdlp_cli = f.read().strip()
@ -357,25 +367,13 @@ class Config:
self._ytdlp_cli_mutable += f"\n--socket-timeout {self.socket_timeout}"
if self.keep_archive:
LOG.info("keep archive option is enabled.")
archive_file: str = os.path.join(self.config_path, "archive.log")
self._ytdlp_cli_mutable += f"\n--download-archive {archive_file}"
archive_file: Path = Path(self.config_path) / "archive.log"
if not archive_file.exists():
LOG.info(f"Creating archive file '{archive_file}'.")
archive_file.touch(exist_ok=True)
if cookies_file := ytdl_options.get("cookiefile", None):
if os.path.exists(cookies_file) and os.path.getsize(cookies_file) > 2:
LOG.info(f"Using cookies from '{cookies_file}'.")
load_cookies(cookies_file)
self._ytdlp_cli_mutable += f"\n--cookies {cookies_file}"
else:
LOG.warning(f"Invalid cookie file '{cookies_file}' specified.")
ytdl_options.pop("cookiefile", None)
if not ytdl_options.get("cookiefile", None):
cookies_file: str = os.path.join(self.config_path, "cookies.txt")
if os.path.exists(cookies_file) and os.path.getsize(cookies_file) > 2:
LOG.info(f"Using cookies from '{cookies_file}'.")
load_cookies(cookies_file)
self._ytdlp_cli_mutable += f"\n--cookies {cookies_file}"
LOG.info(f"keep archive option is enabled. Using archive file '{archive_file}'.")
self._ytdlp_cli_mutable += f"\n--download-archive {archive_file.as_posix()!s}"
if self.temp_keep:
LOG.info("Keep temp files option is enabled.")
@ -392,23 +390,24 @@ class Config:
msg = f"Invalid file log level '{self.log_level_file}' specified."
raise TypeError(msg)
loggingPath = os.path.join(self.config_path, "logs")
if not os.path.exists(loggingPath):
os.makedirs(loggingPath, exist_ok=True)
loggingPath = Path(self.config_path) / "logs"
if not loggingPath.exists():
loggingPath.mkdir(parents=True, exist_ok=True)
handler = TimedRotatingFileHandler(
filename=os.path.join(self.config_path, "logs", "app.log"),
filename=loggingPath / "app.log",
when="midnight",
backupCount=3,
)
handler.setLevel(log_level_file)
formatter = FileLogFormatter("%(asctime)s [%(levelname)s.%(name)s]: %(message)s")
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
key_file: str = os.path.join(self.config_path, "secret.key")
key_file: str = Path(self.config_path) / "secret.key"
if os.path.exists(key_file) and os.path.getsize(key_file) > 5:
if key_file.exists() and key_file.stat().st_size > 2:
with open(key_file, "rb") as f:
self.secret_key = f.read().strip()
else:
@ -421,6 +420,16 @@ class Config:
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.INFO)
# check env
if self.app_env not in ("production", "development"):
msg: str = (
f"Invalid application environment '{self.app_env}' specified. Must be 'production' or 'development'."
)
raise ValueError(msg)
if self.version == "dev-master":
self._version_via_git()
def _get_attributes(self) -> dict:
attrs: dict = {}
vClass: str = self.__class__
@ -435,6 +444,26 @@ class Config:
return attrs
def is_dev(self) -> bool:
"""
Check if the application is running in development mode.
Returns:
bool: True if the application is in development mode, False otherwise.
"""
return "development" == self.app_env
def is_prod(self) -> bool:
"""
Check if the application is running in production mode.
Returns:
bool: True if the application is in production mode, False otherwise.
"""
return "production" == self.app_env
def get_ytdlp_args(self) -> dict:
try:
return arg_converter(args=self._ytdlp_cli_mutable, level=True)
@ -454,9 +483,6 @@ class Config:
ytdlp_args = self.get_ytdlp_args()
hasCookies = ytdlp_args.get("cookiefile", None)
data["has_cookies"] = hasCookies is not None and os.path.exists(hasCookies)
if not data.get("keep_archive", False) and ytdlp_args.get("download_archive", None):
data["keep_archive"] = True
@ -471,3 +497,59 @@ class Config:
return YTDLP_VERSION
except ImportError:
return "0.0.0"
def _version_via_git(self):
"""
Updates the version of the application using git tags.
This is used to set the version to the latest git tag.
"""
git_path: Path = Path(__file__).parent / ".." / ".." / ".git"
if not git_path.exists():
logging.info(f"Git directory '{git_path}' does not exist. Cannot determine version.")
return
try:
import subprocess
branch_result = subprocess.run( # noqa: S603
["git", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607
cwd=git_path.parent,
capture_output=True,
text=True,
check=False,
)
if 0 != branch_result.returncode:
logging.error(f"Git rev-parse failed: {branch_result.stderr.strip()}")
return
branch_name: str = branch_result.stdout.strip()
if not branch_name:
logging.warning("Git branch name is empty.")
return
commit_result = subprocess.run( # noqa: S603
["git", "log", "-1", "--format=%ct_%H"], # noqa: S607
cwd=git_path.parent,
capture_output=True,
text=True,
check=False,
)
if 0 != commit_result.returncode:
logging.error(f"Git log failed: {commit_result.stderr.strip()}")
return
commit_info: str = commit_result.stdout.strip()
if not commit_info:
logging.warning("Git commit info is empty.")
return
commit_date, commit_sha = commit_info.split("_", 1)
commit_date = time.strftime("%Y%m%d", time.localtime(int(commit_date)))
commit_sha = commit_sha[:8]
self.version = f"{branch_name}-{commit_date}-{commit_sha}"
logging.info(f"Application version set to '{self.version}' based on git data.")
except Exception as e:
logging.error(f"Error while getting git version: {e!s}")

5
app/library/ffprobe.py
View file

@ -7,6 +7,7 @@ import functools
import json
import operator
import os
from pathlib import Path
import anyio
@ -271,7 +272,7 @@ async def ffprobe(file: str) -> FFProbeResult:
msg = "ffprobe not found."
raise OSError(msg) from e
if not os.path.isfile(file):
if not Path(file).exists():
msg = f"No such media file '{file}'."
raise OSError(msg)
@ -290,7 +291,7 @@ async def ffprobe(file: str) -> FFProbeResult:
if 0 == exitCode:
parsed: dict = json.loads(data.decode("utf-8"))
else:
msg = f"FFProbe return with non-0 exit code. '{err.decode('utf-8')}'"
msg = f"ffprobe returned with non-0 exit code. '{err.decode('utf-8')}'"
raise FFProbeError(msg)
result = FFProbeResult()

62
app/library/presets.json
View file

@ -1,62 +0,0 @@
[
{
"id": "3e163c6c-64eb-4448-924f-814b629b3810",
"name": "default",
"description": "",
"folder": "",
"template": "",
"cookies": "",
"cli": "",
"default": true
},
{
"id": "5bf9c42b-8852-468a-99f5-915622dfba25",
"name": "Best video and audio",
"description": "",
"folder": "",
"template": "",
"cookies": "",
"cli": "--format 'bv+ba/b'",
"default": true
},
{
"id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48b",
"name": "1080p H264/m4a or best available",
"description": "",
"folder": "",
"template": "",
"cookies": "",
"cli": "-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
"default": true
},
{
"id": "9719fcc3-4cf2-4d88-b1e4-74dff3dba00e",
"name": "720p h264/m4a or best available",
"description": "",
"folder": "",
"template": "",
"cookies": "",
"cli": "-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
"default": true
},
{
"id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330",
"name": "Audio only",
"description": "",
"folder": "",
"template": "",
"cookies": "",
"cli": "--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'",
"default": true
},
{
"id": "2ade2c28-cad4-4a06-b7eb-2439fdf46f60",
"name": "ytdlp-info-reader",
"description": "This preset generate specific filename format and metadata to work with yt-dlp info reader plugins for jellyfin/emby and plex and to make play state sync work for WatchState.\n\nThere is one more step you need to do via Other > Terminal if you have it enabled or directly from container shell\n\nyt-dlp -I0 --write-info-json --write-thumbnail --convert-thumbnails jpg --paths /downloads/youtube -o \"%(channel|Unknown_title)s [%(channel_id|Unknown_id)s]/%(title).180B [%(channel_id|Unknown_id)s].%(ext)s\" -- https://www.youtube.com/channel/UCClfFsWcT3N2I7VTXXyt84A\n\nChange the url to the channel you want to download.\n\nFor more information please visit \nhttps://github.com/arabcoders/watchstate/blob/master/FAQ.md#how-to-get-watchstate-working-with-youtube-contentlibrary",
"folder": "youtube",
"template": "%(channel)s %(channel_id|Unknown_id)s/Season %(release_date>%Y,upload_date>%Y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s",
"cookies": "",
"cli": "--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata",
"default": true
}
]

61
app/main.py
View file

@ -2,8 +2,8 @@
import asyncio
import logging
import os
import sqlite3
import sys
from pathlib import Path
import caribou
@ -23,16 +23,22 @@ from library.Tasks import Tasks
LOG = logging.getLogger("app")
MIME = magic.Magic(mime=True)
ROOT_PATH: Path = Path(__file__).parent.absolute()
class Main:
def __init__(self):
self._config = Config.get_instance()
self.rootPath = str(Path(__file__).parent.absolute())
def __init__(self, is_native: bool = False):
self._config = Config.get_instance(is_native=is_native)
self._app = web.Application()
if self._config.debug:
loop = asyncio.get_event_loop()
loop.set_debug(True)
loop.slow_callback_duration = 0.05
self._check_folders()
caribou.upgrade(self._config.db_file, os.path.join(self.rootPath, "migrations"))
caribou.upgrade(self._config.db_file, ROOT_PATH / "migrations")
connection = sqlite3.connect(database=self._config.db_file, isolation_level=None)
connection.row_factory = sqlite3.Row
@ -44,13 +50,14 @@ class Main:
LOG.debug("Database connection closed.")
try:
if "600" != oct(os.stat(self._config.db_file).st_mode)[-3:]:
os.chmod(self._config.db_file, 0o600)
db_file = Path(self._config.db_file)
if "600" != oct(db_file.stat().st_mode)[-3:]:
db_file.chmod(0o600)
except Exception:
pass
self._queue = DownloadQueue(connection=connection)
self._http = HttpAPI(queue=self._queue)
self._http = HttpAPI(root_path=ROOT_PATH, queue=self._queue)
self._socket = HttpSocket(queue=self._queue)
self._app.on_cleanup.append(_close_connection)
@ -60,31 +67,34 @@ class Main:
folders = (self._config.download_path, self._config.temp_path, self._config.config_path)
for folder in folders:
folder = Path(folder)
try:
LOG.debug(f"Checking folder at '{folder}'.")
if not os.path.exists(folder):
if not folder.exists():
LOG.info(f"Creating folder at '{folder}'.")
os.makedirs(folder, exist_ok=True)
folder.mkdir(parents=True, exist_ok=True)
except OSError:
LOG.error(f"Could not create folder at '{folder}'.")
raise
try:
LOG.debug(f"Checking database file at '{self._config.db_file}'.")
if not os.path.exists(self._config.db_file):
LOG.info(f"Creating database file at '{self._config.db_file}'.")
with open(self._config.db_file, "w") as _:
pass
except OSError:
LOG.error(f"Could not create database file at '{self._config.db_file}'.")
db_file = Path(self._config.db_file)
LOG.debug(f"Checking database file at '{db_file}'.")
if not db_file.exists():
LOG.info(f"Creating database file at '{db_file}'.")
db_file.touch(exist_ok=True)
except OSError as e:
LOG.error(f"Could not create database file at '{self._config.db_file}'. {e!s}")
raise
def start(self):
def start(self, host: str | None = None, port: int | None = None, cb=None):
"""
Start the application.
"""
EventBus.get_instance().sync_emit(Events.STARTUP, data={"app": self._app})
host = host or self._config.host
port = port or self._config.port
EventBus.get_instance().sync_emit(Events.STARTUP, data={"app": self._app})
Scheduler.get_instance().attach(self._app)
self._socket.attach(self._app)
@ -100,10 +110,10 @@ class Main:
def started(_):
LOG.info("=" * 40)
LOG.info(
f"YTPTube v{self._config.version} - started on http://{self._config.host}:{self._config.port}{self._config.base_path}"
)
LOG.info(f"YTPTube v{self._config.version} - started on http://{host}:{port}{self._config.base_path}")
LOG.info("=" * 40)
if cb:
cb()
HTTP_LOGGER = None
if self._config.access_log:
@ -113,12 +123,13 @@ class Main:
web.run_app(
self._app,
host=self._config.host,
port=self._config.port,
reuse_port=True,
host=host,
port=port,
reuse_port="win32" != sys.platform,
loop=asyncio.get_event_loop(),
access_log=HTTP_LOGGER,
print=started,
handle_signals=cb is None,
)

114
app/native.py Normal file
View file

@ -0,0 +1,114 @@
import json
import os
import queue
import socket
import threading
from pathlib import Path
ready = threading.Event()
exception_holder = queue.Queue()
APP_NAME = "YTPTube"
try:
import webview # type: ignore
except ImportError as e:
msg = "Please run 'pipenv install pywebview[qt]' package to run YTPTube in native mode."
raise ImportError(msg) from e
def set_env():
import platformdirs
dct = {}
if not os.getenv("YTP_CONFIG_PATH"):
dct["YTP_CONFIG_PATH"] = platformdirs.user_config_dir(APP_NAME.lower(), "arabcoders", ensure_exists=True)
if not os.getenv("YTP_TEMP_PATH"):
dct["YTP_TEMP_PATH"] = platformdirs.user_cache_dir(APP_NAME.lower(), "arabcoders", ensure_exists=True)
if not os.getenv("YTP_DOWNLOAD_PATH"):
dct["YTP_DOWNLOAD_PATH"] = platformdirs.user_downloads_dir()
if os.getenv("YTP_ACCESS_LOG", None) is None:
dct["YTP_ACCESS_LOG"] = "false"
if dct:
os.environ.update(dct)
def app_start(host: str, port: int) -> None:
import asyncio
from main import Main
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
Main(is_native=True).start(host, port, cb=lambda: ready.set())
except Exception as e:
exception_holder.put(e)
ready.set()
if __name__ == "__main__":
host = "127.0.0.1"
set_env()
cfg_path: Path = Path(os.getenv("YTP_CONFIG_PATH")) / "webview.json"
port = None
win_conf: dict[str, int] = {}
if cfg_path.exists():
data = json.loads(cfg_path.read_text())
port = data.get("port")
for key in ("width", "height", "x", "y"):
if key in data:
win_conf[key] = data[key]
if not port:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((host, 0))
port = s.getsockname()[1]
cfg_path.write_text(json.dumps({"port": port}))
threading.Thread(target=app_start, args=(host, port), daemon=True).start()
ready.wait()
if not exception_holder.empty():
raise exception_holder.get()
create_kwargs = {**win_conf, "resizable": True}
webview.settings["ALLOW_DOWNLOADS"] = True
webview.settings["OPEN_DEVTOOLS_IN_DEBUG"] = False
window = webview.create_window(APP_NAME, f"http://{host}:{port}", **create_kwargs)
def save_geometry():
cfg = {
"port": port,
"width": window.width,
"height": window.height,
"x": window.x,
"y": window.y,
}
cfg_path.write_text(json.dumps(cfg))
window.events.resized += lambda *_: save_geometry()
window.events.moved += lambda *_: save_geometry()
gui = os.getenv("YTP_WV_GUI", None)
gui = "edgechromium" if os.name == "nt" else "qt"
webview.start(
gui=gui,
debug=True,
storage_path=str(Path(os.getenv("YTP_TEMP_PATH", os.getcwd())) / "webview"),
private_mode=False,
)

36
app/upgrader.py
View file

@ -12,31 +12,18 @@ LOG = logging.getLogger("upgrader")
class Upgrader:
def __init__(self):
import argparse
config_path: Path = Path(__file__).parent.parent / "var" / "config"
if env_path := os.environ.get("YTP_CONFIG_PATH", None):
config_path = Path(env_path)
parser = argparse.ArgumentParser(
prog="upgrader.py",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Upgrade packages and run the application.",
epilog="Example: upgrader.py --run",
)
parser.add_argument(
"-r", "--run", action="store_true", help="Run the application after upgrading the packages."
)
args, _ = parser.parse_known_args()
rootPath = str(Path(__file__).parent.parent.absolute())
config_path = os.environ.get("YTP_CONFIG_PATH", None) or os.path.join(rootPath, "var", "config")
if not Path(config_path).exists():
if config_path.exists():
envFile: Path = config_path / ".env"
if envFile.exists():
LOG.debug(f"loading environment variables from '{envFile}'.")
load_dotenv(str(envFile))
else:
LOG.error(f"config path '{config_path}' doesn't exists.")
envFile = Path(config_path, ".env")
if envFile.exists():
LOG.debug(f"loading environment variables from '{envFile}'.")
load_dotenv(str(envFile))
pkg_installer = PackageInstaller()
ytdlp_auto_update: bool = os.environ.get("YTP_YTDLP_AUTO_UPDATE", "true").strip().lower() == "true"
@ -65,11 +52,6 @@ class Upgrader:
LOG.exception(e)
LOG.error(f"Failed to check for packages. '{e!s}'.")
if args.run:
from main import Main
Main().start()
if __name__ == "__main__":
try:

8
entrypoint.sh
View file

@ -11,11 +11,11 @@ if [ ! -w "${YTP_CONFIG_PATH}" ]; then
CH_GRP=$(stat -c "%g" "${YTP_CONFIG_PATH}")
echo_err "ERROR: Unable to write to '${YTP_CONFIG_PATH}' data directory. Current user id '${UID}' while directory owner is '${CH_USER}'."
echo_err "[Running under docker]"
echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
echo_err "change compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
echo_err "Run the following command to change the directory ownership"
echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config"
echo_err "[Running under podman]"
echo_err "change docker-compose.yaml user: to user:\"0:0\""
echo_err "change compose.yaml user: to user:\"0:0\""
exit 1
fi
@ -24,11 +24,11 @@ if [ ! -w "${YTP_DOWNLOAD_PATH}" ]; then
CH_GRP=$(stat -c "%g" "${YTP_DOWNLOAD_PATH}")
echo_err "ERROR: Unable to write to '${YTP_DOWNLOAD_PATH}' downloads directory. Current user id '${UID}' while directory owner is '${CH_USER}'."
echo_err "[Running under docker]"
echo_err "change docker-compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
echo_err "change compose.yaml user: to user:\"${CH_USER}:${CH_GRP}\""
echo_err "Run the following command to change the directory ownership"
echo_err "chown -R \"${CH_USER}:${CH_GRP}\" ./config"
echo_err "[Running under podman]"
echo_err "change docker-compose.yaml user: to user:\"0:0\""
echo_err "change compose.yaml user: to user:\"0:0\""
exit 1
fi

6
healthcheck.sh
View file

@ -2,6 +2,7 @@
set -euuo pipefail
LOCAL_PORT="${YTP_PORT:-8081}"
BASE_PATH="${YTP_BASE_PATH:-/}"
curl_args=(-f)
@ -10,5 +11,8 @@ if [[ -n "${YTP_AUTH_USERNAME:-}" && -n "${YTP_AUTH_PASSWORD:-}" ]]; then
curl_args+=(-H "Authorization: Basic ${cred}")
fi
curl_args+=("http://localhost:${LOCAL_PORT}/api/ping")
# strip trailing slashes from BASE_PATH
BASE_PATH=$(echo "${BASE_PATH}" | sed 's:/*$::')
curl_args+=("http://localhost:${LOCAL_PORT}${BASE_PATH}/api/ping")
/usr/bin/curl "${curl_args[@]}"

54
pyproject.toml
View file

@ -1,5 +1,42 @@
[project]
name = "ytptube"
version = "1.0.0"
description = "A WebUI for yt-dlp with concurrent downloads support, presets and scheduled tasks and many more."
readme = "README.md"
requires-python = ">=3.11"
license = { text = "MIT" }
authors = [{ name = "Abdulmohsen", email = "contact@arabcoders.org" }]
dependencies = [
# Windows‐only
"tzdata; sys_platform == 'win32'",
"python-magic-bin; sys_platform == 'win32'",
# Non‐Windows
"python-magic>=0.4.27; sys_platform != 'win32'",
# Cross‐platform
"python-socketio>=5.11.1",
"aiohttp>=3.9.3",
"caribou>=0.3.0",
"coloredlogs>=15.0.1",
"aiocron>=1.8",
"python-dotenv>=1.0.1",
"debugpy>=1.8.1",
"httpx",
"async-timeout",
"pyjson5",
"curl_cffi==0.7.1",
"pysubs2",
"regex",
"mutagen",
"brotli",
"brotlicffi",
"anyio",
"pycryptodome",
"yt-dlp",
"platformdirs",
]
[tool.ruff]
# Exclude a variety of commonly ignored directories.
exclude = [
".bzr",
".direnv",
@ -26,7 +63,6 @@ exclude = [
"node_modules",
"site-packages",
"venv",
".venv",
]
# Same as Black.
@ -36,6 +72,16 @@ indent-width = 4
# Assume Python 3.11
target-version = "py311"
[project.optional-dependencies]
webview = [
#windows only
"qtpy; sys_platform == 'win32'",
"pyside6; sys_platform == 'win32'",
# all
"pywebview",
"pyinstaller",
]
[tool.ruff.lint]
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
@ -109,6 +155,7 @@ ignore = [
"PLW2901",
"ERA001",
"S101",
"LOG015",
]
# Allow fix for all enabled rules (when `--fix`) is provided.
@ -147,3 +194,6 @@ docstring-code-line-length = "dynamic"
[tool.autopep8]
--max-line-length = 120
[tool.uv]
link-mode = "copy"

13
ui/components/ConditionForm.vue
View file

@ -71,6 +71,11 @@
<label class="label is-inline" for="filter">
<span class="icon"><i class="fa-solid fa-filter" /></span>
Condition Filter
<template v-if="!addInProgress || form.filter">
- <NuxtLink @click="test_data.show = true" class="is-bold">
Test filter logic
</NuxtLink>
</template>
</label>
<div class="control">
<input type="text" class="input" id="filter" v-model="form.filter" :disabled="addInProgress"
@ -78,13 +83,7 @@
</div>
<span class="help">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>
The yt-dlp <code>[--match-filters]</code> filter logic.
<NuxtLink @click="test_data.show = true" :disabled="addInProgress || !form.filter"
class="is-bold">
Test filter logic
</NuxtLink>
</span>
<span>yt-dlp <code>[--match-filters]</code> logic.</span>
</span>
</div>
</div>

7
ui/components/Queue.vue
View file

@ -66,6 +66,9 @@
</label>
</td>
<td class="is-text-overflow is-vcentered">
<div class="is-inline is-pulled-right" v-if="item.downloaded_bytes">
<span class="tag">{{ formatBytes(item.downloaded_bytes) }}</span>
</div>
<div v-if="showThumbnails && item.extras?.thumbnail">
<FloatingImage :image="uri('/api/thumbnail?url=' + encodePath(item.extras.thumbnail))"
:title="item.title">
@ -180,6 +183,10 @@
<span v-tooltip="moment(item.datetime).format('MMMM Do YYYY, h:mm:ss a')" :data-datetime="item.datetime"
v-rtime="item.datetime" />
</div>
<div class="column is-half-mobile has-text-centered is-text-overflow is-unselectable"
v-if="item.downloaded_bytes">
{{ formatBytes(item.downloaded_bytes) }}
</div>
<div class="column is-half-mobile has-text-centered is-text-overflow is-unselectable">
<label class="checkbox is-block">
<input class="completed-checkbox" type="checkbox" v-model="selectedElms" :id="'checkbox-' + item._id"

2
ui/layouts/default.vue
View file

@ -74,7 +74,7 @@
</div>
</div>
<div class="navbar-item is-hidden-mobile">
<div class="navbar-item is-hidden-mobile" v-if="false === config.app.is_native">
<button class="button is-dark" @click="reloadPage">
<span class="icon"><i class="fas fa-refresh" /></span>
</button>

90
ui/pages/changelog.vue
View file

@ -23,19 +23,19 @@
<template v-if="!isLoading">
<div class="logs-container">
<div class="columns is-multiline">
<div class="column is-12" v-for="(log, index) in logs" :key="log.tag">
<div class="content">
<div class="column p-0 m-0 is-12" v-for="(log, index) in logs" :key="log.tag">
<div class="content p-0 m-0">
<h1 class="is-4">
<span class="icon"><i class="fas fa-code-branch" /></span>
{{ formatTag(log.tag) }} <span class="tag has-text-success" v-if="isInstalled(log.tag)">Installed</span>
{{ formatTag(log) }} <span class="tag has-text-success" v-if="isInstalled(log)">Installed</span>
</h1>
<hr>
<ul>
<li v-for="commit in log.commits" :key="commit.sha">
<strong>{{ ucFirst(commit.message).replace(/\.$/, "") }}.</strong> -
<small>
<NuxtLink :to="`https://github.com/arabcoders/ytptube/commit/${commit.sha}`" target="_blank">
<span class="has-tooltip" v-tooltip="`SHA: ${commit.sha} - Date: ${commit.date}`">
<NuxtLink :to="`${REPO}/commit/${commit.full_sha}`" target="_blank">
<span class="has-tooltip" v-tooltip="`SHA: ${commit.full_sha} - Date: ${commit.date}`">
{{ moment(commit.date).fromNow() }}
</span>
</NuxtLink>
@ -54,44 +54,47 @@
<script setup>
import moment from 'moment'
useHead({ title: 'CHANGELOG' })
const REPO_URL = "https://arabcoders.github.io/ytptube/CHANGELOG-{branch}.json?version={version}";
const toast = useNotification();
const toast = useNotification()
const config = useConfigStore()
const logs = ref([]);
useHead({ title: 'CHANGELOG' })
const PROJECT = 'ytptube'
const REPO = `https://github.com/arabcoders/${PROJECT}`
const REPO_URL = `https://arabcoders.github.io/${PROJECT}/CHANGELOG-{branch}.json?version={version}`
const logs = ref([])
const api_version = ref('')
const isLoading = ref(false);
const isLoading = ref(false)
const hashLength = ref(7)
const branch = computed(() => {
const branch = String(api_version.value).split('-')[0] ?? 'master';
return ['master', 'dev'].includes(branch) ? branch : 'master';
});
const branch = String(api_version.value).split('-')[0] ?? 'master'
return ['master', 'dev'].includes(branch) ? branch : 'master'
})
const formatTag = tag => {
const parts = tag.split('-');
const formatTag = log => {
const parts = log.tag.split('-')
if (parts.length < 3) {
return tag;
return log.tag
}
const branch = parts[0];
const date = parts[1];
const shortSha = parts[2].substring(0, 7);
return `${ucFirst(branch)}: ${moment(date, 'YYYYMMDD').format('YYYY-MM-DD')} - ${shortSha}`;
const branch = parts[0]
const date = parts[1]
const shortSha = log.full_sha.substring(0, hashLength.value)
return `${ucFirst(branch)}: ${moment(date, 'YYYYMMDD').format('YYYY-MM-DD')} - ${shortSha}`
}
watch(() => config.app.version, async value => {
if (!value) {
return
}
api_version.value = value
hashLength.value = value.split('-').pop().length
await nextTick();
await loadContent();
await nextTick()
await loadContent()
}, { immediate: true })
const loadContent = async () => {
@ -99,25 +102,35 @@ const loadContent = async () => {
return
}
isLoading.value = true;
isLoading.value = true
try {
const changes = await fetch(r(REPO_URL, { branch: branch.value, version: api_version.value }));
logs.value = await changes.json();
const changes = await fetch(REPO_URL.replace('{branch}', branch.value).replace('{version}', api_version.value))
logs.value = await changes.json()
} catch (e) {
console.error(e);
toast.error('error', 'Error', `Failed to fetch changelog. ${e.message}`);
console.error(e)
toast.error('error', 'Error', `Failed to fetch changelog. ${e.message}`)
} finally {
isLoading.value = false;
isLoading.value = false
}
}
const isInstalled = tag => {
const installed = String(api_version.value).split('-').pop();
const tagHash = tag.split('-').pop();
return tagHash.startsWith(installed);
const isInstalled = log => {
const installed = String(api_version.value).split('-').pop()
if (log.full_sha.startsWith(installed)) {
return true
}
for (const commit of log?.commits ?? []) {
if (commit.full_sha.startsWith(installed)) {
return true
}
}
return false
}
onMounted(() => loadContent());
onMounted(() => loadContent())
</script>
<style scoped>
@ -130,6 +143,7 @@ onMounted(() => loadContent());
}
hr {
border-bottom: 1px solid #b2d1ff;
background-color: unset;
border-bottom: 1px solid var(--bulma-grey-light) !important
}
</style>

2
ui/stores/ConfigStore.js
View file

@ -11,7 +11,6 @@ const CONFIG_KEYS = {
ytdlp_version: '',
max_workers: 1,
version: '',
has_cookies: false,
basic_mode: true,
default_preset: 'default',
instance_title: null,
@ -20,6 +19,7 @@ const CONFIG_KEYS = {
browser_enabled: false,
ytdlp_cli: '',
file_logging: false,
is_native: false,
},
presets: [
{

5
ui/stores/SocketStore.js
View file

@ -11,7 +11,10 @@ export const useSocketStore = defineStore('socket', () => {
const isConnected = ref(false)
const connect = () => {
let opts = { withCredentials: true }
let opts = {
transports: ['websocket'],
withCredentials: true,
}
let url = runtimeConfig.public.wss

107
ui/utils/ytdlp.ts Normal file
View file

@ -0,0 +1,107 @@
const NUMBER_RE = /\d+(?:\.\d+)?/;
function match_str(filterStr: string, dct: Record<string, any>, incomplete: boolean | Set<string> = false): boolean {
return filterStr
.split(/(?<!\\)&/)
.every(filterPart => _match_one(filterPart.replace(/\\&/g, '&'), dct, incomplete));
}
function lookup_unit_table(unitTable: Record<string, number>, s: string, strict = false): number | null {
const numRe = strict ? NUMBER_RE : /\d+[,.]?\d*/;
const unitsRe = Object.keys(unitTable).map(u => u.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
const re = new RegExp(`^(?<num>${numRe.source})\s*(?<unit>${unitsRe})\b`, strict ? '' : 'i');
const m = s.match(re);
if (!m || !m.groups) return null;
const num = parseFloat(m.groups.num.replace(',', '.'));
const mult = unitTable[m.groups.unit];
return Math.round(num * mult);
}
function parse_filesize(s: string | null): number | null {
if (!s) return null;
const UNIT_TABLE: Record<string, number> = {
B: 1, bytes: 1, b: 1,
KiB: 1024, KB: 1000, kB: 1024, kb: 1000, kilobytes: 1000, kibibytes: 1024,
MiB: 1024 ** 2, MB: 1000 ** 2, megabytes: 1000 ** 2, mebibytes: 1024 ** 2,
GiB: 1024 ** 3, GB: 1000 ** 3, gigabytes: 1000 ** 3, gibibytes: 1024 ** 3,
TiB: 1024 ** 4, TB: 1000 ** 4, terabytes: 1000 ** 4, tebibytes: 1024 ** 4,
};
return lookup_unit_table(UNIT_TABLE, s);
}
function parse_duration(s: string): number | null {
if (!s.trim()) return null;
const durationRe = /^(?:(?:(?<days>\d+):)?(?<hours>\d+):)?(?<mins>\d+):(?<secs>\d{1,2})(?<ms>[.:]\d+)?Z?$/;
const m = s.match(durationRe);
if (!m || !m.groups) return null;
const { days, hours, mins, secs, ms } = m.groups;
return (
(parseFloat(days || '0') * 86400) +
(parseFloat(hours || '0') * 3600) +
(parseFloat(mins || '0') * 60) +
parseFloat(secs || '0') +
parseFloat((ms || '0').replace(':', '.'))
);
}
function _match_one(filterPart: string, dct: Record<string, any>, incomplete: boolean | Set<string>): boolean {
const STRING_OPERATORS: Record<string, Function> = {
'*=': (a: string, v: string) => a.includes(v),
'^=': (a: string, v: string) => a.startsWith(v),
'$=': (a: string, v: string) => a.endsWith(v),
'~=': (a: string, v: string) => new RegExp(v).test(a),
};
const COMPARISON_OPERATORS: Record<string, Function> = {
...STRING_OPERATORS,
'<=': (a: any, b: any) => a <= b,
'<': (a: any, b: any) => a < b,
'>=': (a: any, b: any) => a >= b,
'>': (a: any, b: any) => a > b,
'=': (a: any, b: any) => a == b,
};
const match = filterPart.trim().match(/^([a-z_]+)\s*(!)?\s*([*^$~=<>]+)\s*(?:"(.+)"|'(.+)'|(.+))$/);
if (match) {
const [, key, negation, op, qstr, sstr, ustr] = match;
const value = qstr || sstr || ustr;
const actual = dct[key];
const compValue = parse_filesize(value) || parse_duration(value) || value;
const _cmp = COMPARISON_OPERATORS[op];
if (actual === undefined) return incomplete === true || (incomplete instanceof Set && incomplete.has(key));
return negation ? !_cmp(actual, compValue) : _cmp(actual, compValue);
}
const UNARY_OPERATORS: Record<string, Function> = {
'': (v: any) => v != null,
'!': (v: any) => v == null,
};
const unaryMatch = filterPart.trim().match(/^(!)?([a-z_]+)$/);
if (unaryMatch) {
const [, op, key] = unaryMatch;
const actual = dct[key];
if (actual === undefined && (incomplete === true || (incomplete instanceof Set && incomplete.has(key)))) return true;
return UNARY_OPERATORS[op || ''](actual);
}
throw new Error(`Invalid filter part '${filterPart}'`);
}
export { match_str, lookup_unit_table, parse_filesize, parse_duration, _match_one }

1409
uv.lock Normal file
View file

File diff suppressed because it is too large Load diff