updates API endpoints, fix webhooks not firing due to an error.

This commit is contained in:
ArabCoders 2025-01-03 19:49:45 +03:00
parent 7633f8ffe2
commit 812e92cf23
19 changed files with 142 additions and 85 deletions

2
.vscode/launch.json vendored
View file

@ -40,7 +40,7 @@
"YTP_PIP_IGNORE_UPDATES": "true",
"YTP_LOG_LEVEL": "DEBUG",
"YTP_DEBUG": "true",
"YTP_YTDL_DEBUG": "true",
"YTP_YTDL_DEBUG": "false",
}
},
{

View file

@ -8,7 +8,7 @@ YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since
# YTPTube Features.
* A built in video player that can play any video file regardless of the format. **With support for sidecar external subtitles**.
* New `/add_batch` endpoint that allow multiple links to be sent.
* New `/api/add_batch` endpoint that allow multiple links to be sent.
* Completely redesigned the frontend UI.
* Switched out of binary file storage in favor of SQLite.
* Handle live streams.

View file

@ -14,6 +14,7 @@ from .config import Config
from .Emitter import Emitter
from .ItemDTO import ItemDTO
from .Utils import get_opts, jsonCookie, mergeConfig
from .ffprobe import ffprobe
LOG = logging.getLogger("download")
@ -161,10 +162,10 @@ class Download:
)
LOG.info(
f'Downloading pid: {os.getpid()} id="{self.info.id}" title="{self.info.title}" preset="{self.preset}".'
f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.'
)
LOG.debug(f"Params before passing to yt-dlp: {params}")
LOG.debug("Params before passing to yt-dlp.", extra=params)
cls = yt_dlp.YoutubeDL(params=params)
@ -180,7 +181,7 @@ class Download:
except Exception as exc:
self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)})
LOG.info(f'Finished {os.getpid()=} id="{self.info.id}" title="{self.info.title}".')
LOG.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.')
async def start(self, emitter: Emitter):
self.manager = multiprocessing.Manager()
@ -364,4 +365,13 @@ class Download:
self.info.file_size = None
pass
try:
ff = await ffprobe(status.get("filename"))
self.info.extras['is_video'] = ff.has_video()
self.info.extras['is_audio'] = ff.has_audio()
except Exception as e:
self.info.extras['is_video'] = True
self.info.extras['is_audio'] = True
LOG.exception(f"Failed to ffprobe: {status.get('filename')}. {e}")
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")

View file

@ -22,7 +22,8 @@ from .M3u8 import M3u8
from .Playlist import Playlist
from .Segments import Segments
from .Subtitle import Subtitle
from .Utils import validate_url, StreamingError
from .Utils import validate_url, calcDownloadPath, StreamingError
from .ffprobe import ffprobe
LOG = logging.getLogger("http_api")
MIME = magic.Magic(mime=True)
@ -161,7 +162,7 @@ class HttpAPI(common):
self.routes.route("GET", "/")(lambda _: web.HTTPFound(self.config.url_prefix))
self.routes.get(self.config.url_prefix[:-1])(lambda _: web.HTTPFound(self.config.url_prefix))
self.routes.static(f"{self.config.url_prefix}download/", self.config.download_path)
self.routes.static(f"{self.config.url_prefix}api/download/", self.config.download_path)
self.preloadStatic(app)
try:
@ -209,12 +210,12 @@ class HttpAPI(common):
return middleware_handler
@route("GET", "ping")
@route("GET", "api/ping")
async def ping(self, _) -> Response:
await self.queue.test()
return web.json_response(data={"status": "pong"}, status=web.HTTPOk.status_code)
@route("POST", "add")
@route("POST", "api/add")
async def add_url(self, request: Request) -> Response:
post = await request.json()
@ -242,7 +243,7 @@ class HttpAPI(common):
return web.json_response(data=status, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
@route("GET", "tasks")
@route("GET", "api/tasks")
async def tasks(self, _: Request) -> Response:
tasks_file: str = os.path.join(self.config.config_path, "tasks.json")
@ -258,7 +259,7 @@ class HttpAPI(common):
return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
@route("POST", "add_batch")
@route("POST", "api/add_batch")
async def add_batch(self, request: Request) -> Response:
status = {}
@ -293,7 +294,7 @@ class HttpAPI(common):
return web.json_response(data=status, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
@route("DELETE", "delete")
@route("DELETE", "api/delete")
async def delete(self, request: Request) -> Response:
post = await request.json()
ids = post.get("ids")
@ -311,7 +312,7 @@ class HttpAPI(common):
dumps=self.encoder.encode,
)
@route("POST", "item/{id}")
@route("POST", "api/history/{id}")
async def update_item(self, request: Request) -> Response:
id: str = request.match_info.get("id")
if not id:
@ -348,7 +349,7 @@ class HttpAPI(common):
dumps=self.encoder.encode,
)
@route("GET", "history")
@route("GET", "api/history")
async def history(self, _: Request) -> Response:
data: dict = {"queue": [], "history": []}
@ -359,7 +360,7 @@ class HttpAPI(common):
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=self.encoder.encode)
@route("GET", "workers")
@route("GET", "api/workers")
async def workers(self, _) -> Response:
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
@ -387,7 +388,7 @@ class HttpAPI(common):
dumps=lambda obj: json.dumps(obj, default=lambda o: f"<<non-serializable: {type(o).__qualname__}>>"),
)
@route("POST", "workers")
@route("POST", "api/workers")
async def restart_pool(self, _) -> Response:
if self.queue.pool is None:
return web.json_response({"error": "Worker pool not initialized."}, status=web.HTTPNotFound.status_code)
@ -396,7 +397,7 @@ class HttpAPI(common):
return web.json_response({"message": "Workers pool being restarted."}, status=web.HTTPOk.status_code)
@route("PATCH", "workers/{id}")
@route("PATCH", "api/workers/{id}")
async def restart_worker(self, request: Request) -> Response:
id: str = request.match_info.get("id")
if not id:
@ -409,7 +410,7 @@ class HttpAPI(common):
return web.json_response({"status": "restarted" if status else "in_error_state"}, status=web.HTTPOk.status_code)
@route("delete", "workers/{id}")
@route("delete", "api/workers/{id}")
async def stop_worker(self, request: Request) -> Response:
id: str = request.match_info.get("id")
if not id:
@ -422,7 +423,7 @@ class HttpAPI(common):
return web.json_response({"status": "stopped" if status else "in_error_state"}, status=web.HTTPOk.status_code)
@route("GET", "player/playlist/{file:.*}.m3u8")
@route("GET", "api/player/playlist/{file:.*}.m3u8")
async def playlist(self, request: Request) -> Response:
file: str = request.match_info.get("file")
@ -448,7 +449,7 @@ class HttpAPI(common):
status=web.HTTPOk.status_code,
)
@route("GET", "player/m3u8/{mode}/{file:.*}.m3u8")
@route("GET", "api/player/m3u8/{mode}/{file:.*}.m3u8")
async def m3u8(self, request: Request) -> Response:
file: str = request.match_info.get("file")
mode: str = request.match_info.get("mode")
@ -489,7 +490,7 @@ class HttpAPI(common):
status=web.HTTPOk.status_code,
)
@route("GET", r"player/segments/{segment:\d+}/{file:.*}.ts")
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts")
async def segments(self, request: Request) -> Response:
file: str = request.match_info.get("file")
segment: int = request.match_info.get("segment")
@ -538,7 +539,7 @@ class HttpAPI(common):
status=web.HTTPOk.status_code,
)
@route("GET", "player/subtitle/{file:.*}.vtt")
@route("GET", "api/player/subtitle/{file:.*}.vtt")
async def subtitles(self, request: Request) -> Response:
file: str = request.match_info.get("file")
file_path: str = os.path.normpath(os.path.join(self.config.download_path, file))
@ -573,11 +574,11 @@ class HttpAPI(common):
status=web.HTTPOk.status_code,
)
@route("OPTIONS", "/add")
@route("OPTIONS", "api/add")
async def add_cors(self, _: Request) -> Response:
return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
@route("OPTIONS", "/delete")
@route("OPTIONS", "api/delete")
async def delete_cors(self, _: Request) -> Response:
return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
@ -597,7 +598,7 @@ class HttpAPI(common):
status=web.HTTPOk.status_code,
)
@route("GET", "/thumbnail")
@route("GET", "api/thumbnail")
async def get_thumbnail(self, request: Request) -> Response:
url = request.query.get("url")
if not url:
@ -638,3 +639,19 @@ class HttpAPI(common):
return web.json_response(
data={"error": "failed to retrieve the thumbnail."}, status=web.HTTPInternalServerError.status_code
)
@route("GET", "api/ffprobe/{file:.*}")
async def get_ffprobe(self, request: Request) -> Response:
file: str = request.match_info.get("file")
try:
realFile: str = calcDownloadPath(basePath=self.config.download_path, folder=file, createPath=False)
if not os.path.exists(realFile):
return web.json_response(
data={"error": f"File '{file}' does not exist."}, status=web.HTTPNotFound.status_code
)
return web.json_response(
data=await ffprobe(realFile), status=web.HTTPOk.status_code, dumps=self.encoder.encode
)
except Exception as e:
return web.json_response(data={"error": str(e)}, status=web.HTTPInternalServerError.status_code)

View file

@ -3,7 +3,7 @@ import time
import uuid
from dataclasses import dataclass, field
from email.utils import formatdate
from typing import Any
@dataclass(kw_only=True)
class ItemDTO:
@ -67,3 +67,6 @@ class ItemDTO:
def json(self) -> str:
return json.dumps(self.serialize(), default=lambda o: o.__dict__, sort_keys=True, indent=4)
def get(self, key: str, default: Any = None) -> Any:
return self.__dict__.get(key, default)

View file

@ -67,7 +67,7 @@ class M3u8:
m3u8.append(f"#EXTINF:{segmentSize},")
url = f"{self.url}player/segments/{i}/{quote(file)}.ts"
url = f"{self.url}api/player/segments/{i}/{quote(file)}.ts"
if len(segmentParams) > 0:
url += "?" + "&".join([f"{key}={value}" for key, value in segmentParams.items()])
@ -91,7 +91,7 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
m3u8.append(f"#EXTINF:{duration},")
m3u8.append(f"{self.url}player/subtitle/{quote(file)}.vtt")
m3u8.append(f"{self.url}api/player/subtitle/{quote(file)}.vtt")
m3u8.append("#EXT-X-ENDLIST")
return "\n".join(m3u8)

View file

@ -25,7 +25,7 @@ class Playlist:
return Response(
status=302,
headers={
"Location": f"{self.url}player/playlist/{quote(str(possibleFile).replace(download_path, '').strip('/'))}.m3u8"
"Location": f"{self.url}api/player/playlist/{quote(str(possibleFile).replace(download_path, '').strip('/'))}.m3u8"
},
)
@ -57,16 +57,14 @@ class Playlist:
subs = ',SUBTITLES="subs"'
url = (
f"{self.url}player/m3u8/subtitle/{quote(str(Path(file).with_name(item.name)))}.m3u8?duration={duration}"
)
url = f"{self.url}api/player/m3u8/subtitle/{quote(str(Path(file).with_name(item.name)))}.m3u8?duration={duration}"
name = f"{item.suffix[1:].upper()} ({index}) - {lang}"
playlist.append(
f'#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="{name}",DEFAULT=NO,AUTOSELECT=NO,FORCED=NO,LANGUAGE="{lang}",URI="{url}"'
)
playlist.append(f"#EXT-X-STREAM-INF:PROGRAM-ID=1{subs}")
playlist.append(f"{self.url}player/m3u8/video/{quote(file)}.m3u8")
playlist.append(f"{self.url}api/player/m3u8/video/{quote(file)}.m3u8")
return "\n".join(playlist)

View file

@ -64,7 +64,12 @@ class Webhooks:
from .config import Config
req: dict = target.get("request")
itemId = item.get("id", item.get("_id", "??"))
try:
itemId = item.get("id", item.get("_id", "??"))
except Exception:
itemId = "??"
try:
LOG.info(f"Sending event '{event}' id '{itemId}' to '{target.get('name')}'.")
async with httpx.AsyncClient() as client:

View file

@ -244,7 +244,7 @@ class Config:
try:
import debugpy
debugpy.listen(("0.0.0.0", self.debugpy_port))
debugpy.listen(("0.0.0.0", self.debugpy_port), in_process_debug_adapter=True)
LOG.info(f"starting debugpy server on '0.0.0.0:{self.debugpy_port}'.")
except ImportError:
LOG.error("debugpy package not found, please install it with 'pip install debugpy'.")

View file

@ -200,6 +200,14 @@ class FFProbeResult:
self.subtitle: list[FFStream] = []
self.attachment: list[FFStream] = []
@property
def is_video(self):
return self.has_video()
@property
def is_audio(self):
return self.has_audio()
def get(self, key: str, default=None):
return getattr(self, key) if hasattr(self, key) else default
@ -222,6 +230,24 @@ class FFProbeResult:
def __repr__(self):
return "<FFprobe: {metadata}, {video}, {audio}, {subtitle}, {attachment}>".format(**vars(self))
def unserialize(self, data: dict):
self.metadata = data.get("metadata", {})
self.video = [FFStream(v) for v in data.get("video", [])]
self.audio = [FFStream(a) for a in data.get("audio", [])]
self.subtitle = [FFStream(s) for s in data.get("subtitle", [])]
self.attachment = [FFStream(a) for a in data.get("attachment", [])]
def serialize(self) -> dict:
return {
"metadata": self.metadata,
"video": [v.__dict__ for v in self.video],
"audio": [a.__dict__ for a in self.audio],
"subtitle": [s.__dict__ for s in self.subtitle],
"attachment": [a.__dict__ for a in self.attachment],
"is_video": self.has_video(),
"is_audio": self.has_audio(),
}
@async_lru_cache(maxsize=512)
async def ffprobe(file: str) -> FFProbeResult:

View file

@ -4,4 +4,4 @@ set -eu pipefail
LOCAL_PORT="${YTP_PORT:-8081}"
/usr/bin/curl -f "http://localhost:${LOCAL_PORT}/ping"
/usr/bin/curl -f "http://localhost:${LOCAL_PORT}/api/ping"

View file

@ -53,7 +53,7 @@ const defaultLoader = async () => {
return
}
const response = await fetch(config.app.url_host + config.app.url_prefix + 'thumbnail?url=' + encodePath(props.image), {
const response = await fetch(config.app.url_host + config.app.url_prefix + 'api/thumbnail?url=' + encodePath(props.image), {
signal: cancelRequest.signal
})

View file

@ -102,13 +102,13 @@
<span v-if="'finished' === item.status" @click="playVideo(item)" class="play-overlay">
<div class="play-icon"></div>
<img
:src="config.app.url_host + config.app.url_prefix + 'thumbnail?url=' + encodePath(item.extras.thumbnail)"
:src="config.app.url_host + config.app.url_prefix + 'api/thumbnail?url=' + encodePath(item.extras.thumbnail)"
:alt="item.title" v-if="item.extras?.thumbnail" />
<img v-else src="/images/placeholder.png" :alt="item.title" />
</span>
<NuxtLink v-else target="_blank" :href="item.url" v-tooltip="`Open: ${item.title} link`">
<img :alt="item.title" v-if="item.extras?.thumbnail"
:src="config.app.url_host + config.app.url_prefix + 'thumbnail?url=' + encodePath(item.extras.thumbnail)" />
:src="config.app.url_host + config.app.url_prefix + 'api/thumbnail?url=' + encodePath(item.extras.thumbnail)" />
<img v-else src="/images/placeholder.png" :alt="item.title" />
</NuxtLink>
</figure>
@ -173,18 +173,14 @@
<div class="columns is-mobile is-multiline">
<div class="column is-half-mobile" v-if="item.status != 'finished'">
<a class="button is-warning is-fullwidth" v-tooltip="'Re-queue item.'" @click="reQueueItem(item)">
<span class="icon-text is-block">
<span class="icon"><i class="fa-solid fa-rotate-right" /></span>
<span>Re-queue</span>
</span>
<span class="icon"><i class="fa-solid fa-rotate-right" /></span>
<span class="is-hidden-mobile">Re-queue</span>
</a>
</div>
<div class="column is-half-mobile">
<a class="button is-danger is-fullwidth" @click="removeItem(item)">
<span class="icon-text is-block">
<span class="icon"><i class="fa-solid fa-trash-can" /></span>
<span>Remove</span>
</span>
<span class="icon"><i class="fa-solid fa-trash-can" /></span>
<span class="is-hidden-mobile">Remove</span>
</a>
</div>
<div class="column is-half-mobile" v-if="config.app?.keep_archive && item.status != 'finished'">
@ -192,18 +188,16 @@
@click="archiveItem(item)">
<span class="icon-text is-block">
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
<span>Archive</span>
<span class="is-hidden-mobile">Archive</span>
</span>
</a>
</div>
<div class="column is-half-mobile">
<button class="button is-link is-fullwidth" target="_blank" @click="copyText(item.url)"
v-tooltip="'Copy url to clipboard.'">
<span class="icon-text is-block">
<span class="icon"><i class="fa-solid fa-copy" /></span>
<span>Copy</span>
</span>
</button>
<div class="column is-half-mobile" v-if="item.filename && item.status === 'finished'">
<a class="button is-link is-fullwidth" :href="makeDownload(config, item)"
:download="item.filename?.split('/').reverse()[0]">
<span class="icon"><i class="fa-solid fa-download" /></span>
<span class="is-hidden-mobile">Download</span>
</a>
</div>
</div>
</div>
@ -231,13 +225,13 @@
</div>
<div class="modal is-active" v-if="video_link">
<div class="modal-background"></div>
<div class="modal-background" @click="closeVideo"></div>
<div class="modal-content">
<VideoPlayer type="default" :link="video_link" :isMuted="false" autoplay="true" :isControls="true"
:title="video_title" :thumbnail="video_thumbnail" :artist="video_artist" class="is-fullwidth"
@closeModel="video_link = ''; video_title = ''; video_thumbnail = ''; video_artist = '';" />
@closeModel="closeVideo" />
</div>
<button class="modal-close is-large" aria-label="close" @click="video_link = ''"></button>
<button class="modal-close is-large" aria-label="close" @click="closeVideo"></button>
</div>
</div>
</template>
@ -269,7 +263,7 @@ const playVideo = item => {
video_link.value = makeDownload(config, item, 'm3u8')
video_title.value = item.title
if (item.extras?.thumbnail) {
video_thumbnail.value = config.app.url_host + config.app.url_prefix + 'thumbnail?url=' + encodePath(item.extras.thumbnail)
video_thumbnail.value = config.app.url_host + config.app.url_prefix + 'api/thumbnail?url=' + encodePath(item.extras.thumbnail)
}
if (item.extras?.channel) {
video_artist.value = item.extras.channel
@ -277,6 +271,16 @@ const playVideo = item => {
if (!video_artist.value && item.extras?.uploader) {
video_artist.value = item.extras.uploader
}
if (!item.extras?.is_video && item.extras?.is_audio) {
video_audio.value = true
}
}
const closeVideo = () => {
video_link.value = ''
video_title.value = ''
video_thumbnail.value = ''
video_artist.value = ''
}
watch(masterSelectAll, (value) => {

View file

@ -52,7 +52,7 @@
<figure class="image is-3by1" v-if="item.extras?.thumbnail">
<NuxtLink v-tooltip="item.title" :href="item.url" target="_blank">
<img
:src="config.app.url_host + config.app.url_prefix + 'thumbnail?url=' + encodePath(item.extras.thumbnail)"
:src="config.app.url_host + config.app.url_prefix + 'api/thumbnail?url=' + encodePath(item.extras.thumbnail)"
:alt="item.title" />
</NuxtLink>
</figure>
@ -202,7 +202,7 @@ const ETAPipe = value => {
}
const speedPipe = value => {
if (value === null || 0 === value) {
if (null === value || 0 === value) {
return '0KB/s';
}
@ -231,13 +231,13 @@ const updateProgress = (item) => {
return 'Preparing';
}
if (item.status != null) {
if (null != item.status) {
string += item.percent && !item.is_live ? percentPipe(item.percent) + '%' : 'Live';
}
string += item.speed ? ' - ' + speedPipe(item.speed) : ' - Waiting..';
if (item.status != null && item.eta) {
if (null != item.status && item.eta) {
string += ' - ' + ETAPipe(item.eta);
}

View file

@ -18,7 +18,7 @@
<template>
<div>
<video ref="video" :data-poster="thumbnail" :controls="isControls" :title="title" playsinline>
<video ref="video" :poster="thumbnail" :controls="isControls" :title="title" playsinline>
<source :src="link" type="application/x-mpegURL" />
</video>
</div>
@ -141,7 +141,6 @@ const prepareVideoPlayer = () => {
if (props.thumbnail) {
opts.poster = props.thumbnail
}
player = new Plyr(video.value, opts);
hls = new Hls({
@ -161,7 +160,5 @@ const prepareVideoPlayer = () => {
if (props.title) {
window.document.title = `YTPTube - Playing: ${props.title}`
}
}
</script>

View file

@ -67,9 +67,9 @@
<div class="column is-8-mobile">
<div class="has-text-left" v-if="config.app?.version">
© {{ Year }} - <NuxtLink href="https://github.com/ArabCoders/ytptube" target="_blank">YTPTube</NuxtLink>
<span class="is-hidden-mobile">&nbsp;({{ config.app?.version || 'unknown' }})</span>
<span class="is-hidden-mobile">&nbsp;({{ config?.app?.version || 'unknown' }})</span>
- <NuxtLink target="_blank" href="https://github.com/yt-dlp/yt-dlp">yt-dlp</NuxtLink>
<span class="is-hidden-mobile">&nbsp;({{ config?.app.ytdlp_version || 'unknown' }})</span>
<span class="is-hidden-mobile">&nbsp;({{ config?.app?.ytdlp_version || 'unknown' }})</span>
</div>
</div>
<div class="column is-4-mobile" v-if="config.app?.started">

View file

@ -59,11 +59,8 @@ import "@xterm/xterm/css/xterm.css"
import { Terminal } from "@xterm/xterm"
import { FitAddon } from "@xterm/addon-fit"
const emitter = defineEmits(['runCommand', 'cli_clear']);
const terminal = ref()
const terminalFit = ref()
const command = ref('')
const terminal_window = ref()
const command_input = ref()

View file

@ -1,3 +1,13 @@
<style scoped>
table.is-fixed {
table-layout: fixed;
}
div.table-container {
overflow: hidden;
}
</style>
<template>
<div class="mt-1 columns is-multiline">
<div class="column is-12 is-clearfix">
@ -55,13 +65,3 @@ import moment from 'moment'
import { parseExpression } from 'cron-parser'
const config = useConfigStore()
</script>
<style scoped>
table.is-fixed {
table-layout: fixed;
}
div.table-container {
overflow: hidden;
}
</style>

View file

@ -345,8 +345,8 @@ const getQueryParams = (url = window.location.search) => Object.fromEntries(new
*
* @returns {string} The download URL
*/
const makeDownload = (config, item, base = 'download') => {
let baseDir = 'download' === base ? `${base}/` : 'player/playlist/';
const makeDownload = (config, item, base = 'api/download') => {
let baseDir = 'api/download' === base ? `${base}/` : 'api/player/playlist/';
if (item.folder) {
item.folder = item.folder.replace(/#/g, '%23');