From 46e7d28fc878044dfb66968589124b2e46ce13d7 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Tue, 26 Mar 2024 00:16:00 +0300 Subject: [PATCH 1/3] Cleanup the video player. --- app/main.py | 6 +++--- app/player/M3u8.py | 14 +++++++------- app/player/Segments.py | 26 +++++++++++++------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/app/main.py b/app/main.py index 1c374cae..93f5e97e 100644 --- a/app/main.py +++ b/app/main.py @@ -443,14 +443,14 @@ class Main: raise web.HTTPBadRequest(reason='segment is required') segmenter = Segments( - segment_index=int(segment), - segment_duration=float('{:.6f}'.format(float(sd if sd else M3u8.segment_duration))), + index=int(segment), + duration=float('{:.6f}'.format(float(sd if sd else M3u8.duration))), vconvert=True if vc == 1 else False, aconvert=True if ac == 1 else False ) return web.Response( - body=await segmenter.stream(download_path=self.config.download_path, file=file), + body=await segmenter.stream(path=self.config.download_path, file=file), headers={ 'Content-Type': 'video/mpegts', 'Cache-Control': 'no-cache', diff --git a/app/player/M3u8.py b/app/player/M3u8.py index 8980e4d4..22790217 100644 --- a/app/player/M3u8.py +++ b/app/player/M3u8.py @@ -9,12 +9,12 @@ class M3u8: ok_vcodecs: tuple = ('h264', 'x264', 'avc',) ok_acodecs: tuple = ('aac', 'mp3',) - segment_duration: float = 10.000000 url: str = None + duration: float = 6.000000 - def __init__(self, url: str, segment_duration: float = 6.000000): + def __init__(self, url: str, segment_duration: float = None): self.url = url - self.segment_duration = float(segment_duration) + self.duration = float(segment_duration) if segment_duration is not None else self.duration async def make_stream(self, download_path: str, file: str): realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) @@ -35,12 +35,12 @@ class M3u8: m3u8 = "#EXTM3U\n" m3u8 += "#EXT-X-VERSION:3\n" - m3u8 += f"#EXT-X-TARGETDURATION:{int(self.segment_duration)}\n" + m3u8 += f"#EXT-X-TARGETDURATION:{int(self.duration)}\n" m3u8 += "#EXT-X-MEDIA-SEQUENCE:0\n" m3u8 += "#EXT-X-PLAYLIST-TYPE:VOD\n" - segmentSize: float = '{:.6f}'.format(self.segment_duration) - splits: int = math.ceil(duration / self.segment_duration) + segmentSize: float = '{:.6f}'.format(self.duration) + splits: int = math.ceil(duration / self.duration) segmentParams: dict = {} @@ -54,7 +54,7 @@ class M3u8: for i in range(splits): if (i + 1) == splits: - segmentParams.update({'sd': '{:.6f}'.format(duration - (i * self.segment_duration))}) + segmentParams.update({'sd': '{:.6f}'.format(duration - (i * self.duration))}) m3u8 += f"#EXTINF:{segmentParams['sd']}, nodesc\n" else: m3u8 += f"#EXTINF:{segmentSize}, nodesc\n" diff --git a/app/player/Segments.py b/app/player/Segments.py index b862717c..f9f7243f 100644 --- a/app/player/Segments.py +++ b/app/player/Segments.py @@ -10,19 +10,19 @@ LOG = logging.getLogger('segments') class Segments: - segment_duration: int - segment_index: int + duration: int + index: int vconvert: bool aconvert: bool - def __init__(self, segment_index: int, segment_duration: float, vconvert: bool, aconvert: bool): - self.segment_duration = float(segment_duration) - self.segment_index = int(segment_index) + def __init__(self, index: int, duration: float, vconvert: bool, aconvert: bool): + self.index = int(index) + self.duration = float(duration) self.vconvert = bool(vconvert) self.aconvert = bool(aconvert) - async def stream(self, download_path: str, file: str) -> bytes: - realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) + async def stream(self, path: str, file: str) -> bytes: + realFile: str = calcDownloadPath(basePath=path, folder=file, createPath=False) if not os.path.exists(realFile): raise Exception(f"File {realFile} does not exist.") @@ -33,10 +33,10 @@ class Segments: if not os.path.exists(tmpFile): os.symlink(realFile, tmpFile) - if self.segment_index == 0: + if self.index == 0: startTime: float = '{:.6f}'.format(0) else: - startTime: float = '{:.6f}'.format((self.segment_duration * self.segment_index)) + startTime: float = '{:.6f}'.format((self.duration * self.index)) fargs = [] fargs.append('-xerror') @@ -47,7 +47,7 @@ class Segments: fargs.append('-ss') fargs.append(str(startTime if startTime else '0.00000')) fargs.append('-t') - fargs.append(str('{:.6f}'.format(self.segment_duration))) + fargs.append(str('{:.6f}'.format(self.duration))) fargs.append('-copyts') @@ -99,7 +99,7 @@ class Segments: fargs.append('mpegts') fargs.append('pipe:1') - LOG.debug(f"Streaming '{realFile}' segment '{self.segment_index}'. " + " ".join(fargs)) + LOG.debug(f"Streaming '{realFile}' segment '{self.index}'. " + " ".join(fargs)) proc = await asyncio.subprocess.create_subprocess_exec( 'ffmpeg', *fargs, @@ -111,7 +111,7 @@ class Segments: data, err = await proc.communicate() if 0 != proc.returncode: - LOG.error(f'Failed to stream {realFile} segment {self.segment_index}. {err.decode("utf-8")}') - raise Exception(f'Failed to stream {realFile} segment {self.segment_index}.') + LOG.error(f'Failed to stream {realFile} segment {self.index}. {err.decode("utf-8")}') + raise Exception(f'Failed to stream {realFile} segment {self.index}.') return data From 5d57ae25abe567839e077b73a9ff188be70f79fb Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Thu, 28 Mar 2024 22:55:24 +0300 Subject: [PATCH 2/3] Fixes --- app/DataStore.py | 1 - app/main.py | 12 ++++++++---- app/player/Segments.py | 1 - app/player/ffprobe.py | 3 --- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/DataStore.py b/app/DataStore.py index 0de83233..0c811f0b 100644 --- a/app/DataStore.py +++ b/app/DataStore.py @@ -4,7 +4,6 @@ from datetime import datetime, timezone from email.utils import formatdate import json from sqlite3 import Connection -from Utils import calcDownloadPath from Config import Config from Download import Download from ItemDTO import ItemDTO diff --git a/app/main.py b/app/main.py index 93f5e97e..65cc68f1 100644 --- a/app/main.py +++ b/app/main.py @@ -11,7 +11,6 @@ from Utils import ObjectSerializer, Notifier from aiohttp import web, client from aiohttp.web import Request, Response from Webhooks import Webhooks -from Download import Download from player.M3u8 import M3u8 from player.Segments import Segments import socketio @@ -416,11 +415,16 @@ class Main: if not file: raise web.HTTPBadRequest(reason='file is required.') - return web.Response( - text=await M3u8(url=f"{self.config.url_host}{self.config.url_prefix}").make_stream( + try: + text = await M3u8(url=f"{self.config.url_host}{self.config.url_prefix}").make_stream( download_path=self.config.download_path, file=file - ), + ) + except Exception as e: + return web.HTTPNotFound(reason=str(e)) + + return web.Response( + text=text, headers={ 'Content-Type': 'application/x-mpegURL', 'Cache-Control': 'no-cache', diff --git a/app/player/Segments.py b/app/player/Segments.py index f9f7243f..b89f781d 100644 --- a/app/player/Segments.py +++ b/app/player/Segments.py @@ -2,7 +2,6 @@ import asyncio import hashlib import logging import os -import subprocess import tempfile from Utils import calcDownloadPath diff --git a/app/player/ffprobe.py b/app/player/ffprobe.py index e40c7796..66e3da01 100644 --- a/app/player/ffprobe.py +++ b/app/player/ffprobe.py @@ -4,11 +4,8 @@ Python wrapper for ffprobe command line tool. ffprobe must exist in the path. import asyncio import functools import json -import logging import operator import os -import pipes -import subprocess class FFProbeError(Exception): From 39e594c394b59babaf131e5df8637f908c767b53 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Fri, 29 Mar 2024 00:08:16 +0300 Subject: [PATCH 3/3] upgrade webhooks to support both form and json request. --- .vscode/settings.json | 1 + Pipfile | 1 + Pipfile.lock | 47 ++++++++++++++++++++++++++++++++------ README.md | 2 ++ app/Webhooks.py | 53 +++++++++++++++++++++++++++++++------------ 5 files changed, 82 insertions(+), 22 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f1a33979..4f87eae2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,7 @@ "dotenv", "finaldir", "getpid", + "httpx", "libcurl", "libx", "mpegts", diff --git a/Pipfile b/Pipfile index 372a888d..b9eeafc2 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ aiocron = ">=1.8" python-dotenv = ">=1.0.1" python-magic = ">=0.4.27" debugpy = ">=1.8.1" +httpx = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 45b43f32..2b5921ba 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6f7d266189a02316b0a7f6d1958351963de5dfa35b6e3860c1da5da875629843" + "sha256": "b8c580658ddc19c696fab69a5317e66edf477f5560957ab549357b25b1b70212" }, "pipfile-spec": 6, "requires": { @@ -115,6 +115,14 @@ "markers": "python_version >= '3.7'", "version": "==1.3.1" }, + "anyio": { + "hashes": [ + "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", + "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6" + ], + "markers": "python_version >= '3.8'", + "version": "==4.3.0" + }, "argparse": { "hashes": [ "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", @@ -349,11 +357,11 @@ }, "croniter": { "hashes": [ - "sha256:78bf110a2c7dbbfdd98b926318ae6c64a731a4c637c7befe3685755110834746", - "sha256:8bff16c9af4ef1fb6f05416973b8f7cb54997c02f2f8365251f9bf1dded91866" + "sha256:28763ad39c404e159140874f08010cfd8a18f4c2a7cea1ce73e9506a4380cfc1", + "sha256:84dc95b2eb6760144cc01eca65a6b9cc1619c93b2dc37d8a27f4319b3eb740de" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.0.2" + "version": "==2.0.3" }, "debugpy": { "hashes": [ @@ -475,6 +483,23 @@ "markers": "python_version >= '3.7'", "version": "==0.14.0" }, + "httpcore": { + "hashes": [ + "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61", + "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5" + ], + "markers": "python_version >= '3.8'", + "version": "==1.0.5" + }, + "httpx": { + "hashes": [ + "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5", + "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==0.27.0" + }, "humanfriendly": { "hashes": [ "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", @@ -669,12 +694,12 @@ }, "python-socketio": { "hashes": [ - "sha256:bbcbd758ed8c183775cb2853ba001361e2fa018babf5cbe11a5b77e91c2ec2a2", - "sha256:f1a0228b8b1fbdbd93fbbedd821ebce0ef54b2b5bf6e98fcf710deaa7c574259" + "sha256:ae6a1de5c5209ca859dc574dccc8931c4be17ee003e74ce3b8d1306162bb4a37", + "sha256:b9f22a8ff762d7a6e123d16a43ddb1a27d50f07c3c88ea999334f2f89b0ad52b" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==5.11.1" + "version": "==5.11.2" }, "pytz": { "hashes": [ @@ -707,6 +732,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, + "sniffio": { + "hashes": [ + "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", + "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.1" + }, "tzlocal": { "hashes": [ "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8", diff --git a/README.md b/README.md index 7607c7ee..4c4f325a 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,8 @@ The `config/webhooks.json`, is a json file, which can be used to add webhook end "request":{ // (url: string) - REQUIRED- The webhook url "url": "https://mysecert.webhook.com/endpoint", + // (type: string) - OPTIONAL - The request type, it can be json or form. + "type": "json", // (method: string) - OPTIONAL - The request method, it can be POST or PUT "method": "POST", // (headers: dictionary) - OPTIONAL - Extra headers to include. diff --git a/app/Webhooks.py b/app/Webhooks.py index 2887f98c..86c0cde2 100644 --- a/app/Webhooks.py +++ b/app/Webhooks.py @@ -3,8 +3,8 @@ import json import logging import os from ItemDTO import ItemDTO -from aiohttp import client - +import httpx +from version import APP_VERSION LOG = logging.getLogger('Webhooks') @@ -49,20 +49,43 @@ class Webhooks: req: dict = target.get('request') try: LOG.info(f"Sending {event=} {item.id=} to [{target.get('name')}]") - async with client.ClientSession() as session: - headers = req.get('headers', {}) if 'headers' in req else {} - async with session.request(method=req.get('method', 'POST'), url=req.get('url'), json=item.__dict__, headers=headers) as response: - respData = { - 'url': req.get('url'), - 'status': response.status, - 'text': await response.text() - } - msg = f"[{target.get('name')}] Response to [{event=} {item.id=}] [status: {response.status}]." - if respData.get('text'): - msg += f" [Body: {respData.get('text','??')}]" - LOG.info(msg) + async with httpx.AsyncClient() as client: + request_type = req.get('type', 'json') - return respData + reqBody = { + 'method': req.get('method', 'POST'), + 'url': req.get('url'), + 'headers': { + 'User-Agent': f"YTPTube/{APP_VERSION}" + }, + } + + if req.get('headers', None): + reqBody['headers'].update(req.get('headers')) + + match(request_type): + case 'json': + reqBody['json'] = item.__dict__ + reqBody['headers']['Content-Type'] = 'application/json' + case _: + reqBody['data'] = item.__dict__ + reqBody['headers']['Content-Type'] = 'application/x-www-form-urlencoded' + + response = await client.request(**reqBody) + + respData = { + 'url': req.get('url'), + 'status': response.status_code, + 'text': response.text + } + + msg = f"[{target.get('name')}] Response to [{event=} {item.id=}] [status: {response.status_code}]." + if respData.get('text'): + msg += f" [Body: {respData.get('text','??')}]" + + LOG.info(msg) + + return respData except Exception as e: return { 'url': req.get('url'),