469 lines
17 KiB
Python
469 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import asyncio
|
|
from datetime import datetime
|
|
import json
|
|
import os
|
|
import random
|
|
from Config import Config
|
|
from DownloadQueue import DownloadQueue
|
|
from Utils import ObjectSerializer, Notifier
|
|
from aiohttp import web, client
|
|
from aiohttp.web import Request, Response
|
|
from Webhooks import Webhooks
|
|
from player.M3u8 import M3u8
|
|
from player.Segments import Segments
|
|
import socketio
|
|
import logging
|
|
import caribou
|
|
import sqlite3
|
|
from aiocron import crontab
|
|
import magic
|
|
|
|
LOG = logging.getLogger('app')
|
|
MIME = magic.Magic(mime=True)
|
|
|
|
|
|
class Main:
|
|
config: Config = None
|
|
serializer: ObjectSerializer = None
|
|
app: web.Application = None
|
|
sio: socketio.AsyncServer = None
|
|
routes: web.RouteTableDef = None
|
|
connection: sqlite3.Connection = None
|
|
queue: DownloadQueue = None
|
|
loop: asyncio.AbstractEventLoop = None
|
|
appLoader: str = None
|
|
|
|
staticHolder: dict = {}
|
|
|
|
extToMime: dict = {
|
|
'.html': 'text/html',
|
|
'.css': 'text/css',
|
|
'.js': 'application/javascript',
|
|
'.json': 'application/json',
|
|
'.ico': 'image/x-icon',
|
|
}
|
|
|
|
def __init__(self):
|
|
self.config = Config.get_instance()
|
|
|
|
try:
|
|
if not os.path.exists(self.config.download_path):
|
|
LOG.info(f'Creating download folder at [{self.config.download_path}]')
|
|
os.makedirs(self.config.download_path, exist_ok=True)
|
|
except OSError as e:
|
|
LOG.error(f'Could not create download folder at [{self.config.download_path}]')
|
|
raise e
|
|
|
|
try:
|
|
if not os.path.exists(self.config.temp_path):
|
|
LOG.info(f'Creating temp folder at [{self.config.temp_path}]')
|
|
os.makedirs(self.config.temp_path, exist_ok=True)
|
|
except OSError as e:
|
|
LOG.error(f'Could not create temp folder at [{self.config.temp_path}]')
|
|
raise e
|
|
|
|
try:
|
|
if not os.path.exists(self.config.config_path):
|
|
LOG.info(f'Creating config folder at [{self.config.config_path}]')
|
|
os.makedirs(self.config.config_path, exist_ok=True)
|
|
except OSError as e:
|
|
LOG.error(f'Could not create config folder at [{self.config.config_path}]')
|
|
raise e
|
|
|
|
try:
|
|
if not os.path.exists(self.config.db_file):
|
|
LOG.info(f'Creating database file at [{self.config.db_file}]')
|
|
with open(self.config.db_file, 'w') as _:
|
|
pass
|
|
except OSError as e:
|
|
LOG.error(f'Could not create database file at [{self.config.db_file}]')
|
|
raise e
|
|
|
|
caribou.upgrade(self.config.db_file, os.path.join(os.path.realpath(os.path.dirname(__file__)), 'migrations'))
|
|
|
|
self.webhooks = Webhooks(os.path.join(self.config.config_path, 'webhooks.json'))
|
|
self.loop = asyncio.get_event_loop()
|
|
self.serializer = ObjectSerializer()
|
|
|
|
self.app = web.Application()
|
|
self.sio = socketio.AsyncServer(cors_allowed_origins='*')
|
|
self.sio.attach(self.app, socketio_path=self.config.url_prefix + 'socket.io')
|
|
|
|
self.routes = web.RouteTableDef()
|
|
|
|
self.connection = sqlite3.connect(self.config.db_file, isolation_level=None)
|
|
self.connection.row_factory = sqlite3.Row
|
|
self.connection.execute('PRAGMA journal_mode=wal')
|
|
|
|
self.queue = DownloadQueue(
|
|
notifier=Notifier(sio=self.sio, serializer=self.serializer, webhooks=self.webhooks),
|
|
connection=self.connection,
|
|
serializer=self.serializer,
|
|
)
|
|
|
|
self.app.on_startup.append(lambda _: self.queue.initialize())
|
|
self.addRoutes()
|
|
self.addTasks()
|
|
self.start()
|
|
|
|
def start(self):
|
|
web.run_app(
|
|
self.app,
|
|
host=self.config.host,
|
|
port=self.config.port,
|
|
reuse_port=True,
|
|
loop=self.loop,
|
|
access_log=None,
|
|
print=lambda _: print(
|
|
f'YTPTube v{self.config.version} - listening on http://{self.config.host}:{self.config.port}'),
|
|
)
|
|
|
|
async def connect(self, sid, _):
|
|
data: dict = {
|
|
**self.queue.get(),
|
|
"config": self.config,
|
|
"tasks": [],
|
|
}
|
|
|
|
if os.path.exists(os.path.join(self.config.config_path, 'tasks.json')):
|
|
try:
|
|
with open(os.path.join(self.config.config_path, 'tasks.json'), 'r') as f:
|
|
data['tasks'] = json.load(f)
|
|
except Exception as e:
|
|
pass
|
|
|
|
await self.sio.emit('initial_data', self.serializer.encode(data), to=sid)
|
|
|
|
async def version_check(self):
|
|
try:
|
|
with client.ClientSession() as session:
|
|
async with session.get('https://api.github.com/repos/ytptube/ytptube/tags') as r:
|
|
if r.status != 200:
|
|
return
|
|
|
|
tags = await r.json()
|
|
|
|
for tag in tags:
|
|
tagName = tag.get('name')
|
|
splitTagName = tagName.split('-')
|
|
if len(splitTagName) > 1:
|
|
tagName = splitTagName[0]
|
|
cvDate = self.config.version.split('-')
|
|
if tagName > cvDate[0]:
|
|
self.config.new_version_available = True
|
|
LOG.info(f'New version [{tagName}] is available.')
|
|
break
|
|
except Exception as e:
|
|
pass
|
|
|
|
def addTasks(self):
|
|
tasks_file: str = os.path.join(self.config.config_path, 'tasks.json')
|
|
if not os.path.exists(tasks_file):
|
|
LOG.info(
|
|
f'No tasks file found at {tasks_file}. Skipping Tasks.')
|
|
return
|
|
|
|
try:
|
|
with open(tasks_file, 'r') as f:
|
|
tasks = json.load(f)
|
|
except Exception as e:
|
|
LOG.error(f'Could not load tasks file [{tasks_file}]. Error message [{str(e)}]. Skipping Tasks.')
|
|
return
|
|
|
|
for task in tasks:
|
|
if not task.get('url'):
|
|
LOG.warning(f'Invalid task {task}.')
|
|
continue
|
|
|
|
cron_timer: str = task.get('timer', f'{random.randint(1,59)} */1 * * *')
|
|
|
|
async def cron_runner(task: dict):
|
|
taskName = task.get("name", task.get("url"))
|
|
timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
LOG.info(f'Started [Task: {taskName}] at [{timeNow}].')
|
|
|
|
await self.add(
|
|
url=task.get('url'),
|
|
quality=task.get('quality', 'best'),
|
|
format=task.get('format', 'any'),
|
|
folder=task.get('folder'),
|
|
ytdlp_cookies=task.get('ytdlp_cookies'),
|
|
ytdlp_config=task.get('ytdlp_config'),
|
|
output_template=task.get('output_template')
|
|
)
|
|
|
|
timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
LOG.info(f'Completed [Task: {taskName}] at [{timeNow}].')
|
|
|
|
crontab(
|
|
spec=cron_timer,
|
|
func=cron_runner,
|
|
args=(task,),
|
|
start=True,
|
|
loop=self.loop
|
|
)
|
|
|
|
LOG.info(f'Added [Task: {task.get("name", task.get("url"))}] executed every [{cron_timer}].')
|
|
|
|
def addRoutes(self):
|
|
@self.routes.get(self.config.url_prefix + 'ping')
|
|
async def ping(_) -> Response:
|
|
self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone()
|
|
return web.Response(text='pong')
|
|
|
|
@self.routes.post(self.config.url_prefix + 'add')
|
|
async def add(request: Request) -> Response:
|
|
post = await request.json()
|
|
|
|
url: str = post.get('url')
|
|
quality: str = post.get('quality')
|
|
|
|
if not url:
|
|
raise web.HTTPBadRequest()
|
|
|
|
format: str = post.get('format')
|
|
folder: str = post.get('folder')
|
|
ytdlp_cookies: str = post.get('ytdlp_cookies')
|
|
ytdlp_config: dict = post.get('ytdlp_config')
|
|
output_template: str = post.get('output_template')
|
|
if ytdlp_config is None:
|
|
ytdlp_config = {}
|
|
|
|
status = await self.add(
|
|
url=url,
|
|
quality=quality,
|
|
format=format,
|
|
folder=folder,
|
|
ytdlp_cookies=ytdlp_cookies,
|
|
ytdlp_config=ytdlp_config,
|
|
output_template=output_template
|
|
)
|
|
|
|
return web.Response(text=self.serializer.encode(status))
|
|
|
|
@self.routes.get(self.config.url_prefix + 'tasks')
|
|
async def tasks(_: Request) -> Response:
|
|
tasks_file: str = os.path.join(self.config.config_path, 'tasks.json')
|
|
|
|
if not os.path.exists(tasks_file):
|
|
return web.json_response({"error": "No tasks defined."}, status=404)
|
|
|
|
try:
|
|
with open(tasks_file, 'r') as f:
|
|
tasks = json.load(f)
|
|
except Exception as e:
|
|
return web.json_response({"error": str(e)}, status=500)
|
|
|
|
return web.json_response(tasks)
|
|
|
|
@self.routes.post(self.config.url_prefix + 'add_batch')
|
|
async def add_batch(request: Request) -> Response:
|
|
status = {}
|
|
|
|
post = await request.json()
|
|
if not isinstance(post, list):
|
|
raise web.HTTPBadRequest()
|
|
|
|
for item in post:
|
|
if not isinstance(item, dict):
|
|
raise web.HTTPBadRequest(reason='Invalid request body expecting list with dicts.')
|
|
if not item.get('url'):
|
|
raise web.HTTPBadRequest(reason='url is required.')
|
|
|
|
status[item.get('url')] = await self.add(
|
|
url=item.get('url'),
|
|
quality=item.get('quality'),
|
|
format=item.get('format'),
|
|
folder=item.get('folder'),
|
|
ytdlp_cookies=item.get('ytdlp_cookies'),
|
|
ytdlp_config=item.get('ytdlp_config'),
|
|
output_template=item.get('output_template')
|
|
)
|
|
|
|
return web.Response(text=self.serializer.encode(status))
|
|
|
|
@self.routes.delete(self.config.url_prefix + 'delete')
|
|
async def delete(request: Request) -> Response:
|
|
post = await request.json()
|
|
ids = post.get('ids')
|
|
where = post.get('where')
|
|
|
|
if not ids or where not in ['queue', 'done']:
|
|
raise web.HTTPBadRequest()
|
|
|
|
status = await (self.queue.cancel(ids) if where == 'queue' else self.queue.clear(ids))
|
|
|
|
return web.Response(text=self.serializer.encode(status))
|
|
|
|
@self.routes.get(self.config.url_prefix + 'history')
|
|
async def history(_) -> Response:
|
|
history = {'done': [], 'queue': []}
|
|
|
|
for _, v in self.queue.queue.saved_items():
|
|
history['queue'].append(v)
|
|
for _, v in self.queue.done.saved_items():
|
|
history['done'].append(v)
|
|
|
|
return web.Response(text=self.serializer.encode(history))
|
|
|
|
@self.routes.get(self.config.url_prefix + 'm3u8/{file:.*}')
|
|
async def m3u8(request: Request) -> Response:
|
|
file: str = request.match_info.get('file')
|
|
|
|
if not file:
|
|
raise web.HTTPBadRequest(reason='file is required.')
|
|
|
|
return web.Response(
|
|
text=M3u8(url=f"{self.config.url_host}{self.config.url_prefix}").make_stream(
|
|
download_path=self.config.download_path,
|
|
file=file
|
|
),
|
|
headers={
|
|
'Content-Type': 'application/x-mpegURL',
|
|
'Cache-Control': 'no-cache',
|
|
'Access-Control-Max-Age': "300",
|
|
}
|
|
)
|
|
|
|
@self.routes.get(self.config.url_prefix + 'segments/{segment:\d+}/{file:.*}')
|
|
async def segments(request: Request) -> Response:
|
|
file: str = request.match_info.get('file')
|
|
segment: int = request.match_info.get('segment')
|
|
sd: int = request.query.get('sd')
|
|
vc: int = int(request.query.get('vc', 0))
|
|
ac: int = int(request.query.get('ac', 0))
|
|
|
|
if not file:
|
|
raise web.HTTPBadRequest(reason='file is required')
|
|
|
|
if not segment:
|
|
raise web.HTTPBadRequest(reason='segment is required')
|
|
|
|
segmenter = Segments(
|
|
segment_index=int(segment),
|
|
segment_duration=float('{:.6f}'.format(float(sd if sd else M3u8.segment_duration))),
|
|
vconvert=True if vc == 1 else False,
|
|
aconvert=True if ac == 1 else False
|
|
)
|
|
|
|
return web.Response(
|
|
body=await segmenter.stream(download_path=self.config.download_path, file=file),
|
|
headers={
|
|
'Content-Type': 'video/mpegts',
|
|
'Cache-Control': 'no-cache',
|
|
'X-Accel-Buffering': 'no',
|
|
'Access-Control-Allow-Origin': '*',
|
|
'Access-Control-Max-Age': "300",
|
|
}
|
|
)
|
|
|
|
@self.routes.get(self.config.url_prefix)
|
|
async def index(_) -> Response:
|
|
if not self.appLoader:
|
|
with open(os.path.join(
|
|
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
|
|
'frontend/dist/index.html'
|
|
), 'r') as f:
|
|
self.appLoader = f.read()
|
|
|
|
return web.Response(text=self.appLoader, content_type='text/html', charset='utf-8', status=200)
|
|
|
|
@self.sio.event()
|
|
async def connect(sid, environ):
|
|
await self.connect(sid, environ)
|
|
|
|
if self.config.url_prefix != '/':
|
|
@self.routes.get('/')
|
|
async def redirect_d(_) -> Response:
|
|
return web.HTTPFound(self.config.url_prefix)
|
|
|
|
@self.routes.get(self.config.url_prefix[:-1])
|
|
async def redirect_w(_) -> Response:
|
|
return web.HTTPFound(self.config.url_prefix)
|
|
|
|
@self.routes.options(self.config.url_prefix + 'add')
|
|
async def add_cors(_) -> Response:
|
|
return web.Response(text=self.serializer.encode({"status": "ok"}))
|
|
|
|
@self.routes.options(f'{self.config.url_prefix}delete')
|
|
async def delete_cors(_) -> Response:
|
|
return web.Response(text=self.serializer.encode({"status": "ok"}))
|
|
|
|
self.routes.static(f'{self.config.url_prefix}download/', self.config.download_path)
|
|
|
|
async def staticFile(req: Request) -> Response:
|
|
if req.path not in self.staticHolder:
|
|
return web.HTTPNotFound()
|
|
|
|
return web.FileResponse(path=self.staticHolder[req.path].get('file'), headers={
|
|
'Cache-Control': 'public, max-age=31536000',
|
|
'Pragma': 'public',
|
|
})
|
|
|
|
staticDir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'frontend/dist')
|
|
for root, _, files in os.walk(staticDir):
|
|
for file in files:
|
|
if file.endswith('.map'):
|
|
continue
|
|
|
|
file = os.path.join(root, file)
|
|
urlPath = f"{self.config.url_prefix}{file.replace(f'{staticDir}/', '')}"
|
|
|
|
self.staticHolder[urlPath] = {
|
|
'file': file,
|
|
# 'content': open(file, 'rb').read(),
|
|
'content_type': self.extToMime.get(os.path.splitext(file)[1], MIME.from_file(file)),
|
|
}
|
|
|
|
self.app.router.add_get(urlPath, staticFile)
|
|
LOG.debug(f'Preloading: [{urlPath}].')
|
|
|
|
try:
|
|
self.app.add_routes(self.routes)
|
|
|
|
async def on_prepare(request, response):
|
|
if 'Origin' in request.headers:
|
|
response.headers['Access-Control-Allow-Origin'] = request.headers['Origin']
|
|
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
|
|
response.headers['Access-Control-Allow-Methods'] = 'PUT, POST, DELETE'
|
|
|
|
self.app.on_response_prepare.append(on_prepare)
|
|
except ValueError as e:
|
|
if 'frontend/dist' in str(e):
|
|
raise RuntimeError('Could not find the frontend UI static assets.') from e
|
|
raise e
|
|
|
|
async def add(
|
|
self,
|
|
url: str,
|
|
quality: str,
|
|
format: str,
|
|
folder: str,
|
|
ytdlp_cookies: str,
|
|
ytdlp_config: dict,
|
|
output_template: str
|
|
) -> dict[str, str]:
|
|
if ytdlp_config is None:
|
|
ytdlp_config = {}
|
|
|
|
if ytdlp_cookies and isinstance(ytdlp_cookies, dict):
|
|
ytdlp_cookies = self.serializer.encode(ytdlp_cookies)
|
|
|
|
status = await self.queue.add(
|
|
url=url,
|
|
format=format if format else 'any',
|
|
quality=quality if quality else 'best',
|
|
folder=folder,
|
|
ytdlp_cookies=ytdlp_cookies,
|
|
ytdlp_config=ytdlp_config,
|
|
output_template=output_template
|
|
)
|
|
|
|
return status
|
|
|
|
|
|
if __name__ == '__main__':
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
main = Main()
|