danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #40 from ArabCoders/dev

Browse source 
Added Task Runner.
This commit is contained in:
Abdulmohsen 2023-12-04 23:34:22 +03:00 committed by GitHub
commit 02796942f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 141 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Pipfile
View file

@ -10,6 +10,7 @@ yt-dlp = "*"
caribou = "*"
coloredlogs = "*"
ffprobe-python = "*"
aiocron = "*"
[dev-packages]

49
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "258ec26d54c61c901fb67cb207bfb76d71d60ae0866f2ff3ab9f5e2e97940c84"
"sha256": "9256047e3c0597d835039eadb3c589948874236a04f2392e5293ee7dd29084bc"
},
"pipfile-spec": 6,
"requires": {
@ -16,6 +16,14 @@
]
},
"default": {
"aiocron": {
"hashes": [
"sha256:48546513faf2eb7901e65a64eba7b653c80106ed00ed9ca3419c3d10b6555a01",
"sha256:b6313214c311b62aa2220e872b94139b648631b3103d062ef29e5d3230ddce6d"
],
"index": "pypi",
"version": "==1.8"
},
"aiohttp": {
"hashes": [
"sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f",
@ -345,6 +353,14 @@
"index": "pypi",
"version": "==15.0.1"
},
"croniter": {
"hashes": [
"sha256:4cb064ce2d8f695b3b078be36ff50115cf8ac306c10a7e8653ee2a5b534673d7",
"sha256:d199b2ec3ea5e82988d1f72022433c5f9302b3b3ea9e6bfd6a1518f6ea5e700a"
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.0.1"
},
"ffprobe-python": {
"hashes": [
"sha256:b02576228f6da0b0bb973e15e5dadb132013a4ded202dabb89ac77779e9634d6",
@ -570,6 +586,14 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.19.0"
},
"python-dateutil": {
"hashes": [
"sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
"sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.8.2"
},
"python-engineio": {
"hashes": [
"sha256:2a32585d8fecd0118264fe0c39788670456ca9aa466d7c026d995cfff68af164",
@ -586,6 +610,13 @@
"index": "pypi",
"version": "==5.10.0"
},
"pytz": {
"hashes": [
"sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b",
"sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"
],
"version": "==2023.3.post1"
},
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
@ -602,6 +633,22 @@
"markers": "python_version >= '3.6'",
"version": "==1.0.0"
},
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
},
"tzlocal": {
"hashes": [
"sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8",
"sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"
],
"markers": "python_version >= '3.8'",
"version": "==5.2"
},
"urllib3": {
"hashes": [
"sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3",

29
README.md
View file

@ -13,6 +13,7 @@ YTPTube started as a fork of [meTube](https://github.com/alexta69/metube) projec
* Switched out of binary file storage in favor of SQLite.
* Handle live streams.
* Support per link, `yt-dlp config` and `cookies`. and `output format`
* Tasks Runner. It allow you to queue channels for downloading using simple `json` file for configuration.
### Tips
Your `yt-dlp` config should include the following options for optimal working conditions.
@ -144,6 +145,34 @@ A Docker image can be built locally (it will build the UI too):
docker build . -t ytptube
```
### tasks.json File
The `config/tasks.json`, is a json file, which can be used to queue URLs for downloading, it's mainly useful if you follow specific channels and you want it downloaded automatically, The schema for the file is as the following, Only the `URL` key is required.
```json
[
{
"url": "", // (URL: string) **REQUIRED**, URL to the content.
"name": "My super secret channel", // (Name: string) Optional field. Mainly used for logging. If omitted, random GUID will be shown.
"timer": "1 */1 * * *", // (Timer: string) Optional field. Using regular cronjob timer, if the field is omitted, it will run every hour in random minute.
"ytdlp_cookies": {}, // (yt-dlp cookies: object) Optional field. A JSON cookies exported by flagCookies.
"ytdlp_config": {}, // (yt-dlp config: object) Optional field. A JSON yt-dlp config.
"output_template": "", // (Output Template: string) Optional field. A File output format,
"folder":"", // (Folder: string) Optional field. Where to store the downloads relative to the main download path.
"format": "", // (Format: string) Optional field. Format as specified in Web GUI. Defaults to "any".
"quality": "", // (Quality: string), Optional field. Quality as specified in Web GUI. Defaults to "best".
},
{
"url": "https://..." // This is valid config, it will queue the channel for downloading every hour at random minute.
},
...
]
```
The task runner is doing what you are doing when you click the add button on the WebGUI, this just fancy way to automate that.
**WARNING**: We strongly advice turning on `YTP_KEEP_ARCHIVE` option. Otherwise, you will keep re-downloading the items, and you will eventually get banned from the source or or you will waste space, bandwidth re-downloading content over and over.
# Donation
If you feel like donating and appreciate my work, you can do so by donating to children charity. For example [Make-A-Wish](https://worldwish.org).

67
app/main.py
View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3
import asyncio
from datetime import datetime
import json
import os
import random
from src.Config import Config
from version import APP_VERSION
from src.Notifier import Notifier
@ -14,6 +18,7 @@ import socketio
import logging
import caribou
import sqlite3
from aiocron import crontab
class Main:
@ -24,6 +29,7 @@ class Main:
routes: web.RouteTableDef = None
connection: sqlite3.Connection = None
dqueue: DownloadQueue = None
loop: asyncio.AbstractEventLoop = None
def __init__(self):
self.config = Config()
@ -71,6 +77,7 @@ class Main:
caribou.upgrade(self.config.db_file, './app/migrations')
self.loop = asyncio.get_event_loop()
self.serializer = ObjectSerializer()
self.app = web.Application()
self.sio = socketio.AsyncServer(cors_allowed_origins='*')
@ -84,8 +91,9 @@ class Main:
connection=self.connection,
serializer=self.serializer
)
self.app.on_startup.append(lambda app: self.dqueue.initialize())
self.app.on_startup.append(lambda _: self.dqueue.initialize())
self.addRoutes()
self.addTasks()
self.start()
def start(self):
@ -94,13 +102,61 @@ class Main:
host=self.config.host,
port=self.config.port,
reuse_port=True,
print=lambda _: print(f'YTPTube v{self.config.version} - listening on http://{self.config.host}:{self.config.port}')
loop=self.loop,
print=lambda _: print(
f'YTPTube v{self.config.version} - listening on http://{self.config.host}:{self.config.port}'),
)
async def connect(self, sid, environ):
await self.sio.emit('all', self.serializer.encode(self.dqueue.get()), to=sid)
await self.sio.emit('configuration', self.serializer.encode(self.config), to=sid)
def addTasks(self):
tasks_file: str = os.path.join(self.config.config_path, 'tasks.json')
if not os.path.exists(tasks_file):
logging.info(
f'No tasks file found at {tasks_file}. Skipping Tasks.')
return
with open(tasks_file, 'r') as f:
tasks = json.load(f)
for task in tasks:
if not task.get('url'):
logging.warning(f'Invalid task {task}.')
continue
cron_timer: str = task.get(
'timer', f'{random.randint(1,59)} */1 * * *')
async def cron_runner(task: dict):
logging.info(
f'Running task [{task.get("name",task.get("url"))}] at [{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}].')
await self.add(
url=task.get('url'),
quality=task.get('quality', 'best'),
format=task.get('format', 'any'),
folder=task.get('folder'),
ytdlp_cookies=task.get('ytdlp_cookies'),
ytdlp_config=task.get('ytdlp_config'),
output_template=task.get('output_template')
)
logging.info(
f'Finished Running task [{task.get("name",task.get("url"))}] at [{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}].')
crontab(
spec=cron_timer,
func=cron_runner,
args=(task,),
start=True,
loop=self.loop
)
logging.info(
f'Added task to grab {task.get("name",task.get("url"))} content every [{cron_timer}].')
def addRoutes(self):
@self.routes.post(self.config.url_prefix + 'add')
async def add(request: Request) -> Response:
@ -301,10 +357,13 @@ class Main:
if ytdlp_config is None:
ytdlp_config = {}
if ytdlp_cookies and isinstance(ytdlp_cookies, dict):
ytdlp_cookies = self.serializer.encode(ytdlp_cookies)
status = await self.dqueue.add(
url=url,
quality=quality,
format=format,
format=format if format else 'any',
quality=quality if quality else 'best',
folder=folder,
ytdlp_cookies=ytdlp_cookies,
ytdlp_config=ytdlp_config,