danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

updated some type checks

Browse source
This commit is contained in:
ArabCoders 2024-12-19 22:58:33 +03:00
parent 252970f66f
commit df86cc0001
6 changed files with 333 additions and 232 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.editorconfig
View file

@ -15,3 +15,4 @@ trim_trailing_whitespace = false
[*.py]
indent_size = 4
max_line_length = 120

233
app/library/DownloadQueue.py
View file

@ -10,13 +10,13 @@ from .config import Config
from .Download import Download
from .ItemDTO import ItemDTO
from .DataStore import DataStore
from .Utils import ag, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig
from .Utils import calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig
from .AsyncPool import AsyncPool
from .Emitter import Emitter
LOG = logging.getLogger('DownloadQueue')
TYPE_DONE: str = 'done'
TYPE_QUEUE: str = 'queue'
LOG = logging.getLogger("DownloadQueue")
TYPE_DONE: str = "done"
TYPE_QUEUE: str = "queue"
class DownloadQueue:
@ -41,11 +41,11 @@ class DownloadQueue:
async def initialize(self):
self.event = asyncio.Event()
LOG.info(f'Using {self.config.max_workers} workers for downloading.')
LOG.info(f"Using {self.config.max_workers} workers for downloading.")
asyncio.create_task(
self.__download_pool() if self.config.max_workers > 1 else self.__download(),
name='download_pool' if self.config.max_workers > 1 else 'download_worker'
name="download_pool" if self.config.max_workers > 1 else "download_worker",
)
async def __add_entry(
@ -54,29 +54,29 @@ class DownloadQueue:
preset: str,
folder: str,
ytdlp_config: dict = {},
ytdlp_cookies: str = '',
output_template: str = '',
already=None
ytdlp_cookies: str = "",
output_template: str = "",
already=None,
):
if not entry:
return {'status': 'error', 'msg': 'Invalid/empty data was given.'}
return {"status": "error", "msg": "Invalid/empty data was given."}
options: dict = {}
error: str = None
live_in: str = None
eventType = entry.get('_type') or 'video'
if eventType == 'playlist':
entries = entry['entries']
eventType = entry.get("_type") or "video"
if eventType == "playlist":
entries = entry["entries"]
playlist_index_digits = len(str(len(entries)))
results = []
for i, etr in enumerate(entries, start=1):
etr['playlist'] = entry.get('id')
etr['playlist_index'] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(i)
for property in ('id', 'title', 'uploader', 'uploader_id'):
etr["playlist"] = entry.get("id")
etr["playlist_index"] = "{{0:0{0:d}d}}".format(playlist_index_digits).format(i)
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
etr[f'playlist_{property}'] = entry.get(property)
etr[f"playlist_{property}"] = entry.get(property)
results.append(
await self.__add_entry(
@ -86,60 +86,63 @@ class DownloadQueue:
ytdlp_config=ytdlp_config,
ytdlp_cookies=ytdlp_cookies,
output_template=output_template,
already=already
already=already,
)
)
if any(res['status'] == 'error' for res in results):
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
if any(res["status"] == "error" for res in results):
return {
"status": "error",
"msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
}
return {'status': 'ok'}
elif (eventType == 'video' or eventType.startswith('url')) and 'id' in entry and 'title' in entry:
return {"status": "ok"}
elif (eventType == "video" or eventType.startswith("url")) and "id" in entry and "title" in entry:
# check if the video is live stream.
if 'live_status' in entry and entry.get('live_status') == 'is_upcoming':
if 'release_timestamp' in entry and entry.get('release_timestamp'):
live_in = formatdate(entry.get('release_timestamp'))
if "live_status" in entry and entry.get("live_status") == "is_upcoming":
if "release_timestamp" in entry and entry.get("release_timestamp"):
live_in = formatdate(entry.get("release_timestamp"))
else:
error = 'Live stream not yet started. And no date is set.'
error = "Live stream not yet started. And no date is set."
else:
error = entry.get('msg', None)
error = entry.get("msg", None)
LOG.debug(
f"Entry id '{entry.get('id', None)}' url '{entry.get('webpage_url', None)} - {entry.get('url', None)}'."
)
if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry.get('url')):
item = self.done.get(key=entry['id'], url=entry.get('webpage_url') or entry['url'])
if self.done.exists(key=entry["id"], url=entry.get("webpage_url") or entry.get("url")):
item = self.done.get(key=entry["id"], url=entry.get("webpage_url") or entry["url"])
LOG.warning(f"Item '{item.info.id}' - '{item.info.title}' already downloaded. Removing from history.")
await self.clear([item.info._id])
try:
item = self.queue.get(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url'))
item = self.queue.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
if item is not None:
err_message = f"Item ID '{item.info.id}' - '{item.info.title}' already in download queue."
LOG.info(err_message)
return {'status': 'error', 'msg': err_message}
return {"status": "error", "msg": err_message}
except KeyError:
pass
is_manifestless = entry.get('is_manifestless', False)
options.update({'is_manifestless': is_manifestless})
is_manifestless = entry.get("is_manifestless", False)
options.update({"is_manifestless": is_manifestless})
live_status: list = ['is_live', 'is_upcoming']
is_live = entry.get('is_live', None) or live_in or entry.get('live_status', None) in live_status
live_status: list = ["is_live", "is_upcoming"]
is_live = entry.get("is_live", None) or live_in or entry.get("live_status", None) in live_status
try:
download_dir = calcDownloadPath(basePath=self.config.download_path, folder=folder)
except Exception as e:
LOG.exception(e)
return {'status': 'error', 'msg': str(e)}
return {"status": "error", "msg": str(e)}
dl = ItemDTO(
id=entry.get('id'),
title=entry.get('title'),
url=entry.get('webpage_url') or entry.get('url'),
id=entry.get("id"),
title=entry.get("title"),
url=entry.get("webpage_url") or entry.get("url"),
preset=preset,
thumbnail=entry.get('thumbnail', None),
thumbnail=entry.get("thumbnail", None),
folder=folder,
download_dir=download_dir,
temp_dir=self.config.temp_path,
@ -160,42 +163,37 @@ class DownloadQueue:
dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug))
if dlInfo.info.live_in or 'is_upcoming' == entry.get('live_status', None):
dlInfo.info.status = 'not_live'
if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status", None):
dlInfo.info.status = "not_live"
itemDownload = self.done.put(dlInfo)
NotifyEvent = 'completed'
NotifyEvent = "completed"
elif self.config.allow_manifestless is False and is_manifestless is True:
dlInfo.info.status = 'error'
dlInfo.info.error = 'Video is in post-live manifestless mode.'
dlInfo.info.status = "error"
dlInfo.info.error = "Video is in post-live manifestless mode."
itemDownload = self.done.put(dlInfo)
NotifyEvent = 'completed'
NotifyEvent = "completed"
else:
NotifyEvent = 'added'
NotifyEvent = "added"
itemDownload = self.queue.put(dlInfo)
self.event.set()
asyncio.create_task(
self.emitter.emit(NotifyEvent, itemDownload.info),
name=f'notifier-{NotifyEvent}-{itemDownload.info.id}')
self.emitter.emit(NotifyEvent, itemDownload.info), name=f"notifier-{NotifyEvent}-{itemDownload.info.id}"
)
return {
'status': 'ok'
}
elif eventType.startswith('url'):
return {"status": "ok"}
elif eventType.startswith("url"):
return await self.add(
url=entry.get('url'),
url=entry.get("url"),
preset=preset,
folder=folder,
ytdlp_config=ytdlp_config,
ytdlp_cookies=ytdlp_cookies,
output_template=output_template,
already=already
already=already,
)
return {
'status': 'error',
'msg': f'Unsupported resource "{eventType}"'
}
return {"status": "error", "msg": f'Unsupported resource "{eventType}"'}
async def add(
self,
@ -203,26 +201,28 @@ class DownloadQueue:
preset: str,
folder: str,
ytdlp_config: dict = {},
ytdlp_cookies: str = '',
output_template: str = '',
already=None
ytdlp_cookies: str = "",
output_template: str = "",
already=None,
):
ytdlp_config = ytdlp_config if ytdlp_config else {}
LOG.info(f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'.")
LOG.info(
f"Adding url '{url}' to folder '{folder}' with the following options 'Preset: {preset}' 'Naming: {output_template}', 'Cookies: {ytdlp_cookies}' 'YTConfig: {ytdlp_config}'."
)
if isinstance(ytdlp_config, str):
try:
ytdlp_config = json.loads(ytdlp_config)
except Exception as e:
LOG.error(f"Unable to load '{ytdlp_config=}'. {str(e)}")
return {'status': 'error', 'msg': f"Failed to parse json yt-dlp config. {str(e)}"}
return {"status": "error", "msg": f"Failed to parse json yt-dlp config. {str(e)}"}
already = set() if already is None else already
if url in already:
LOG.warning(f"Recursion detected with url '{url}' skipping.")
return {'status': 'ok'}
return {"status": "ok"}
already.add(url)
@ -231,10 +231,10 @@ class DownloadQueue:
if downloaded is True:
message = f"This url with ID '{id_dict.get('id')}' has been downloaded already and recorded in archive."
LOG.info(message)
return {'status': 'error', 'msg': message}
return {"status": "error", "msg": message}
started = time.perf_counter()
LOG.debug(f'extract_info: checking {url=}')
LOG.debug(f"extract_info: checking {url=}")
entry = await asyncio.wait_for(
fut=asyncio.get_running_loop().run_in_executor(
@ -242,24 +242,29 @@ class DownloadQueue:
ExtractInfo,
mergeConfig(self.config.ytdl_options, ytdlp_config),
url,
bool(self.config.ytdl_debug)
bool(self.config.ytdl_debug),
),
timeout=self.config.extract_info_timeout)
timeout=self.config.extract_info_timeout,
)
if not entry:
return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
return {"status": "error", "msg": "Unable to extract info check logs."}
LOG.debug(
f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'.")
f"extract_info: for 'URL: {url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}'."
)
except yt_dlp.utils.ExistingVideoReached as exc:
LOG.error(f'Video has been downloaded already and recorded in archive.log file. {str(exc)}')
return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
LOG.error(f"Video has been downloaded already and recorded in archive.log file. {str(exc)}")
return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
except yt_dlp.utils.YoutubeDLError as exc:
LOG.error(f'YoutubeDLError: Unable to extract info. {str(exc)}')
return {'status': 'error', 'msg': str(exc)}
LOG.error(f"YoutubeDLError: Unable to extract info. {str(exc)}")
return {"status": "error", "msg": str(exc)}
except asyncio.exceptions.TimeoutError as exc:
LOG.error(f'TimeoutError: Unable to extract info. {str(exc)}')
return {'status': 'error', 'msg': f'TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.'}
LOG.error(f"TimeoutError: Unable to extract info. {str(exc)}")
return {
"status": "error",
"msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.",
}
return await self.__add_entry(
entry=entry,
@ -279,27 +284,27 @@ class DownloadQueue:
item = self.queue.get(key=id)
except KeyError as e:
status[id] = str(e)
status['status'] = 'error'
LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}')
status["status"] = "error"
LOG.warning(f"Requested cancel for non-existent download {id=}. {str(e)}")
continue
itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
if item.running():
LOG.debug(f'Canceling {itemMessage}')
LOG.debug(f"Canceling {itemMessage}")
item.cancel()
LOG.info(f'Cancelled {itemMessage}')
LOG.info(f"Cancelled {itemMessage}")
await item.close()
else:
await item.close()
LOG.debug(f'Deleting from queue {itemMessage}')
LOG.debug(f"Deleting from queue {itemMessage}")
self.queue.delete(id)
asyncio.create_task(self.emitter.canceled(id=id, dl=item), name=f'notifier-c-{id}')
asyncio.create_task(self.emitter.canceled(id=id, dl=item), name=f"notifier-c-{id}")
self.done.put(item)
asyncio.create_task(self.emitter.completed(dl=item), name=f'notifier-d-{id}')
LOG.info(f'Deleted from queue {itemMessage}')
asyncio.create_task(self.emitter.completed(dl=item), name=f"notifier-d-{id}")
LOG.info(f"Deleted from queue {itemMessage}")
status[id] = 'ok'
status[id] = "ok"
return status
@ -311,35 +316,35 @@ class DownloadQueue:
item = self.done.get(key=id)
except KeyError as e:
status[id] = str(e)
status['status'] = 'error'
LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}')
status["status"] = "error"
LOG.warning(f"Requested delete for non-existent download {id=}. {str(e)}")
continue
itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
LOG.debug(f'Deleting completed download {itemMessage}')
LOG.debug(f"Deleting completed download {itemMessage}")
self.done.delete(id)
asyncio.create_task(self.emitter.cleared(id, dl=item), name=f'notifier-c-{id}')
LOG.info(f'Deleted completed download {itemMessage}')
status[id] = 'ok'
asyncio.create_task(self.emitter.cleared(id, dl=item), name=f"notifier-c-{id}")
LOG.info(f"Deleted completed download {itemMessage}")
status[id] = "ok"
return status
def get(self) -> dict[str, list[dict[str, ItemDTO]]]:
items = {'queue': {}, 'done': {}}
items = {"queue": {}, "done": {}}
for k, v in self.queue.saved_items():
items['queue'][k] = v
items["queue"][k] = v
for k, v in self.done.saved_items():
items['done'][k] = v
items["done"][k] = v
for k, v in self.queue.items():
if not k in items['queue']:
items['queue'][k] = v.info
if k not in items["queue"]:
items["queue"][k] = v.info
for k, v in self.done.items():
if not k in items['done']:
items['done'][k] = v.info
if k not in items["done"]:
items["done"][k] = v.info
return items
@ -348,8 +353,8 @@ class DownloadQueue:
loop=asyncio.get_running_loop(),
num_workers=self.config.max_workers,
worker_co=self.__downloadFile,
name='download_pool',
logger=logging.getLogger('WorkerPool'),
name="download_pool",
logger=logging.getLogger("WorkerPool"),
)
self.pool.start()
@ -362,14 +367,14 @@ class DownloadQueue:
break
if time.time() - lastLog > 120:
lastLog = time.time()
LOG.info(f'Waiting for worker to be free. {self.pool.get_workers_status()}')
LOG.info(f"Waiting for worker to be free. {self.pool.get_workers_status()}")
await asyncio.sleep(1)
while not self.queue.hasDownloads():
LOG.info(f"Waiting for item to download. '{self.pool.get_available_workers()}' free workers.")
await self.event.wait()
self.event.clear()
LOG.debug(f"Cleared wait event.")
LOG.debug("Cleared wait event.")
entry = self.queue.getNextDownload()
await asyncio.sleep(0.2)
@ -387,7 +392,7 @@ class DownloadQueue:
async def __download(self):
while True:
while self.queue.empty():
LOG.info('Waiting for item to download.')
LOG.info("Waiting for item to download.")
await self.event.wait()
self.event.clear()
@ -396,34 +401,34 @@ class DownloadQueue:
async def __downloadFile(self, id: str, entry: Download):
LOG.info(
f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'folder: {entry.info.folder}'.")
f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' to 'folder: {entry.info.folder}'."
)
try:
await entry.start(self.emitter)
if entry.info.status != 'finished':
if entry.info.status != "finished":
if entry.tmpfilename and os.path.isfile(entry.tmpfilename):
try:
os.remove(entry.tmpfilename)
entry.tmpfilename = None
except:
except Exception:
pass
entry.info.status = 'error'
entry.info.status = "error"
finally:
await entry.close()
if self.queue.exists(key=id):
self.queue.delete(key=id)
if entry.is_canceled() is True:
asyncio.create_task(self.emitter.canceled(id, dl=entry.info), name=f'notifier-c-{id}')
entry.info.status = 'canceled'
entry.info.error = 'Canceled by user.'
asyncio.create_task(self.emitter.canceled(id, dl=entry.info), name=f"notifier-c-{id}")
entry.info.status = "canceled"
entry.info.error = "Canceled by user."
self.done.put(value=entry)
asyncio.create_task(self.emitter.completed(dl=entry.info), name=f'notifier-d-{id}')
asyncio.create_task(self.emitter.completed(dl=entry.info), name=f"notifier-d-{id}")
self.event.set()
@ -431,4 +436,4 @@ class DownloadQueue:
if not url or not self.config.keep_archive:
return False, None
return isDownloaded(self.config.ytdl_options.get('download_archive', None), url)
return isDownloaded(self.config.ytdl_options.get("download_archive", None), url)

2
app/library/HttpAPI.py
View file

@ -448,7 +448,7 @@ class HttpAPI(common):
'Access-Control-Max-Age': "300",
})
@route('GET', 'player/segments/{segment:\d+}/{file:.*}.ts')
@route('GET', 'player/segments/{segment:d+}/{file:.*}.ts')
async def segments(self, request: Request) -> Response:
file: str = request.match_info.get('file')
segment: int = request.match_info.get('segment')

198
app/library/config.py
View file

@ -13,31 +13,31 @@ from pathlib import Path
class Config:
__instance = None
config_path: str = '.'
download_path: str = '.'
temp_path: str = '/tmp'
config_path: str = "."
download_path: str = "."
temp_path: str = "/tmp"
temp_keep: bool = False
db_file: str = '{config_path}/ytptube.db'
manual_archive: str = '{config_path}/archive.manual.log'
db_file: str = "{config_path}/ytptube.db"
manual_archive: str = "{config_path}/archive.manual.log"
url_host: str = ''
url_prefix: str = ''
url_socketio: str = '{url_prefix}socket.io'
url_host: str = ""
url_prefix: str = ""
url_socketio: str = "{url_prefix}socket.io"
output_template: str = '%(title)s.%(ext)s'
output_template_chapter: str = '%(title)s - %(section_number)s %(section_title)s.%(ext)s'
output_template: str = "%(title)s.%(ext)s"
output_template_chapter: str = "%(title)s - %(section_number)s %(section_title)s.%(ext)s"
ytdl_options: dict | str = {}
ytdl_debug: bool = False
host: str = '0.0.0.0'
host: str = "0.0.0.0"
port: int = 8081
keep_archive: bool = True
base_path: str = ''
base_path: str = ""
log_level: str = 'info'
log_level: str = "info"
allow_manifestless: bool = False
@ -57,8 +57,8 @@ class Config:
started: int = 0
streamer_vcodec = 'libx264'
streamer_acodec = 'aac'
streamer_vcodec = "libx264"
streamer_acodec = "aac"
auth_username: str = None
auth_password: str = None
@ -66,64 +66,87 @@ class Config:
ytdlp_version: str = YTDLP_VERSION
tasks: list = []
presets: list = [
{'name': 'default', 'format': 'default', 'postprocessors': [],
'args': {}},
{'name': 'Video - 1080p h264/acc', 'format': 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]',
'args': {'format_sort': ['vcodec:h264'], }, },
{'name': 'Video - 720p h264/acc', 'format': 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]',
'args': {'format_sort': ['vcodec:h264'], }, },
{"name": "default", "format": "default", "postprocessors": [], "args": {}},
{
'name': 'Audio - Best audio [MP3]',
'format': 'bestaudio/best',
'postprocessors': [
"name": "Video - 1080p h264/acc",
"format": "bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]",
"args": {
"format_sort": ["vcodec:h264"],
},
},
{
"name": "Video - 720p h264/acc",
"format": "bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]",
"args": {
"format_sort": ["vcodec:h264"],
},
},
{
"name": "Audio - Best audio [MP3]",
"format": "bestaudio/best",
"postprocessors": [
{
"key": "FFmpegExtractAudio",
"preferredcodec": "mp3",
"preferredquality": "5",
"nopostoverwrites": False
"nopostoverwrites": False,
},
{
"key": "FFmpegMetadata",
"add_chapters": True,
"add_metadata": True,
"add_infojson": "if_exists"
},
{
"key": "EmbedThumbnail",
"already_have_thumbnail": False
},
{
"key": "FFmpegConcat",
"only_multi_video": True,
"when": "playlist"
}
{"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": True, "add_infojson": "if_exists"},
{"key": "EmbedThumbnail", "already_have_thumbnail": False},
{"key": "FFmpegConcat", "only_multi_video": True, "when": "playlist"},
],
'args': {
"outtmpl": {
"pl_thumbnail": ""
},
"args": {
"outtmpl": {"pl_thumbnail": ""},
"ignoreerrors": "only_download",
"retries": 10,
"fragment_retries": 10,
"writethumbnail": True,
"extract_flat": "discard_in_playlist",
"final_ext": "mp3",
}
}
},
},
]
_manual_vars: tuple = ('temp_path', 'config_path', 'download_path',)
_manual_vars: tuple = (
"temp_path",
"config_path",
"download_path",
)
_immutable: tuple = (
'version', '__instance', 'ytdl_options', 'tasks',
'new_version_available', 'ytdlp_version', 'started', 'presets',
"version",
"__instance",
"ytdl_options",
"tasks",
"new_version_available",
"ytdlp_version",
"started",
"presets",
)
_int_vars: tuple = ('port', 'max_workers', 'socket_timeout', 'extract_info_timeout', 'debugpy_port',)
_boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',)
_int_vars: tuple = (
"port",
"max_workers",
"socket_timeout",
"extract_info_timeout",
"debugpy_port",
)
_boolean_vars: tuple = (
"keep_archive",
"ytdl_debug",
"debug",
"temp_keep",
"allow_manifestless",
)
_frontend_vars: tuple = (
'download_path', 'keep_archive', 'output_template',
'ytdlp_version', 'version', 'url_host', 'started', 'url_prefix',
"download_path",
"keep_archive",
"output_template",
"ytdlp_version",
"version",
"url_host",
"started",
"url_prefix",
)
@staticmethod
@ -140,20 +163,20 @@ class Config:
baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute())
self.temp_path = os.environ.get('YTP_TEMP_PATH', None) or os.path.join(baseDefaultPath, 'var', 'tmp')
self.config_path = os.environ.get('YTP_CONFIG_PATH', None) or os.path.join(baseDefaultPath, 'var', 'config')
self.download_path = os.environ.get(
'YTP_DOWNLOAD_PATH', None) or os.path.join(
baseDefaultPath, 'var', 'downloads')
self.temp_path = os.environ.get("YTP_TEMP_PATH", None) or os.path.join(baseDefaultPath, "var", "tmp")
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or os.path.join(baseDefaultPath, "var", "config")
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or os.path.join(
baseDefaultPath, "var", "downloads"
)
envFile: str = os.path.join(self.config_path, '.env')
envFile: str = os.path.join(self.config_path, ".env")
if os.path.exists(envFile):
logging.info(f"Loading environment variables from '{envFile}'.")
load_dotenv(envFile)
for k, v in self._getAttributes().items():
if k.startswith('_') or k in self._manual_vars:
if k.startswith("_") or k in self._manual_vars:
continue
# If the variable declared as immutable, set it to the default value.
@ -161,15 +184,15 @@ class Config:
setattr(self, k, v)
continue
lookUpKey: str = f'YTP_{k}'.upper()
lookUpKey: str = f"YTP_{k}".upper()
setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v)
for k, v in self.__dict__.items():
if k.startswith('_') or k in self._immutable or k in self._manual_vars:
if k.startswith("_") or k in self._immutable or k in self._manual_vars:
continue
if isinstance(v, str) and '{' in v and '}' in v:
for key in re.findall(r'\{.*?\}', v):
if isinstance(v, str) and "{" in v and "}" in v:
for key in re.findall(r"\{.*?\}", v):
localKey: str = key[1:-1]
if localKey not in self.__dict__:
logging.error(f"Config variable '{k}' had non-existing config reference '{key}'.")
@ -180,32 +203,31 @@ class Config:
setattr(self, k, v)
if k in self._boolean_vars:
if str(v).lower() not in (True, False, 'true', 'false', 'on', 'off', '1', '0'):
if str(v).lower() not in (True, False, "true", "false", "on", "off", "1", "0"):
raise ValueError(f"Config variable '{k}' is set to a non-boolean value '{v}'.")
setattr(self, k, str(v).lower() in (True, 'true', 'on', '1'))
setattr(self, k, str(v).lower() in (True, "true", "on", "1"))
if k in self._int_vars:
setattr(self, k, int(v))
if not self.url_prefix.endswith('/'):
self.url_prefix += '/'
if not self.url_prefix.endswith("/"):
self.url_prefix += "/"
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f"Invalid log level '{self.log_level}' specified.")
coloredlogs.install(
level=numeric_level,
fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s",
datefmt='%H:%M:%S'
level=numeric_level, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S"
)
LOG = logging.getLogger('config')
LOG = logging.getLogger("config")
if self.debug:
try:
import debugpy
debugpy.listen(("0.0.0.0", self.debugpy_port))
LOG.info(f"starting debugpy server on '0.0.0.0:{self.debugpy_port}'.")
except ImportError:
@ -213,7 +235,7 @@ class Config:
except Exception as e:
LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}")
optsFile: str = os.path.join(self.config_path, 'ytdlp.json')
optsFile: str = os.path.join(self.config_path, "ytdlp.json")
if os.path.exists(optsFile) and os.path.getsize(optsFile) > 4:
LOG.info(f"Loading yt-dlp custom options from '{optsFile}'.")
@ -226,7 +248,7 @@ class Config:
else:
LOG.info(f"No yt-dlp custom options found at '{optsFile}'.")
tasksFile = os.path.join(self.config_path, 'tasks.json')
tasksFile = os.path.join(self.config_path, "tasks.json")
if os.path.exists(tasksFile) and os.path.getsize(tasksFile) > 0:
LOG.info(f"Loading tasks from '{tasksFile}'.")
try:
@ -235,10 +257,10 @@ class Config:
LOG.error(f"Could not load tasks file from '{tasksFile}'. '{error}'.")
sys.exit(1)
self.tasks.extend(tasks)
except Exception as e:
except Exception:
pass
presetsFile = os.path.join(self.config_path, 'presets.json')
presetsFile = os.path.join(self.config_path, "presets.json")
if os.path.exists(presetsFile) and os.path.getsize(presetsFile) > 0:
LOG.info(f"Loading extra presets from '{presetsFile}'.")
try:
@ -248,32 +270,32 @@ class Config:
sys.exit(1)
for preset in presets:
if 'name' not in preset:
if "name" not in preset:
LOG.error(f"Missing 'name' key in preset '{preset}'.")
continue
if 'format' not in preset:
if "format" not in preset:
LOG.error(f"Missing 'format' key in preset '{preset}'.")
continue
if 'args' not in preset:
preset['args'] = {}
if "args" not in preset:
preset["args"] = {}
if 'postprocessors' not in preset:
preset['postprocessors'] = []
if "postprocessors" not in preset:
preset["postprocessors"] = []
self.presets.append(preset)
except Exception as e:
except Exception:
pass
self.ytdl_options['socket_timeout'] = self.socket_timeout
self.ytdl_options["socket_timeout"] = self.socket_timeout
if self.keep_archive:
LOG.info(f"keep archive option is enabled.")
self.ytdl_options['download_archive'] = os.path.join(self.config_path, 'archive.log')
LOG.info("keep archive option is enabled.")
self.ytdl_options["download_archive"] = os.path.join(self.config_path, "archive.log")
if self.temp_keep:
LOG.info(f'Keep temp files option is enabled.')
LOG.info("Keep temp files option is enabled.")
if self.auth_password and self.auth_username:
LOG.warning(f"Basic authentication enabled with username '{self.auth_username}'.")
@ -285,7 +307,7 @@ class Config:
vClass: str = self.__class__
for attribute in vClass.__dict__.keys():
if attribute.startswith('_'):
if attribute.startswith("_"):
continue
value = getattr(vClass, attribute)

47
app/main.py
View file

@ -1,33 +1,34 @@
#!/usr/bin/env python3
import asyncio
import logging
import os
import random
import logging
import caribou
import sqlite3
import magic
from datetime import datetime
from aiohttp import web
from pathlib import Path
from library.Emitter import Emitter
from library.config import Config
from library.encoder import Encoder
from library.DownloadQueue import DownloadQueue
from library.Webhooks import Webhooks
from library.HttpSocket import HttpSocket
from library.HttpAPI import HttpAPI
import caribou
import magic
from aiocron import crontab
from aiohttp import web
from library.config import Config
from library.DownloadQueue import DownloadQueue
from library.Emitter import Emitter
from library.encoder import Encoder
from library.HttpAPI import HttpAPI
from library.HttpSocket import HttpSocket
from library.Webhooks import Webhooks
LOG = logging.getLogger("app")
MIME = magic.Magic(mime=True)
class main:
config: Config = None
app: web.Application = None
http: HttpAPI = None
socket: HttpSocket = None
class Main:
config: Config
app: web.Application
http: HttpAPI
socket: HttpSocket
def __init__(self):
self.config = Config.get_instance()
@ -61,9 +62,7 @@ class main:
LOG.info(f"Creating download folder at '{self.config.download_path}'.")
os.makedirs(self.config.download_path, exist_ok=True)
except OSError as e:
LOG.error(
f"Could not create download folder at '{self.config.download_path}'."
)
LOG.error(f"Could not create download folder at '{self.config.download_path}'.")
raise e
try:
@ -111,9 +110,7 @@ class main:
LOG.info(f"Completed 'Task: {taskName}' at '{timeNow}'.")
except Exception as e:
timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
LOG.error(
f"Failed 'Task: {taskName}' at '{timeNow}'. Error message '{str(e)}'."
)
LOG.error(f"Failed 'Task: {taskName}' at '{timeNow}'. Error message '{str(e)}'.")
def load_tasks(self):
for task in self.config.tasks:
@ -131,9 +128,7 @@ class main:
loop=asyncio.get_event_loop(),
)
LOG.info(
f"Added 'Task: {task.get('name', task.get('url'))}' to be executed every '{cron_timer}'."
)
LOG.info(f"Added 'Task: {task.get('name', task.get('url'))}' to be executed every '{cron_timer}'.")
def start(self):
self.socket.attach(self.app)
@ -155,4 +150,4 @@ class main:
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
main().start()
Main().start()

78
pyproject.toml
View file

@ -1,2 +1,80 @@
[tool.ruff]
# Exclude a variety of commonly ignored directories.
exclude = [
".bzr",
".direnv",
".eggs",
".git",
".git-rewrite",
".hg",
".mypy_cache",
".nox",
".pants.d",
".pyenv",
".pytest_cache",
".pytype",
".ruff_cache",
".svn",
".tox",
".venv",
".vscode",
"__pypackages__",
"_build",
"buck-out",
"build",
"dist",
"node_modules",
"site-packages",
"venv",
]
# Same as Black.
line-length = 120
indent-width = 4
# Assume Python 3.11
target-version = "py311"
[tool.ruff.lint]
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
# McCabe complexity (`C901`) by default.
select = ["E4", "E7", "E9", "F", "G", "W"]
ignore = ["PTH", "G0", "G1", "G201"]
# Allow fix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]
unfixable = []
# Allow unused variables when underscore-prefixed.
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
[tool.ruff.format]
# Like Black, use double quotes for strings.
quote-style = "double"
# Like Black, indent with spaces, rather than tabs.
indent-style = "space"
# Like Black, respect magic trailing commas.
skip-magic-trailing-comma = false
# Like Black, automatically detect the appropriate line ending.
line-ending = "auto"
# Enable auto-formatting of code examples in docstrings. Markdown,
# reStructuredText code/literal blocks and doctests are all supported.
#
# This is currently disabled by default, but it is planned for this
# to be opt-out in the future.
docstring-code-format = false
# Set the line length limit used when formatting code snippets in
# docstrings.
#
# This only has an effect when the `docstring-code-format` setting is
# enabled.
docstring-code-line-length = "dynamic"
[tool.autopep8]
--max-line-length = 120