diff --git a/.gitignore b/.gitignore index adbafeaf..647126e0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json +!.vscode/settings.json .history/* # Keep Var dir diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..f1a33979 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,34 @@ +{ + "files.exclude": { + "**/__pycache__": true + }, + "autopep8.args": [ + "--max-line-length", + "120", + "--experimental" + ], + "editor.rulers": [ + 120 + ], + "cSpell.words": [ + "aconvert", + "aiocron", + "copyts", + "dotenv", + "finaldir", + "getpid", + "libcurl", + "libx", + "mpegts", + "muxdelay", + "nodesc", + "noprogress", + "postprocessor", + "preferredcodec", + "preferredquality", + "tmpfilename", + "vconvert", + "writedescription", + "xerror" + ], +} diff --git a/app/AsyncPool.py b/app/AsyncPool.py index 62f6aff2..d9cea779 100644 --- a/app/AsyncPool.py +++ b/app/AsyncPool.py @@ -103,7 +103,7 @@ class AsyncPool: """ return self._active_workers < self._num_workers - def get_avalialbe_workers(self) -> int: + def get_available_workers(self) -> int: """ :return: number of available workers. """ diff --git a/app/Config.py b/app/Config.py index 1bf0ba77..31d4b929 100644 --- a/app/Config.py +++ b/app/Config.py @@ -41,12 +41,11 @@ class Config: version: str = APP_VERSION - _boolean_vars: tuple = ( - 'keep_archive', 'ytdl_debug', - 'temp_keep', 'allow_manifestless', - ) + new_version_available: bool = False + _int_vars: tuple = ('port', 'max_workers',) - _immutable: tuple = ('version', '__instance', 'ytdl_options',) + _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available') + _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'temp_keep', 'allow_manifestless',) @staticmethod def get_instance(): @@ -56,19 +55,15 @@ class Config: def __init__(self): """ Virtually private constructor. """ if Config.__instance is not None: - raise Exception( - "This class is a singleton!. Use Config.getInstance() instead.") + raise Exception("This class is a singleton!. Use Config.getInstance() instead.") else: Config.__instance = self - baseDefualtPath: str = os.path.dirname(__file__) + baseDefaultPath: str = os.path.dirname(__file__) - self.config_path = os.environ.get('YTP_CONFIG_PATH', None) or os.path.join( - baseDefualtPath, 'var', 'config') - self.download_path = os.environ.get('YTP_DOWNLOAD_PATH', None) or os.path.join( - baseDefualtPath, 'var', 'downloads') - self.temp_path = os.environ.get('YTP_TEMP_PATH', None) or os.path.join( - baseDefualtPath, 'var', 'tmp') + self.temp_path = os.environ.get('YTP_TEMP_PATH', None) or os.path.join(baseDefaultPath, 'var', 'tmp') + self.config_path = os.environ.get('YTP_CONFIG_PATH', None) or os.path.join(baseDefaultPath, 'var', 'config') + self.download_path = os.environ.get('YTP_DOWNLOAD_PATH', None) or os.path.join(baseDefaultPath, 'var', 'downloads') envFile: str = os.path.join(self.config_path, '.env') @@ -80,16 +75,13 @@ class Config: if k.startswith('_'): continue - # If the variable is not updateable, set it to the default value. + # If the variable declared as immutable, set it to the default value. if k in self._immutable: setattr(self, k, v) continue lookUpKey: str = f'YTP_{k}'.upper() - setattr( - self, k, - os.environ[lookUpKey] if lookUpKey in os.environ else v - ) + setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v) for k, v in self.__dict__.items(): if k.startswith('_') or k in self._immutable: @@ -99,17 +91,16 @@ class Config: for key in re.findall(r'\{.*?\}', v): localKey: str = key[1:-1] if localKey not in self.__dict__: - logging.error( - f'Config variable "{k}" had non exisitng config reference "{key}"') + logging.error(f'Config variable "{k}" had non existing config reference "{key}"') sys.exit(1) + v = v.replace(key, getattr(self, localKey)) setattr(self, k, v) if k in self._boolean_vars: if str(v).lower() not in (True, False, 'true', 'false', 'on', 'off', '1', '0'): - raise ValueError( - f'Config variable "{k}" is set to a non-boolean value "{v}".') + raise ValueError(f'Config variable "{k}" is set to a non-boolean value "{v}".') setattr(self, k, str(v).lower() in (True, 'true', 'on', '1')) @@ -119,25 +110,21 @@ class Config: if not self.url_prefix.endswith('/'): self.url_prefix += '/' - numeric_level = getattr( - logging, self.logging_level.upper(), None) - + numeric_level = getattr(logging, self.logging_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError(f"Invalid log level: {self.logging_level}") - logging.basicConfig( - force=True, + coloredlogs.install( level=numeric_level, - format="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s" + fmt="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s", + datefmt='%H:%M:%S' ) - log = logging.getLogger('config') - - coloredlogs.install() + LOG = logging.getLogger('config') optsFile: str = os.path.join(self.config_path, 'ytdlp.json') if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0: - log.info(f'Loading yt-dlp custom options from "{optsFile}"') + LOG.info(f'Loading yt-dlp custom options from "{optsFile}"') try: with open(optsFile) as json_data: @@ -145,28 +132,27 @@ class Config: assert isinstance(opts, dict) self.ytdl_options.update(opts) except (json.decoder.JSONDecodeError, AssertionError) as e: - log.error( - f'JSON error in "{optsFile}": {e}') + LOG.error(f'JSON error in "{optsFile}": {e}') sys.exit(1) else: - log.info(f'No custom yt-dlp options found in "{self.config_path}"') + LOG.info(f'No custom yt-dlp options found in "{self.config_path}"') if self.keep_archive: - log.info(f'keep archive: {self.keep_archive}') + LOG.info(f'keep archive: {self.keep_archive}') self.ytdl_options['download_archive'] = os.path.join( self.config_path, 'archive.log') - log.info(f'Keep temp: {self.temp_keep}') + LOG.info(f'Keep temp: {self.temp_keep}') def _getAttributes(self) -> dict: attrs: dict = {} - vclass: str = self.__class__ + vClass: str = self.__class__ - for attribute in vclass.__dict__.keys(): + for attribute in vClass.__dict__.keys(): if attribute.startswith('_'): continue - value = getattr(vclass, attribute) + value = getattr(vClass, attribute) if not callable(value): attrs[attribute] = value diff --git a/app/DataStore.py b/app/DataStore.py index b7d37f4a..c3089da5 100644 --- a/app/DataStore.py +++ b/app/DataStore.py @@ -3,7 +3,7 @@ import copy from datetime import datetime, timezone from email.utils import formatdate import json -import sqlite3 +from sqlite3 import Connection from Utils import calcDownloadPath from Config import Config from Download import Download @@ -15,24 +15,22 @@ class DataStore: Persistent queue. """ type: str = None - db_file: str = None dict: OrderedDict[str, Download] = None config: Config = None - def __init__(self, type: str, config: Config): + connection: Connection + + def __init__(self, type: str, connection: Connection): self.dict = OrderedDict() self.type = type - self.config = config - self.db_file = self.config.db_file + self.config = Config.get_instance() + self.connection = connection def load(self) -> None: for id, item in self.saved_items(): self.dict.update({id: Download( info=item, - download_dir=calcDownloadPath( - basePath=self.config.download_path, - folder=item.folder - ), + download_dir=calcDownloadPath(basePath=self.config.download_path, folder=item.folder), temp_dir=self.config.temp_path, output_template_chapter=self.config.output_template_chapter, default_ytdl_opts=self.config.ytdl_options) @@ -40,7 +38,7 @@ class DataStore: def exists(self, key: str = None, url: str = None) -> bool: if not key and not url: - raise KeyError('key or url must be provided') + raise KeyError('key or url must be provided.') if key and key in self.dict: return True @@ -66,31 +64,24 @@ class DataStore: def saved_items(self) -> list[tuple[str, ItemDTO]]: items: list[tuple[str, ItemDTO]] = [] - with sqlite3.connect(self.db_file) as db: - db.row_factory = sqlite3.Row - cursor = db.execute( - f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', - (self.type,) - ) - for row in cursor: - data: dict = json.loads(row['data']) - key: str = data.pop('_id') - item: ItemDTO = ItemDTO(**data) - item._id = key - item.datetime = formatdate(datetime.strptime( - row['created_at'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp() - ) - items.append((row['id'], item)) + cursor = self.connection.execute( + f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', + (self.type,) + ) + + for row in cursor: + rowDate = datetime.strptime(row['created_at'], '%Y-%m-%d %H:%M:%S') + data: dict = json.loads(row['data']) + key: str = data.pop('_id') + item: ItemDTO = ItemDTO(**data) + item._id = key + item.datetime = formatdate(rowDate.replace(tzinfo=timezone.utc).timestamp()) + items.append((row['id'], item)) return items def put(self, value: Download) -> Download: - # for key in self.dict: - # if self.dict[key].info.url == value.info.url: - # value.info._id = key - # return - self.dict.update({value.info._id: value}) self._updateStoreItem(self.type, value.info) @@ -144,21 +135,10 @@ class DataStore: except AttributeError: pass - with sqlite3.connect(self.db_file) as db: - db.execute(sqlStatement.strip(), ( - stored._id, - type, - stored.url, - stored.json(), - type, - stored.url, - stored.json(), - datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') - )) + self.connection.execute(sqlStatement.strip(), ( + stored._id, type, stored.url, stored.json(), type, stored.url, + stored.json(), datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') + )) def _deleteStoreItem(self, key: str) -> None: - with sqlite3.connect(self.db_file) as db: - db.execute( - 'DELETE FROM "history" WHERE "type" = ? AND "id" = ?', - (self.type, key,) - ) + self.connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (self.type, key,)) diff --git a/app/Download.py b/app/Download.py index 19e5bff1..cd10785a 100644 --- a/app/Download.py +++ b/app/Download.py @@ -11,7 +11,7 @@ from ItemDTO import ItemDTO from Config import Config import hashlib -log = logging.getLogger('download') +LOG = logging.getLogger('download') class Download: @@ -52,7 +52,10 @@ class Download: 'speed', 'eta', ) + "Fields to be extracted from yt-dlp progress hook." + tempKeep: bool = False + "Keep temp directory after download." def __init__( self, @@ -68,8 +71,7 @@ class Download: self.output_template_chapter = output_template_chapter self.output_template = info.output_template self.format = get_format(info.format, info.quality) - self.ytdl_opts = get_opts( - info.format, info.quality, info.ytdlp_config if info.ytdlp_config else {}) + self.ytdl_opts = get_opts(info.format, info.quality, info.ytdlp_config if info.ytdlp_config else {}) self.info = info self.id = info._id self.default_ytdl_opts = default_ytdl_opts @@ -81,26 +83,21 @@ class Download: self.proc = None self.loop = None self.notifier = None - config = Config.get_instance() - self.max_workers = int(config.max_workers) - self.tempKeep = bool(config.temp_keep) + self.max_workers = int(Config.get_instance().max_workers) + self.tempKeep = bool(Config.get_instance().temp_keep) self.is_live = bool(info.is_live) or info.live_in is not None - self.is_manifestless = 'is_manifestless' in self.info.options and self.info.options[ - 'is_manifestless'] is True + self.is_manifestless = 'is_manifestless' in self.info.options and self.info.options['is_manifestless'] is True def _progress_hook(self, data: dict): - dicto = {k: v for k, v in data.items() if k in self._ytdlp_fields} - self.status_queue.put({'id': self.id, **dicto}) + dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} + self.status_queue.put({'id': self.id, **dataDict}) def _postprocessor_hook(self, data: dict): if data.get('postprocessor') != 'MoveFiles' or data.get('status') != 'finished': return if '__finaldir' in data['info_dict']: - filename = os.path.join( - data['info_dict']['__finaldir'], - os.path.basename(data['info_dict']['filepath']) - ) + filename = os.path.join(data['info_dict']['__finaldir'], os.path.basename(data['info_dict']['filepath'])) else: filename = data['info_dict']['filepath'] @@ -141,14 +138,14 @@ class Download: try: data = jsonCookie(json.loads(self.info.ytdlp_cookies)) if not data: - log.warning( + LOG.warning( f'The cookie string that was provided for {self.info.title} is empty or not in expected spec.') with open(os.path.join(self.tempPath, f'cookie_{self.info._id}.txt'), 'w') as f: f.write(data) params['cookiefile'] = f.name except ValueError as e: - log.error( + LOG.error( f'Invalid cookies: was provided for {self.info.title} - {str(e)}') if self.is_live or self.is_manifestless: @@ -161,18 +158,14 @@ class Download: deletedOpts.append(opt) if hasDeletedOptions: - log.warning( + LOG.warning( f'Live stream detected for [{self.info.title}], The following opts [{deletedOpts=}] have been deleted which are known to cause issues with live stream and post stream manifestless mode.') - log.info( - f'Downloading {os.getpid()=} id="{self.info.id}" title="{self.info.title}".') + LOG.info(f'Downloading {os.getpid()=} id="{self.info.id}" title="{self.info.title}".') ret = yt_dlp.YoutubeDL(params=params).download([self.info.url]) - self.status_queue.put({ - 'id': self.id, - 'status': 'finished' if ret == 0 else 'error' - }) + self.status_queue.put({'id': self.id, 'status': 'finished' if ret == 0 else 'error'}) except Exception as exc: self.status_queue.put({ 'id': self.id, @@ -181,22 +174,17 @@ class Download: 'error': str(exc) }) - log.info( - f'Finished {os.getpid()=} id="{self.info.id}" title="{self.info.title}".') + LOG.info(f'Finished {os.getpid()=} id="{self.info.id}" title="{self.info.title}".') async def start(self, notifier: Notifier): - if self.manager is None: - self.manager = multiprocessing.Manager() + self.manager = multiprocessing.Manager() if self.manager is None else self.manager self.status_queue = self.manager.Queue() self.loop = asyncio.get_running_loop() self.notifier = notifier # Create temp dir for each download. - self.tempPath = os.path.join( - self.temp_dir, - hashlib.shake_256(self.info.id.encode('utf-8')).hexdigest(5) - ) + self.tempPath = os.path.join(self.temp_dir, hashlib.shake_256(self.info.id.encode('utf-8')).hexdigest(5)) if not os.path.exists(self.tempPath): os.makedirs(self.tempPath, exist_ok=True) @@ -229,7 +217,7 @@ class Download: def kill(self): if self.running(): - log.info(f'Killing download process: {self.proc.ident}') + LOG.info(f'Killing download process: {self.proc.ident}') self.proc.kill() def delete_temp(self): @@ -237,19 +225,19 @@ class Download: return if self.info.status != 'finished' and self.is_live: - log.warning( - f'Keeping live temp directory: {self.tempPath}, as the reported status is not finished [{self.info.status}].') + LOG.warning( + f'Keeping live temp directory [{self.tempPath}], as the reported status is not finished [{self.info.status}].') return if not os.path.exists(self.tempPath): return if self.tempPath == self.temp_dir: - log.warning( + LOG.warning( f'Attempted to delete video temp directory: {self.tempPath}, but it is the same as main temp directory.') return - log.info(f'Deleting Temp directory: {self.tempPath}') + LOG.info(f'Deleting Temp directory: {self.tempPath}') shutil.rmtree(self.tempPath, ignore_errors=True) async def progress_update(self): @@ -262,7 +250,7 @@ class Download: return if self.debug: - log.debug(f'Status Update: {self.info._id=} {status=}') + LOG.debug(f'Status Update: {self.info._id=} {status=}') if isinstance(status, str): await self.notifier.updated(self.info) @@ -271,13 +259,11 @@ class Download: self.tmpfilename = status.get('tmpfilename') if 'filename' in status: - self.info.filename = os.path.relpath( - status.get('filename'), self.download_dir) + self.info.filename = os.path.relpath(status.get('filename'), self.download_dir) # Set correct file extension for thumbnails if self.info.format == 'thumbnail': - self.info.filename = re.sub( - r'\.webm$', '.jpg', self.info.filename) + self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) self.info.status = status.get('status', self.info.status) self.info.msg = status.get('msg') @@ -287,11 +273,9 @@ class Download: await self.notifier.error(self.info, self.info.error) if 'downloaded_bytes' in status: - total = status.get('total_bytes') or status.get( - 'total_bytes_estimate') + total = status.get('total_bytes') or status.get('total_bytes_estimate') if total: - perc = status['downloaded_bytes'] / total * 100 - self.info.percent = perc + self.info.percent = status['downloaded_bytes'] / total * 100 self.info.total_bytes = total self.info.speed = status.get('speed') @@ -299,11 +283,9 @@ class Download: if self.info.status == 'finished' and 'filename' in status and self.info.format != 'thumbnail': try: - self.info.file_size = os.path.getsize( - status.get('filename')) + self.info.file_size = os.path.getsize(status.get('filename')) except FileNotFoundError: - log.warning( - f'File not found: {status.get("filename")}') + LOG.warning(f'File not found: {status.get("filename")}') self.info.file_size = None pass diff --git a/app/DownloadQueue.py b/app/DownloadQueue.py index e1d91065..486442a2 100644 --- a/app/DownloadQueue.py +++ b/app/DownloadQueue.py @@ -12,32 +12,31 @@ from DataStore import DataStore from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig from AsyncPool import AsyncPool -log = logging.getLogger('DownloadQueue') -dl_queue = asyncio.Queue() +LOG = logging.getLogger('DownloadQueue') +TYPE_DONE: str = 'done' +TYPE_QUEUE: str = 'queue' class DownloadQueue: config: Config = None notifier: Notifier = None - connection: Connection = None serializer: ObjectSerializer = None queue: DataStore = None done: DataStore = None event: asyncio.Event = None - def __init__(self, config: Config, notifier: Notifier, connection: Connection, serializer: ObjectSerializer): - self.config = config + def __init__(self, notifier: Notifier, connection: Connection, serializer: ObjectSerializer): + self.config = Config.get_instance() self.notifier = notifier - self.connection = connection self.serializer = serializer - self.done = DataStore(type='done', config=self.config) - self.queue = DataStore(type='queue', config=self.config) + self.done = DataStore(type=TYPE_DONE, connection=connection) + self.queue = DataStore(type=TYPE_QUEUE, connection=connection) self.done.load() self.queue.load() async def initialize(self): self.event = asyncio.Event() - log.info(f'Using {self.config.max_workers} workers for downloading.') + LOG.info(f'Using {self.config.max_workers} workers for downloading.') asyncio.create_task( self.__download_pool() if self.config.max_workers > 1 else self.__download() @@ -62,15 +61,14 @@ class DownloadQueue: error: str = None live_in: str = None - etype = entry.get('_type') or 'video' - if etype == 'playlist': + eventType = entry.get('_type') or 'video' + if eventType == 'playlist': entries = entry['entries'] playlist_index_digits = len(str(len(entries))) results = [] for index, etr in enumerate(entries, start=1): etr['playlist'] = entry.get('id') - etr['playlist_index'] = '{{0:0{0:d}d}}'.format( - playlist_index_digits).format(index) + etr['playlist_index'] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index) for property in ('id', 'title', 'uploader', 'uploader_id'): if property in entry: etr[f'playlist_{property}'] = entry.get(property) @@ -92,7 +90,7 @@ class DownloadQueue: return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} return {'status': 'ok'} - elif (etype == 'video' or etype.startswith('url')) and 'id' in entry and 'title' in entry: + elif (eventType == 'video' or eventType.startswith('url')) and 'id' in entry and 'title' in entry: # check if the video is live stream. if 'live_status' in entry and entry.get('live_status') == 'is_upcoming': if 'release_timestamp' in entry and entry.get('release_timestamp'): @@ -102,30 +100,25 @@ class DownloadQueue: else: error = entry.get('msg', None) - log.debug( - f"entry: {entry.get('id', None)} - {entry.get('webpage_url', None)} - {entry.get('url', None) }") + LOG.debug(f"entry: {entry.get('id', None)} - {entry.get('webpage_url', None)} - {entry.get('url', None)}") if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry.get('url')): - item = self.done.get(key=entry['id'], url=entry.get( - 'webpage_url') or entry['url']) + item = self.done.get(key=entry['id'], url=entry.get('webpage_url') or entry['url']) - log.debug( - f'Item [{item.info.title}] already downloaded. Removing from history.') + LOG.debug(f'Item [{item.info.title}] already downloaded. Removing from history.') await self.clear([item.info._id]) try: - item = self.queue.get(key=entry.get('id'), url=entry.get( - 'webpage_url') or entry.get('url')) + item = self.queue.get(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url')) if item is not None: err_message = f'Item [{item.info.id}: {item.info.title}] already in download queue.' - log.info(err_message) + LOG.info(err_message) return {'status': 'error', 'msg': err_message} except KeyError: pass - is_manifestless = 'post_live' == entry.get( - 'live_status') if 'live_status' in entry else None + is_manifestless = 'post_live' == entry.get('live_status') if 'live_status' in entry else None options.update({'is_manifestless': is_manifestless}) @@ -147,23 +140,20 @@ class DownloadQueue: ) try: - dldirectory = calcDownloadPath( - basePath=self.config.download_path, - folder=folder - ) + download_dir = calcDownloadPath(basePath=self.config.download_path, folder=folder) except Exception as e: + LOG.exception(e) return {'status': 'error', 'msg': str(e)} output_chapter = self.config.output_template_chapter for property, value in entry.items(): if property.startswith("playlist"): - dl.output_template = dl.output_template.replace( - f"%({property})s", str(value)) + dl.output_template = dl.output_template.replace(f"%({property})s", str(value)) dlInfo: Download = Download( info=dl, - download_dir=dldirectory, + download_dir=download_dir, temp_dir=self.config.temp_path, output_template_chapter=output_chapter, default_ytdl_opts=self.config.ytdl_options, @@ -173,23 +163,23 @@ class DownloadQueue: if dlInfo.info.live_in: dlInfo.info.status = 'not_live' itemDownload = self.done.put(dlInfo) - NotifiyEvent = 'completed' + NotifyEvent = 'completed' elif self.config.allow_manifestless is False and is_manifestless is True: dlInfo.info.status = 'error' dlInfo.info.error = 'Video is in Post-Live Manifestless mode.' itemDownload = self.done.put(dlInfo) - NotifiyEvent = 'completed' + NotifyEvent = 'completed' else: - NotifiyEvent = 'added' + NotifyEvent = 'added' itemDownload = self.queue.put(dlInfo) self.event.set() - await self.notifier.emit(NotifiyEvent, itemDownload.info) + await self.notifier.emit(NotifyEvent, itemDownload.info) return { 'status': 'ok' } - elif etype.startswith('url'): + elif eventType.startswith('url'): return await self.add( entry=entry.get('url'), quality=quality, @@ -203,7 +193,7 @@ class DownloadQueue: return { 'status': 'error', - 'msg': f'Unsupported resource "{etype}"' + 'msg': f'Unsupported resource "{eventType}"' } async def add( @@ -219,12 +209,11 @@ class DownloadQueue: ): ytdlp_config = ytdlp_config if ytdlp_config else {} - log.info( - f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}') + LOG.info(f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}') already = set() if already is None else already if url in already: - log.info('recursion detected, skipping') + LOG.info('recursion detected, skipping') return {'status': 'ok'} else: already.add(url) @@ -252,8 +241,7 @@ class DownloadQueue: return {'status': 'error', 'msg': 'Unable to extract info check logs.'} if self.isDownloaded(entry): - log.info( - f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.') + LOG.info(f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.') return { 'status': 'error', 'msg': f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.' @@ -267,7 +255,7 @@ class DownloadQueue: if self.isDownloaded(entry): raise yt_dlp.utils.ExistingVideoReached() - log.debug(f'entry: extract info says: {entry}') + LOG.debug(f'entry: extract info says: {entry}') except yt_dlp.utils.ExistingVideoReached: return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'} except yt_dlp.utils.YoutubeDLError as exc: @@ -289,16 +277,15 @@ class DownloadQueue: try: item = self.queue.get(key=id) except KeyError as e: - log.warn( - f'Requested cancel for non-existent download {id=}. {str(e)}') + LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}') continue if item.started() is True: - log.info(f'Canceling {id=} {item.info.title=}') + LOG.info(f'Canceling {id=} {item.info.title=}') item.cancel() else: item.close() - log.info(f'Deleting from queue {id=} {item.info.title=}') + LOG.info(f'Deleting from queue {id=} {item.info.title=}') self.queue.delete(id) await self.notifier.canceled(id) self.done.put(item) @@ -312,11 +299,10 @@ class DownloadQueue: try: item = self.done.get(key=id) except KeyError as e: - log.warn( - f'Requested delete for non-existent download {id=}. {str(e)}') + LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}') continue - log.info(f'Deleting completed download {id=} {item.info.title=}') + LOG.info(f'Deleting completed download {id=} {item.info.title=}') self.done.delete(id) await self.notifier.cleared(id) @@ -347,7 +333,7 @@ class DownloadQueue: num_workers=self.config.max_workers, worker_co=self.__downloadFile, name='WorkerPool', - logger=log, + logger=logging.getLogger('WorkerPool'), ) as executor: while True: while True: @@ -357,8 +343,7 @@ class DownloadQueue: break while not self.queue.hasDownloads(): - log.info( - f'Waiting for item to download. [{executor.get_avalialbe_workers()}] Workers available.') + LOG.info(f'Waiting for item to download. [{executor.get_available_workers()}] workers available.') await self.event.wait() self.event.clear() @@ -372,7 +357,7 @@ class DownloadQueue: async def __download(self): while True: while self.queue.empty(): - log.info('waiting for item to download.') + LOG.info('waiting for item to download.') await self.event.wait() self.event.clear() @@ -380,8 +365,7 @@ class DownloadQueue: await self.__downloadFile(id, entry) async def __downloadFile(self, id: str, entry: Download): - log.info( - f'Queuing {id=} - {entry.info.title=} - {entry.info.url=} - {entry.info.folder=}.') + LOG.info(f'Queuing {id=} - {entry.info.title=} - {entry.info.url=} - {entry.info.folder=}.') await entry.start(self.notifier) diff --git a/app/Utils.py b/app/Utils.py index 85b05e0b..37790259 100644 --- a/app/Utils.py +++ b/app/Utils.py @@ -9,19 +9,10 @@ import yt_dlp from socketio import AsyncServer from Webhooks import Webhooks -log = logging.getLogger('Utils') +LOG = logging.getLogger('Utils') -IGNORED_KEYS: tuple = ( - 'cookiefile', - 'paths', - 'outtmpl', - 'progress_hooks', - 'postprocessor_hooks', -) - -AUDIO_FORMATS: tuple = ( - 'm4a', 'mp3', 'opus', 'wav' -) +AUDIO_FORMATS: tuple = ('m4a', 'mp3', 'opus', 'wav') +IGNORED_KEYS: tuple = ('cookiefile', 'paths', 'outtmpl', 'progress_hooks', 'postprocessor_hooks',) def get_format(format: str, quality: str) -> str: @@ -55,16 +46,15 @@ def get_format(format: str, quality: str) -> str: if quality == 'audio': return "bestaudio/best" - vfmt, afmt = ( - '[ext=mp4]', '[ext=m4a]') if format == "mp4" else ('', '') + videoFormat, audioFormat = ('[ext=mp4]', '[ext=m4a]') if format == "mp4" else ('', '') - vres = f'[height<={quality}]' if quality != 'best' else '' + videoResolution = f'[height<={quality}]' if quality != 'best' else '' - vcombo = vres + vfmt + videoCombo = videoResolution + videoFormat - return f'bestvideo{vcombo}+bestaudio{afmt}/best{vcombo}' + return f'bestvideo{videoCombo}+bestaudio{audioFormat}/best{videoCombo}' - raise Exception(f"Unkown format {format}") + raise Exception(f"Unknown format {format}") def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict: @@ -98,19 +88,16 @@ def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict: # Audio formats without thumbnail if format not in ("wav") and "writethumbnail" not in opts: opts["writethumbnail"] = True - postprocessors.append( - {"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}) + postprocessors.append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}) postprocessors.append({"key": "FFmpegMetadata"}) postprocessors.append({"key": "EmbedThumbnail"}) if format == "thumbnail": opts["skip_download"] = True opts["writethumbnail"] = True - postprocessors.append( - {"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}) + postprocessors.append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}) - opts["postprocessors"] = postprocessors + \ - (opts["postprocessors"] if "postprocessors" in opts else []) + opts["postprocessors"] = postprocessors + (opts["postprocessors"] if "postprocessors" in opts else []) return opts @@ -127,26 +114,11 @@ def getVideoInfo(url: str, ytdlp_opts: dict = None) -> (Any | dict[str, Any] | N return yt_dlp.YoutubeDL().extract_info(url, download=False) -def getAttributes(vclass: str | type) -> dict: - attrs: dict = {} - - if not isinstance(vclass, type): - vclass = vclass.__class__ - - for attribute in vclass.__dict__.keys(): - if not attribute.startswith('_'): - value = getattr(vclass, attribute) - if not callable(value): - attrs[attribute] = value - - return attrs - - def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) -> str: """Calculates download path and prevents directory traversal. Returns: - Dir with base dir factored in. + Download path with base directory factored in. """ if not folder: return basePath @@ -155,8 +127,7 @@ def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) download_path = os.path.realpath(os.path.join(basePath, folder)) if not download_path.startswith(realBasePath): - raise Exception( - f'Folder "{folder}" must resolve inside the base download directory "{realBasePath}"') + raise Exception(f'Folder "{folder}" must resolve inside the base download directory "{realBasePath}".') if not os.path.isdir(download_path) and createPath: os.makedirs(download_path, exist_ok=True) @@ -198,10 +169,7 @@ def mergeDict(source: dict, destination: dict) -> dict: for key, value in source.items(): destination_key_value = destination_copy.get(key) if isinstance(value, dict) and isinstance(destination_key_value, dict): - destination_copy[key] = mergeDict( - source=value, - destination=destination_copy.setdefault(key, {}) - ) + destination_copy[key] = mergeDict(source=value, destination=destination_copy.setdefault(key, {})) elif isinstance(value, list) and isinstance(destination_key_value, list): destination_copy[key] = destination_key_value + value else: @@ -238,7 +206,7 @@ def isDownloaded(archive_file: str, info: dict) -> bool: if not id: return False except Exception as e: - log.error(f'Error generating archive id: {e}') + LOG.error(f'Error generating archive id: {e}') return False with open(archive_file, 'r') as f: @@ -270,22 +238,20 @@ def jsonCookie(cookies: dict[dict[str, any]]) -> str | None: if not isinstance(cookies[domain][subDomain][cookie], dict): continue - dicto = cookies[domain][subDomain][cookie] + cookieDict = cookies[domain][subDomain][cookie] - if 0 == int(dicto['expirationDate']): - dicto['expirationDate']: float = datetime.now( - timezone.utc).timestamp() + (86400 * 1000) + if 0 == int(cookieDict['expirationDate']): + cookieDict['expirationDate']: float = datetime.now(timezone.utc).timestamp() + (86400 * 1000) hasCookies = True netscapeCookies += "\t".join([ - dicto['domain'] if str(dicto['domain']).startswith( - '.') else '.' + dicto['domain'], + cookieDict['domain'] if str(cookieDict['domain']).startswith('.') else '.' + cookieDict['domain'], 'TRUE', - dicto['path'], - 'TRUE' if dicto['secure'] else 'FALSE', - str(int(dicto['expirationDate'])), - dicto['name'], - dicto['value'] + cookieDict['path'], + 'TRUE' if cookieDict['secure'] else 'FALSE', + str(int(cookieDict['expirationDate'])), + cookieDict['name'], + cookieDict['value'] ])+"\n" return netscapeCookies if hasCookies else None @@ -309,10 +275,13 @@ class Notifier: sio: AsyncServer = None "SocketIO server instance" + serializer: ObjectSerializer = None "Serializer used to serialize objects to JSON" + webhooks: Webhooks = None "Send webhooks events." + webhooks_allowed_events: tuple = ( 'added', 'completed', 'error', 'not_live' ) diff --git a/app/Webhooks.py b/app/Webhooks.py index f9d594fe..5228b459 100644 --- a/app/Webhooks.py +++ b/app/Webhooks.py @@ -5,7 +5,7 @@ import os from ItemDTO import ItemDTO from aiohttp import client -log = logging.getLogger('Webhooks') +LOG = logging.getLogger('Webhooks') class Webhooks: @@ -13,16 +13,19 @@ class Webhooks: def __init__(self, file: str): if os.path.exists(file): - try: - if os.path.getsize(file) < 2: - raise Exception(f'file is empty.') + self.load(file) - log.info(f'Loading webhooks from {file}') - with open(file, 'r') as f: - self.targets = json.load(f) - except Exception as e: - log.error(f'Error loading webhooks from {file}: {e}') - pass + def load(self, file: str): + try: + if os.path.getsize(file) < 2: + raise Exception(f'file is empty.') + + LOG.info(f'Loading webhooks from {file}') + with open(file, 'r') as f: + self.targets = json.load(f) + except Exception as e: + LOG.error(f'Error loading webhooks from {file}: {e}') + pass async def send(self, event: str, item: ItemDTO) -> list[dict]: if len(self.targets) == 0: @@ -45,11 +48,11 @@ class Webhooks: async def __send(self, event: str, target: dict, item: ItemDTO) -> dict: req: dict = target.get('request') try: - log.info(f"Sending {event=} {item.id=} to [{target.get('name')}]") + LOG.info(f"Sending {event=} {item.id=} to [{target.get('name')}]") async with client.ClientSession() as session: headers = req.get('headers', {}) if 'headers' in req else {} async with session.request(method=req.get('method', 'POST'), url=req.get('url'), json=item.__dict__, headers=headers) as response: - log.info(f"Sent {event=} {item.id=} to [{target.get('name')}] and responsed with [status: {response.status}].") + LOG.info(f"[{target.get('name')}] Response to [{event=} {item.id=}] [status: {response.status}].") return { 'url': req.get('url'), 'status': response.status, diff --git a/app/main.py b/app/main.py index 5ddeae12..095eccb4 100644 --- a/app/main.py +++ b/app/main.py @@ -19,6 +19,8 @@ import caribou import sqlite3 from aiocron import crontab +LOG = logging.getLogger('app') + class Main: config: Config = None @@ -27,75 +29,68 @@ class Main: sio: socketio.AsyncServer = None routes: web.RouteTableDef = None connection: sqlite3.Connection = None - dqueue: DownloadQueue = None + queue: DownloadQueue = None loop: asyncio.AbstractEventLoop = None - logger: logging.Logger = None def __init__(self): self.config = Config.get_instance() - self.logger = logging.getLogger('main') try: if not os.path.exists(self.config.download_path): - self.logger.info( - f'Creating download folder at {self.config.download_path}') + LOG.info(f'Creating download folder at [{self.config.download_path}]') os.makedirs(self.config.download_path, exist_ok=True) except OSError as e: - self.logger.error( - f'Could not create download folder at {self.config.download_path}') + LOG.error(f'Could not create download folder at [{self.config.download_path}]') raise e try: if not os.path.exists(self.config.temp_path): - self.logger.info( - f'Creating temp folder at {self.config.temp_path}') + LOG.info(f'Creating temp folder at [{self.config.temp_path}]') os.makedirs(self.config.temp_path, exist_ok=True) except OSError as e: - self.logger.error( - f'Could not create temp folder at {self.config.temp_path}') + LOG.error(f'Could not create temp folder at [{self.config.temp_path}]') raise e try: if not os.path.exists(self.config.config_path): - self.logger.info( - f'Creating config folder at {self.config.config_path}') + LOG.info(f'Creating config folder at [{self.config.config_path}]') os.makedirs(self.config.config_path, exist_ok=True) except OSError as e: - self.logger.error( - f'Could not create config folder at {self.config.config_path}') + LOG.error(f'Could not create config folder at [{self.config.config_path}]') raise e try: if not os.path.exists(self.config.db_file): - self.logger.info( - f'Creating database file at {self.config.db_file}') + LOG.info(f'Creating database file at [{self.config.db_file}]') with open(self.config.db_file, 'w') as _: pass except OSError as e: - self.logger.error( - f'Could not create database file at {self.config.db_file}') + LOG.error(f'Could not create database file at [{self.config.db_file}]') raise e - caribou.upgrade(self.config.db_file, os.path.join( - os.path.realpath(os.path.dirname(__file__)), 'migrations')) - self.webhooks = Webhooks(os.path.join( - self.config.config_path, 'webhooks.json')) + caribou.upgrade(self.config.db_file, os.path.join(os.path.realpath(os.path.dirname(__file__)), 'migrations')) + + self.webhooks = Webhooks(os.path.join(self.config.config_path, 'webhooks.json')) self.loop = asyncio.get_event_loop() self.serializer = ObjectSerializer() + self.app = web.Application() self.sio = socketio.AsyncServer(cors_allowed_origins='*') - self.sio.attach( - self.app, socketio_path=self.config.url_prefix + 'socket.io') + self.sio.attach(self.app, socketio_path=self.config.url_prefix + 'socket.io') + self.routes = web.RouteTableDef() - self.connection = sqlite3.connect(self.config.db_file) - self.dqueue = DownloadQueue( - config=self.config, - notifier=Notifier( - sio=self.sio, serializer=self.serializer, webhooks=self.webhooks), + + self.connection = sqlite3.connect(self.config.db_file, isolation_level=None) + self.connection.row_factory = sqlite3.Row + self.connection.execute('PRAGMA journal_mode=wal') + + self.queue = DownloadQueue( + notifier=Notifier(sio=self.sio, serializer=self.serializer, webhooks=self.webhooks), connection=self.connection, serializer=self.serializer, ) - self.app.on_startup.append(lambda _: self.dqueue.initialize()) + + self.app.on_startup.append(lambda _: self.queue.initialize()) self.addRoutes() self.addTasks() self.start() @@ -114,7 +109,7 @@ class Main: async def connect(self, sid, _): data: dict = { - **self.dqueue.get(), + **self.queue.get(), "config": self.config, "tasks": [], } @@ -145,17 +140,15 @@ class Main: cvDate = self.config.version.split('-') if tagName > cvDate[0]: self.config.new_version_available = True - self.logger.info( - f'New version {tagName} available at') + LOG.info(f'New version [{tagName}] is available.') break - except Exception as e: pass def addTasks(self): tasks_file: str = os.path.join(self.config.config_path, 'tasks.json') if not os.path.exists(tasks_file): - self.logger.info( + LOG.info( f'No tasks file found at {tasks_file}. Skipping Tasks.') return @@ -163,21 +156,20 @@ class Main: with open(tasks_file, 'r') as f: tasks = json.load(f) except Exception as e: - self.logger.error( - f'Could not load tasks file [{tasks_file}]. Error message [{str(e)}]. Skipping Tasks.') + LOG.error(f'Could not load tasks file [{tasks_file}]. Error message [{str(e)}]. Skipping Tasks.') return for task in tasks: if not task.get('url'): - self.logger.warning(f'Invalid task {task}.') + LOG.warning(f'Invalid task {task}.') continue - cron_timer: str = task.get( - 'timer', f'{random.randint(1,59)} */1 * * *') + cron_timer: str = task.get('timer', f'{random.randint(1,59)} */1 * * *') async def cron_runner(task: dict): - self.logger.info( - f'Running task [{task.get("name",task.get("url"))}] at [{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}].') + taskName = task.get("name", task.get("url")) + timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + LOG.info(f'Started [Task: {taskName}] at [{timeNow}].') await self.add( url=task.get('url'), @@ -189,8 +181,8 @@ class Main: output_template=task.get('output_template') ) - self.logger.info( - f'Finished Running task [{task.get("name",task.get("url"))}] at [{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}].') + timeNow = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + LOG.info(f'Completed [Task: {taskName}] at [{timeNow}].') crontab( spec=cron_timer, @@ -200,12 +192,12 @@ class Main: loop=self.loop ) - self.logger.info( - f'Added task to grab {task.get("name",task.get("url"))} content every [{cron_timer}].') + LOG.info(f'Added [Task: {task.get("name", task.get("url"))}] executed every [{cron_timer}].') def addRoutes(self): @self.routes.get(self.config.url_prefix + 'ping') async def ping(_) -> Response: + self.connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() return web.Response(text='pong') @self.routes.post(self.config.url_prefix + 'add') @@ -240,8 +232,7 @@ class Main: @self.routes.get(self.config.url_prefix + 'tasks') async def tasks(_: Request) -> Response: - tasks_file: str = os.path.join( - self.config.config_path, 'tasks.json') + tasks_file: str = os.path.join(self.config.config_path, 'tasks.json') if not os.path.exists(tasks_file): return web.json_response({"error": "No tasks defined."}, status=404) @@ -264,10 +255,9 @@ class Main: for item in post: if not isinstance(item, dict): - raise web.HTTPBadRequest( - 'Invalid request body expecting list with dicts.') + raise web.HTTPBadRequest(reason='Invalid request body expecting list with dicts.') if not item.get('url'): - raise web.HTTPBadRequest(reason='url is required') + raise web.HTTPBadRequest(reason='url is required.') status[item.get('url')] = await self.add( url=item.get('url'), @@ -290,7 +280,7 @@ class Main: if not ids or where not in ['queue', 'done']: raise web.HTTPBadRequest() - status = await (self.dqueue.cancel(ids) if where == 'queue' else self.dqueue.clear(ids)) + status = await (self.queue.cancel(ids) if where == 'queue' else self.queue.clear(ids)) return web.Response(text=self.serializer.encode(status)) @@ -298,9 +288,9 @@ class Main: async def history(_) -> Response: history = {'done': [], 'queue': []} - for _, v in self.dqueue.queue.saved_items(): + for _, v in self.queue.queue.saved_items(): history['queue'].append(v) - for _, v in self.dqueue.done.saved_items(): + for _, v in self.queue.done.saved_items(): history['done'].append(v) return web.Response(text=self.serializer.encode(history)) @@ -310,7 +300,7 @@ class Main: file: str = request.match_info.get('file') if not file: - raise web.HTTPBadRequest(reason='file is required') + raise web.HTTPBadRequest(reason='file is required.') return web.Response( text=M3u8(url=f"{self.config.url_host}{self.config.url_prefix}").make_stream( @@ -340,8 +330,7 @@ class Main: segmenter = Segments( segment_index=int(segment), - segment_duration=float('{:.6f}'.format( - float(sd if sd else M3u8.segment_duration))), + segment_duration=float('{:.6f}'.format(float(sd if sd else M3u8.segment_duration))), vconvert=True if vc == 1 else False, aconvert=True if ac == 1 else False ) @@ -408,8 +397,7 @@ class Main: self.app.on_response_prepare.append(on_prepare) except ValueError as e: if 'frontend/dist' in str(e): - raise RuntimeError( - 'Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the frontend folder.') from e + raise RuntimeError('Could not find the frontend UI static assets.') from e raise e async def add( @@ -428,7 +416,7 @@ class Main: if ytdlp_cookies and isinstance(ytdlp_cookies, dict): ytdlp_cookies = self.serializer.encode(ytdlp_cookies) - status = await self.dqueue.add( + status = await self.queue.add( url=url, format=format if format else 'any', quality=quality if quality else 'best', diff --git a/app/player/M3u8.py b/app/player/M3u8.py index 28ef74fa..adcde458 100644 --- a/app/player/M3u8.py +++ b/app/player/M3u8.py @@ -17,11 +17,7 @@ class M3u8: self.segment_duration = float(segment_duration) def make_stream(self, download_path: str, file: str): - realFile: str = calcDownloadPath( - basePath=download_path, - folder=file, - createPath=False - ) + realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) if not os.path.exists(realFile): raise Exception(f"File {realFile} does not exist.") @@ -45,8 +41,7 @@ class M3u8: segmentSize: float = '{:.6f}'.format(self.segment_duration) splits: int = math.ceil(duration / self.segment_duration) - segmentParams = { - } + segmentParams: dict = {} for stream in ffprobe.streams: if stream.is_video(): @@ -58,17 +53,14 @@ class M3u8: for i in range(splits): if (i + 1) == splits: - segmentParams.update({ - 'sd': '{:.6f}'.format(duration - (i * self.segment_duration)) - }) + segmentParams.update({'sd': '{:.6f}'.format(duration - (i * self.segment_duration))}) m3u8 += f"#EXTINF:{segmentParams['sd']}, nodesc\n" else: m3u8 += f"#EXTINF:{segmentSize}, nodesc\n" m3u8 += f"{self.url}segments/{i}/{quote(file)}" if len(segmentParams) > 0: - m3u8 += '?'+'&'.join([f"{key}={value}" for key, - value in segmentParams.items()]) + m3u8 += '?'+'&'.join([f"{key}={value}" for key, value in segmentParams.items()]) m3u8 += "\n" m3u8 += "#EXT-X-ENDLIST\n" @@ -79,8 +71,7 @@ class M3u8: if duration.find(':') > -1: duration = duration.split(':') duration.reverse() - duration = sum([float(duration[i]) * (60 ** i) - for i in range(len(duration))]) + duration = sum([float(duration[i]) * (60 ** i) for i in range(len(duration))]) else: duration = float(duration) diff --git a/app/player/Segments.py b/app/player/Segments.py index 9fa4de46..01e65d57 100644 --- a/app/player/Segments.py +++ b/app/player/Segments.py @@ -4,9 +4,8 @@ import os import subprocess import tempfile from Utils import calcDownloadPath -from Config import Config -log = logging.getLogger('segments') +LOG = logging.getLogger('segments') class Segments: @@ -22,18 +21,13 @@ class Segments: self.aconvert = bool(aconvert) async def stream(self, download_path: str, file: str) -> bytes: - realFile: str = calcDownloadPath( - basePath=download_path, - folder=file, - createPath=False - ) + realFile: str = calcDownloadPath(basePath=download_path, folder=file, createPath=False) if not os.path.exists(realFile): raise Exception(f"File {realFile} does not exist.") tmpDir: str = tempfile.gettempdir() - tmpFile = os.path.join( - tmpDir, f'ytptube_stream.{hashlib.md5(realFile.encode("utf-8")).hexdigest()}') + tmpFile = os.path.join(tmpDir, f'ytptube_stream.{hashlib.md5(realFile.encode("utf-8")).hexdigest()}') if not os.path.exists(tmpFile): os.symlink(realFile, tmpFile) @@ -41,9 +35,7 @@ class Segments: if self.segment_index == 0: startTime: float = '{:.6f}'.format(0) else: - startTime: float = '{:.6f}'.format( - (self.segment_duration * self.segment_index - )) + startTime: float = '{:.6f}'.format((self.segment_duration * self.segment_index)) ffmpegCmd = [] ffmpegCmd.append('ffmpeg') @@ -107,8 +99,7 @@ class Segments: ffmpegCmd.append('mpegts') ffmpegCmd.append('pipe:1') - log.debug( - f'Streaming {realFile} segment {self.segment_index}.' + ' '.join(ffmpegCmd)) + LOG.debug(f'Streaming {realFile} segment {self.segment_index}.' + ' '.join(ffmpegCmd)) proc = subprocess.run(ffmpegCmd, stdout=subprocess.PIPE) return proc.stdout diff --git a/app/player/ffprobe.py b/app/player/ffprobe.py index d7ed76c4..ac52afb8 100644 --- a/app/player/ffprobe.py +++ b/app/player/ffprobe.py @@ -12,6 +12,7 @@ import subprocess class FFProbeError(Exception): pass + class FFStream: """ An object representation of an individual stream in a multimedia file. @@ -24,8 +25,7 @@ class FFStream: try: self.__dict__['framerate'] = round( functools.reduce( - operator.truediv, map(int, self.__dict__.get( - 'avg_frame_rate', '').split('/')) + operator.truediv, map(int, self.__dict__.get('avg_frame_rate', '').split('/')) ) ) except ValueError: @@ -41,8 +41,7 @@ class FFStream: return f"" if self.is_audio(): - return f" " + return f" " if self.is_subtitle() or self.is_attachment(): return f"" @@ -87,8 +86,7 @@ class FFStream: try: size = (int(width), int(height)) except ValueError: - raise FFProbeError( - "None integer size {}:{}".format(width, height)) + raise FFProbeError("None integer size {}:{}".format(width, height)) else: return None @@ -163,6 +161,7 @@ class FFStream: except ValueError: raise FFProbeError('None integer bit_rate') + class FFProbe: """ FFProbe wraps the ffprobe command and pulls the data into an object form:: @@ -181,8 +180,7 @@ class FFProbe: try: with open(os.devnull, 'w') as tempf: - subprocess.check_call( - ["ffprobe", "-h"], stdout=tempf, stderr=tempf) + subprocess.check_call(["ffprobe", "-h"], stdout=tempf, stderr=tempf) except FileNotFoundError: raise IOError('ffprobe not found.') @@ -232,15 +230,3 @@ class FFProbe: def __repr__(self): return "".format(**vars(self)) - - - -if __name__ == '__main__': - import sys - - if len(sys.argv) < 2: - print("Usage: ffprobe.py ") - sys.exit(1) - - ff = FFProbe(sys.argv[1]) - print(ff)