diff --git a/app/main.py b/app/main.py index 43dd3eb..e72f125 100644 --- a/app/main.py +++ b/app/main.py @@ -21,26 +21,6 @@ from yt_dlp.version import __version__ as yt_dlp_version log = logging.getLogger('main') -def parseLogLevel(logLevel): - match logLevel: - case 'DEBUG': - return logging.DEBUG - case 'INFO': - return logging.INFO - case 'WARNING': - return logging.WARNING - case 'ERROR': - return logging.ERROR - case 'CRITICAL': - return logging.CRITICAL - case _: - return None - -# Configure logging before Config() uses it so early messages are not dropped. -# Only configure if no handlers are set (avoid clobbering hosting app settings). -if not logging.getLogger().hasHandlers(): - logging.basicConfig(level=parseLogLevel(os.environ.get('LOGLEVEL', 'INFO')) or logging.INFO) - class Config: _DEFAULTS = { 'DOWNLOAD_DIR': '.', @@ -56,10 +36,12 @@ class Config: 'PUBLIC_HOST_URL': 'download/', 'PUBLIC_HOST_AUDIO_URL': 'audio_download/', 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', - 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s', + 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s', 'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false', 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0', + 'RETRY_FAILED_DOWNLOADS': 'false', + 'MAX_RETRY_ATTEMPTS': '3', 'YTDL_OPTIONS': '{}', 'YTDL_OPTIONS_FILE': '', 'ROBOTS_TXT': '', @@ -76,7 +58,7 @@ class Config: 'ENABLE_ACCESSLOG': 'false', } - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG') + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'RETRY_FAILED_DOWNLOADS', 'HTTPS', 'ENABLE_ACCESSLOG') def __init__(self): for k, v in self._DEFAULTS.items(): @@ -132,10 +114,6 @@ class Config: return (True, '') config = Config() -# Align root logger level with Config (keeps a single source of truth). -# This re-applies the log level after Config loads, in case LOGLEVEL was -# overridden by config file settings or differs from the environment variable. -logging.getLogger().setLevel(parseLogLevel(str(config.LOGLEVEL)) or logging.INFO) class ObjectSerializer(json.JSONEncoder): def default(self, obj): @@ -247,8 +225,6 @@ async def add(request): playlist_strict_mode = post.get('playlist_strict_mode') playlist_item_limit = post.get('playlist_item_limit') auto_start = post.get('auto_start') - split_by_chapters = post.get('split_by_chapters') - chapter_template = post.get('chapter_template') if custom_name_prefix is None: custom_name_prefix = '' @@ -258,14 +234,10 @@ async def add(request): playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE if playlist_item_limit is None: playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT - if split_by_chapters is None: - split_by_chapters = False - if chapter_template is None: - chapter_template = config.OUTPUT_TEMPLATE_CHAPTER playlist_item_limit = int(playlist_item_limit) - status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template) + status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start) return web.Response(text=serializer.encode(status)) @routes.post(config.URL_PREFIX + 'delete') @@ -416,6 +388,21 @@ def supports_reuse_port(): except (AttributeError, OSError): return False +def parseLogLevel(logLevel): + match logLevel: + case 'DEBUG': + return logging.DEBUG + case 'INFO': + return logging.INFO + case 'WARNING': + return logging.WARNING + case 'ERROR': + return logging.ERROR + case 'CRITICAL': + return logging.CRITICAL + case _: + return None + def isAccessLogEnabled(): if config.ENABLE_ACCESSLOG: return access_logger @@ -423,7 +410,7 @@ def isAccessLogEnabled(): return None if __name__ == '__main__': - logging.getLogger().setLevel(parseLogLevel(config.LOGLEVEL) or logging.INFO) + logging.basicConfig(level=parseLogLevel(config.LOGLEVEL)) log.info(f"Listening on {config.HOST}:{config.PORT}") if config.HTTPS: diff --git a/app/ytdl.py b/app/ytdl.py index e88bcd5..5eecb01 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -43,7 +43,7 @@ class DownloadQueueNotifier: raise NotImplementedError class DownloadInfo: - def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template): + def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit): self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' self.url = url @@ -59,8 +59,7 @@ class DownloadInfo: # Convert generators to lists to make entry pickleable self.entry = _convert_generators_to_lists(entry) if entry is not None else None self.playlist_item_limit = playlist_item_limit - self.split_by_chapters = split_by_chapters - self.chapter_template = chapter_template + self.retry_count = 0 class Download: manager = None @@ -85,7 +84,6 @@ class Download: def _download(self): log.info(f"Starting download for: {self.info.title} ({self.info.url})") try: - debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG) def put_status(st): self.status_queue.put({k: v for k, v in st.items() if k in ( 'tmpfilename', @@ -106,21 +104,9 @@ class Download: else: filename = d['info_dict']['filepath'] self.status_queue.put({'status': 'finished', 'filename': filename}) - - # Capture all chapter files when SplitChapters finishes - elif d.get('postprocessor') == 'SplitChapters' and d.get('status') == 'finished': - chapters = d.get('info_dict', {}).get('chapters', []) - if chapters: - for chapter in chapters: - if isinstance(chapter, dict) and 'filepath' in chapter: - log.info(f"Captured chapter file: {chapter['filepath']}") - self.status_queue.put({'chapter_file': chapter['filepath']}) - else: - log.warning("SplitChapters finished but no chapter files found in info_dict") - ytdl_params = { - 'quiet': not debug_logging, - 'verbose': debug_logging, + ret = yt_dlp.YoutubeDL(params={ + 'quiet': True, 'no_color': True, 'paths': {"home": self.download_dir, "temp": self.temp_dir}, 'outtmpl': { "default": self.output_template, "chapter": self.output_template_chapter }, @@ -130,19 +116,7 @@ class Download: 'progress_hooks': [put_status], 'postprocessor_hooks': [put_status_postprocessor], **self.ytdl_opts, - } - - # Add chapter splitting options if enabled - if self.info.split_by_chapters: - ytdl_params['outtmpl']['chapter'] = self.info.chapter_template - if 'postprocessors' not in ytdl_params: - ytdl_params['postprocessors'] = [] - ytdl_params['postprocessors'].append({ - 'key': 'FFmpegSplitChapters', - 'force_keyframes': False - }) - - ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url]) + }).download([self.info.url]) self.status_queue.put({'status': 'finished' if ret == 0 else 'error'}) log.info(f"Finished download for: {self.info.title}") except yt_dlp.utils.YoutubeDLError as exc: @@ -206,22 +180,6 @@ class Download: self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None if self.info.format == 'thumbnail': self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename) - - # Handle chapter files - log.debug(f"Update status for {self.info.title}: {status}") - if 'chapter_file' in status: - chapter_file = status.get('chapter_file') - if not hasattr(self.info, 'chapter_files'): - self.info.chapter_files = [] - rel_path = os.path.relpath(chapter_file, self.download_dir) - file_size = os.path.getsize(chapter_file) if os.path.exists(chapter_file) else None - #Postprocessor hook called multiple times with chapters. Only insert if not already present. - existing = next((cf for cf in self.info.chapter_files if cf['filename'] == rel_path), None) - if not existing: - self.info.chapter_files.append({'filename': rel_path, 'size': file_size}) - # Skip the rest of status processing for chapter files - continue - self.info.status = status['status'] self.info.msg = status.get('msg') if 'downloaded_bytes' in status: @@ -353,14 +311,44 @@ class DownloadQueue: if download.canceled: asyncio.create_task(self.notifier.canceled(download.info.url)) else: - self.done.put(download) - asyncio.create_task(self.notifier.completed(download.info)) + # Check if we should retry failed downloads + if (download.info.status == 'error' and + self.config.RETRY_FAILED_DOWNLOADS and + download.info.retry_count < int(self.config.MAX_RETRY_ATTEMPTS)): + # Increment retry count and retry the download + download.info.retry_count += 1 + log.info(f"Retrying download {download.info.title} (attempt {download.info.retry_count}/{self.config.MAX_RETRY_ATTEMPTS})") + download.info.status = 'pending' + download.info.msg = f'Retrying (attempt {download.info.retry_count}/{self.config.MAX_RETRY_ATTEMPTS})' + download.info.percent = None + download.info.speed = None + download.info.eta = None + # Create a new download with the same info + dldirectory, _ = self.__calc_download_path(download.info.quality, download.info.format, download.info.folder) + output = self.config.OUTPUT_TEMPLATE if len(download.info.custom_name_prefix) == 0 else f'{download.info.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' + output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER + entry = getattr(download.info, 'entry', None) + if entry is not None and 'playlist' in entry and entry['playlist'] is not None: + if len(self.config.OUTPUT_TEMPLATE_PLAYLIST): + output = self.config.OUTPUT_TEMPLATE_PLAYLIST + for property, value in entry.items(): + if property.startswith("playlist"): + output = output.replace(f"%({property})s", str(value)) + ytdl_options = dict(self.config.YTDL_OPTIONS) + playlist_item_limit = getattr(download.info, 'playlist_item_limit', 0) + if playlist_item_limit > 0: + ytdl_options['playlistend'] = playlist_item_limit + new_download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, download.info.quality, download.info.format, ytdl_options, download.info) + self.queue.put(new_download) + asyncio.create_task(self.__start_download(new_download)) + else: + # No more retries, mark as completed (failed) + self.done.put(download) + asyncio.create_task(self.notifier.completed(download.info)) def __extract_info(self, url, playlist_strict_mode): - debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG) return yt_dlp.YoutubeDL(params={ - 'quiet': not debug_logging, - 'verbose': debug_logging, + 'quiet': True, 'no_color': True, 'extract_flat': True, 'ignore_no_formats_error': True, @@ -413,7 +401,7 @@ class DownloadQueue: self.pending.put(download) await self.notifier.added(dl) - async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already): + async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already): if not entry: return {'status': 'error', 'msg': "Invalid/empty data was given."} @@ -429,7 +417,7 @@ class DownloadQueue: if etype.startswith('url'): log.debug('Processing as an url') - return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already) + return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) elif etype == 'playlist': log.debug('Processing as a playlist') entries = entry['entries'] @@ -449,7 +437,7 @@ class DownloadQueue: for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry[property] - results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)) + results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)) if any(res['status'] == 'error' for res in results): return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} return {'status': 'ok'} @@ -457,13 +445,13 @@ class DownloadQueue: log.debug('Processing as a video') key = entry.get('webpage_url') or entry['url'] if not self.queue.exists(key): - dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template) + dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit) await self.__add_download(dl, auto_start) return {'status': 'ok'} return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} - async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None): - log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}') + async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, already=None): + log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=}') already = set() if already is None else already if url in already: log.info('recursion detected, skipping') @@ -474,7 +462,7 @@ class DownloadQueue: entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} - return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already) + return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already) async def start_pending(self, ids): for id in ids: diff --git a/ui/src/app/app.html b/ui/src/app/app.html index 8456d9f..dd4353d 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -232,26 +232,33 @@
-
-
-
- - -
-
- @if (splitByChapters) { -
-
- Template - -
-
- } +
+ + +
+
+
+
+ Max Retry Attempts +
@@ -448,32 +455,7 @@ - @if (download.value.chapter_files && download.value.chapter_files.length > 0) { - @for (chapterFile of download.value.chapter_files; track chapterFile.filename) { - - - -
- - {{ - getChapterFileName(chapterFile.filename) }} -
- - - @if (chapterFile.size) { - {{ chapterFile.size | fileSize }} - } - - -
- -
- - } - } - } diff --git a/ui/src/app/app.ts b/ui/src/app/app.ts index 7f7a959..d4903d6 100644 --- a/ui/src/app/app.ts +++ b/ui/src/app/app.ts @@ -48,8 +48,8 @@ export class App implements AfterViewInit, OnInit { autoStart: boolean; playlistStrictMode!: boolean; playlistItemLimit!: number; - splitByChapters: boolean; - chapterTemplate: string; + enableRetryFailed: boolean; + maxRetryAttempts: number; addInProgress = false; themes: Theme[] = Themes; activeTheme: Theme | undefined; @@ -105,9 +105,8 @@ export class App implements AfterViewInit, OnInit { this.setQualities() this.quality = this.cookieService.get('metube_quality') || 'best'; this.autoStart = this.cookieService.get('metube_auto_start') !== 'false'; - this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true'; - // Will be set from backend configuration, use empty string as placeholder - this.chapterTemplate = this.cookieService.get('metube_chapter_template') || ''; + this.enableRetryFailed = this.cookieService.get('metube_retry_failed') === 'true'; + this.maxRetryAttempts = parseInt(this.cookieService.get('metube_max_retry_attempts') || '3', 10); this.activeTheme = this.getPreferredTheme(this.cookieService); @@ -226,10 +225,6 @@ export class App implements AfterViewInit, OnInit { if (playlistItemLimit !== '0') { this.playlistItemLimit = playlistItemLimit; } - // Set chapter template from backend config if not already set by cookie - if (!this.chapterTemplate) { - this.chapterTemplate = config['OUTPUT_TEMPLATE_CHAPTER']; - } } }); } @@ -269,16 +264,12 @@ export class App implements AfterViewInit, OnInit { this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: 3650 }); } - splitByChaptersChanged() { - this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: 3650 }); + retryFailedChanged() { + this.cookieService.set('metube_retry_failed', this.enableRetryFailed ? 'true' : 'false', { expires: 3650 }); } - chapterTemplateChanged() { - // Restore default if template is cleared - get from configuration - if (!this.chapterTemplate || this.chapterTemplate.trim() === '') { - this.chapterTemplate = this.downloads.configuration['OUTPUT_TEMPLATE_CHAPTER']; - } - this.cookieService.set('metube_chapter_template', this.chapterTemplate, { expires: 3650 }); + maxRetryAttemptsChanged() { + this.cookieService.set('metube_max_retry_attempts', this.maxRetryAttempts.toString(), { expires: 3650 }); } queueSelectionChanged(checked: number) { @@ -301,7 +292,7 @@ export class App implements AfterViewInit, OnInit { } } - addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) { + addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean) { url = url ?? this.addUrl quality = quality ?? this.quality format = format ?? this.format @@ -310,18 +301,10 @@ export class App implements AfterViewInit, OnInit { playlistStrictMode = playlistStrictMode ?? this.playlistStrictMode playlistItemLimit = playlistItemLimit ?? this.playlistItemLimit autoStart = autoStart ?? this.autoStart - splitByChapters = splitByChapters ?? this.splitByChapters - chapterTemplate = chapterTemplate ?? this.chapterTemplate - // Validate chapter template if chapter splitting is enabled - if (splitByChapters && !chapterTemplate.includes('%(section_number)')) { - alert('Chapter template must include %(section_number)'); - return; - } - - console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate); + console.debug('Downloading: url='+url+' quality='+quality+' format='+format+' folder='+folder+' customNamePrefix='+customNamePrefix+' playlistStrictMode='+playlistStrictMode+' playlistItemLimit='+playlistItemLimit+' autoStart='+autoStart); this.addInProgress = true; - this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => { + this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart).subscribe((status: Status) => { if (status.status === 'error') { alert(`Error adding URL: ${status.msg}`); } else { @@ -336,7 +319,7 @@ export class App implements AfterViewInit, OnInit { } retryDownload(key: string, download: Download) { - this.addDownload(download.url, download.quality, download.format, download.folder, download.custom_name_prefix, download.playlist_strict_mode, download.playlist_item_limit, true, download.split_by_chapters, download.chapter_template); + this.addDownload(download.url, download.quality, download.format, download.folder, download.custom_name_prefix, download.playlist_strict_mode, download.playlist_item_limit, true); this.downloads.delById('done', [key]).subscribe(); } @@ -407,25 +390,6 @@ export class App implements AfterViewInit, OnInit { return parts.join(' | '); } - buildChapterDownloadLink(download: Download, chapterFilename: string) { - let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"]; - if (download.quality == 'audio' || chapterFilename.endsWith('.mp3')) { - baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"]; - } - - if (download.folder) { - baseDir += download.folder + '/'; - } - - return baseDir + encodeURIComponent(chapterFilename); - } - - getChapterFileName(filepath: string) { - // Extract just the filename from the path - const parts = filepath.split('/'); - return parts[parts.length - 1]; - } - isNumber(event: KeyboardEvent) { const charCode = +event.code || event.keyCode; if (charCode > 31 && (charCode < 48 || charCode > 57)) { @@ -482,7 +446,7 @@ export class App implements AfterViewInit, OnInit { this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`; // Now pass the selected quality, format, folder, etc. to the add() method this.downloads.add(url, this.quality, this.format, this.folder, this.customNamePrefix, - this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate) + this.playlistStrictMode, this.playlistItemLimit, this.autoStart) .subscribe({ next: (status: Status) => { if (status.status === 'error') {