diff --git a/app/main.py b/app/main.py index 43dd3eb..4a23911 100644 --- a/app/main.py +++ b/app/main.py @@ -60,6 +60,8 @@ class Config: 'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false', 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0', + 'RETRY_FAILED_DOWNLOADS': 'false', + 'MAX_RETRY_ATTEMPTS': '3', 'YTDL_OPTIONS': '{}', 'YTDL_OPTIONS_FILE': '', 'ROBOTS_TXT': '', @@ -76,7 +78,7 @@ class Config: 'ENABLE_ACCESSLOG': 'false', } - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG') + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'RETRY_FAILED_DOWNLOADS', 'HTTPS', 'ENABLE_ACCESSLOG') def __init__(self): for k, v in self._DEFAULTS.items(): @@ -249,6 +251,8 @@ async def add(request): auto_start = post.get('auto_start') split_by_chapters = post.get('split_by_chapters') chapter_template = post.get('chapter_template') + retry_failed = post.get('retry_failed') + max_retry_attempts = post.get('max_retry_attempts') if custom_name_prefix is None: custom_name_prefix = '' @@ -262,10 +266,15 @@ async def add(request): split_by_chapters = False if chapter_template is None: chapter_template = config.OUTPUT_TEMPLATE_CHAPTER + if retry_failed is None: + retry_failed = config.RETRY_FAILED_DOWNLOADS + if max_retry_attempts is None: + max_retry_attempts = config.MAX_RETRY_ATTEMPTS playlist_item_limit = int(playlist_item_limit) + max_retry_attempts = int(max_retry_attempts) - status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template) + status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts) return web.Response(text=serializer.encode(status)) @routes.post(config.URL_PREFIX + 'delete') diff --git a/app/ytdl.py b/app/ytdl.py index e88bcd5..6fda885 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -43,7 +43,7 @@ class DownloadQueueNotifier: raise NotImplementedError class DownloadInfo: - def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template): + def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template, retry_failed=False, max_retry_attempts=3): self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' self.url = url @@ -61,6 +61,9 @@ class DownloadInfo: self.playlist_item_limit = playlist_item_limit self.split_by_chapters = split_by_chapters self.chapter_template = chapter_template + self.retry_failed = retry_failed + self.max_retry_attempts = max_retry_attempts + self.retry_count = 0 class Download: manager = None @@ -353,8 +356,40 @@ class DownloadQueue: if download.canceled: asyncio.create_task(self.notifier.canceled(download.info.url)) else: - self.done.put(download) - asyncio.create_task(self.notifier.completed(download.info)) + # Check if we should retry failed downloads + if (download.info.status == 'error' and + download.info.retry_failed and + download.info.retry_count < download.info.max_retry_attempts): + # Increment retry count and retry the download + download.info.retry_count += 1 + log.info(f"Retrying download {download.info.title} (attempt {download.info.retry_count}/{download.info.max_retry_attempts})") + download.info.status = 'pending' + download.info.msg = f'Retrying (attempt {download.info.retry_count}/{download.info.max_retry_attempts})' + download.info.percent = None + download.info.speed = None + download.info.eta = None + # Create a new download with the same info + dldirectory, _ = self.__calc_download_path(download.info.quality, download.info.format, download.info.folder) + output = self.config.OUTPUT_TEMPLATE if len(download.info.custom_name_prefix) == 0 else f'{download.info.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' + output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER if not download.info.split_by_chapters else download.info.chapter_template + entry = getattr(download.info, 'entry', None) + if entry is not None and 'playlist' in entry and entry['playlist'] is not None: + if len(self.config.OUTPUT_TEMPLATE_PLAYLIST): + output = self.config.OUTPUT_TEMPLATE_PLAYLIST + for property, value in entry.items(): + if property.startswith("playlist"): + output = output.replace(f"%({property})s", str(value)) + ytdl_options = dict(self.config.YTDL_OPTIONS) + playlist_item_limit = getattr(download.info, 'playlist_item_limit', 0) + if playlist_item_limit > 0: + ytdl_options['playlistend'] = playlist_item_limit + new_download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, download.info.quality, download.info.format, ytdl_options, download.info) + self.queue.put(new_download) + asyncio.create_task(self.__start_download(new_download)) + else: + # No more retries, mark as completed (failed) + self.done.put(download) + asyncio.create_task(self.notifier.completed(download.info)) def __extract_info(self, url, playlist_strict_mode): debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG) @@ -413,7 +448,7 @@ class DownloadQueue: self.pending.put(download) await self.notifier.added(dl) - async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already): + async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already): if not entry: return {'status': 'error', 'msg': "Invalid/empty data was given."} @@ -429,7 +464,7 @@ class DownloadQueue: if etype.startswith('url'): log.debug('Processing as an url') - return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already) + return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already) elif etype == 'playlist': log.debug('Processing as a playlist') entries = entry['entries'] @@ -449,7 +484,7 @@ class DownloadQueue: for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry[property] - results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)) + results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already)) if any(res['status'] == 'error' for res in results): return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} return {'status': 'ok'} @@ -457,13 +492,13 @@ class DownloadQueue: log.debug('Processing as a video') key = entry.get('webpage_url') or entry['url'] if not self.queue.exists(key): - dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template) + dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template, retry_failed, max_retry_attempts) await self.__add_download(dl, auto_start) return {'status': 'ok'} return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} - async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None): - log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}') + async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, retry_failed=False, max_retry_attempts=3, already=None): + log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} {retry_failed=} {max_retry_attempts=}') already = set() if already is None else already if url in already: log.info('recursion detected, skipping') @@ -474,7 +509,7 @@ class DownloadQueue: entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} - return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already) + return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already) async def start_pending(self, ids): for id in ids: diff --git a/ui/src/app/app.html b/ui/src/app/app.html index 8456d9f..0395c82 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -231,6 +231,36 @@ +
+
+ + +
+
+
+
+ Max Retry Attempts + +
+
diff --git a/ui/src/app/app.ts b/ui/src/app/app.ts index 7f7a959..38a39aa 100644 --- a/ui/src/app/app.ts +++ b/ui/src/app/app.ts @@ -50,6 +50,8 @@ export class App implements AfterViewInit, OnInit { playlistItemLimit!: number; splitByChapters: boolean; chapterTemplate: string; + enableRetryFailed: boolean; + maxRetryAttempts: number; addInProgress = false; themes: Theme[] = Themes; activeTheme: Theme | undefined; @@ -108,6 +110,8 @@ export class App implements AfterViewInit, OnInit { this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true'; // Will be set from backend configuration, use empty string as placeholder this.chapterTemplate = this.cookieService.get('metube_chapter_template') || ''; + this.enableRetryFailed = this.cookieService.get('metube_retry_failed') === 'true'; + this.maxRetryAttempts = parseInt(this.cookieService.get('metube_max_retry_attempts') || '3', 10); this.activeTheme = this.getPreferredTheme(this.cookieService); @@ -281,6 +285,20 @@ export class App implements AfterViewInit, OnInit { this.cookieService.set('metube_chapter_template', this.chapterTemplate, { expires: 3650 }); } + retryFailedChanged() { + this.cookieService.set('metube_retry_failed', this.enableRetryFailed ? 'true' : 'false', { expires: 3650 }); + } + + maxRetryAttemptsChanged() { + // Ensure value is between 1 and 10 + if (this.maxRetryAttempts < 1) { + this.maxRetryAttempts = 1; + } else if (this.maxRetryAttempts > 10) { + this.maxRetryAttempts = 10; + } + this.cookieService.set('metube_max_retry_attempts', this.maxRetryAttempts.toString(), { expires: 3650 }); + } + queueSelectionChanged(checked: number) { this.queueDelSelected().nativeElement.disabled = checked == 0; this.queueDownloadSelected().nativeElement.disabled = checked == 0; @@ -301,7 +319,7 @@ export class App implements AfterViewInit, OnInit { } } - addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) { + addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string, retryFailed?: boolean, maxRetryAttempts?: number) { url = url ?? this.addUrl quality = quality ?? this.quality format = format ?? this.format @@ -312,6 +330,8 @@ export class App implements AfterViewInit, OnInit { autoStart = autoStart ?? this.autoStart splitByChapters = splitByChapters ?? this.splitByChapters chapterTemplate = chapterTemplate ?? this.chapterTemplate + retryFailed = retryFailed ?? this.enableRetryFailed + maxRetryAttempts = maxRetryAttempts ?? this.maxRetryAttempts // Validate chapter template if chapter splitting is enabled if (splitByChapters && !chapterTemplate.includes('%(section_number)')) { @@ -319,9 +339,9 @@ export class App implements AfterViewInit, OnInit { return; } - console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate); + console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate + ' retryFailed=' + retryFailed + ' maxRetryAttempts=' + maxRetryAttempts); this.addInProgress = true; - this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => { + this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate, retryFailed, maxRetryAttempts).subscribe((status: Status) => { if (status.status === 'error') { alert(`Error adding URL: ${status.msg}`); } else { @@ -482,7 +502,7 @@ export class App implements AfterViewInit, OnInit { this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`; // Now pass the selected quality, format, folder, etc. to the add() method this.downloads.add(url, this.quality, this.format, this.folder, this.customNamePrefix, - this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate) + this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate, this.enableRetryFailed, this.maxRetryAttempts) .subscribe({ next: (status: Status) => { if (status.status === 'error') { diff --git a/ui/src/app/services/downloads.service.ts b/ui/src/app/services/downloads.service.ts index 5a360f8..48bae00 100644 --- a/ui/src/app/services/downloads.service.ts +++ b/ui/src/app/services/downloads.service.ts @@ -107,8 +107,8 @@ export class DownloadsService { return of({status: 'error', msg: msg}) } - public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) { - return this.http.post('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe( + public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string, retryFailed: boolean, maxRetryAttempts: number) { + return this.http.post('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate, retry_failed: retryFailed, max_retry_attempts: maxRetryAttempts }).pipe( catchError(this.handleHTTPError) ); } @@ -152,9 +152,11 @@ export class DownloadsService { const defaultAutoStart = true; const defaultSplitByChapters = false; const defaultChapterTemplate = this.configuration['OUTPUT_TEMPLATE_CHAPTER']; + const defaultRetryFailed = false; + const defaultMaxRetryAttempts = 3; return new Promise((resolve, reject) => { - this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate) + this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate, defaultRetryFailed, defaultMaxRetryAttempts) .subscribe({ next: (response) => resolve(response), error: (error) => reject(error)