diff --git a/README.md b/README.md index b8301d1..a497ca3 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,13 @@ Certain values can be set via environment variables, using the `-e` parameter on * __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` . * __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit). +* __RETRY_FAILED_DOWNLOADS__: When set to `true`, MeTube will automatically retry failed downloads up to a configured number of attempts. This option is opt-in and defaults to `false`. + * When enabled, retries are performed per-download and shown in the UI as "Retrying (attempt X/Y)". + * The UI also exposes a toggle in Advanced Options to enable/disable retries and will persist the preference in a browser cookie. + +* __MAX_RETRY_ATTEMPTS__: The maximum number of automatic retry attempts for a failed download when `RETRY_FAILED_DOWNLOADS` is enabled. Must be an integer between `1` and `10`. Defaults to `3`. + * This value can be configured globally via environment variable or set per-download via the UI. The frontend enforces the 1–10 range; the backend validates the value as well. + ### 📁 Storage & Directories * __DOWNLOAD_DIR__: Path to where the downloads will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise. diff --git a/app/main.py b/app/main.py index 43dd3eb..357cd61 100644 --- a/app/main.py +++ b/app/main.py @@ -60,6 +60,8 @@ class Config: 'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false', 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0', + 'RETRY_FAILED_DOWNLOADS': 'false', + 'MAX_RETRY_ATTEMPTS': '3', 'YTDL_OPTIONS': '{}', 'YTDL_OPTIONS_FILE': '', 'ROBOTS_TXT': '', @@ -76,7 +78,7 @@ class Config: 'ENABLE_ACCESSLOG': 'false', } - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG') + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'RETRY_FAILED_DOWNLOADS', 'HTTPS', 'ENABLE_ACCESSLOG') def __init__(self): for k, v in self._DEFAULTS.items(): @@ -249,6 +251,8 @@ async def add(request): auto_start = post.get('auto_start') split_by_chapters = post.get('split_by_chapters') chapter_template = post.get('chapter_template') + retry_failed = post.get('retry_failed') + max_retry_attempts = post.get('max_retry_attempts') if custom_name_prefix is None: custom_name_prefix = '' @@ -262,10 +266,22 @@ async def add(request): split_by_chapters = False if chapter_template is None: chapter_template = config.OUTPUT_TEMPLATE_CHAPTER + if retry_failed is None: + retry_failed = config.RETRY_FAILED_DOWNLOADS + if max_retry_attempts is None: + max_retry_attempts = config.MAX_RETRY_ATTEMPTS playlist_item_limit = int(playlist_item_limit) + try: + max_retry_attempts = int(max_retry_attempts) + except (TypeError, ValueError): + log.error("Bad request: invalid 'max_retry_attempts' value (must be an integer)") + raise web.HTTPBadRequest() - status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template) + if not 1 <= max_retry_attempts <= 10: + log.error("Bad request: 'max_retry_attempts' out of allowed range (1-10)") + raise web.HTTPBadRequest() + status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts) return web.Response(text=serializer.encode(status)) @routes.post(config.URL_PREFIX + 'delete') diff --git a/app/ytdl.py b/app/ytdl.py index b84cc8c..4523e27 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -46,7 +46,7 @@ class DownloadQueueNotifier: raise NotImplementedError class DownloadInfo: - def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template): + def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template, retry_failed=False, max_retry_attempts=3): self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' self.url = url @@ -64,6 +64,9 @@ class DownloadInfo: self.playlist_item_limit = playlist_item_limit self.split_by_chapters = split_by_chapters self.chapter_template = chapter_template + self.retry_failed = retry_failed + self.max_retry_attempts = max_retry_attempts + self.retry_count = 0 class Download: manager = None @@ -394,7 +397,8 @@ class DownloadQueue: async with self.seq_lock: log.info("Starting sequential download.") await download.start(self.notifier) - self._post_download_cleanup(download) + # lock released here + self._post_download_cleanup(download) elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None: await self.__limited_concurrent_download(download) else: @@ -430,8 +434,31 @@ class DownloadQueue: if download.canceled: asyncio.create_task(self.notifier.canceled(download.info.url)) else: - self.done.put(download) - asyncio.create_task(self.notifier.completed(download.info)) + # Check if we should retry failed downloads + if (download.info.status == 'error' and + download.info.retry_failed and + download.info.retry_count < download.info.max_retry_attempts): + # Increment retry count and retry the download + download.info.retry_count += 1 + log.info(f"Retrying download {download.info.title} (attempt {download.info.retry_count}/{download.info.max_retry_attempts})") + download.info.status = 'pending' + download.info.msg = f'Retrying (attempt {download.info.retry_count}/{download.info.max_retry_attempts})' + download.info.percent = None + download.info.speed = None + download.info.eta = None + # Create a new download with the same info via helper + new_download, err = self._create_download_object(download.info) + if err is not None: + log.error(f"Retry failed: cannot create download object: {err}") + self.done.put(download) + asyncio.create_task(self.notifier.completed(download.info)) + else: + self.queue.put(new_download) + asyncio.create_task(self.__start_download(new_download)) + else: + # No more retries, mark as completed (failed) + self.done.put(download) + asyncio.create_task(self.notifier.completed(download.info)) def __extract_info(self, url, playlist_strict_mode): debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG) @@ -464,13 +491,17 @@ class DownloadQueue: dldirectory = base_directory return dldirectory, None - async def __add_download(self, dl, auto_start): - dldirectory, error_message = self.__calc_download_path(dl.quality, dl.format, dl.folder) + def _create_download_object(self, info): + """Create a Download object from a DownloadInfo-like object. + + Returns (download, error_message). On success error_message is None. + """ + dldirectory, error_message = self.__calc_download_path(info.quality, info.format, info.folder) if error_message is not None: - return error_message - output = self.config.OUTPUT_TEMPLATE if len(dl.custom_name_prefix) == 0 else f'{dl.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' - output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER - entry = getattr(dl, 'entry', None) + return None, error_message + output = self.config.OUTPUT_TEMPLATE if len(info.custom_name_prefix) == 0 else f'{info.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}' + output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER if not info.split_by_chapters else info.chapter_template + entry = getattr(info, 'entry', None) if entry is not None and 'playlist' in entry and entry['playlist'] is not None: if len(self.config.OUTPUT_TEMPLATE_PLAYLIST): output = self.config.OUTPUT_TEMPLATE_PLAYLIST @@ -478,11 +509,16 @@ class DownloadQueue: if property.startswith("playlist"): output = output.replace(f"%({property})s", str(value)) ytdl_options = dict(self.config.YTDL_OPTIONS) - playlist_item_limit = getattr(dl, 'playlist_item_limit', 0) + playlist_item_limit = getattr(info, 'playlist_item_limit', 0) if playlist_item_limit > 0: - log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries') ytdl_options['playlistend'] = playlist_item_limit - download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl) + download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, info.quality, info.format, ytdl_options, info) + return download, None + + async def __add_download(self, dl, auto_start): + download, error_message = self._create_download_object(dl) + if error_message is not None: + return error_message if auto_start is True: self.queue.put(download) asyncio.create_task(self.__start_download(download)) @@ -490,7 +526,7 @@ class DownloadQueue: self.pending.put(download) await self.notifier.added(dl) - async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already): + async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already): if not entry: return {'status': 'error', 'msg': "Invalid/empty data was given."} @@ -506,7 +542,7 @@ class DownloadQueue: if etype.startswith('url'): log.debug('Processing as an url') - return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already) + return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already) elif etype == 'playlist': log.debug('Processing as a playlist') entries = entry['entries'] @@ -526,7 +562,7 @@ class DownloadQueue: for property in ("id", "title", "uploader", "uploader_id"): if property in entry: etr[f"playlist_{property}"] = entry[property] - results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)) + results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already)) if any(res['status'] == 'error' for res in results): return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)} return {'status': 'ok'} @@ -534,13 +570,13 @@ class DownloadQueue: log.debug('Processing as a video') key = entry.get('webpage_url') or entry['url'] if not self.queue.exists(key): - dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template) + dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template, retry_failed, max_retry_attempts) await self.__add_download(dl, auto_start) return {'status': 'ok'} return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} - async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None): - log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}') + async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, retry_failed=False, max_retry_attempts=3, already=None): + log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} {retry_failed=} {max_retry_attempts=}') already = set() if already is None else already if url in already: log.info('recursion detected, skipping') @@ -551,7 +587,7 @@ class DownloadQueue: entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} - return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already) + return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already) async def start_pending(self, ids): for id in ids: diff --git a/ui/src/app/app.html b/ui/src/app/app.html index 8456d9f..0395c82 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -231,6 +231,36 @@ +
+
+ + +
+
+
+
+ Max Retry Attempts + +
+
diff --git a/ui/src/app/app.ts b/ui/src/app/app.ts index 7f7a959..be186d0 100644 --- a/ui/src/app/app.ts +++ b/ui/src/app/app.ts @@ -50,6 +50,8 @@ export class App implements AfterViewInit, OnInit { playlistItemLimit!: number; splitByChapters: boolean; chapterTemplate: string; + enableRetryFailed: boolean; + maxRetryAttempts: number; addInProgress = false; themes: Theme[] = Themes; activeTheme: Theme | undefined; @@ -108,6 +110,14 @@ export class App implements AfterViewInit, OnInit { this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true'; // Will be set from backend configuration, use empty string as placeholder this.chapterTemplate = this.cookieService.get('metube_chapter_template') || ''; + this.enableRetryFailed = this.cookieService.get('metube_retry_failed') === 'true'; + const maxRetryCookie = this.cookieService.get('metube_max_retry_attempts'); + const parsedMaxRetry = parseInt(maxRetryCookie, 10); + if (isNaN(parsedMaxRetry) || parsedMaxRetry < 1 || parsedMaxRetry > 10) { + this.maxRetryAttempts = 3; + } else { + this.maxRetryAttempts = parsedMaxRetry; + } this.activeTheme = this.getPreferredTheme(this.cookieService); @@ -281,6 +291,30 @@ export class App implements AfterViewInit, OnInit { this.cookieService.set('metube_chapter_template', this.chapterTemplate, { expires: 3650 }); } + retryFailedChanged() { + this.cookieService.set('metube_retry_failed', this.enableRetryFailed ? 'true' : 'false', { expires: 3650 }); + } + + maxRetryAttemptsChanged() { + // Ensure value is a valid integer between 1 and 10 + let attempts = Number(this.maxRetryAttempts); + + if (!Number.isFinite(attempts)) { + attempts = 1; + } + + attempts = Math.round(attempts); + + if (attempts < 1) { + attempts = 1; + } else if (attempts > 10) { + attempts = 10; + } + + this.maxRetryAttempts = attempts; + this.cookieService.set('metube_max_retry_attempts', this.maxRetryAttempts.toString(), { expires: 3650 }); + } + queueSelectionChanged(checked: number) { this.queueDelSelected().nativeElement.disabled = checked == 0; this.queueDownloadSelected().nativeElement.disabled = checked == 0; @@ -301,7 +335,7 @@ export class App implements AfterViewInit, OnInit { } } - addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) { + addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string, retryFailed?: boolean, maxRetryAttempts?: number) { url = url ?? this.addUrl quality = quality ?? this.quality format = format ?? this.format @@ -312,6 +346,8 @@ export class App implements AfterViewInit, OnInit { autoStart = autoStart ?? this.autoStart splitByChapters = splitByChapters ?? this.splitByChapters chapterTemplate = chapterTemplate ?? this.chapterTemplate + retryFailed = retryFailed ?? this.enableRetryFailed + maxRetryAttempts = maxRetryAttempts ?? this.maxRetryAttempts // Validate chapter template if chapter splitting is enabled if (splitByChapters && !chapterTemplate.includes('%(section_number)')) { @@ -319,9 +355,9 @@ export class App implements AfterViewInit, OnInit { return; } - console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate); + console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate + ' retryFailed=' + retryFailed + ' maxRetryAttempts=' + maxRetryAttempts); this.addInProgress = true; - this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => { + this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate, retryFailed, maxRetryAttempts).subscribe((status: Status) => { if (status.status === 'error') { alert(`Error adding URL: ${status.msg}`); } else { @@ -482,7 +518,7 @@ export class App implements AfterViewInit, OnInit { this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`; // Now pass the selected quality, format, folder, etc. to the add() method this.downloads.add(url, this.quality, this.format, this.folder, this.customNamePrefix, - this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate) + this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate, this.enableRetryFailed, this.maxRetryAttempts) .subscribe({ next: (status: Status) => { if (status.status === 'error') { diff --git a/ui/src/app/services/downloads.service.ts b/ui/src/app/services/downloads.service.ts index 5a360f8..b5375fb 100644 --- a/ui/src/app/services/downloads.service.ts +++ b/ui/src/app/services/downloads.service.ts @@ -107,8 +107,8 @@ export class DownloadsService { return of({status: 'error', msg: msg}) } - public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) { - return this.http.post('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe( + public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string, retryFailed: boolean, maxRetryAttempts: number) { + return this.http.post('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate, retry_failed: retryFailed, max_retry_attempts: maxRetryAttempts }).pipe( catchError(this.handleHTTPError) ); } @@ -152,9 +152,13 @@ export class DownloadsService { const defaultAutoStart = true; const defaultSplitByChapters = false; const defaultChapterTemplate = this.configuration['OUTPUT_TEMPLATE_CHAPTER']; + const configuredRetryFailed = this.configuration['DEFAULT_RETRY_FAILED']; + const defaultRetryFailed = typeof configuredRetryFailed === 'boolean' ? configuredRetryFailed : false; + const configuredMaxRetryAttempts = this.configuration['DEFAULT_MAX_RETRY_ATTEMPTS']; + const defaultMaxRetryAttempts = typeof configuredMaxRetryAttempts === 'number' ? configuredMaxRetryAttempts : 3; return new Promise((resolve, reject) => { - this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate) + this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate, defaultRetryFailed, defaultMaxRetryAttempts) .subscribe({ next: (response) => resolve(response), error: (error) => reject(error)