feat: Add retry failed downloads feature with RETRY_FAILED_DOWNLOADS config (default: false), MAX_RETRY_ATTEMPTS config (default: 3) - #767
This commit is contained in:
Qwertyuiop@824030
parent
a002af9bf2
commit
e4c4ad4381
5 changed files with 115 additions and 19 deletions
13
app/main.py
13
app/main.py
|
|
@ -60,6 +60,8 @@ class Config:
|
|||
'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
|
||||
'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false',
|
||||
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
|
||||
'RETRY_FAILED_DOWNLOADS': 'false',
|
||||
'MAX_RETRY_ATTEMPTS': '3',
|
||||
'YTDL_OPTIONS': '{}',
|
||||
'YTDL_OPTIONS_FILE': '',
|
||||
'ROBOTS_TXT': '',
|
||||
|
|
@ -76,7 +78,7 @@ class Config:
|
|||
'ENABLE_ACCESSLOG': 'false',
|
||||
}
|
||||
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG')
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'RETRY_FAILED_DOWNLOADS', 'HTTPS', 'ENABLE_ACCESSLOG')
|
||||
|
||||
def __init__(self):
|
||||
for k, v in self._DEFAULTS.items():
|
||||
|
|
@ -249,6 +251,8 @@ async def add(request):
|
|||
auto_start = post.get('auto_start')
|
||||
split_by_chapters = post.get('split_by_chapters')
|
||||
chapter_template = post.get('chapter_template')
|
||||
retry_failed = post.get('retry_failed')
|
||||
max_retry_attempts = post.get('max_retry_attempts')
|
||||
|
||||
if custom_name_prefix is None:
|
||||
custom_name_prefix = ''
|
||||
|
|
@ -262,10 +266,15 @@ async def add(request):
|
|||
split_by_chapters = False
|
||||
if chapter_template is None:
|
||||
chapter_template = config.OUTPUT_TEMPLATE_CHAPTER
|
||||
if retry_failed is None:
|
||||
retry_failed = config.RETRY_FAILED_DOWNLOADS
|
||||
if max_retry_attempts is None:
|
||||
max_retry_attempts = config.MAX_RETRY_ATTEMPTS
|
||||
|
||||
playlist_item_limit = int(playlist_item_limit)
|
||||
max_retry_attempts = int(max_retry_attempts)
|
||||
|
||||
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template)
|
||||
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts)
|
||||
return web.Response(text=serializer.encode(status))
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'delete')
|
||||
|
|
|
|||
55
app/ytdl.py
55
app/ytdl.py
|
|
@ -43,7 +43,7 @@ class DownloadQueueNotifier:
|
|||
raise NotImplementedError
|
||||
|
||||
class DownloadInfo:
|
||||
def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template):
|
||||
def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template, retry_failed=False, max_retry_attempts=3):
|
||||
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
||||
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
||||
self.url = url
|
||||
|
|
@ -61,6 +61,9 @@ class DownloadInfo:
|
|||
self.playlist_item_limit = playlist_item_limit
|
||||
self.split_by_chapters = split_by_chapters
|
||||
self.chapter_template = chapter_template
|
||||
self.retry_failed = retry_failed
|
||||
self.max_retry_attempts = max_retry_attempts
|
||||
self.retry_count = 0
|
||||
|
||||
class Download:
|
||||
manager = None
|
||||
|
|
@ -353,8 +356,40 @@ class DownloadQueue:
|
|||
if download.canceled:
|
||||
asyncio.create_task(self.notifier.canceled(download.info.url))
|
||||
else:
|
||||
self.done.put(download)
|
||||
asyncio.create_task(self.notifier.completed(download.info))
|
||||
# Check if we should retry failed downloads
|
||||
if (download.info.status == 'error' and
|
||||
download.info.retry_failed and
|
||||
download.info.retry_count < download.info.max_retry_attempts):
|
||||
# Increment retry count and retry the download
|
||||
download.info.retry_count += 1
|
||||
log.info(f"Retrying download {download.info.title} (attempt {download.info.retry_count}/{download.info.max_retry_attempts})")
|
||||
download.info.status = 'pending'
|
||||
download.info.msg = f'Retrying (attempt {download.info.retry_count}/{download.info.max_retry_attempts})'
|
||||
download.info.percent = None
|
||||
download.info.speed = None
|
||||
download.info.eta = None
|
||||
# Create a new download with the same info
|
||||
dldirectory, _ = self.__calc_download_path(download.info.quality, download.info.format, download.info.folder)
|
||||
output = self.config.OUTPUT_TEMPLATE if len(download.info.custom_name_prefix) == 0 else f'{download.info.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}'
|
||||
output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER if not download.info.split_by_chapters else download.info.chapter_template
|
||||
entry = getattr(download.info, 'entry', None)
|
||||
if entry is not None and 'playlist' in entry and entry['playlist'] is not None:
|
||||
if len(self.config.OUTPUT_TEMPLATE_PLAYLIST):
|
||||
output = self.config.OUTPUT_TEMPLATE_PLAYLIST
|
||||
for property, value in entry.items():
|
||||
if property.startswith("playlist"):
|
||||
output = output.replace(f"%({property})s", str(value))
|
||||
ytdl_options = dict(self.config.YTDL_OPTIONS)
|
||||
playlist_item_limit = getattr(download.info, 'playlist_item_limit', 0)
|
||||
if playlist_item_limit > 0:
|
||||
ytdl_options['playlistend'] = playlist_item_limit
|
||||
new_download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, download.info.quality, download.info.format, ytdl_options, download.info)
|
||||
self.queue.put(new_download)
|
||||
asyncio.create_task(self.__start_download(new_download))
|
||||
else:
|
||||
# No more retries, mark as completed (failed)
|
||||
self.done.put(download)
|
||||
asyncio.create_task(self.notifier.completed(download.info))
|
||||
|
||||
def __extract_info(self, url, playlist_strict_mode):
|
||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||
|
|
@ -413,7 +448,7 @@ class DownloadQueue:
|
|||
self.pending.put(download)
|
||||
await self.notifier.added(dl)
|
||||
|
||||
async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already):
|
||||
async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already):
|
||||
if not entry:
|
||||
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
||||
|
||||
|
|
@ -429,7 +464,7 @@ class DownloadQueue:
|
|||
|
||||
if etype.startswith('url'):
|
||||
log.debug('Processing as an url')
|
||||
return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
|
||||
return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already)
|
||||
elif etype == 'playlist':
|
||||
log.debug('Processing as a playlist')
|
||||
entries = entry['entries']
|
||||
|
|
@ -449,7 +484,7 @@ class DownloadQueue:
|
|||
for property in ("id", "title", "uploader", "uploader_id"):
|
||||
if property in entry:
|
||||
etr[f"playlist_{property}"] = entry[property]
|
||||
results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
|
||||
results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already))
|
||||
if any(res['status'] == 'error' for res in results):
|
||||
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
|
||||
return {'status': 'ok'}
|
||||
|
|
@ -457,13 +492,13 @@ class DownloadQueue:
|
|||
log.debug('Processing as a video')
|
||||
key = entry.get('webpage_url') or entry['url']
|
||||
if not self.queue.exists(key):
|
||||
dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template)
|
||||
dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template, retry_failed, max_retry_attempts)
|
||||
await self.__add_download(dl, auto_start)
|
||||
return {'status': 'ok'}
|
||||
return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
|
||||
|
||||
async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None):
|
||||
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}')
|
||||
async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, retry_failed=False, max_retry_attempts=3, already=None):
|
||||
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} {retry_failed=} {max_retry_attempts=}')
|
||||
already = set() if already is None else already
|
||||
if url in already:
|
||||
log.info('recursion detected, skipping')
|
||||
|
|
@ -474,7 +509,7 @@ class DownloadQueue:
|
|||
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode)
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
return {'status': 'error', 'msg': str(exc)}
|
||||
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
|
||||
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, retry_failed, max_retry_attempts, already)
|
||||
|
||||
async def start_pending(self, ids):
|
||||
for id in ids:
|
||||
|
|
|
|||
|
|
@ -231,6 +231,36 @@
|
|||
<label class="form-check-label" for="checkbox-strict-mode">Strict Playlist Mode</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<div class="form-check form-switch">
|
||||
<input class="form-check-input"
|
||||
type="checkbox"
|
||||
role="switch"
|
||||
id="checkbox-retry-failed"
|
||||
name="enableRetryFailed"
|
||||
[(ngModel)]="enableRetryFailed"
|
||||
(change)="retryFailedChanged()"
|
||||
[disabled]="addInProgress || downloads.loading"
|
||||
ngbTooltip="Automatically retry failed downloads">
|
||||
<label class="form-check-label" for="checkbox-retry-failed">Retry Failed Downloads</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text">Max Retry Attempts</span>
|
||||
<input type="number"
|
||||
min="1"
|
||||
max="10"
|
||||
class="form-control"
|
||||
placeholder="3"
|
||||
name="maxRetryAttempts"
|
||||
(keydown)="isNumber($event)"
|
||||
[(ngModel)]="maxRetryAttempts"
|
||||
(change)="maxRetryAttemptsChanged()"
|
||||
[disabled]="addInProgress || downloads.loading || !enableRetryFailed"
|
||||
ngbTooltip="Maximum number of times to retry a failed download (1-10)">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<div class="row g-2 align-items-center">
|
||||
<div class="col-auto">
|
||||
|
|
|
|||
|
|
@ -50,6 +50,8 @@ export class App implements AfterViewInit, OnInit {
|
|||
playlistItemLimit!: number;
|
||||
splitByChapters: boolean;
|
||||
chapterTemplate: string;
|
||||
enableRetryFailed: boolean;
|
||||
maxRetryAttempts: number;
|
||||
addInProgress = false;
|
||||
themes: Theme[] = Themes;
|
||||
activeTheme: Theme | undefined;
|
||||
|
|
@ -108,6 +110,8 @@ export class App implements AfterViewInit, OnInit {
|
|||
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
||||
// Will be set from backend configuration, use empty string as placeholder
|
||||
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
||||
this.enableRetryFailed = this.cookieService.get('metube_retry_failed') === 'true';
|
||||
this.maxRetryAttempts = parseInt(this.cookieService.get('metube_max_retry_attempts') || '3', 10);
|
||||
|
||||
this.activeTheme = this.getPreferredTheme(this.cookieService);
|
||||
|
||||
|
|
@ -281,6 +285,20 @@ export class App implements AfterViewInit, OnInit {
|
|||
this.cookieService.set('metube_chapter_template', this.chapterTemplate, { expires: 3650 });
|
||||
}
|
||||
|
||||
retryFailedChanged() {
|
||||
this.cookieService.set('metube_retry_failed', this.enableRetryFailed ? 'true' : 'false', { expires: 3650 });
|
||||
}
|
||||
|
||||
maxRetryAttemptsChanged() {
|
||||
// Ensure value is between 1 and 10
|
||||
if (this.maxRetryAttempts < 1) {
|
||||
this.maxRetryAttempts = 1;
|
||||
} else if (this.maxRetryAttempts > 10) {
|
||||
this.maxRetryAttempts = 10;
|
||||
}
|
||||
this.cookieService.set('metube_max_retry_attempts', this.maxRetryAttempts.toString(), { expires: 3650 });
|
||||
}
|
||||
|
||||
queueSelectionChanged(checked: number) {
|
||||
this.queueDelSelected().nativeElement.disabled = checked == 0;
|
||||
this.queueDownloadSelected().nativeElement.disabled = checked == 0;
|
||||
|
|
@ -301,7 +319,7 @@ export class App implements AfterViewInit, OnInit {
|
|||
}
|
||||
}
|
||||
|
||||
addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) {
|
||||
addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string, retryFailed?: boolean, maxRetryAttempts?: number) {
|
||||
url = url ?? this.addUrl
|
||||
quality = quality ?? this.quality
|
||||
format = format ?? this.format
|
||||
|
|
@ -312,6 +330,8 @@ export class App implements AfterViewInit, OnInit {
|
|||
autoStart = autoStart ?? this.autoStart
|
||||
splitByChapters = splitByChapters ?? this.splitByChapters
|
||||
chapterTemplate = chapterTemplate ?? this.chapterTemplate
|
||||
retryFailed = retryFailed ?? this.enableRetryFailed
|
||||
maxRetryAttempts = maxRetryAttempts ?? this.maxRetryAttempts
|
||||
|
||||
// Validate chapter template if chapter splitting is enabled
|
||||
if (splitByChapters && !chapterTemplate.includes('%(section_number)')) {
|
||||
|
|
@ -319,9 +339,9 @@ export class App implements AfterViewInit, OnInit {
|
|||
return;
|
||||
}
|
||||
|
||||
console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate);
|
||||
console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate + ' retryFailed=' + retryFailed + ' maxRetryAttempts=' + maxRetryAttempts);
|
||||
this.addInProgress = true;
|
||||
this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => {
|
||||
this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate, retryFailed, maxRetryAttempts).subscribe((status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
alert(`Error adding URL: ${status.msg}`);
|
||||
} else {
|
||||
|
|
@ -482,7 +502,7 @@ export class App implements AfterViewInit, OnInit {
|
|||
this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`;
|
||||
// Now pass the selected quality, format, folder, etc. to the add() method
|
||||
this.downloads.add(url, this.quality, this.format, this.folder, this.customNamePrefix,
|
||||
this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate)
|
||||
this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate, this.enableRetryFailed, this.maxRetryAttempts)
|
||||
.subscribe({
|
||||
next: (status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
|
|
|
|||
|
|
@ -107,8 +107,8 @@ export class DownloadsService {
|
|||
return of({status: 'error', msg: msg})
|
||||
}
|
||||
|
||||
public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) {
|
||||
return this.http.post<Status>('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe(
|
||||
public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string, retryFailed: boolean, maxRetryAttempts: number) {
|
||||
return this.http.post<Status>('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate, retry_failed: retryFailed, max_retry_attempts: maxRetryAttempts }).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
|
@ -152,9 +152,11 @@ export class DownloadsService {
|
|||
const defaultAutoStart = true;
|
||||
const defaultSplitByChapters = false;
|
||||
const defaultChapterTemplate = this.configuration['OUTPUT_TEMPLATE_CHAPTER'];
|
||||
const defaultRetryFailed = false;
|
||||
const defaultMaxRetryAttempts = 3;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate)
|
||||
this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate, defaultRetryFailed, defaultMaxRetryAttempts)
|
||||
.subscribe({
|
||||
next: (response) => resolve(response),
|
||||
error: (error) => reject(error)
|
||||
|
|
|
|||
Loading…
Reference in a new issue