diff --git a/README.md b/README.md index d6f80aa..f7bfcc9 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,16 @@ Key capabilities: * Download videos, audio, captions, and thumbnails from a browser UI. * Download playlists and channels, with configurable output and download options. * Subscribe to channels and playlists, periodically check for new items, and queue new uploads automatically. +* Download only part of a video or audio track using **Clip start** / **Clip end** (see below). ![screenshot1](https://github.com/alexta69/metube/raw/master/screenshot.gif) +## ✂️ Time clips (partial downloads) + +For **Video** or **Audio**, use **Clip start** / **Clip end** (`90`, `1:30`, `H:MM:SS`), or **Batch clips** in Advanced Options (several ranges for the same URL — separate files or one merged file). Prefer a URL without `&t=` when setting times manually. Not available for subscriptions. + +Bookmarklets or other tools can prefill the add form via `?url=…&clips=…` or `#mt=…` (see `ui/src/app/metube-import-query.ts`). + ## 🐳 Run using Docker ```bash diff --git a/app/main.py b/app/main.py index 61d7e6e..4d22452 100644 --- a/app/main.py +++ b/app/main.py @@ -665,12 +665,14 @@ def parse_download_options(post: dict) -> dict: clip_end = None else: cleaned_url, url_t = _extract_t_query_from_url(url) - if url_t is not None: - url = cleaned_url explicit_start = _optional_clip_field(clip_start_raw) explicit_end = _optional_clip_field(clip_end_raw) explicit_start_provided = _clip_field_provided_in_post(clip_start_raw) explicit_end_provided = _clip_field_provided_in_post(clip_end_raw) + if explicit_start_provided or explicit_end_provided: + url = cleaned_url + elif url_t is not None: + url = cleaned_url if explicit_start_provided: clip_start = explicit_start elif explicit_end_provided: @@ -706,6 +708,43 @@ def parse_download_options(post: dict) -> dict: } +def parse_clips_list(raw) -> list[tuple[float, float]]: + if not isinstance(raw, list) or len(raw) == 0: + raise web.HTTPBadRequest(reason='clips must be a non-empty array') + if len(raw) > 50: + raise web.HTTPBadRequest(reason='clips supports at most 50 ranges per batch') + parsed: list[tuple[float, float]] = [] + for index, item in enumerate(raw): + if not isinstance(item, dict): + raise web.HTTPBadRequest(reason=f'clips[{index}] must be an object with start and end') + try: + start = _parse_clip_timestamp_value(item.get('start')) + end = _parse_clip_timestamp_value(item.get('end')) + except web.HTTPBadRequest as exc: + raise web.HTTPBadRequest(reason=f'clips[{index}]: {exc.reason}') from exc + if end <= start: + raise web.HTTPBadRequest(reason=f'clips[{index}]: end must be greater than start') + parsed.append((start, end)) + return parsed + + +@routes.post(config.URL_PREFIX + 'add-batch') +async def add_batch(request): + log.info('Received request to add batch clips') + post = await _read_json_request(request) + try: + merge_clips = bool(post.get('merge_clips', False)) + clips = parse_clips_list(post.get('clips')) + base_post = {k: v for k, v in post.items() if k not in ('clips', 'merge_clips')} + base_post['clip_start'] = None + base_post['clip_end'] = None + o = parse_download_options(base_post) + except web.HTTPBadRequest: + raise + status = await dqueue.add_batch(o, clips, merge_clips) + return web.Response(text=serializer.encode(status)) + + @routes.post(config.URL_PREFIX + 'add') async def add(request): log.info("Received request to add download") @@ -1074,6 +1113,7 @@ async def add_cors(request): return web.Response(text=serializer.encode({"status": "ok"})) app.router.add_route('OPTIONS', config.URL_PREFIX + 'add', add_cors) +app.router.add_route('OPTIONS', config.URL_PREFIX + 'add-batch', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'cancel-add', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscribe', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions', add_cors) diff --git a/app/tests/test_batch_clips.py b/app/tests/test_batch_clips.py new file mode 100644 index 0000000..ca109ab --- /dev/null +++ b/app/tests/test_batch_clips.py @@ -0,0 +1,35 @@ +import unittest + +from aiohttp import web + +from main import parse_clips_list +from ytdl import clip_batch_autoname_prefix, download_queue_key + + +class ParseClipsListTests(unittest.TestCase): + def test_parses_mm_ss_ranges(self): + clips = parse_clips_list([ + {'start': '1:30', 'end': '1:45'}, + {'start': '10:00', 'end': '10:05'}, + ]) + self.assertEqual(clips, [(90.0, 105.0), (600.0, 605.0)]) + + def test_rejects_invalid_range(self): + with self.assertRaises(web.HTTPBadRequest): + parse_clips_list([{'start': '2:00', 'end': '1:00'}]) + + +class BatchQueueKeyTests(unittest.TestCase): + def test_merged_batch_key(self): + url = 'https://example.com/v' + ranges = [(10.0, 20.0), (30.0, 40.0)] + key = download_queue_key(url, clip_ranges=ranges, merge_clips=True) + self.assertIn('clipmerged', key) + self.assertEqual( + clip_batch_autoname_prefix(ranges, merge=True), + 'clipmerged_10-20_30-40', + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/app/tests/test_clip_queue_keys.py b/app/tests/test_clip_queue_keys.py new file mode 100644 index 0000000..2270ff3 --- /dev/null +++ b/app/tests/test_clip_queue_keys.py @@ -0,0 +1,34 @@ +import unittest + +from ytdl import clip_autoname_prefix, download_queue_key, merge_custom_name_prefix + + +class ClipQueueKeyTests(unittest.TestCase): + def test_same_url_different_clips_get_different_keys(self): + url = "https://www.youtube.com/watch?v=abc" + k1 = download_queue_key(url, 10.0, 20.0) + k2 = download_queue_key(url, 30.0, 40.0) + self.assertNotEqual(k1, k2) + self.assertTrue(k1.startswith(url)) + self.assertTrue(k2.startswith(url)) + + def test_full_video_uses_url_only(self): + url = "https://example.com/v" + self.assertEqual(download_queue_key(url), url) + + def test_merge_custom_name_prefix_adds_clip_suffix(self): + self.assertEqual( + merge_custom_name_prefix("", 90.0, 120.0), + "clip_90-120", + ) + self.assertEqual( + merge_custom_name_prefix("mine", 1.0, 5.0), + "mine_clip_1-5", + ) + + def test_clip_autoname_prefix_empty_without_bounds(self): + self.assertEqual(clip_autoname_prefix(), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_download_queue.py b/app/tests/test_download_queue.py index 20bf863..62e5986 100644 --- a/app/tests/test_download_queue.py +++ b/app/tests/test_download_queue.py @@ -383,6 +383,45 @@ async def test_add_sets_clip_bounds_on_download_info(dq_env): ) assert result["status"] == "ok" - download = dq.pending.get("https://example.com/clip") + from ytdl import download_queue_key + + url = "https://example.com/clip" + download = dq.pending.get(download_queue_key(url, 10.0, 99.5)) assert download.info.clip_start == 10.0 assert download.info.clip_end == 99.5 + assert download.info.url == url + + +@pytest.mark.asyncio +async def test_add_allows_multiple_clips_for_same_url(dq_env): + notifier = AsyncMock() + url = "https://example.com/watch?v=same" + + def fake_extract(self, _url, ytdl_options_presets=None, ytdl_options_overrides=None): + return { + "_type": "video", + "id": "vid1", + "title": "Test Video", + "url": _url, + "webpage_url": _url, + } + + dq = DownloadQueue(dq_env, notifier) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract): + first = await dq.add( + url, "video", "auto", "any", "best", "", "", 0, + auto_start=False, clip_start=10.0, clip_end=20.0, + ) + second = await dq.add( + url, "video", "auto", "any", "best", "", "", 0, + auto_start=False, clip_start=30.0, clip_end=40.0, + ) + + assert first["status"] == "ok" + assert second["status"] == "ok" + from ytdl import download_queue_key + + assert dq.pending.exists(download_queue_key(url, 10.0, 20.0)) + assert dq.pending.exists(download_queue_key(url, 30.0, 40.0)) + assert dq.pending.get(download_queue_key(url, 10.0, 20.0)).info.clip_start == 10.0 + assert dq.pending.get(download_queue_key(url, 30.0, 40.0)).info.clip_start == 30.0 diff --git a/app/ytdl.py b/app/ytdl.py index d928c0f..a57dfa7 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -1,5 +1,7 @@ +import glob import os import shutil +import subprocess import yt_dlp import collections import collections.abc @@ -45,6 +47,64 @@ def _sanitize_path_component(value: Any) -> Any: return _WINDOWS_INVALID_PATH_CHARS.sub('_', value) +def _format_clip_bound(value: Any) -> str: + if value is None: + return "none" + if isinstance(value, float): + if value == float("inf"): + return "inf" + if value == int(value): + return str(int(value)) + return format(value, "g") + return str(value) + + +def download_queue_key( + url: str, + clip_start=None, + clip_end=None, + clip_ranges: list[tuple[float, float]] | None = None, + merge_clips: bool = False, +) -> str: + """Stable queue/history key; allows multiple clips from the same video URL.""" + if clip_ranges: + parts = "_".join( + f"{_format_clip_bound(s)}-{_format_clip_bound(e)}" for s, e in clip_ranges + ) + tag = "clipmerged" if merge_clips else "clipbatch" + return f"{url}#{tag}:{parts}" + if clip_start is None and clip_end is None: + return url + return f"{url}#clip:{_format_clip_bound(clip_start)}-{_format_clip_bound(clip_end)}" + + +def clip_batch_autoname_prefix( + clip_ranges: list[tuple[float, float]], + *, + merge: bool, +) -> str: + parts = "_".join( + f"{_format_clip_bound(s)}-{_format_clip_bound(e)}" for s, e in clip_ranges + ) + return f"clipmerged_{parts}" if merge else f"clipbatch_{parts}" + + +def clip_autoname_prefix(clip_start=None, clip_end=None) -> str: + """Filename prefix segment so clips from the same title do not overwrite each other.""" + if clip_start is None and clip_end is None: + return "" + return f"clip_{_format_clip_bound(clip_start)}-{_format_clip_bound(clip_end)}" + + +def merge_custom_name_prefix(custom_name_prefix: str, clip_start=None, clip_end=None) -> str: + suffix = clip_autoname_prefix(clip_start, clip_end) + if not suffix: + return custom_name_prefix or "" + if custom_name_prefix: + return f"{custom_name_prefix}_{suffix}" + return suffix + + # Regex matching yt-dlp output-template field references, e.g. ``%(title)s`` # or ``%(playlist_index)03d``. Built from yt-dlp's own ``STR_FORMAT_RE_TMPL`` # so that it stays in sync with upstream changes to the template syntax. @@ -194,10 +254,21 @@ class DownloadInfo: ytdl_options_overrides=None, clip_start=None, clip_end=None, + clip_ranges=None, + merge_clips=False, ): self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' self.url = url + self.clip_ranges = list(clip_ranges or []) + self.merge_clips = bool(merge_clips) + self.queue_key = download_queue_key( + url, + clip_start, + clip_end, + clip_ranges=self.clip_ranges if self.merge_clips else None, + merge_clips=self.merge_clips, + ) self.quality = quality self.download_type = download_type self.codec = codec @@ -292,6 +363,20 @@ class DownloadInfo: self.clip_start = None if not hasattr(self, "clip_end"): self.clip_end = None + if not hasattr(self, "clip_ranges"): + self.clip_ranges = [] + elif self.clip_ranges and isinstance(self.clip_ranges[0], (list, tuple)): + self.clip_ranges = [tuple(float(x) for x in r) for r in self.clip_ranges] + if not hasattr(self, "merge_clips"): + self.merge_clips = False + if not hasattr(self, "queue_key"): + self.queue_key = download_queue_key( + self.url, + getattr(self, "clip_start", None), + getattr(self, "clip_end", None), + clip_ranges=self.clip_ranges if self.merge_clips else None, + merge_clips=self.merge_clips, + ) _PERSISTED_DOWNLOAD_FIELDS = ( @@ -313,6 +398,9 @@ _PERSISTED_DOWNLOAD_FIELDS = ( "ytdl_options_overrides", "clip_start", "clip_end", + "clip_ranges", + "merge_clips", + "queue_key", "status", "timestamp", "error", @@ -413,6 +501,77 @@ class Download: self.loop = None self.notifier = None + def _download_merged_clips(self, ytdl_params: dict) -> int: + """Download multiple time ranges and concatenate into one file with ffmpeg.""" + ranges = list(self.info.clip_ranges or []) + if not ranges: + return 1 + batch_dir = os.path.join(self.temp_dir, f'merge_{self.info.id}') + os.makedirs(batch_dir, exist_ok=True) + part_paths: list[str] = [] + try: + for i, (start, end) in enumerate(ranges): + part_tmpl = os.path.join(batch_dir, f'part_{i:03d}.%(ext)s') + part_params = { + **ytdl_params, + 'outtmpl': {'default': part_tmpl, 'chapter': self.output_template_chapter}, + 'download_ranges': yt_dlp.utils.download_range_func( + None, + [(float(start), float(end))], + ), + } + code = yt_dlp.YoutubeDL(params=part_params).download([self.info.url]) + if code != 0: + return code + matches = sorted(glob.glob(os.path.join(batch_dir, f'part_{i:03d}.*'))) + if not matches: + log.error('Merged clip part %s produced no file', i) + return 1 + part_paths.append(matches[0]) + + list_path = os.path.join(batch_dir, 'concat.txt') + with open(list_path, 'w', encoding='utf-8') as fh: + for path in part_paths: + escaped = path.replace("'", "'\\''") + fh.write(f"file '{escaped}'\n") + + ext = os.path.splitext(part_paths[0])[1] or '.mp4' + name_info = dict(self.info.entry) if isinstance(self.info.entry, dict) else {} + name_info.setdefault('title', self.info.title) + name_info.setdefault('id', self.info.id) + name_info['ext'] = ext.lstrip('.') or 'mp4' + merged_name = yt_dlp.YoutubeDL({ + 'quiet': True, + 'paths': {'home': self.download_dir}, + }).prepare_filename(name_info, outtmpl=self.output_template) + if not os.path.isabs(merged_name): + merged_name = os.path.join(self.download_dir, merged_name) + os.makedirs(os.path.dirname(merged_name) or self.download_dir, exist_ok=True) + + cmd = [ + 'ffmpeg', '-y', '-f', 'concat', '-safe', '0', + '-i', list_path, '-c', 'copy', merged_name, + ] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + log.warning('ffmpeg concat -c copy failed, re-encoding: %s', proc.stderr) + cmd = [ + 'ffmpeg', '-y', '-f', 'concat', '-safe', '0', + '-i', list_path, '-c:v', 'libx264', '-c:a', 'aac', merged_name, + ] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + log.error('ffmpeg merge failed: %s', proc.stderr) + return 1 + + rel_name = os.path.relpath(merged_name, self.download_dir) + self.info.filename = rel_name + self.info.size = os.path.getsize(merged_name) if os.path.exists(merged_name) else None + self.status_queue.put({'status': 'finished', 'filename': merged_name}) + return 0 + finally: + shutil.rmtree(batch_dir, ignore_errors=True) + def _download(self): log.info(f"Starting download for: {self.info.title} ({self.info.url})") try: @@ -483,17 +642,21 @@ class Download: 'force_keyframes': False }) - clip_start = getattr(self.info, 'clip_start', None) - clip_end = getattr(self.info, 'clip_end', None) - if clip_start is not None or clip_end is not None: - start = float(clip_start) if clip_start is not None else 0.0 - end = float(clip_end) if clip_end is not None else float('inf') - ytdl_params['download_ranges'] = yt_dlp.utils.download_range_func( - None, - [(start, end)], - ) - - ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url]) + clip_ranges = getattr(self.info, 'clip_ranges', None) or [] + merge_clips = getattr(self.info, 'merge_clips', False) + if merge_clips and clip_ranges: + ret = self._download_merged_clips(ytdl_params) + else: + clip_start = getattr(self.info, 'clip_start', None) + clip_end = getattr(self.info, 'clip_end', None) + if clip_start is not None or clip_end is not None: + start = float(clip_start) if clip_start is not None else 0.0 + end = float(clip_end) if clip_end is not None else float('inf') + ytdl_params['download_ranges'] = yt_dlp.utils.download_range_func( + None, + [(start, end)], + ) + ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url]) self.status_queue.put({'status': 'finished' if ret == 0 else 'error'}) log.info(f"Finished download for: {self.info.title}") except yt_dlp.utils.YoutubeDLError as exc: @@ -639,8 +802,11 @@ class PersistentQueue: self.dict = OrderedDict() def load(self): - for k, v in self.saved_items(): - self.dict[k] = Download(None, None, None, None, getattr(v, 'quality', 'best'), getattr(v, 'format', 'any'), {}, v) + for file_key, v in self.saved_items(): + key = getattr(v, "queue_key", None) or file_key + if key != file_key: + log.info("PersistentQueue:%s migrating key %s -> %s", self.identifier, file_key, key) + self.dict[key] = Download(None, None, None, None, getattr(v, 'quality', 'best'), getattr(v, 'format', 'any'), {}, v) def exists(self, key): return key in self.dict @@ -716,7 +882,7 @@ class PersistentQueue: return items def put(self, value): - key = value.info.url + key = value.info.queue_key old = self.dict.get(key) self.dict[key] = value try: @@ -795,10 +961,10 @@ class DownloadQueue: pass download.info.status = 'error' download.close() - if self.queue.exists(download.info.url): - self.queue.delete(download.info.url) + if self.queue.exists(download.info.queue_key): + self.queue.delete(download.info.queue_key) if download.canceled: - asyncio.create_task(self.notifier.canceled(download.info.url)) + asyncio.create_task(self.notifier.canceled(download.info.queue_key)) else: self.done.put(download) asyncio.create_task(self.notifier.completed(download.info)) @@ -808,7 +974,7 @@ class DownloadQueue: log.error(f'CLEAR_COMPLETED_AFTER is set to an invalid value "{self.config.CLEAR_COMPLETED_AFTER}", expected an integer number of seconds') clear_after = 0 if clear_after > 0: - task = asyncio.create_task(self.__auto_clear_after_delay(download.info.url, clear_after)) + task = asyncio.create_task(self.__auto_clear_after_delay(download.info.queue_key, clear_after)) task.add_done_callback(lambda t: log.error(f'Auto-clear task failed: {t.exception()}') if not t.cancelled() and t.exception() else None) async def __auto_clear_after_delay(self, url, delay_seconds): @@ -914,6 +1080,8 @@ class DownloadQueue: clip_end, already, _add_gen=None, + clip_ranges=None, + merge_clips=False, ): if not entry: return {'status': 'error', 'msg': "Invalid/empty data was given."} @@ -1003,6 +1171,8 @@ class DownloadQueue: clip_end, already, _add_gen, + clip_ranges, + merge_clips, ) ) if any(res['status'] == 'error' for res in results): @@ -1014,7 +1184,20 @@ class DownloadQueue: if key in self._canceled_urls: log.info(f'Skipping canceled URL: {entry.get("title") or key}') return {'status': 'ok'} - if not self.queue.exists(key): + clip_ranges = clip_ranges or [] + merge_clips = bool(merge_clips) + if merge_clips and clip_ranges: + queue_key = download_queue_key( + key, clip_ranges=clip_ranges, merge_clips=True, + ) + if self.queue.exists(queue_key) or self.pending.exists(queue_key): + log.info('Download already queued for %s', queue_key) + return {'status': 'ok', 'msg': 'This merged clip batch is already in the queue'} + effective_prefix = custom_name_prefix or '' + batch_prefix = clip_batch_autoname_prefix(clip_ranges, merge=True) + effective_prefix = ( + f"{effective_prefix}_{batch_prefix}" if effective_prefix else batch_prefix + ) dl = DownloadInfo( id=entry['id'], title=entry.get('title') or entry['id'], @@ -1024,7 +1207,37 @@ class DownloadQueue: codec=codec, format=format, folder=folder, - custom_name_prefix=custom_name_prefix, + custom_name_prefix=effective_prefix, + error=error, + entry=entry, + playlist_item_limit=playlist_item_limit, + split_by_chapters=False, + chapter_template=chapter_template, + subtitle_language=subtitle_language, + subtitle_mode=subtitle_mode, + ytdl_options_presets=ytdl_options_presets, + ytdl_options_overrides=ytdl_options_overrides, + clip_ranges=clip_ranges, + merge_clips=True, + ) + else: + queue_key = download_queue_key(key, clip_start, clip_end) + if self.queue.exists(queue_key) or self.pending.exists(queue_key): + log.info('Download already queued for %s', queue_key) + return {'status': 'ok', 'msg': 'This clip is already in the queue'} + effective_prefix = merge_custom_name_prefix( + custom_name_prefix, clip_start, clip_end, + ) + dl = DownloadInfo( + id=entry['id'], + title=entry.get('title') or entry['id'], + url=key, + quality=quality, + download_type=download_type, + codec=codec, + format=format, + folder=folder, + custom_name_prefix=effective_prefix, error=error, entry=entry, playlist_item_limit=playlist_item_limit, @@ -1037,7 +1250,7 @@ class DownloadQueue: clip_start=clip_start, clip_end=clip_end, ) - await self.__add_download(dl, auto_start) + await self.__add_download(dl, auto_start) return {'status': 'ok'} return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'} @@ -1062,13 +1275,16 @@ class DownloadQueue: clip_end=None, already=None, _add_gen=None, + clip_ranges=None, + merge_clips=False, ): if ytdl_options_presets is None: ytdl_options_presets = [] log.info( f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} ' f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} ' - f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=}' + f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=} ' + f'{merge_clips=} {clip_ranges=}' ) if already is None: _add_gen = self._add_generation @@ -1106,8 +1322,65 @@ class DownloadQueue: clip_end, already, _add_gen, + clip_ranges, + merge_clips, ) + async def add_batch(self, options: dict, clips: list[tuple[float, float]], merge_clips: bool): + if options['download_type'] in ('captions', 'thumbnail'): + return {'status': 'error', 'msg': 'Batch clips only support video and audio downloads'} + if not clips: + return {'status': 'error', 'msg': 'No clip ranges provided'} + if merge_clips: + return await self.add( + options['url'], + options['download_type'], + options['codec'], + options['format'], + options['quality'], + options['folder'], + options['custom_name_prefix'], + options['playlist_item_limit'], + options['auto_start'], + False, + options['chapter_template'], + options['subtitle_language'], + options['subtitle_mode'], + options['ytdl_options_presets'], + options['ytdl_options_overrides'], + clip_start=None, + clip_end=None, + clip_ranges=clips, + merge_clips=True, + ) + errors: list[str] = [] + for start, end in clips: + result = await self.add( + options['url'], + options['download_type'], + options['codec'], + options['format'], + options['quality'], + options['folder'], + options['custom_name_prefix'], + options['playlist_item_limit'], + options['auto_start'], + options['split_by_chapters'], + options['chapter_template'], + options['subtitle_language'], + options['subtitle_mode'], + options['ytdl_options_presets'], + options['ytdl_options_overrides'], + clip_start=start, + clip_end=end, + ) + if result.get('status') == 'error': + msg = result.get('msg') or 'unknown error' + errors.append(str(msg)) + if errors: + return {'status': 'error', 'msg': '; '.join(errors)} + return {'status': 'ok', 'msg': f'Queued {len(clips)} separate clips'} + async def add_entry( self, entry, diff --git a/ui/src/app/app.html b/ui/src/app/app.html index b2ae0db..7b7e1d3 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -436,7 +436,7 @@ class="form-control" name="clipStart" [(ngModel)]="clipStart" - (change)="clipStartChanged()" + (ngModelChange)="clipStartChanged()" placeholder="e.g. 2:26" [disabled]="addInProgress || subscribeInProgress || downloads.loading" ngbTooltip="Optional start time (seconds, M:SS, or H:MM:SS). Blank = from start or YouTube &t= in URL."> @@ -449,13 +449,81 @@ class="form-control" name="clipEnd" [(ngModel)]="clipEnd" - (change)="clipEndChanged()" + (ngModelChange)="clipEndChanged()" placeholder="e.g. 3:24" [disabled]="addInProgress || subscribeInProgress || downloads.loading" ngbTooltip="Optional end time. Blank = until end of media."> } + @if (downloadType === 'video' || downloadType === 'audio') { +
+
Batch clips (same URL)
+

+ Add multiple time ranges for the URL above. Use M:SS or H:MM:SS (seconds also work). + @if (importMergeHint) { + Extension suggested a merged download — use Download merged below. + } +

+
+ + + + + + + + + + @for (row of batchClipRows; track $index; let i = $index) { + + + + + + } + +
StartEnd
+ + + + + +
+
+
+ + + +
+
+ } diff --git a/ui/src/app/app.ts b/ui/src/app/app.ts index a73f879..d17ef06 100644 --- a/ui/src/app/app.ts +++ b/ui/src/app/app.ts @@ -32,6 +32,7 @@ import { } from './interfaces'; import { EtaPipe, SpeedPipe, FileSizePipe } from './pipes'; import { SelectAllCheckboxComponent, ItemCheckboxComponent } from './components/'; +import { parseMetubeImportFromLocation } from './metube-import-query'; @Component({ selector: 'app-root', @@ -83,6 +84,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy { chapterTemplate: string; clipStart = ''; clipEnd = ''; + batchClipRows: { start: string; end: string }[] = [{ start: '', end: '' }]; + /** Set when opened via extension deep link with merge=1 */ + importMergeHint = false; + private urlImportDone = false; subtitleLanguage: string; subtitleMode: string; ytdlOptionsPresets: string[] = []; @@ -301,6 +306,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy { } ngOnInit() { + this.applyImportFromUrl(); this.downloads.getCookieStatus().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(data => { this.hasCookies = !!(data && typeof data === 'object' && 'has_cookies' in data && data.has_cookies); this.cdr.markForCheck(); @@ -315,6 +321,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy { } ngAfterViewInit() { + this.applyImportFromUrl(); this.downloads.queueChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { this.queueMasterCheckbox()?.selectionChanged(); this.cdr.markForCheck(); @@ -367,6 +374,35 @@ export class App implements AfterViewInit, OnInit, OnDestroy { this.saveSelection(this.downloadType); } + applyImportFromUrl() { + if (this.urlImportDone) { + return; + } + const imp = parseMetubeImportFromLocation(window.location.search, window.location.hash); + if (!imp) { + return; + } + this.urlImportDone = true; + this.addUrl = imp.url; + this.importMergeHint = !!imp.mergeClips; + if (imp.clips.length > 0) { + this.batchClipRows = imp.clips.map((c) => ({ start: c.start, end: c.end })); + if (imp.clips.length === 1) { + this.clipStart = imp.clips[0].start; + this.clipEnd = imp.clips[0].end; + } else { + this.clipStart = ''; + this.clipEnd = ''; + } + } + if (imp.clips.length > 0 && (this.downloadType === 'video' || this.downloadType === 'audio')) { + this.isAdvancedOpen = true; + } + const path = window.location.pathname || '/'; + window.history.replaceState(null, '', path); + this.cdr.markForCheck(); + } + showAdvanced() { return this.downloads.configuration['CUSTOM_DIRS']; } @@ -1057,6 +1093,57 @@ export class App implements AfterViewInit, OnInit, OnDestroy { }; } + addBatchClipRow() { + this.batchClipRows.push({ start: '', end: '' }); + } + + removeBatchClipRow(index: number) { + if (this.batchClipRows.length > 1) { + this.batchClipRows.splice(index, 1); + } + } + + hasValidBatchClips(): boolean { + return this.batchClipRows.some( + (row) => Boolean(row.start?.trim()) && Boolean(row.end?.trim()), + ); + } + + addBatchDownload(mergeClips: boolean) { + if (!this.addUrl?.trim()) { + alert('Enter a video URL first.'); + return; + } + const clips = this.batchClipRows + .filter((row) => row.start.trim() && row.end.trim()) + .map((row) => ({ start: row.start.trim(), end: row.end.trim() })); + if (!clips.length) { + alert('Add at least one row with start and end times.'); + return; + } + const payload = this.buildAddPayload(); + if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) { + return; + } + this.addInProgress = true; + this.cancelRequested = false; + this.addRequestSub?.unsubscribe(); + this.addRequestSub = this.downloads + .addBatch(payload, clips, mergeClips) + .subscribe((status: Status) => { + if (status.status === 'error' && !this.cancelRequested) { + alert(`Error adding batch: ${status.msg}`); + } else if (status.status !== 'error') { + if (status.msg) { + alert(status.msg); + } else { + this.addUrl = ''; + } + } + this.resetAddState(); + }); + } + addDownload(overrides: Partial = {}) { const payload = this.buildAddPayload(overrides); @@ -1077,7 +1164,11 @@ export class App implements AfterViewInit, OnInit, OnDestroy { if (status.status === 'error' && !this.cancelRequested) { alert(`Error adding URL: ${status.msg}`); } else if (status.status !== 'error') { - this.addUrl = ''; + if (status.msg) { + alert(status.msg); + } else { + this.addUrl = ''; + } } this.resetAddState(); }); diff --git a/ui/src/app/interfaces/download.ts b/ui/src/app/interfaces/download.ts index 935d4da..493b953 100644 --- a/ui/src/app/interfaces/download.ts +++ b/ui/src/app/interfaces/download.ts @@ -3,6 +3,8 @@ export interface Download { id: string; title: string; url: string; + /** Backend queue/history id; required for clips (differs from url). */ + queue_key?: string; download_type: string; codec?: string; quality: string; diff --git a/ui/src/app/metube-import-query.spec.ts b/ui/src/app/metube-import-query.spec.ts new file mode 100644 index 0000000..0f8fe31 --- /dev/null +++ b/ui/src/app/metube-import-query.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { + buildMetubeImportHash, + buildMetubeImportSearch, + parseMetubeImportFromHash, + parseMetubeImportFromLocation, + parseMetubeImportFromSearch, +} from './metube-import-query'; + +describe('parseMetubeImportFromSearch', () => { + it('parses url and single clip from JSON clips', () => { + const clips = encodeURIComponent(JSON.stringify([{ start: '1:30', end: '2:00' }])); + const result = parseMetubeImportFromSearch(`?url=${encodeURIComponent('https://youtu.be/x')}&clips=${clips}`); + expect(result?.url).toBe('https://youtu.be/x'); + expect(result?.clips).toEqual([{ start: '1:30', end: '2:00' }]); + }); + + it('parses compact clips syntax', () => { + const result = parseMetubeImportFromSearch( + `?url=https://example.com/v&clips=${encodeURIComponent('1:30-2:00;3:10-3:25')}`, + ); + expect(result?.clips).toHaveLength(2); + expect(result?.clips[1]).toEqual({ start: '3:10', end: '3:25' }); + }); + + it('returns null without url', () => { + expect(parseMetubeImportFromSearch('?clips=1:00-2:00')).toBeNull(); + }); +}); + +describe('parseMetubeImportFromHash', () => { + it('round-trips via buildMetubeImportHash', () => { + const clips = [{ start: '1:05', end: '2:10' }]; + const hash = buildMetubeImportHash('https://www.youtube.com/watch?v=abc', clips, { mergeClips: true }); + const result = parseMetubeImportFromHash(hash); + expect(result?.url).toContain('watch?v=abc'); + expect(result?.clips).toEqual(clips); + expect(result?.mergeClips).toBe(true); + }); +}); + +describe('parseMetubeImportFromLocation', () => { + it('prefers hash over search', () => { + const hash = buildMetubeImportHash('https://a.test/v', [{ start: '0:01', end: '0:02' }]); + const result = parseMetubeImportFromLocation('?url=https://other.test', hash); + expect(result?.url).toBe('https://a.test/v'); + }); +}); + +describe('buildMetubeImportSearch', () => { + it('round-trips clips', () => { + const clips = [{ start: '0:05', end: '0:10' }]; + const search = buildMetubeImportSearch('https://a.test/v', clips, { mergeClips: true }); + const parsed = parseMetubeImportFromSearch(`?${search}`); + expect(parsed?.mergeClips).toBe(true); + expect(parsed?.clips).toEqual(clips); + }); +}); diff --git a/ui/src/app/metube-import-query.ts b/ui/src/app/metube-import-query.ts new file mode 100644 index 0000000..10aee76 --- /dev/null +++ b/ui/src/app/metube-import-query.ts @@ -0,0 +1,134 @@ +export interface ImportClip { + start: string; + end: string; +} + +export interface MetubeImportParams { + url: string; + clips: ImportClip[]; + mergeClips?: boolean; +} + +/** Parse `?url=…&clips=…` from extension deep links. */ +export function parseMetubeImportFromSearch(search: string): MetubeImportParams | null { + const raw = search.startsWith('?') ? search.slice(1) : search; + if (!raw.trim()) { + return null; + } + const params = new URLSearchParams(raw); + const url = params.get('url')?.trim(); + if (!url) { + return null; + } + const clips = parseClipsParam(params.get('clips')); + const mergeRaw = params.get('merge')?.trim().toLowerCase(); + const mergeClips = mergeRaw === '1' || mergeRaw === 'true' || mergeRaw === 'yes'; + return { url, clips, mergeClips: mergeClips || undefined }; +} + +/** Parse `#mt=…` (base64url JSON) — preferred for extension handoff. */ +export function parseMetubeImportFromHash(hash: string): MetubeImportParams | null { + const raw = hash.startsWith('#') ? hash.slice(1) : hash; + const match = raw.match(/(?:^|&)mt=([^&]+)/); + if (!match) { + return null; + } + try { + const b64 = match[1].replace(/-/g, '+').replace(/_/g, '/'); + const padded = b64 + '==='.slice((b64.length + 3) % 4); + const json = decodeURIComponent( + Array.from(atob(padded), (c) => '%' + c.charCodeAt(0).toString(16).padStart(2, '0')).join(''), + ); + const data = JSON.parse(json) as Record; + const url = String(data['url'] ?? '').trim(); + if (!url) { + return null; + } + const clips = parseClipsArray(data['clips']); + const mergeClips = data['merge'] === true || data['merge'] === '1'; + return { url, clips, mergeClips: mergeClips || undefined }; + } catch { + return null; + } +} + +export function parseMetubeImportFromLocation(search: string, hash: string): MetubeImportParams | null { + return parseMetubeImportFromHash(hash) ?? parseMetubeImportFromSearch(search); +} + +function parseClipsArray(raw: unknown): ImportClip[] { + if (!Array.isArray(raw)) { + return []; + } + return raw + .filter((item): item is Record => item != null && typeof item === 'object') + .map((item) => ({ + start: String(item['start'] ?? '').trim(), + end: String(item['end'] ?? '').trim(), + })) + .filter((c) => c.start && c.end); +} + +function parseClipsParam(raw: string | null): ImportClip[] { + if (!raw?.trim()) { + return []; + } + const trimmed = raw.trim(); + if (trimmed.startsWith('[')) { + try { + return parseClipsArray(JSON.parse(trimmed) as unknown); + } catch { + return []; + } + } + return trimmed + .split(';') + .map((part) => part.trim()) + .filter(Boolean) + .map((part) => { + const dash = part.indexOf('-'); + if (dash <= 0) { + return { start: '', end: '' }; + } + return { + start: part.slice(0, dash).trim(), + end: part.slice(dash + 1).trim(), + }; + }) + .filter((c) => c.start && c.end); +} + +export function buildMetubeImportSearch( + url: string, + clips: ImportClip[], + options?: { mergeClips?: boolean }, +): string { + const params = new URLSearchParams(); + params.set('url', url); + if (clips.length) { + params.set('clips', JSON.stringify(clips)); + } + if (options?.mergeClips) { + params.set('merge', '1'); + } + return params.toString(); +} + +/** Build `#mt=…` fragment for extension → MeTube (survives service worker / long URLs). */ +export function buildMetubeImportHash( + url: string, + clips: ImportClip[], + options?: { mergeClips?: boolean }, +): string { + const payload = { + url, + clips, + merge: options?.mergeClips ?? false, + }; + const json = JSON.stringify(payload); + const bytes = encodeURIComponent(json).replace(/%([0-9A-F]{2})/gi, (_, hex) => + String.fromCharCode(parseInt(hex, 16)), + ); + const b64 = btoa(bytes).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); + return `#mt=${b64}`; +} diff --git a/ui/src/app/services/downloads.service.spec.ts b/ui/src/app/services/downloads.service.spec.ts index 2fbc4be..fcbcf47 100644 --- a/ui/src/app/services/downloads.service.spec.ts +++ b/ui/src/app/services/downloads.service.spec.ts @@ -197,6 +197,35 @@ describe('DownloadsService', () => { expect(service.queue.has('u1')).toBe(true); }); + it('delByFilter uses map keys for clip downloads', () => { + const clipKey = 'https://youtu.be/x#clip:10-20'; + const dl: Download = { + id: '1', + title: 't', + url: 'https://youtu.be/x', + queue_key: clipKey, + download_type: 'video', + quality: 'best', + format: 'any', + folder: '', + custom_name_prefix: '', + playlist_item_limit: 0, + status: 'finished', + msg: '', + percent: 0, + speed: 0, + eta: 0, + filename: '', + checked: false, + deleting: false, + }; + service.done.set(clipKey, dl); + service.delByFilter('done', () => true).subscribe(); + const req = httpMock.expectOne('delete'); + expect(req.request.body).toEqual({ where: 'done', ids: [clipKey] }); + req.flush({}); + }); + it('socket updated preserves checked and deleting', () => { service.queue.set('u1', { id: '1', @@ -227,10 +256,12 @@ describe('DownloadsService', () => { }); it('socket completed moves entry to done', () => { - service.queue.set('u1', { + const clipKey = 'u1#clip:10-20'; + service.queue.set(clipKey, { id: '1', title: 't', url: 'u1', + queue_key: clipKey, download_type: 'video', quality: 'best', format: 'any', @@ -245,9 +276,12 @@ describe('DownloadsService', () => { filename: '', checked: false, }); - socket.emit('completed', JSON.stringify({ url: 'u1', title: 't', status: 'finished' })); - expect(service.queue.has('u1')).toBe(false); - expect(service.done.has('u1')).toBe(true); + socket.emit( + 'completed', + JSON.stringify({ url: 'u1', queue_key: clipKey, title: 't', status: 'finished' }), + ); + expect(service.queue.has(clipKey)).toBe(false); + expect(service.done.has(clipKey)).toBe(true); }); it('socket canceled removes from queue', () => { diff --git a/ui/src/app/services/downloads.service.ts b/ui/src/app/services/downloads.service.ts index 60c3df6..73619cd 100644 --- a/ui/src/app/services/downloads.service.ts +++ b/ui/src/app/services/downloads.service.ts @@ -25,6 +25,17 @@ export interface AddDownloadPayload { clipStart?: string; clipEnd?: string; } + +export interface BatchClipInput { + start: string; + end: string; +} + +/** Map key for queue/done — matches backend PersistentQueue key. */ +export function downloadMapKey(dl: Pick): string { + return dl.queue_key ?? dl.url; +} + @Injectable({ providedIn: 'root' }) @@ -61,25 +72,36 @@ export class DownloadsService { .pipe(takeUntilDestroyed()) .subscribe((strdata: string) => { const data: Download = JSON.parse(strdata); - this.queue.set(data.url, data); + const key = downloadMapKey(data); + if (key !== data.url) { + this.queue.delete(data.url); + } + this.queue.set(key, data); this.queueChanged.next(); }); this.socket.fromEvent('updated') .pipe(takeUntilDestroyed()) .subscribe((strdata: string) => { const data: Download = JSON.parse(strdata); - const dl: Download | undefined = this.queue.get(data.url); + const key = downloadMapKey(data); + const dl: Download | undefined = this.queue.get(key) ?? this.queue.get(data.url); data.checked = !!dl?.checked; data.deleting = !!dl?.deleting; - this.queue.set(data.url, data); + if (key !== data.url) { + this.queue.delete(data.url); + } + this.queue.set(key, data); this.updated.next(); }); this.socket.fromEvent('completed') .pipe(takeUntilDestroyed()) .subscribe((strdata: string) => { const data: Download = JSON.parse(strdata); + const key = downloadMapKey(data); this.queue.delete(data.url); - this.done.set(data.url, data); + this.queue.delete(key); + this.done.delete(data.url); + this.done.set(key, data); this.queueChanged.next(); this.doneChanged.next(); }); @@ -150,13 +172,46 @@ export class DownloadsService { }; const cs = payload.clipStart?.trim(); const ce = payload.clipEnd?.trim(); - if (cs) body['clip_start'] = cs; - if (ce) body['clip_end'] = ce; + if (cs) { + body['clip_start'] = cs; + } + if (ce) { + body['clip_end'] = ce; + } + // If only end is set, backend needs an explicit start of 0 — send it when end is present. + if (ce && !cs) { + body['clip_start'] = '0'; + } return this.http.post('add', body).pipe( catchError(this.handleHTTPError) ); } + public addBatch(payload: AddDownloadPayload, clips: BatchClipInput[], mergeClips: boolean) { + const body: Record = { + url: payload.url, + download_type: payload.downloadType, + codec: payload.codec, + quality: payload.quality, + format: payload.format, + folder: payload.folder, + custom_name_prefix: payload.customNamePrefix, + playlist_item_limit: payload.playlistItemLimit, + auto_start: payload.autoStart, + split_by_chapters: payload.splitByChapters, + chapter_template: payload.chapterTemplate, + subtitle_language: payload.subtitleLanguage, + subtitle_mode: payload.subtitleMode, + ytdl_options_presets: payload.ytdlOptionsPresets, + ytdl_options_overrides: payload.ytdlOptionsOverrides, + merge_clips: mergeClips, + clips: clips.map((c) => ({ start: c.start.trim(), end: c.end.trim() })), + }; + return this.http.post('add-batch', body).pipe( + catchError(this.handleHTTPError) + ); + } + public getPresets() { return this.http.get<{ presets: string[] }>('presets').pipe( catchError(() => of({ presets: [] })) @@ -182,13 +237,21 @@ export class DownloadsService { public startByFilter(where: State, filter: (dl: Download) => boolean) { const ids: string[] = []; - this[where].forEach((dl: Download) => { if (filter(dl)) ids.push(dl.url) }); + this[where].forEach((dl: Download, key: string) => { + if (filter(dl)) { + ids.push(key); + } + }); return this.startById(ids); } public delByFilter(where: State, filter: (dl: Download) => boolean) { const ids: string[] = []; - this[where].forEach((dl: Download) => { if (filter(dl)) ids.push(dl.url) }); + this[where].forEach((dl: Download, key: string) => { + if (filter(dl)) { + ids.push(key); + } + }); return this.delById(where, ids); } public cancelAdd() {