diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 795a201e..fb9c2a0f 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -35,6 +35,7 @@ from .Utils import ( dt_delta, extract_info, extract_ytdlp_logs, + get_extras, merge_dict, str_to_dt, ytdlp_reject, @@ -365,8 +366,9 @@ class DownloadQueue(metaclass=Singleton): if property in entry: extras[f"playlist_{property}"] = entry.get(property) - if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""): - extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg" + extractor_key = entry.get("ie_key") or entry.get("extractor_key") or entry.get("extractor") or "" + if "thumbnail" not in etr and "youtube" in str(extractor_key).lower(): + extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**etr) newItem: Item = item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras) @@ -521,7 +523,7 @@ class DownloadQueue(metaclass=Singleton): options=options, cli=item.cli, auto_start=item.auto_start, - extras=item.extras, + extras=merge_dict(item.extras, get_extras(entry)), ) try: @@ -731,7 +733,7 @@ class DownloadQueue(metaclass=Singleton): # Early archive check to avoid unnecessary extraction calls # This sometimes can be different from the final extracted ID, so we need to verify again after extraction. if archive_id and item.is_archived(): - store_type, _ = await self.get_item(archive_id=archive_id) + store_type, dlInfo = await self.get_item(archive_id=archive_id) if not store_type: dlInfo = Download( info=ItemDTO( @@ -821,7 +823,7 @@ class DownloadQueue(metaclass=Singleton): if len(archive_ids) > 0: store_type = None for n in archive_ids: - store_type, _ = await self.get_item(archive_id=n) + store_type, dlInfo = await self.get_item(archive_id=n) if store_type: break @@ -838,7 +840,7 @@ class DownloadQueue(metaclass=Singleton): cookies=item.cookies, template=item.template, msg="URL is already downloaded.", - extras=item.extras, + extras=merge_dict(item.extras, get_extras(entry)), ) ) @@ -879,7 +881,7 @@ class DownloadQueue(metaclass=Singleton): _name = entry.get("title", entry.get("id")) log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}." - store_type, _ = await self.get_item(archive_id=archive_id) + store_type, dlInfo = await self.get_item(archive_id=archive_id) if not store_type: dlInfo = Download( info=ItemDTO( @@ -892,7 +894,7 @@ class DownloadQueue(metaclass=Singleton): cookies=item.cookies, template=item.template, msg=log_message, - extras=item.extras, + extras=merge_dict(item.extras, get_extras(entry)), ) ) await self.done.put(dlInfo) diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index cb75e43f..05b5000c 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -102,6 +102,20 @@ class Item: """ return self.extras and len(self.extras) > 0 + def add_extras(self, key: str, value: Any) -> None: + """ + Add an extra data to the item. + + Args: + key (str): The key of the extra data. + value (Any): The value of the extra data. + + """ + if not self.extras: + self.extras = {} + + self.extras[key] = value + def has_cli(self) -> bool: """ Check if the item has any command options for yt-dlp associated with it. diff --git a/app/library/Presets.py b/app/library/Presets.py index 9ddfbd74..7f68346c 100644 --- a/app/library/Presets.py +++ b/app/library/Presets.py @@ -103,6 +103,9 @@ class Preset: default: bool = False """If True, the preset is a default preset.""" + priority: int = 0 + """Priority of the preset.""" + _cookies_file: Path | None = field(init=False, default=None) """The path to the cookies file.""" @@ -219,7 +222,7 @@ class Presets(metaclass=Singleton): def get_all(self) -> list[Preset]: """Return the items.""" - return self._default + self._items + return sorted(self._default + self._items, key=lambda x: x.priority, reverse=True) def load(self) -> "Presets": """ @@ -249,6 +252,10 @@ class Presets(metaclass=Singleton): for i, preset in enumerate(presets): try: + if "priority" not in preset: + preset["priority"] = 0 + need_save = True + if "id" not in preset: preset["id"] = str(uuid.uuid4()) need_save = True @@ -272,7 +279,7 @@ class Presets(metaclass=Singleton): continue if need_save: - LOG.info(f"Saving '{self._file}'.") + LOG.warning("Saving presets due to schema changes.") self.save(self._items) return self @@ -328,6 +335,15 @@ class Presets(metaclass=Singleton): msg = f"Invalid command options for yt-dlp. '{e!s}'." raise ValueError(msg) from e + if item.get("priority") is not None: + priority = item.get("priority") + if not isinstance(priority, int): + msg = "Priority must be an integer." + raise ValueError(msg) + if priority < 0: + msg = "Priority must be >= 0." + raise ValueError(msg) + return True def save(self, items: list[Preset | dict]) -> "Presets": diff --git a/app/library/Utils.py b/app/library/Utils.py index 66c38925..f0e69cbb 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -11,6 +11,7 @@ import socket import time import uuid from datetime import UTC, datetime, timedelta +from email.utils import formatdate from functools import lru_cache, wraps from http.cookiejar import MozillaCookieJar from pathlib import Path @@ -1847,3 +1848,78 @@ def create_cookies_file(cookies: str, file: Path | None = None) -> Path: load_cookies(file) return file + + +def get_thumbnail(thumbnails: list) -> str | None: + """ + Extract thumbnail URL from a yt-dlp entry. + + Args: + thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry. + + Returns: + str | None: The thumbnail URL if available, otherwise None. + + """ + if not thumbnails or not isinstance(thumbnails, list): + return None + + def _thumb_sort_key(thumb: dict) -> tuple: + return ( + thumb.get("preference") if thumb.get("preference") is not None else -1, + thumb.get("width") if thumb.get("width") is not None else -1, + thumb.get("height") if thumb.get("height") is not None else -1, + thumb.get("id") if thumb.get("id") is not None else "", + thumb.get("url"), + ) + + return max(thumbnails, key=_thumb_sort_key, default=None) + + +def get_extras(entry: dict, kind: str = "video") -> dict: + """ + Extract useful information from a yt-dlp entry. + + Args: + entry (dict): The entry data from yt-dlp. + kind (str): The type of the item (e.g., "video", "playlist"). + + Returns: + dict: The extracted data. + + """ + extras = {} + + if not entry or not isinstance(entry, dict): + return extras + + if "playlist" == kind: + for property in ("id", "title", "uploader", "uploader_id"): + if val := entry.get(property): + extras[f"playlist_{property}"] = val + + if thumbnail := get_thumbnail(entry.get("thumbnails", [])): + extras["thumbnail"] = thumbnail.get("url") + elif thumbnail := entry.get("thumbnail"): + extras["thumbnail"] = thumbnail + + for property in ("uploader", "channel"): + if val := entry.get(property): + extras[property] = val + + if release_in := entry.get("release_timestamp"): + extras["release_in"] = formatdate(release_in, usegmt=True) + + if release_in and "is_upcoming" == entry.get("live_status"): + extras["is_live"] = release_in + + if duration := entry.get("duration"): + extras["duration"] = duration + + extras["is_premiere"] = bool(entry.get("is_premiere", False)) + + extractor_key = entry.get("ie_key") or entry.get("extractor_key") or entry.get("extractor") or "" + if "thumbnail" not in extras and "youtube" in str(extractor_key).lower(): + extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**entry) + + return extras diff --git a/app/schema/presets.json b/app/schema/presets.json index a751d97c..13f74806 100644 --- a/app/schema/presets.json +++ b/app/schema/presets.json @@ -43,6 +43,12 @@ "type": "boolean", "description": "Whether this preset is a default preset.", "default": false + }, + "priority": { + "type": "integer", + "description": "Priority of the preset for sorting. Higher priority presets appear first.", + "default": 0, + "minimum": 0 } }, "required": [ @@ -54,12 +60,14 @@ "id": "3e163c6c-64eb-4448-924f-814b629b3810", "name": "default", "default": true, + "priority": 0, "cli": "--socket-timeout 30 --download-archive %(config_path)s/archive.log", "description": "Default preset for yt-dlp. It will download whatever yt-dlp decides is the best quality for the video and audio." }, { "id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330", "name": "Audio Only", + "priority": 5, "cli": "--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'", "description": "This preset is designed to download only the audio of the video. It will extract the audio, add chapters, metadata, and thumbnail." } diff --git a/app/tests/test_itemdto.py b/app/tests/test_itemdto.py index 0f192a72..c2e7e7fa 100644 --- a/app/tests/test_itemdto.py +++ b/app/tests/test_itemdto.py @@ -329,3 +329,57 @@ class TestItemDTO: mock_presets.return_value.get.assert_called_once_with("nonexistent") assert result is None + + +class TestItemAddExtras: + def test_add_extras_to_empty_dict(self): + """Test adding extras when extras dict is empty.""" + item = Item(url="https://example.com") + item.extras = {} + + item.add_extras("key1", "value1") + + assert item.extras["key1"] == "value1" + + def test_add_extras_when_extras_is_none(self): + """Test adding extras when extras is None.""" + item = Item(url="https://example.com") + item.extras = None + + item.add_extras("key1", "value1") + + assert item.extras == {"key1": "value1"} + + def test_add_extras_to_existing_dict(self): + """Test adding extras to an existing extras dict.""" + item = Item(url="https://example.com", extras={"existing": "data"}) + + item.add_extras("new_key", "new_value") + + assert item.extras["existing"] == "data" + assert item.extras["new_key"] == "new_value" + + def test_add_extras_overwrites_existing_key(self): + """Test that adding extras overwrites existing keys.""" + item = Item(url="https://example.com", extras={"key1": "old_value"}) + + item.add_extras("key1", "new_value") + + assert item.extras["key1"] == "new_value" + + def test_add_extras_with_various_types(self): + """Test adding extras with various data types.""" + item = Item(url="https://example.com") + item.extras = {} + + item.add_extras("string", "value") + item.add_extras("number", 42) + item.add_extras("boolean", True) + item.add_extras("list", [1, 2, 3]) + item.add_extras("dict", {"nested": "data"}) + + assert item.extras["string"] == "value" + assert item.extras["number"] == 42 + assert item.extras["boolean"] is True + assert item.extras["list"] == [1, 2, 3] + assert item.extras["dict"] == {"nested": "data"} diff --git a/app/tests/test_presets.py b/app/tests/test_presets.py index 95464e8f..1da4c6dc 100644 --- a/app/tests/test_presets.py +++ b/app/tests/test_presets.py @@ -39,6 +39,7 @@ class TestPreset: cookies="test_cookies", cli="--test-option", default=True, + priority=10, ) assert preset.id == preset_id @@ -48,6 +49,7 @@ class TestPreset: assert preset.template == "test_template" assert preset.cookies == "test_cookies" assert preset.cli == "--test-option" + assert preset.priority == 10 assert preset.default is True def test_preset_serialize(self): @@ -597,3 +599,100 @@ class TestPresets: # Should have logged errors for invalid presets mock_log.error.assert_called() + + @patch("app.library.Presets.Config") + @patch("app.library.Presets.EventBus") + def test_preset_priority_validation_integer(self, mock_eventbus, mock_config): + """Test that priority must be an integer.""" + mock_config_instance = Mock(config_path="/tmp", default_preset="default") + mock_config.get_instance.return_value = mock_config_instance + mock_eventbus.get_instance.return_value = Mock() + + with tempfile.TemporaryDirectory() as temp_dir: + presets_file = Path(temp_dir) / "presets.json" + presets = Presets(file=presets_file, config=mock_config_instance) + + # Test with non-integer priority + invalid_preset = {"id": str(uuid.uuid4()), "name": "test", "priority": "not_an_int"} + + with pytest.raises(ValueError, match="Priority must be an integer"): + presets.validate(invalid_preset) + + @patch("app.library.Presets.Config") + @patch("app.library.Presets.EventBus") + def test_preset_priority_validation_negative(self, mock_eventbus, mock_config): + """Test that priority must be >= 0.""" + mock_config_instance = Mock(config_path="/tmp", default_preset="default") + mock_config.get_instance.return_value = mock_config_instance + mock_eventbus.get_instance.return_value = Mock() + + with tempfile.TemporaryDirectory() as temp_dir: + presets_file = Path(temp_dir) / "presets.json" + presets = Presets(file=presets_file, config=mock_config_instance) + + # Test with negative priority + invalid_preset = {"id": str(uuid.uuid4()), "name": "test", "priority": -5} + + with pytest.raises(ValueError, match="Priority must be >= 0"): + presets.validate(invalid_preset) + + @patch("app.library.Presets.Config") + @patch("app.library.Presets.EventBus") + def test_preset_priority_sorting(self, mock_eventbus, mock_config): + """Test that presets are sorted by priority in descending order.""" + mock_config_instance = Mock(config_path="/tmp", default_preset="default") + mock_config.get_instance.return_value = mock_config_instance + mock_eventbus.get_instance.return_value = Mock() + + with tempfile.TemporaryDirectory() as temp_dir: + presets_file = Path(temp_dir) / "presets.json" + presets = Presets(file=presets_file, config=mock_config_instance) + + # Create presets with different priorities + preset1 = Preset(name="low", priority=1) + preset2 = Preset(name="high", priority=10) + preset3 = Preset(name="medium", priority=5) + + presets.save([preset1, preset2, preset3]).load() + + all_presets = presets.get_all() + non_default = [p for p in all_presets if not p.default] + + # Should be sorted by priority descending + assert non_default[0].name == "high" + assert non_default[0].priority == 10 + assert non_default[1].name == "medium" + assert non_default[1].priority == 5 + assert non_default[2].name == "low" + assert non_default[2].priority == 1 + + @patch("app.library.Presets.Config") + @patch("app.library.Presets.EventBus") + def test_preset_priority_default_zero(self, mock_eventbus, mock_config): + """Test that priority defaults to 0 if not specified.""" + preset = Preset(name="test") + assert preset.priority == 0 + + @patch("app.library.Presets.Config") + @patch("app.library.Presets.EventBus") + def test_preset_priority_migration(self, mock_eventbus, mock_config): + """Test that old presets without priority get it added on load.""" + mock_config_instance = Mock(config_path="/tmp", default_preset="default") + mock_config.get_instance.return_value = mock_config_instance + mock_eventbus.get_instance.return_value = Mock() + + with tempfile.TemporaryDirectory() as temp_dir: + presets_file = Path(temp_dir) / "presets.json" + + # Create a preset file without priority field + old_preset_data = [{"id": str(uuid.uuid4()), "name": "old_preset", "cli": ""}] + presets_file.write_text(json.dumps(old_preset_data)) + + # Load presets - should migrate by adding priority + presets = Presets(file=presets_file, config=mock_config_instance) + presets.load() + + # Check that priority was added + loaded_data = json.loads(presets_file.read_text()) + assert "priority" in loaded_data[0] + assert loaded_data[0]["priority"] == 0 diff --git a/app/tests/test_utils.py b/app/tests/test_utils.py index aa5bc1f9..7c1221f5 100644 --- a/app/tests/test_utils.py +++ b/app/tests/test_utils.py @@ -2487,3 +2487,191 @@ class TestMoveFile: # Original file should still exist assert test_file.exists() + + +class TestGetThumbnail: + def test_returns_none_for_empty_list(self): + """Test that None is returned for an empty thumbnail list.""" + from app.library.Utils import get_thumbnail + + assert get_thumbnail([]) is None + + def test_returns_none_for_non_list(self): + """Test that None is returned for non-list input.""" + from app.library.Utils import get_thumbnail + + assert get_thumbnail(None) is None + assert get_thumbnail("not a list") is None + assert get_thumbnail({"not": "list"}) is None + + def test_returns_highest_preference_thumbnail(self): + """Test that the thumbnail with highest preference is returned.""" + from app.library.Utils import get_thumbnail + + thumbnails = [ + {"url": "low.jpg", "preference": 1, "width": 100, "height": 100}, + {"url": "high.jpg", "preference": 10, "width": 200, "height": 200}, + {"url": "medium.jpg", "preference": 5, "width": 150, "height": 150}, + ] + + result = get_thumbnail(thumbnails) + assert result == {"url": "high.jpg", "preference": 10, "width": 200, "height": 200} + + def test_returns_highest_width_when_preference_equal(self): + """Test that the thumbnail with highest width is returned when preference is equal.""" + from app.library.Utils import get_thumbnail + + thumbnails = [ + {"url": "small.jpg", "preference": 1, "width": 100, "height": 100}, + {"url": "large.jpg", "preference": 1, "width": 200, "height": 200}, + {"url": "medium.jpg", "preference": 1, "width": 150, "height": 150}, + ] + + result = get_thumbnail(thumbnails) + assert result == {"url": "large.jpg", "preference": 1, "width": 200, "height": 200} + + def test_handles_missing_attributes(self): + """Test that thumbnails with missing attributes are handled correctly.""" + from app.library.Utils import get_thumbnail + + thumbnails = [ + {"url": "no_pref.jpg", "width": 100}, + {"url": "with_pref.jpg", "preference": 5, "width": 50}, + ] + + result = get_thumbnail(thumbnails) + assert result["url"] == "with_pref.jpg" + + def test_returns_first_when_all_equal(self): + """Test that any thumbnail is returned when all attributes are equal.""" + from app.library.Utils import get_thumbnail + + thumbnails = [ + {"url": "first.jpg"}, + {"url": "second.jpg"}, + ] + + result = get_thumbnail(thumbnails) + assert result is not None + assert result["url"] in ["first.jpg", "second.jpg"] + + +class TestGetExtras: + def test_returns_empty_dict_for_none(self): + """Test that empty dict is returned for None input.""" + from app.library.Utils import get_extras + + assert get_extras(None) == {} + + def test_returns_empty_dict_for_non_dict(self): + """Test that empty dict is returned for non-dict input.""" + from app.library.Utils import get_extras + + assert get_extras("not a dict") == {} + assert get_extras([]) == {} + + def test_extracts_video_information(self): + """Test extracting information from a video entry.""" + from app.library.Utils import get_extras + + entry = { + "id": "test123", + "title": "Test Video", + "uploader": "Test Uploader", + "channel": "Test Channel", + "thumbnails": [{"url": "thumb.jpg", "preference": 1}], + "duration": 120, + } + + result = get_extras(entry, kind="video") + + assert result["uploader"] == "Test Uploader" + assert result["channel"] == "Test Channel" + assert result["thumbnail"] == "thumb.jpg" + assert result["duration"] == 120 + assert result["is_premiere"] is False + + def test_extracts_playlist_information(self): + """Test extracting information from a playlist entry.""" + from app.library.Utils import get_extras + + entry = { + "id": "playlist123", + "title": "Test Playlist", + "uploader": "Playlist Owner", + "uploader_id": "owner123", + } + + result = get_extras(entry, kind="playlist") + + assert result["playlist_id"] == "playlist123" + assert result["playlist_title"] == "Test Playlist" + assert result["playlist_uploader"] == "Playlist Owner" + assert result["playlist_uploader_id"] == "owner123" + + def test_handles_release_timestamp(self): + """Test handling of release_timestamp for upcoming content.""" + from app.library.Utils import get_extras + + entry = { + "release_timestamp": 1234567890, + } + + result = get_extras(entry) + + assert "release_in" in result + assert result["release_in"] == "Fri, 13 Feb 2009 23:31:30 GMT" + + def test_handles_upcoming_live_stream(self): + """Test handling of upcoming live stream.""" + from app.library.Utils import get_extras + + entry = { + "release_timestamp": 1234567890, + "live_status": "is_upcoming", + } + + result = get_extras(entry) + + assert result["is_live"] == 1234567890 + assert "release_in" in result + + def test_handles_premiere_flag(self): + """Test handling of is_premiere flag.""" + from app.library.Utils import get_extras + + entry = { + "is_premiere": True, + } + + result = get_extras(entry) + assert result["is_premiere"] is True + + entry2 = {"is_premiere": False} + result2 = get_extras(entry2) + assert result2["is_premiere"] is False + + def test_youtube_fallback_thumbnail(self): + """Test fallback thumbnail generation for YouTube videos.""" + from app.library.Utils import get_extras + + entry = { + "id": "dQw4w9WgXcQ", + "ie_key": "Youtube", + } + + result = get_extras(entry) + + assert result["thumbnail"] == "https://img.youtube.com/vi/dQw4w9WgXcQ/maxresdefault.jpg" + + def test_thumbnail_string_fallback(self): + """Test fallback to thumbnail string when thumbnails list not available.""" + from app.library.Utils import get_extras + + entry = { + "thumbnail": "https://example.com/thumb.jpg", + } + + result = get_extras(entry) + + assert result["thumbnail"] == "https://example.com/thumb.jpg" diff --git a/ui/app/components/History.vue b/ui/app/components/History.vue index a83fa427..4c3842ab 100644 --- a/ui/app/components/History.vue +++ b/ui/app/components/History.vue @@ -340,7 +340,7 @@ v-tooltip="moment(item.datetime).format('YYYY-M-DD H:mm Z')" v-rtime="item.datetime" />
- {{ formatBytes(item.file_size) }} + {{ formatBytes(item.file_size) }}
diff --git a/ui/app/components/PresetForm.vue b/ui/app/components/PresetForm.vue index 2f95e03e..0ab71337 100644 --- a/ui/app/components/PresetForm.vue +++ b/ui/app/components/PresetForm.vue @@ -86,8 +86,7 @@ - -
+
+
+
+ +
+ +
+ + + Higher priority presets appear first in the list. + +
+
+