diff --git a/lib/pinchflat/downloading/media_downloader.ex b/lib/pinchflat/downloading/media_downloader.ex index 249752c..22c1370 100644 --- a/lib/pinchflat/downloading/media_downloader.ex +++ b/lib/pinchflat/downloading/media_downloader.ex @@ -152,7 +152,7 @@ defmodule Pinchflat.Downloading.MediaDownloader do defp download_with_options(url, item_with_preloads, output_filepath, override_opts) do {:ok, options} = DownloadOptionBuilder.build(item_with_preloads, override_opts) - # TODO: test + use_cookies = Sources.use_cookies?(item_with_preloads.source, :downloading) runner_opts = [output_filepath: output_filepath, use_cookies: use_cookies] diff --git a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex index bda355e..15a4342 100644 --- a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex +++ b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex @@ -89,13 +89,15 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do defp create_media_item_from_media_id(source, media_id) do url = "https://www.youtube.com/watch?v=#{media_id}" + # This is set to :metadata instead of :indexing since this happens _after_ the + # actual indexing process. In reality, slow indexing is the only thing that + # should be using :indexing. should_use_cookies = Sources.use_cookies?(source, :metadata) command_opts = [output: DownloadOptionBuilder.build_output_path_for(source)] ++ DownloadOptionBuilder.build_quality_options_for(source) - # TODO: test case YtDlpMedia.get_media_attributes(url, command_opts, use_cookies: should_use_cookies) do {:ok, media_attrs} -> Media.create_media_item_from_backend_attrs(source, media_attrs) diff --git a/lib/pinchflat/metadata/metadata_file_helpers.ex b/lib/pinchflat/metadata/metadata_file_helpers.ex index bb8e4ea..842f6ef 100644 --- a/lib/pinchflat/metadata/metadata_file_helpers.ex +++ b/lib/pinchflat/metadata/metadata_file_helpers.ex @@ -67,7 +67,6 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do yt_dlp_filepath = generate_filepath_for(media_item_with_preloads, "thumbnail.%(ext)s") real_filepath = generate_filepath_for(media_item_with_preloads, "thumbnail.jpg") command_opts = [output: yt_dlp_filepath] - # TODO: test addl_opts = [use_cookies: Sources.use_cookies?(media_item_with_preloads.source, :metadata)] case YtDlpMedia.download_thumbnail(media_item_with_preloads.original_url, command_opts, addl_opts) do diff --git a/lib/pinchflat/metadata/source_metadata_storage_worker.ex b/lib/pinchflat/metadata/source_metadata_storage_worker.ex index adaae95..feb793b 100644 --- a/lib/pinchflat/metadata/source_metadata_storage_worker.ex +++ b/lib/pinchflat/metadata/source_metadata_storage_worker.ex @@ -113,7 +113,6 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do defp fetch_metadata_for_source(source) do tmp_output_path = "#{tmp_directory()}/#{StringUtils.random_string(16)}/source_image.%(ext)S" base_opts = [convert_thumbnails: "jpg", output: tmp_output_path] - # TODO: test should_use_cookies = Sources.use_cookies?(source, :metadata) opts = @@ -123,7 +122,6 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do base_opts ++ [:write_thumbnail, playlist_items: 1] end - # TODO: test MediaCollection.get_source_metadata(source.original_url, opts, use_cookies: should_use_cookies) end diff --git a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex index 3b24b3e..8721b21 100644 --- a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex +++ b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex @@ -132,7 +132,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do {:ok, pid} = FileFollowerServer.start_link() handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end - # TODO: test should_use_cookies = Sources.use_cookies?(source, :indexing) command_opts = diff --git a/lib/pinchflat/sources/source.ex b/lib/pinchflat/sources/source.ex index d595ef0..aa4966a 100644 --- a/lib/pinchflat/sources/source.ex +++ b/lib/pinchflat/sources/source.ex @@ -78,6 +78,7 @@ defmodule Pinchflat.Sources.Source do field :collection_type, Ecto.Enum, values: [:channel, :playlist] field :index_frequency_minutes, :integer, default: 60 * 24 field :fast_index, :boolean, default: false + # TODO: consider renaming `indexing_only` to something like `when_needed` (but better) field :cookie_behaviour, Ecto.Enum, values: [:disabled, :indexing_only, :all_operations], default: :disabled field :download_media, :boolean, default: true field :last_indexed_at, :utc_datetime diff --git a/lib/pinchflat/sources/sources.ex b/lib/pinchflat/sources/sources.ex index a0f1313..40c0ad0 100644 --- a/lib/pinchflat/sources/sources.ex +++ b/lib/pinchflat/sources/sources.ex @@ -32,7 +32,11 @@ defmodule Pinchflat.Sources do source.output_path_template_override || media_profile.output_path_template end - # TODO: test + @doc """ + Returns a boolean indicating whether or not cookies should be used for a given operation. + + Returns boolean() + """ def use_cookies?(source, operation) when operation in [:indexing, :downloading, :metadata] do case source.cookie_behaviour do :disabled -> false @@ -190,7 +194,6 @@ defmodule Pinchflat.Sources do defp add_source_details_to_changeset(source, changeset) do original_url = changeset.changes.original_url - # TODO: test should_use_cookies = Ecto.Changeset.get_field(changeset, :cookie_behaviour) == :all_operations # Skipping sleep interval since this is UI blocking and we want to keep this as fast as possible addl_opts = [use_cookies: should_use_cookies, skip_sleep_interval: true] diff --git a/test/pinchflat/downloading/media_downloader_test.exs b/test/pinchflat/downloading/media_downloader_test.exs index 5bda8cc..a2b9a9d 100644 --- a/test/pinchflat/downloading/media_downloader_test.exs +++ b/test/pinchflat/downloading/media_downloader_test.exs @@ -157,15 +157,16 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do {:ok, ""} end) - source = source_fixture(%{use_cookies: true}) + source = source_fixture(%{cookie_behaviour: :all_operations}) media_item = media_item_fixture(%{source_id: source.id}) assert {:ok, _} = MediaDownloader.download_for_media_item(media_item) end - test "does not set use_cookies if the source does not use cookies" do + test "does not set use_cookies if the source only uses cookies when indexing" do expect(YtDlpRunnerMock, :run, 3, fn - _url, :get_downloadable_status, _opts, _ot, _addl -> + _url, :get_downloadable_status, _opts, _ot, addl -> + assert {:use_cookies, false} in addl {:ok, "{}"} _url, :download, _opts, _ot, addl -> @@ -176,7 +177,27 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do {:ok, ""} end) - source = source_fixture(%{use_cookies: false}) + source = source_fixture(%{cookie_behaviour: :indexing_only}) + media_item = media_item_fixture(%{source_id: source.id}) + + assert {:ok, _} = MediaDownloader.download_for_media_item(media_item) + end + + test "does not set use_cookies if the source does not use cookies" do + expect(YtDlpRunnerMock, :run, 3, fn + _url, :get_downloadable_status, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + {:ok, "{}"} + + _url, :download, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + {:ok, render_metadata(:media_metadata)} + + _url, :download_thumbnail, _opts, _ot, _addl -> + {:ok, ""} + end) + + source = source_fixture(%{cookie_behaviour: :disabled}) media_item = media_item_fixture(%{source_id: source.id}) assert {:ok, _} = MediaDownloader.download_for_media_item(media_item) diff --git a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs index a798275..0cfcb77 100644 --- a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs +++ b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs @@ -104,34 +104,6 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do FastIndexingHelpers.index_and_kickoff_downloads(source) end - test "sets use_cookies if the source uses cookies" do - expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) - - stub(YtDlpRunnerMock, :run, fn _url, :get_media_attributes, _opts, _ot, addl -> - assert {:use_cookies, true} in addl - - {:ok, media_attributes_return_fixture()} - end) - - source = source_fixture(%{use_cookies: true}) - - assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source) - end - - test "does not set use_cookies if the source does not use cookies" do - expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) - - stub(YtDlpRunnerMock, :run, fn _url, :get_media_attributes, _opts, _ot, addl -> - assert {:use_cookies, false} in addl - - {:ok, media_attributes_return_fixture()} - end) - - source = source_fixture(%{use_cookies: false}) - - assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source) - end - test "does not enqueue a download job if the media item does not match the format rules" do expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) @@ -180,6 +152,50 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do end end + describe "index_and_kickoff_downloads/1 when testing cookies" do + test "sets use_cookies if the source uses cookies" do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + stub(YtDlpRunnerMock, :run, fn _url, :get_media_attributes, _opts, _ot, addl -> + assert {:use_cookies, true} in addl + + {:ok, media_attributes_return_fixture()} + end) + + source = source_fixture(%{cookie_behaviour: :all_operations}) + + assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source) + end + + test "does not set use_cookies if the source only uses cookies when indexing" do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + stub(YtDlpRunnerMock, :run, fn _url, :get_media_attributes, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + + {:ok, media_attributes_return_fixture()} + end) + + source = source_fixture(%{cookie_behaviour: :indexing_only}) + + assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source) + end + + test "does not set use_cookies if the source does not use cookies" do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + stub(YtDlpRunnerMock, :run, fn _url, :get_media_attributes, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + + {:ok, media_attributes_return_fixture()} + end) + + source = source_fixture(%{cookie_behaviour: :disabled}) + + assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source) + end + end + describe "index_and_kickoff_downloads/1 when testing backends" do test "uses the YouTube API if it is enabled", %{source: source} do expect(HTTPClientMock, :get, fn url, _headers -> diff --git a/test/pinchflat/metadata/metadata_file_helpers_test.exs b/test/pinchflat/metadata/metadata_file_helpers_test.exs index b193d9a..1b4961b 100644 --- a/test/pinchflat/metadata/metadata_file_helpers_test.exs +++ b/test/pinchflat/metadata/metadata_file_helpers_test.exs @@ -88,13 +88,35 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do Helpers.download_and_store_thumbnail_for(media_item) end + test "returns nil if yt-dlp fails", %{media_item: media_item} do + stub(YtDlpRunnerMock, :run, fn _url, :download_thumbnail, _opts, _ot, _addl -> {:error, "error"} end) + + filepath = Helpers.download_and_store_thumbnail_for(media_item) + + assert filepath == nil + end + end + + describe "download_and_store_thumbnail_for/2 when testing cookie usage" do test "sets use_cookies if the source uses cookies" do expect(YtDlpRunnerMock, :run, fn _url, :download_thumbnail, _opts, _ot, addl -> assert {:use_cookies, true} in addl {:ok, ""} end) - source = source_fixture(%{use_cookies: true}) + source = source_fixture(%{cookie_behaviour: :all_operations}) + media_item = Repo.preload(media_item_fixture(%{source_id: source.id}), :source) + + Helpers.download_and_store_thumbnail_for(media_item) + end + + test "does not set use_cookies if the source only uses cookies when indexing" do + expect(YtDlpRunnerMock, :run, fn _url, :download_thumbnail, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + {:ok, ""} + end) + + source = source_fixture(%{cookie_behaviour: :indexing_only}) media_item = Repo.preload(media_item_fixture(%{source_id: source.id}), :source) Helpers.download_and_store_thumbnail_for(media_item) @@ -106,19 +128,11 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do {:ok, ""} end) - source = source_fixture(%{use_cookies: false}) + source = source_fixture(%{cookie_behaviour: :disabled}) media_item = Repo.preload(media_item_fixture(%{source_id: source.id}), :source) Helpers.download_and_store_thumbnail_for(media_item) end - - test "returns nil if yt-dlp fails", %{media_item: media_item} do - stub(YtDlpRunnerMock, :run, fn _url, :download_thumbnail, _opts, _ot, _addl -> {:error, "error"} end) - - filepath = Helpers.download_and_store_thumbnail_for(media_item) - - assert filepath == nil - end end describe "parse_upload_date/1" do diff --git a/test/pinchflat/metadata/source_metadata_storage_worker_test.exs b/test/pinchflat/metadata/source_metadata_storage_worker_test.exs index 2b08a2f..e363e65 100644 --- a/test/pinchflat/metadata/source_metadata_storage_worker_test.exs +++ b/test/pinchflat/metadata/source_metadata_storage_worker_test.exs @@ -254,7 +254,23 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do end) profile = media_profile_fixture(%{download_source_images: true}) - source = source_fixture(media_profile_id: profile.id, use_cookies: true) + source = source_fixture(media_profile_id: profile.id, cookie_behaviour: :all_operations) + + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + end + + test "does not set use_cookies if the source only uses cookies when indexing" do + expect(YtDlpRunnerMock, :run, 2, fn + _url, :get_source_details, _opts, _ot, _addl -> + {:ok, source_details_return_fixture()} + + _url, :get_source_metadata, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + {:ok, render_metadata(:channel_source_metadata)} + end) + + profile = media_profile_fixture(%{download_source_images: true}) + source = source_fixture(media_profile_id: profile.id, cookie_behaviour: :indexing_only) perform_job(SourceMetadataStorageWorker, %{id: source.id}) end @@ -270,7 +286,7 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do end) profile = media_profile_fixture(%{download_source_images: true}) - source = source_fixture(media_profile_id: profile.id, use_cookies: false) + source = source_fixture(media_profile_id: profile.id, cookie_behaviour: :disabled) perform_job(SourceMetadataStorageWorker, %{id: source.id}) end @@ -323,7 +339,21 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do {:ok, "{}"} end) - source = source_fixture(%{series_directory: nil, use_cookies: true}) + source = source_fixture(%{series_directory: nil, cookie_behaviour: :all_operations}) + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + end + + test "does not set use_cookies if the source only uses cookies when indexing" do + expect(YtDlpRunnerMock, :run, 2, fn + _url, :get_source_details, _opts, _ot, addl -> + assert {:use_cookies, false} in addl + {:ok, source_details_return_fixture()} + + _url, :get_source_metadata, _opts, _ot, _addl -> + {:ok, "{}"} + end) + + source = source_fixture(%{series_directory: nil, cookie_behaviour: :indexing_only}) perform_job(SourceMetadataStorageWorker, %{id: source.id}) end @@ -337,7 +367,7 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do {:ok, "{}"} end) - source = source_fixture(%{series_directory: nil, use_cookies: false}) + source = source_fixture(%{series_directory: nil, cookie_behaviour: :disabled}) perform_job(SourceMetadataStorageWorker, %{id: source.id}) end end diff --git a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs index 6e00622..e74613f 100644 --- a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs +++ b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs @@ -302,14 +302,27 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source) end + end + describe "index_and_enqueue_download_for_media_items/2 when testing cookies" do test "sets use_cookies if the source uses cookies" do expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, _opts, _ot, addl_opts -> assert {:use_cookies, true} in addl_opts {:ok, source_attributes_return_fixture()} end) - source = source_fixture(%{use_cookies: true}) + source = source_fixture(%{cookie_behaviour: :all_operations}) + + SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source) + end + + test "sets use_cookies if the source only uses cookies when indexing" do + expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, _opts, _ot, addl_opts -> + assert {:use_cookies, true} in addl_opts + {:ok, source_attributes_return_fixture()} + end) + + source = source_fixture(%{cookie_behaviour: :indexing_only}) SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source) end @@ -320,7 +333,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do {:ok, source_attributes_return_fixture()} end) - source = source_fixture(%{use_cookies: false}) + source = source_fixture(%{cookie_behaviour: :disabled}) SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source) end diff --git a/test/pinchflat/sources_test.exs b/test/pinchflat/sources_test.exs index e8e02ae..531cae1 100644 --- a/test/pinchflat/sources_test.exs +++ b/test/pinchflat/sources_test.exs @@ -60,6 +60,28 @@ defmodule Pinchflat.SourcesTest do end end + describe "use_cookies?/2" do + test "returns true if the source has been set to use cookies" do + source = source_fixture(%{cookie_behaviour: :all_operations}) + assert Sources.use_cookies?(source, :downloading) + end + + test "returns false if the source has not been set to use cookies" do + source = source_fixture(%{cookie_behaviour: :disabled}) + refute Sources.use_cookies?(source, :downloading) + end + + test "returns true if the action is indexing and the source is set to :indexing_only" do + source = source_fixture(%{cookie_behaviour: :indexing_only}) + assert Sources.use_cookies?(source, :indexing) + end + + test "returns false if the action is downloading and the source is set to :indexing_only" do + source = source_fixture(%{cookie_behaviour: :indexing_only}) + refute Sources.use_cookies?(source, :downloading) + end + end + describe "list_sources/0" do test "it returns all sources" do source = source_fixture() @@ -393,13 +415,13 @@ defmodule Pinchflat.SourcesTest do valid_attrs = %{ media_profile_id: media_profile_fixture().id, original_url: "https://www.youtube.com/channel/abc123", - use_cookies: true + cookie_behaviour: :all_operations } assert {:ok, %Source{}} = Sources.create_source(valid_attrs) end - test "does not set use_cookies to false if the source has not been set to use cookies" do + test "does not set use_cookies if the source only uses cookies when indexing" do expect(YtDlpRunnerMock, :run, fn _url, :get_source_details, _opts, _ot, addl -> refute Keyword.get(addl, :use_cookies) @@ -409,7 +431,23 @@ defmodule Pinchflat.SourcesTest do valid_attrs = %{ media_profile_id: media_profile_fixture().id, original_url: "https://www.youtube.com/channel/abc123", - use_cookies: false + cookie_behaviour: :indexing_only + } + + assert {:ok, %Source{}} = Sources.create_source(valid_attrs) + end + + test "does not set use_cookies if the source has not been set to use cookies" do + expect(YtDlpRunnerMock, :run, fn _url, :get_source_details, _opts, _ot, addl -> + refute Keyword.get(addl, :use_cookies) + + {:ok, playlist_return()} + end) + + valid_attrs = %{ + media_profile_id: media_profile_fixture().id, + original_url: "https://www.youtube.com/channel/abc123", + cookie_behaviour: :disabled } assert {:ok, %Source{}} = Sources.create_source(valid_attrs)