diff --git a/lib/pinchflat/downloading/downloading_helpers.ex b/lib/pinchflat/downloading/downloading_helpers.ex index 1e61545..a7f64e3 100644 --- a/lib/pinchflat/downloading/downloading_helpers.ex +++ b/lib/pinchflat/downloading/downloading_helpers.ex @@ -7,9 +7,11 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do require Logger + alias Pinchflat.Repo alias Pinchflat.Media alias Pinchflat.Tasks alias Pinchflat.Sources.Source + alias Pinchflat.Media.MediaItem alias Pinchflat.Downloading.MediaDownloadWorker @doc """ @@ -43,4 +45,23 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do |> Media.list_pending_media_items_for() |> Enum.each(&Tasks.delete_pending_tasks_for/1) end + + @doc """ + Takes a single media item and enqueues a download job if the media should be + downloaded, based on the source's download settings and whether media is + considered pending. + + Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()} + """ + def kickoff_download_if_pending(%MediaItem{} = media_item) do + media_item = Repo.preload(media_item, :source) + + if media_item.source.download_media && Media.pending_download?(media_item) do + Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})") + + MediaDownloadWorker.kickoff_with_task(media_item) + else + {:error, :should_not_download} + end + end end diff --git a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex index b9d5a44..b12eff2 100644 --- a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex +++ b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex @@ -5,64 +5,45 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do Many of these methods are made to be kickoff or be consumed by workers. """ + require Logger + alias Pinchflat.Repo alias Pinchflat.Media alias Pinchflat.Sources.Source alias Pinchflat.Media.MediaQuery alias Pinchflat.FastIndexing.YoutubeRss - alias Pinchflat.Downloading.MediaDownloadWorker - alias Pinchflat.FastIndexing.MediaIndexingWorker + alias Pinchflat.Downloading.DownloadingHelpers alias Pinchflat.YtDlp.Media, as: YtDlpMedia @doc """ - Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks - for any new media items. See comments in `MediaIndexingWorker` for more info on the + Fetches new media IDs from a source's YouTube RSS feed, indexes them, and kicks off downloading + tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the order of operations and how this fits into the indexing process. - Despite the similar name to `kickoff_fast_indexing_task`, this does work differently. - `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this - function starts individual indexing tasks for each new media item. I think it does - make sense grammatically, but I could see how that's confusing. - - Returns [binary()] where each binary is the media ID of a new media item. + Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily + downloaded_. """ - def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do + def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source) existing_media_items = list_media_items_by_media_id_for(source, media_ids) new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id) - Enum.each(new_media_ids, fn media_id -> - url = "https://www.youtube.com/watch?v=#{media_id}" + maybe_new_media_items = + Enum.map(new_media_ids, fn media_id -> + case create_media_item_from_media_id(source, media_id) do + {:ok, media_item} -> + media_item - MediaIndexingWorker.kickoff_with_task(source, url) - end) - - new_media_ids - end - - @doc """ - Indexes a single media item for a source and enqueues a download job if the - media should be downloaded. This method creates the media item record so it's - the one-stop-shop for adding a media item (and possibly downloading it) just - by a URL and source. - - Returns {:ok, media_item} | {:error, any()} - """ - def index_and_enqueue_download_for_media_item(%Source{} = source, url) do - maybe_media_item = create_media_item_from_url(source, url) - - case maybe_media_item do - {:ok, media_item} -> - if source.download_media && Media.pending_download?(media_item) do - MediaDownloadWorker.kickoff_with_task(media_item) + err -> + Logger.error("Error creating media item '#{media_id}' from URL: #{inspect(err)}") + nil end + end) - {:ok, media_item} + DownloadingHelpers.enqueue_pending_download_tasks(source) - err -> - err - end + Enum.filter(maybe_new_media_items, & &1) end defp list_media_items_by_media_id_for(source, media_ids) do @@ -72,9 +53,15 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do |> Repo.all() end - defp create_media_item_from_url(source, url) do - {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url) + defp create_media_item_from_media_id(source, media_id) do + url = "https://www.youtube.com/watch?v=#{media_id}" - Media.create_media_item_from_backend_attrs(source, media_attrs) + case YtDlpMedia.get_media_attributes(url) do + {:ok, media_attrs} -> + Media.create_media_item_from_backend_attrs(source, media_attrs) + + err -> + err + end end end diff --git a/lib/pinchflat/fast_indexing/fast_indexing_worker.ex b/lib/pinchflat/fast_indexing/fast_indexing_worker.ex index 15d287f..0f85d40 100644 --- a/lib/pinchflat/fast_indexing/fast_indexing_worker.ex +++ b/lib/pinchflat/fast_indexing/fast_indexing_worker.ex @@ -10,6 +10,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do alias __MODULE__ alias Pinchflat.Tasks + alias Pinchflat.Media alias Pinchflat.Sources alias Pinchflat.Settings alias Pinchflat.Sources.Source @@ -28,9 +29,21 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do end @doc """ - Kicks off the fast indexing process for a source, reschedules the job to run again - once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments - for more + Similar to `MediaCollectionIndexingWorker`, but for working with RSS feeds. + `MediaCollectionIndexingWorker` should be preferred in general, but this is + useful for downloading small batches of media items via fast indexing. + + Only kicks off downloads for media that _should_ be downloaded + (ie: the source is set to download and the media matches the profile's format preferences) + + Order of operations: + 1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media. + with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed` + 2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed, + it indexes them with a yt-dlp call to create the media item records then kicks off downloading + tasks (MediaDownloadWorker) for any new media items _that should be downloaded_. + 3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable + then reschedules itself to run again in the future. Returns :ok | {:ok, :job_exists} | {:ok, %Task{}} """ @@ -39,7 +52,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do source = Sources.get_source!(source_id) if source.fast_index do - perform_indexing_and_notification(source) + perform_indexing_and_send_notification(source) reschedule_indexing(source) else :ok @@ -49,11 +62,17 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale") end - defp perform_indexing_and_notification(source) do + defp perform_indexing_and_send_notification(source) do apprise_server = Settings.get!(:apprise_server) - new_media_items = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source) - SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items)) + new_media_items = + source + |> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed() + |> Enum.filter(&Media.pending_download?(&1)) + + if source.download_media do + SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items)) + end end defp reschedule_indexing(source) do diff --git a/lib/pinchflat/fast_indexing/media_indexing_worker.ex b/lib/pinchflat/fast_indexing/media_indexing_worker.ex deleted file mode 100644 index c4da1e1..0000000 --- a/lib/pinchflat/fast_indexing/media_indexing_worker.ex +++ /dev/null @@ -1,69 +0,0 @@ -defmodule Pinchflat.FastIndexing.MediaIndexingWorker do - @moduledoc false - - use Oban.Worker, - queue: :media_indexing, - unique: [period: :infinity, states: [:available, :scheduled, :retryable]], - tags: ["media_source", "media_indexing"] - - require Logger - - alias __MODULE__ - alias Pinchflat.Tasks - alias Pinchflat.Sources - alias Pinchflat.FastIndexing.FastIndexingHelpers - - @doc """ - Starts the fast media indexing worker and creates a task for the source. - - Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}} - """ - def kickoff_with_task(source, media_url, opts \\ []) do - %{id: source.id, media_url: media_url} - |> MediaIndexingWorker.new(opts) - |> Tasks.create_job_with_task(source) - end - - @doc """ - Similar to `MediaCollectionIndexingWorker`, but for individual media items. - Does not reschedule or check anything to do with a source's indexing - frequency - only collects initial metadata then kicks off a download. - `MediaCollectionIndexingWorker` should be preferred in general, but this is - useful for downloading one-off media items based on a URL (like for fast indexing). - - Only downloads media that _should_ be downloaded (ie: the source is set to download - and the media matches the profile's format preferences) - - Order of operations: - 1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running - in its own worker) periodically checks the YouTube RSS feed for new media - 2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media - item - 3. This worker fetches the media metadata and uses that to determine if it should be - downloaded. If so, it enqueues a MediaDownloadWorker - - Each is a worker because they all either need to be scheduled periodically or call out to - an external service and will be long-running. They're split into different jobs to separate - retry logic for each step and allow us to better optimize various queues (eg: the indexing - steps can keep running while the slow download steps are worked through). - - Returns :ok - """ - @impl Oban.Worker - def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do - source = Sources.get_source!(source_id) - - case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do - {:ok, media_item} -> - Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})") - - {:error, reason} -> - Logger.debug("Failed to index and enqueue download for url: #{media_url} (reason: #{inspect(reason)})") - end - - :ok - rescue - Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found") - Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale") - end -end diff --git a/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex b/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex index 0e982eb..678a9a5 100644 --- a/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex +++ b/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex @@ -61,9 +61,8 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do 5. If the source uses fast indexing, that job is kicked off as well. It uses RSS to run a smaller, faster, and more frequent index. That job handles rescheduling itself but largely has a similar behaviour to this - job in that it kicks off index and maybe download jobs. The biggest difference - is that an index job is kicked off _for each new media item_ as opposed - to one larger index job. Check out `MediaIndexingWorker` comments for more. + job in that it runs and index and maybe kicks off media download jobs. + Check out `FastIndexingWorker` comments for more. 6. If the job reschedules, the cycle from step 3 repeats until the heat death of the universe. The user changing things like the index frequency can dequeue or reschedule jobs as well diff --git a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex index b6b118a..2434b92 100644 --- a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex +++ b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex @@ -16,7 +16,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do alias Pinchflat.YtDlp.MediaCollection alias Pinchflat.Downloading.DownloadingHelpers alias Pinchflat.SlowIndexing.FileFollowerServer - alias Pinchflat.Downloading.MediaDownloadWorker alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker alias Pinchflat.YtDlp.Media, as: YtDlpMedia @@ -29,7 +28,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do """ def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do Tasks.delete_pending_tasks_for(source, "FastIndexingWorker") - Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker") Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker") MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts) @@ -127,11 +125,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do case Media.create_media_item_from_backend_attrs(source, media_attrs) do {:ok, %MediaItem{} = media_item} -> - if source.download_media && Media.pending_download?(media_item) do - Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}") - - MediaDownloadWorker.kickoff_with_task(media_item) - end + DownloadingHelpers.kickoff_download_if_pending(media_item) {:error, changeset} -> changeset diff --git a/lib/pinchflat/sources/sources.ex b/lib/pinchflat/sources/sources.ex index b57a628..f9a3658 100644 --- a/lib/pinchflat/sources/sources.ex +++ b/lib/pinchflat/sources/sources.ex @@ -288,7 +288,6 @@ defmodule Pinchflat.Sources do %{index_frequency_minutes: _} -> Tasks.delete_pending_tasks_for(source, "FastIndexingWorker") - Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker") Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker") _ -> diff --git a/test/pinchflat/downloading/downloading_helpers_test.exs b/test/pinchflat/downloading/downloading_helpers_test.exs index b54a5af..1c3c95b 100644 --- a/test/pinchflat/downloading/downloading_helpers_test.exs +++ b/test/pinchflat/downloading/downloading_helpers_test.exs @@ -4,6 +4,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do import Mox import Pinchflat.MediaFixtures import Pinchflat.SourcesFixtures + import Pinchflat.ProfilesFixtures alias Pinchflat.Tasks alias Pinchflat.Downloading.DownloadingHelpers @@ -72,4 +73,44 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do assert [] = Tasks.list_tasks_for(media_item) end end + + describe "kickoff_download_if_pending/1" do + setup do + media_item = media_item_fixture(media_filepath: nil) + + {:ok, media_item: media_item} + end + + test "enqueues a download job", %{media_item: media_item} do + assert {:ok, _} = DownloadingHelpers.kickoff_download_if_pending(media_item) + + assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id}) + end + + test "creates and returns a download task record", %{media_item: media_item} do + assert {:ok, task} = DownloadingHelpers.kickoff_download_if_pending(media_item) + + assert [found_task] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker") + assert task.id == found_task.id + end + + test "does not enqueue a download job if the source does not allow it" do + source = source_fixture(%{download_media: false}) + media_item = media_item_fixture(source_id: source.id, media_filepath: nil) + + assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item) + + refute_enqueued(worker: MediaDownloadWorker) + end + + test "does not enqueue a download job if the media item does not match the format rules" do + profile = media_profile_fixture(%{livestream_behaviour: :exclude}) + source = source_fixture(%{media_profile_id: profile.id}) + media_item = media_item_fixture(source_id: source.id, media_filepath: nil, livestream: true) + + assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item) + + refute_enqueued(worker: MediaDownloadWorker) + end + end end diff --git a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs index 23350c0..f065b7e 100644 --- a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs +++ b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs @@ -9,45 +9,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do alias Pinchflat.Tasks alias Pinchflat.Media.MediaItem alias Pinchflat.Downloading.MediaDownloadWorker - alias Pinchflat.FastIndexing.MediaIndexingWorker alias Pinchflat.FastIndexing.FastIndexingHelpers setup :verify_on_exit! - @media_url "https://www.youtube.com/watch?v=test_1" - - describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do - setup do - {:ok, [source: source_fixture()]} - end - - test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do - expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) - - assert [_] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source) - - assert [worker] = all_enqueued(worker: MediaIndexingWorker) - assert worker.args["id"] == source.id - assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1" - end - - test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do - expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) - media_item_fixture(source_id: source.id, media_id: "test_1") - - assert [] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source) - - refute_enqueued(worker: MediaIndexingWorker) - end - - test "returns the IDs of the found media items", %{source: source} do - expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) - - assert ["test_1"] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source) - end - end - - describe "index_and_enqueue_download_for_media_item/2" do + describe "kickoff_download_tasks_from_youtube_rss_feed/1" do setup do stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, media_attributes_return_fixture()} @@ -56,41 +22,50 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do {:ok, [source: source_fixture()]} end - test "creates a new media item based on the URL", %{source: source} do - assert Repo.aggregate(MediaItem, :count) == 0 - assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) - assert Repo.aggregate(MediaItem, :count) == 1 + test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) + + assert [worker] = all_enqueued(worker: MediaDownloadWorker) + assert worker.args["id"] == media_item.id end - test "won't duplicate media_items based on media_id and source", %{source: source} do - assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) - assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) + test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + media_item_fixture(source_id: source.id, media_id: "test_1") - assert Repo.aggregate(MediaItem, :count) == 1 - assert mi_1.id == mi_2.id - end - - test "enqueues a download job", %{source: source} do - assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) - - assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id}) - end - - test "creates a download task record", %{source: source} do - assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) - - assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker") - end - - test "does not enqueue a download job if the source does not allow it" do - source = source_fixture(%{download_media: false}) - - assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) + assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) refute_enqueued(worker: MediaDownloadWorker) end + test "returns the found media items", %{source: source} do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) + end + + test "does not enqueue a download job if the source does not allow it" do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + source = source_fixture(%{download_media: false}) + + assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) + + refute_enqueued(worker: MediaDownloadWorker) + end + + test "creates a download task record", %{source: source} do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) + + assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker") + end + test "does not enqueue a download job if the media item does not match the format rules" do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + profile = media_profile_fixture(%{shorts_behaviour: :exclude}) source = source_fixture(%{media_profile_id: profile.id}) @@ -110,7 +85,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do {:ok, output} end) - assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url) + assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) + refute_enqueued(worker: MediaDownloadWorker) end end diff --git a/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs b/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs index 55007ba..d2772d7 100644 --- a/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs +++ b/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs @@ -87,6 +87,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do source = source_fixture(fast_index: true) expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end) expect(AppriseRunnerMock, :run, fn servers, opts -> assert "server_1" = servers @@ -98,5 +99,34 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do perform_job(FastIndexingWorker, %{id: source.id}) end + + test "doesn't send a notification if new media is not found" do + source = source_fixture(fast_index: true) + + expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end) + expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end) + + perform_job(FastIndexingWorker, %{id: source.id}) + end + + test "doesn't send a notification if the source doesn't download media" do + source = source_fixture(fast_index: true, download_media: false) + + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end) + expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end) + + perform_job(FastIndexingWorker, %{id: source.id}) + end + + test "doesn't send a notification if the media isn't pending download" do + source = source_fixture(fast_index: true, title_filter_regex: "foobar") + + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end) + expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end) + + perform_job(FastIndexingWorker, %{id: source.id}) + end end end diff --git a/test/pinchflat/fast_indexing/media_indexing_worker_test.exs b/test/pinchflat/fast_indexing/media_indexing_worker_test.exs deleted file mode 100644 index 178e66a..0000000 --- a/test/pinchflat/fast_indexing/media_indexing_worker_test.exs +++ /dev/null @@ -1,61 +0,0 @@ -defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do - use Pinchflat.DataCase - - import Mox - import Pinchflat.MediaFixtures - import Pinchflat.SourcesFixtures - - alias Pinchflat.Media.MediaItem - alias Pinchflat.Downloading.MediaDownloadWorker - alias Pinchflat.FastIndexing.MediaIndexingWorker - - @media_url "https://www.youtube.com/watch?v=1234567890" - - setup :verify_on_exit! - - setup do - source = source_fixture() - - {:ok, source: source} - end - - describe "kickoff_with_task/2" do - test "starts the worker", %{source: source} do - assert [] = all_enqueued(worker: MediaIndexingWorker) - assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url) - assert [_] = all_enqueued(worker: MediaIndexingWorker) - end - - test "attaches a task", %{source: source} do - assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url) - assert task.source_id == source.id - end - end - - describe "perform/1" do - test "indexes the media item and saves it to the database", %{source: source} do - expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> - {:ok, media_attributes_return_fixture()} - end) - - before = Repo.aggregate(MediaItem, :count, :id) - perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url}) - - assert Repo.aggregate(MediaItem, :count, :id) == before + 1 - end - - test "enqueues a download job for the media item", %{source: source} do - expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> - {:ok, media_attributes_return_fixture()} - end) - - perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url}) - - assert [_] = all_enqueued(worker: MediaDownloadWorker) - end - - test "does not blow up if the record doesn't exist" do - assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url}) - end - end -end diff --git a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs index 80e7959..4bc72ee 100644 --- a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs +++ b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs @@ -12,7 +12,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do alias Pinchflat.Media.MediaItem alias Pinchflat.FastIndexing.FastIndexingWorker alias Pinchflat.Downloading.MediaDownloadWorker - alias Pinchflat.FastIndexing.MediaIndexingWorker alias Pinchflat.SlowIndexing.SlowIndexingHelpers alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker @@ -47,7 +46,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do test "it deletes any pending media tasks for the source" do source = source_fixture() - {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id})) + {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id})) task = task_fixture(source_id: source.id, job_id: job.id) assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source) diff --git a/test/pinchflat/sources_test.exs b/test/pinchflat/sources_test.exs index 3b8091d..d6589ff 100644 --- a/test/pinchflat/sources_test.exs +++ b/test/pinchflat/sources_test.exs @@ -13,7 +13,6 @@ defmodule Pinchflat.SourcesTest do alias Pinchflat.Downloading.DownloadingHelpers alias Pinchflat.FastIndexing.FastIndexingWorker alias Pinchflat.Downloading.MediaDownloadWorker - alias Pinchflat.FastIndexing.MediaIndexingWorker alias Pinchflat.Metadata.SourceMetadataStorageWorker alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker @@ -415,16 +414,13 @@ defmodule Pinchflat.SourcesTest do {:ok, job_1} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id})) task_1 = task_fixture(source_id: source.id, job_id: job_1.id) - {:ok, job_2} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id})) + {:ok, job_2} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id})) task_2 = task_fixture(source_id: source.id, job_id: job_2.id) - {:ok, job_3} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id})) - task_3 = task_fixture(source_id: source.id, job_id: job_3.id) assert {:ok, %Source{}} = Sources.update_source(source, update_attrs) assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_1) end assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_2) end - assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_3) end end test "not updating the index frequency will not re-schedule the indexing task or delete tasks" do