diff --git a/lib/pinchflat/downloading/downloading_helpers.ex b/lib/pinchflat/downloading/downloading_helpers.ex
index 1e61545..a7f64e3 100644
--- a/lib/pinchflat/downloading/downloading_helpers.ex
+++ b/lib/pinchflat/downloading/downloading_helpers.ex
@@ -7,9 +7,11 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
require Logger
+ alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
+ alias Pinchflat.Media.MediaItem
alias Pinchflat.Downloading.MediaDownloadWorker
@doc """
@@ -43,4 +45,23 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|> Media.list_pending_media_items_for()
|> Enum.each(&Tasks.delete_pending_tasks_for/1)
end
+
+ @doc """
+ Takes a single media item and enqueues a download job if the media should be
+ downloaded, based on the source's download settings and whether media is
+ considered pending.
+
+ Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()}
+ """
+ def kickoff_download_if_pending(%MediaItem{} = media_item) do
+ media_item = Repo.preload(media_item, :source)
+
+ if media_item.source.download_media && Media.pending_download?(media_item) do
+ Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})")
+
+ MediaDownloadWorker.kickoff_with_task(media_item)
+ else
+ {:error, :should_not_download}
+ end
+ end
end
diff --git a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex
index b9d5a44..b12eff2 100644
--- a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex
+++ b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex
@@ -5,64 +5,45 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
Many of these methods are made to be kickoff or be consumed by workers.
"""
+ require Logger
+
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaQuery
alias Pinchflat.FastIndexing.YoutubeRss
- alias Pinchflat.Downloading.MediaDownloadWorker
- alias Pinchflat.FastIndexing.MediaIndexingWorker
+ alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
- Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
- for any new media items. See comments in `MediaIndexingWorker` for more info on the
+ Fetches new media IDs from a source's YouTube RSS feed, indexes them, and kicks off downloading
+ tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the
order of operations and how this fits into the indexing process.
- Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
- `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
- function starts individual indexing tasks for each new media item. I think it does
- make sense grammatically, but I could see how that's confusing.
-
- Returns [binary()] where each binary is the media ID of a new media item.
+ Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
+ downloaded_.
"""
- def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+ def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
{:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
existing_media_items = list_media_items_by_media_id_for(source, media_ids)
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
- Enum.each(new_media_ids, fn media_id ->
- url = "https://www.youtube.com/watch?v=#{media_id}"
+ maybe_new_media_items =
+ Enum.map(new_media_ids, fn media_id ->
+ case create_media_item_from_media_id(source, media_id) do
+ {:ok, media_item} ->
+ media_item
- MediaIndexingWorker.kickoff_with_task(source, url)
- end)
-
- new_media_ids
- end
-
- @doc """
- Indexes a single media item for a source and enqueues a download job if the
- media should be downloaded. This method creates the media item record so it's
- the one-stop-shop for adding a media item (and possibly downloading it) just
- by a URL and source.
-
- Returns {:ok, media_item} | {:error, any()}
- """
- def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
- maybe_media_item = create_media_item_from_url(source, url)
-
- case maybe_media_item do
- {:ok, media_item} ->
- if source.download_media && Media.pending_download?(media_item) do
- MediaDownloadWorker.kickoff_with_task(media_item)
+ err ->
+ Logger.error("Error creating media item '#{media_id}' from URL: #{inspect(err)}")
+ nil
end
+ end)
- {:ok, media_item}
+ DownloadingHelpers.enqueue_pending_download_tasks(source)
- err ->
- err
- end
+ Enum.filter(maybe_new_media_items, & &1)
end
defp list_media_items_by_media_id_for(source, media_ids) do
@@ -72,9 +53,15 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|> Repo.all()
end
- defp create_media_item_from_url(source, url) do
- {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
+ defp create_media_item_from_media_id(source, media_id) do
+ url = "https://www.youtube.com/watch?v=#{media_id}"
- Media.create_media_item_from_backend_attrs(source, media_attrs)
+ case YtDlpMedia.get_media_attributes(url) do
+ {:ok, media_attrs} ->
+ Media.create_media_item_from_backend_attrs(source, media_attrs)
+
+ err ->
+ err
+ end
end
end
diff --git a/lib/pinchflat/fast_indexing/fast_indexing_worker.ex b/lib/pinchflat/fast_indexing/fast_indexing_worker.ex
index 15d287f..0f85d40 100644
--- a/lib/pinchflat/fast_indexing/fast_indexing_worker.ex
+++ b/lib/pinchflat/fast_indexing/fast_indexing_worker.ex
@@ -10,6 +10,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
alias __MODULE__
alias Pinchflat.Tasks
+ alias Pinchflat.Media
alias Pinchflat.Sources
alias Pinchflat.Settings
alias Pinchflat.Sources.Source
@@ -28,9 +29,21 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
end
@doc """
- Kicks off the fast indexing process for a source, reschedules the job to run again
- once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
- for more
+ Similar to `MediaCollectionIndexingWorker`, but for working with RSS feeds.
+ `MediaCollectionIndexingWorker` should be preferred in general, but this is
+ useful for downloading small batches of media items via fast indexing.
+
+ Only kicks off downloads for media that _should_ be downloaded
+ (ie: the source is set to download and the media matches the profile's format preferences)
+
+ Order of operations:
+ 1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media.
+ with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed`
+ 2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed,
+ it indexes them with a yt-dlp call to create the media item records then kicks off downloading
+ tasks (MediaDownloadWorker) for any new media items _that should be downloaded_.
+ 3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable
+ then reschedules itself to run again in the future.
Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
"""
@@ -39,7 +52,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
source = Sources.get_source!(source_id)
if source.fast_index do
- perform_indexing_and_notification(source)
+ perform_indexing_and_send_notification(source)
reschedule_indexing(source)
else
:ok
@@ -49,11 +62,17 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
end
- defp perform_indexing_and_notification(source) do
+ defp perform_indexing_and_send_notification(source) do
apprise_server = Settings.get!(:apprise_server)
- new_media_items = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
- SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items))
+ new_media_items =
+ source
+ |> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed()
+ |> Enum.filter(&Media.pending_download?(&1))
+
+ if source.download_media do
+ SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items))
+ end
end
defp reschedule_indexing(source) do
diff --git a/lib/pinchflat/fast_indexing/media_indexing_worker.ex b/lib/pinchflat/fast_indexing/media_indexing_worker.ex
deleted file mode 100644
index c4da1e1..0000000
--- a/lib/pinchflat/fast_indexing/media_indexing_worker.ex
+++ /dev/null
@@ -1,69 +0,0 @@
-defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
- @moduledoc false
-
- use Oban.Worker,
- queue: :media_indexing,
- unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
- tags: ["media_source", "media_indexing"]
-
- require Logger
-
- alias __MODULE__
- alias Pinchflat.Tasks
- alias Pinchflat.Sources
- alias Pinchflat.FastIndexing.FastIndexingHelpers
-
- @doc """
- Starts the fast media indexing worker and creates a task for the source.
-
- Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
- """
- def kickoff_with_task(source, media_url, opts \\ []) do
- %{id: source.id, media_url: media_url}
- |> MediaIndexingWorker.new(opts)
- |> Tasks.create_job_with_task(source)
- end
-
- @doc """
- Similar to `MediaCollectionIndexingWorker`, but for individual media items.
- Does not reschedule or check anything to do with a source's indexing
- frequency - only collects initial metadata then kicks off a download.
- `MediaCollectionIndexingWorker` should be preferred in general, but this is
- useful for downloading one-off media items based on a URL (like for fast indexing).
-
- Only downloads media that _should_ be downloaded (ie: the source is set to download
- and the media matches the profile's format preferences)
-
- Order of operations:
- 1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
- in its own worker) periodically checks the YouTube RSS feed for new media
- 2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
- item
- 3. This worker fetches the media metadata and uses that to determine if it should be
- downloaded. If so, it enqueues a MediaDownloadWorker
-
- Each is a worker because they all either need to be scheduled periodically or call out to
- an external service and will be long-running. They're split into different jobs to separate
- retry logic for each step and allow us to better optimize various queues (eg: the indexing
- steps can keep running while the slow download steps are worked through).
-
- Returns :ok
- """
- @impl Oban.Worker
- def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
- source = Sources.get_source!(source_id)
-
- case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
- {:ok, media_item} ->
- Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
-
- {:error, reason} ->
- Logger.debug("Failed to index and enqueue download for url: #{media_url} (reason: #{inspect(reason)})")
- end
-
- :ok
- rescue
- Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
- Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
- end
-end
diff --git a/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex b/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex
index 0e982eb..678a9a5 100644
--- a/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex
+++ b/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex
@@ -61,9 +61,8 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
5. If the source uses fast indexing, that job is kicked off as well. It
uses RSS to run a smaller, faster, and more frequent index. That job
handles rescheduling itself but largely has a similar behaviour to this
- job in that it kicks off index and maybe download jobs. The biggest difference
- is that an index job is kicked off _for each new media item_ as opposed
- to one larger index job. Check out `MediaIndexingWorker` comments for more.
+ job in that it runs and index and maybe kicks off media download jobs.
+ Check out `FastIndexingWorker` comments for more.
6. If the job reschedules, the cycle from step 3 repeats until the heat death
of the universe. The user changing things like the index frequency can
dequeue or reschedule jobs as well
diff --git a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex
index b6b118a..2434b92 100644
--- a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex
+++ b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex
@@ -16,7 +16,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
alias Pinchflat.YtDlp.MediaCollection
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.SlowIndexing.FileFollowerServer
- alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@@ -29,7 +28,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
"""
def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
- Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
@@ -127,11 +125,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
{:ok, %MediaItem{} = media_item} ->
- if source.download_media && Media.pending_download?(media_item) do
- Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
-
- MediaDownloadWorker.kickoff_with_task(media_item)
- end
+ DownloadingHelpers.kickoff_download_if_pending(media_item)
{:error, changeset} ->
changeset
diff --git a/lib/pinchflat/sources/sources.ex b/lib/pinchflat/sources/sources.ex
index b57a628..f9a3658 100644
--- a/lib/pinchflat/sources/sources.ex
+++ b/lib/pinchflat/sources/sources.ex
@@ -288,7 +288,6 @@ defmodule Pinchflat.Sources do
%{index_frequency_minutes: _} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
- Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
_ ->
diff --git a/test/pinchflat/downloading/downloading_helpers_test.exs b/test/pinchflat/downloading/downloading_helpers_test.exs
index b54a5af..1c3c95b 100644
--- a/test/pinchflat/downloading/downloading_helpers_test.exs
+++ b/test/pinchflat/downloading/downloading_helpers_test.exs
@@ -4,6 +4,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
import Mox
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
+ import Pinchflat.ProfilesFixtures
alias Pinchflat.Tasks
alias Pinchflat.Downloading.DownloadingHelpers
@@ -72,4 +73,44 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
assert [] = Tasks.list_tasks_for(media_item)
end
end
+
+ describe "kickoff_download_if_pending/1" do
+ setup do
+ media_item = media_item_fixture(media_filepath: nil)
+
+ {:ok, media_item: media_item}
+ end
+
+ test "enqueues a download job", %{media_item: media_item} do
+ assert {:ok, _} = DownloadingHelpers.kickoff_download_if_pending(media_item)
+
+ assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+ end
+
+ test "creates and returns a download task record", %{media_item: media_item} do
+ assert {:ok, task} = DownloadingHelpers.kickoff_download_if_pending(media_item)
+
+ assert [found_task] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
+ assert task.id == found_task.id
+ end
+
+ test "does not enqueue a download job if the source does not allow it" do
+ source = source_fixture(%{download_media: false})
+ media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+ assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item)
+
+ refute_enqueued(worker: MediaDownloadWorker)
+ end
+
+ test "does not enqueue a download job if the media item does not match the format rules" do
+ profile = media_profile_fixture(%{livestream_behaviour: :exclude})
+ source = source_fixture(%{media_profile_id: profile.id})
+ media_item = media_item_fixture(source_id: source.id, media_filepath: nil, livestream: true)
+
+ assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item)
+
+ refute_enqueued(worker: MediaDownloadWorker)
+ end
+ end
end
diff --git a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs
index 23350c0..f065b7e 100644
--- a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs
+++ b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs
@@ -9,45 +9,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
alias Pinchflat.Tasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.Downloading.MediaDownloadWorker
- alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingHelpers
setup :verify_on_exit!
- @media_url "https://www.youtube.com/watch?v=test_1"
-
- describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
- setup do
- {:ok, [source: source_fixture()]}
- end
-
- test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
- expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
-
- assert [_] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
- assert [worker] = all_enqueued(worker: MediaIndexingWorker)
- assert worker.args["id"] == source.id
- assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
- end
-
- test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
- expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
- media_item_fixture(source_id: source.id, media_id: "test_1")
-
- assert [] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
- refute_enqueued(worker: MediaIndexingWorker)
- end
-
- test "returns the IDs of the found media items", %{source: source} do
- expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
-
- assert ["test_1"] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
- end
- end
-
- describe "index_and_enqueue_download_for_media_item/2" do
+ describe "kickoff_download_tasks_from_youtube_rss_feed/1" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, media_attributes_return_fixture()}
@@ -56,41 +22,50 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
{:ok, [source: source_fixture()]}
end
- test "creates a new media item based on the URL", %{source: source} do
- assert Repo.aggregate(MediaItem, :count) == 0
- assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
- assert Repo.aggregate(MediaItem, :count) == 1
+ test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+
+ assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+
+ assert [worker] = all_enqueued(worker: MediaDownloadWorker)
+ assert worker.args["id"] == media_item.id
end
- test "won't duplicate media_items based on media_id and source", %{source: source} do
- assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
- assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+ test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+ media_item_fixture(source_id: source.id, media_id: "test_1")
- assert Repo.aggregate(MediaItem, :count) == 1
- assert mi_1.id == mi_2.id
- end
-
- test "enqueues a download job", %{source: source} do
- assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
-
- assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
- end
-
- test "creates a download task record", %{source: source} do
- assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
-
- assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
- end
-
- test "does not enqueue a download job if the source does not allow it" do
- source = source_fixture(%{download_media: false})
-
- assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+ assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaDownloadWorker)
end
+ test "returns the found media items", %{source: source} do
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+
+ assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+ end
+
+ test "does not enqueue a download job if the source does not allow it" do
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+ source = source_fixture(%{download_media: false})
+
+ assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+
+ refute_enqueued(worker: MediaDownloadWorker)
+ end
+
+ test "creates a download task record", %{source: source} do
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+
+ assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+
+ assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
+ end
+
test "does not enqueue a download job if the media item does not match the format rules" do
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+
profile = media_profile_fixture(%{shorts_behaviour: :exclude})
source = source_fixture(%{media_profile_id: profile.id})
@@ -110,7 +85,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
{:ok, output}
end)
- assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+ assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+
refute_enqueued(worker: MediaDownloadWorker)
end
end
diff --git a/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs b/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs
index 55007ba..d2772d7 100644
--- a/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs
+++ b/test/pinchflat/fast_indexing/fast_indexing_worker_test.exs
@@ -87,6 +87,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
source = source_fixture(fast_index: true)
expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+ expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
expect(AppriseRunnerMock, :run, fn servers, opts ->
assert "server_1" = servers
@@ -98,5 +99,34 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
perform_job(FastIndexingWorker, %{id: source.id})
end
+
+ test "doesn't send a notification if new media is not found" do
+ source = source_fixture(fast_index: true)
+
+ expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+ expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
+
+ perform_job(FastIndexingWorker, %{id: source.id})
+ end
+
+ test "doesn't send a notification if the source doesn't download media" do
+ source = source_fixture(fast_index: true, download_media: false)
+
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+ expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
+ expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
+
+ perform_job(FastIndexingWorker, %{id: source.id})
+ end
+
+ test "doesn't send a notification if the media isn't pending download" do
+ source = source_fixture(fast_index: true, title_filter_regex: "foobar")
+
+ expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end)
+ expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
+ expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
+
+ perform_job(FastIndexingWorker, %{id: source.id})
+ end
end
end
diff --git a/test/pinchflat/fast_indexing/media_indexing_worker_test.exs b/test/pinchflat/fast_indexing/media_indexing_worker_test.exs
deleted file mode 100644
index 178e66a..0000000
--- a/test/pinchflat/fast_indexing/media_indexing_worker_test.exs
+++ /dev/null
@@ -1,61 +0,0 @@
-defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
- use Pinchflat.DataCase
-
- import Mox
- import Pinchflat.MediaFixtures
- import Pinchflat.SourcesFixtures
-
- alias Pinchflat.Media.MediaItem
- alias Pinchflat.Downloading.MediaDownloadWorker
- alias Pinchflat.FastIndexing.MediaIndexingWorker
-
- @media_url "https://www.youtube.com/watch?v=1234567890"
-
- setup :verify_on_exit!
-
- setup do
- source = source_fixture()
-
- {:ok, source: source}
- end
-
- describe "kickoff_with_task/2" do
- test "starts the worker", %{source: source} do
- assert [] = all_enqueued(worker: MediaIndexingWorker)
- assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
- assert [_] = all_enqueued(worker: MediaIndexingWorker)
- end
-
- test "attaches a task", %{source: source} do
- assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
- assert task.source_id == source.id
- end
- end
-
- describe "perform/1" do
- test "indexes the media item and saves it to the database", %{source: source} do
- expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
- {:ok, media_attributes_return_fixture()}
- end)
-
- before = Repo.aggregate(MediaItem, :count, :id)
- perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
-
- assert Repo.aggregate(MediaItem, :count, :id) == before + 1
- end
-
- test "enqueues a download job for the media item", %{source: source} do
- expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
- {:ok, media_attributes_return_fixture()}
- end)
-
- perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
-
- assert [_] = all_enqueued(worker: MediaDownloadWorker)
- end
-
- test "does not blow up if the record doesn't exist" do
- assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url})
- end
- end
-end
diff --git a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs
index 80e7959..4bc72ee 100644
--- a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs
+++ b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs
@@ -12,7 +12,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
alias Pinchflat.Media.MediaItem
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
- alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
@@ -47,7 +46,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
test "it deletes any pending media tasks for the source" do
source = source_fixture()
- {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
+ {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
diff --git a/test/pinchflat/sources_test.exs b/test/pinchflat/sources_test.exs
index 3b8091d..d6589ff 100644
--- a/test/pinchflat/sources_test.exs
+++ b/test/pinchflat/sources_test.exs
@@ -13,7 +13,6 @@ defmodule Pinchflat.SourcesTest do
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
- alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.Metadata.SourceMetadataStorageWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
@@ -415,16 +414,13 @@ defmodule Pinchflat.SourcesTest do
{:ok, job_1} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task_1 = task_fixture(source_id: source.id, job_id: job_1.id)
- {:ok, job_2} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
+ {:ok, job_2} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
task_2 = task_fixture(source_id: source.id, job_id: job_2.id)
- {:ok, job_3} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
- task_3 = task_fixture(source_id: source.id, job_id: job_3.id)
assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_1) end
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_2) end
- assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_3) end
end
test "not updating the index frequency will not re-schedule the indexing task or delete tasks" do