danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Resolved bug where non-pending media would notify when fast indexing (#187)

Browse source
This commit is contained in:
Kieran 2024-04-16 13:47:56 -07:00 committed by GitHub
parent 8ca19ba076
commit b2e5e9b880
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 190 additions and 259 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
lib/pinchflat/downloading/downloading_helpers.ex
View file

@ -7,9 +7,11 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
require Logger
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaItem
alias Pinchflat.Downloading.MediaDownloadWorker
@doc """
@ -43,4 +45,23 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|> Media.list_pending_media_items_for()
|> Enum.each(&Tasks.delete_pending_tasks_for/1)
end
@doc """
Takes a single media item and enqueues a download job if the media should be
downloaded, based on the source's download settings and whether media is
considered pending.
Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()}
"""
def kickoff_download_if_pending(%MediaItem{} = media_item) do
media_item = Repo.preload(media_item, :source)
if media_item.source.download_media && Media.pending_download?(media_item) do
Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})")
MediaDownloadWorker.kickoff_with_task(media_item)
else
{:error, :should_not_download}
end
end
end

69
lib/pinchflat/fast_indexing/fast_indexing_helpers.ex
View file

@ -5,64 +5,45 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
Many of these methods are made to be kickoff or be consumed by workers.
"""
require Logger
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaQuery
alias Pinchflat.FastIndexing.YoutubeRss
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
for any new media items. See comments in `MediaIndexingWorker` for more info on the
Fetches new media IDs from a source's YouTube RSS feed, indexes them, and kicks off downloading
tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the
order of operations and how this fits into the indexing process.
Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
`kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
function starts individual indexing tasks for each new media item. I think it does
make sense grammatically, but I could see how that's confusing.
Returns [binary()] where each binary is the media ID of a new media item.
Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
downloaded_.
"""
def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
{:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
existing_media_items = list_media_items_by_media_id_for(source, media_ids)
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
Enum.each(new_media_ids, fn media_id ->
url = "https://www.youtube.com/watch?v=#{media_id}"
maybe_new_media_items =
Enum.map(new_media_ids, fn media_id ->
case create_media_item_from_media_id(source, media_id) do
{:ok, media_item} ->
media_item
MediaIndexingWorker.kickoff_with_task(source, url)
end)
new_media_ids
end
@doc """
Indexes a single media item for a source and enqueues a download job if the
media should be downloaded. This method creates the media item record so it's
the one-stop-shop for adding a media item (and possibly downloading it) just
by a URL and source.
Returns {:ok, media_item} | {:error, any()}
"""
def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
maybe_media_item = create_media_item_from_url(source, url)
case maybe_media_item do
{:ok, media_item} ->
if source.download_media && Media.pending_download?(media_item) do
MediaDownloadWorker.kickoff_with_task(media_item)
err ->
Logger.error("Error creating media item '#{media_id}' from URL: #{inspect(err)}")
nil
end
end)
{:ok, media_item}
DownloadingHelpers.enqueue_pending_download_tasks(source)
err ->
err
end
Enum.filter(maybe_new_media_items, & &1)
end
defp list_media_items_by_media_id_for(source, media_ids) do
@ -72,9 +53,15 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|> Repo.all()
end
defp create_media_item_from_url(source, url) do
{:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
defp create_media_item_from_media_id(source, media_id) do
url = "https://www.youtube.com/watch?v=#{media_id}"
Media.create_media_item_from_backend_attrs(source, media_attrs)
case YtDlpMedia.get_media_attributes(url) do
{:ok, media_attrs} ->
Media.create_media_item_from_backend_attrs(source, media_attrs)
err ->
err
end
end
end

33
lib/pinchflat/fast_indexing/fast_indexing_worker.ex
View file

@ -10,6 +10,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
alias __MODULE__
alias Pinchflat.Tasks
alias Pinchflat.Media
alias Pinchflat.Sources
alias Pinchflat.Settings
alias Pinchflat.Sources.Source
@ -28,9 +29,21 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
end
@doc """
Kicks off the fast indexing process for a source, reschedules the job to run again
once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
for more
Similar to `MediaCollectionIndexingWorker`, but for working with RSS feeds.
`MediaCollectionIndexingWorker` should be preferred in general, but this is
useful for downloading small batches of media items via fast indexing.
Only kicks off downloads for media that _should_ be downloaded
(ie: the source is set to download and the media matches the profile's format preferences)
Order of operations:
1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media.
with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed`
2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed,
it indexes them with a yt-dlp call to create the media item records then kicks off downloading
tasks (MediaDownloadWorker) for any new media items _that should be downloaded_.
3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable
then reschedules itself to run again in the future.
Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
"""
@ -39,7 +52,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
source = Sources.get_source!(source_id)
if source.fast_index do
perform_indexing_and_notification(source)
perform_indexing_and_send_notification(source)
reschedule_indexing(source)
else
:ok
@ -49,11 +62,17 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
end
defp perform_indexing_and_notification(source) do
defp perform_indexing_and_send_notification(source) do
apprise_server = Settings.get!(:apprise_server)
new_media_items = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items))
new_media_items =
source
|> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed()
|> Enum.filter(&Media.pending_download?(&1))
if source.download_media do
SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items))
end
end
defp reschedule_indexing(source) do

69
lib/pinchflat/fast_indexing/media_indexing_worker.ex
View file

@ -1,69 +0,0 @@
defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
@moduledoc false
use Oban.Worker,
queue: :media_indexing,
unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
tags: ["media_source", "media_indexing"]
require Logger
alias __MODULE__
alias Pinchflat.Tasks
alias Pinchflat.Sources
alias Pinchflat.FastIndexing.FastIndexingHelpers
@doc """
Starts the fast media indexing worker and creates a task for the source.
Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
"""
def kickoff_with_task(source, media_url, opts \\ []) do
%{id: source.id, media_url: media_url}
|> MediaIndexingWorker.new(opts)
|> Tasks.create_job_with_task(source)
end
@doc """
Similar to `MediaCollectionIndexingWorker`, but for individual media items.
Does not reschedule or check anything to do with a source's indexing
frequency - only collects initial metadata then kicks off a download.
`MediaCollectionIndexingWorker` should be preferred in general, but this is
useful for downloading one-off media items based on a URL (like for fast indexing).
Only downloads media that _should_ be downloaded (ie: the source is set to download
and the media matches the profile's format preferences)
Order of operations:
1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
in its own worker) periodically checks the YouTube RSS feed for new media
2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
item
3. This worker fetches the media metadata and uses that to determine if it should be
downloaded. If so, it enqueues a MediaDownloadWorker
Each is a worker because they all either need to be scheduled periodically or call out to
an external service and will be long-running. They're split into different jobs to separate
retry logic for each step and allow us to better optimize various queues (eg: the indexing
steps can keep running while the slow download steps are worked through).
Returns :ok
"""
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
source = Sources.get_source!(source_id)
case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
{:ok, media_item} ->
Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
{:error, reason} ->
Logger.debug("Failed to index and enqueue download for url: #{media_url} (reason: #{inspect(reason)})")
end
:ok
rescue
Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
end
end

5
lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex
View file

@ -61,9 +61,8 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
5. If the source uses fast indexing, that job is kicked off as well. It
uses RSS to run a smaller, faster, and more frequent index. That job
handles rescheduling itself but largely has a similar behaviour to this
job in that it kicks off index and maybe download jobs. The biggest difference
is that an index job is kicked off _for each new media item_ as opposed
to one larger index job. Check out `MediaIndexingWorker` comments for more.
job in that it runs and index and maybe kicks off media download jobs.
Check out `FastIndexingWorker` comments for more.
6. If the job reschedules, the cycle from step 3 repeats until the heat death
of the universe. The user changing things like the index frequency can
dequeue or reschedule jobs as well

8
lib/pinchflat/slow_indexing/slow_indexing_helpers.ex
View file

@ -16,7 +16,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
alias Pinchflat.YtDlp.MediaCollection
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.SlowIndexing.FileFollowerServer
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@ -29,7 +28,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
"""
def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
@ -127,11 +125,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
{:ok, %MediaItem{} = media_item} ->
if source.download_media && Media.pending_download?(media_item) do
Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
MediaDownloadWorker.kickoff_with_task(media_item)
end
DownloadingHelpers.kickoff_download_if_pending(media_item)
{:error, changeset} ->
changeset

1
lib/pinchflat/sources/sources.ex
View file

@ -288,7 +288,6 @@ defmodule Pinchflat.Sources do
%{index_frequency_minutes: _} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
_ ->

41
test/pinchflat/downloading/downloading_helpers_test.exs
View file

@ -4,6 +4,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
import Mox
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
import Pinchflat.ProfilesFixtures
alias Pinchflat.Tasks
alias Pinchflat.Downloading.DownloadingHelpers
@ -72,4 +73,44 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
assert [] = Tasks.list_tasks_for(media_item)
end
end
describe "kickoff_download_if_pending/1" do
setup do
media_item = media_item_fixture(media_filepath: nil)
{:ok, media_item: media_item}
end
test "enqueues a download job", %{media_item: media_item} do
assert {:ok, _} = DownloadingHelpers.kickoff_download_if_pending(media_item)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
test "creates and returns a download task record", %{media_item: media_item} do
assert {:ok, task} = DownloadingHelpers.kickoff_download_if_pending(media_item)
assert [found_task] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
assert task.id == found_task.id
end
test "does not enqueue a download job if the source does not allow it" do
source = source_fixture(%{download_media: false})
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item)
refute_enqueued(worker: MediaDownloadWorker)
end
test "does not enqueue a download job if the media item does not match the format rules" do
profile = media_profile_fixture(%{livestream_behaviour: :exclude})
source = source_fixture(%{media_profile_id: profile.id})
media_item = media_item_fixture(source_id: source.id, media_filepath: nil, livestream: true)
assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item)
refute_enqueued(worker: MediaDownloadWorker)
end
end
end

102
test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs
View file

@ -9,45 +9,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
alias Pinchflat.Tasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingHelpers
setup :verify_on_exit!
@media_url "https://www.youtube.com/watch?v=test_1"
describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
setup do
{:ok, [source: source_fixture()]}
end
test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert [_] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
assert [worker] = all_enqueued(worker: MediaIndexingWorker)
assert worker.args["id"] == source.id
assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
end
test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
media_item_fixture(source_id: source.id, media_id: "test_1")
assert [] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaIndexingWorker)
end
test "returns the IDs of the found media items", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert ["test_1"] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
end
end
describe "index_and_enqueue_download_for_media_item/2" do
describe "kickoff_download_tasks_from_youtube_rss_feed/1" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, media_attributes_return_fixture()}
@ -56,41 +22,50 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
{:ok, [source: source_fixture()]}
end
test "creates a new media item based on the URL", %{source: source} do
assert Repo.aggregate(MediaItem, :count) == 0
assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert Repo.aggregate(MediaItem, :count) == 1
test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
assert [worker] = all_enqueued(worker: MediaDownloadWorker)
assert worker.args["id"] == media_item.id
end
test "won't duplicate media_items based on media_id and source", %{source: source} do
assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
media_item_fixture(source_id: source.id, media_id: "test_1")
assert Repo.aggregate(MediaItem, :count) == 1
assert mi_1.id == mi_2.id
end
test "enqueues a download job", %{source: source} do
assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
test "creates a download task record", %{source: source} do
assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
end
test "does not enqueue a download job if the source does not allow it" do
source = source_fixture(%{download_media: false})
assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaDownloadWorker)
end
test "returns the found media items", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
end
test "does not enqueue a download job if the source does not allow it" do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
source = source_fixture(%{download_media: false})
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaDownloadWorker)
end
test "creates a download task record", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
end
test "does not enqueue a download job if the media item does not match the format rules" do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
profile = media_profile_fixture(%{shorts_behaviour: :exclude})
source = source_fixture(%{media_profile_id: profile.id})
@ -110,7 +85,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
{:ok, output}
end)
assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaDownloadWorker)
end
end

30
test/pinchflat/fast_indexing/fast_indexing_worker_test.exs
View file

@ -87,6 +87,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
source = source_fixture(fast_index: true)
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
expect(AppriseRunnerMock, :run, fn servers, opts ->
assert "server_1" = servers
@ -98,5 +99,34 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
perform_job(FastIndexingWorker, %{id: source.id})
end
test "doesn't send a notification if new media is not found" do
source = source_fixture(fast_index: true)
expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
perform_job(FastIndexingWorker, %{id: source.id})
end
test "doesn't send a notification if the source doesn't download media" do
source = source_fixture(fast_index: true, download_media: false)
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
perform_job(FastIndexingWorker, %{id: source.id})
end
test "doesn't send a notification if the media isn't pending download" do
source = source_fixture(fast_index: true, title_filter_regex: "foobar")
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
perform_job(FastIndexingWorker, %{id: source.id})
end
end
end

61
test/pinchflat/fast_indexing/media_indexing_worker_test.exs
View file

@ -1,61 +0,0 @@
defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
alias Pinchflat.Media.MediaItem
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
@media_url "https://www.youtube.com/watch?v=1234567890"
setup :verify_on_exit!
setup do
source = source_fixture()
{:ok, source: source}
end
describe "kickoff_with_task/2" do
test "starts the worker", %{source: source} do
assert [] = all_enqueued(worker: MediaIndexingWorker)
assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
assert [_] = all_enqueued(worker: MediaIndexingWorker)
end
test "attaches a task", %{source: source} do
assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
assert task.source_id == source.id
end
end
describe "perform/1" do
test "indexes the media item and saves it to the database", %{source: source} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, media_attributes_return_fixture()}
end)
before = Repo.aggregate(MediaItem, :count, :id)
perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
assert Repo.aggregate(MediaItem, :count, :id) == before + 1
end
test "enqueues a download job for the media item", %{source: source} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, media_attributes_return_fixture()}
end)
perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
assert [_] = all_enqueued(worker: MediaDownloadWorker)
end
test "does not blow up if the record doesn't exist" do
assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url})
end
end
end

3
test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs
View file

@ -12,7 +12,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
alias Pinchflat.Media.MediaItem
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
@ -47,7 +46,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
test "it deletes any pending media tasks for the source" do
source = source_fixture()
{:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)

6
test/pinchflat/sources_test.exs
View file

@ -13,7 +13,6 @@ defmodule Pinchflat.SourcesTest do
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.Metadata.SourceMetadataStorageWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
@ -415,16 +414,13 @@ defmodule Pinchflat.SourcesTest do
{:ok, job_1} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task_1 = task_fixture(source_id: source.id, job_id: job_1.id)
{:ok, job_2} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
{:ok, job_2} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
task_2 = task_fixture(source_id: source.id, job_id: job_2.id)
{:ok, job_3} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
task_3 = task_fixture(source_id: source.id, job_id: job_3.id)
assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_1) end
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_2) end
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_3) end
end
test "not updating the index frequency will not re-schedule the indexing task or delete tasks" do