[WIP] break out a few contexts, start refactoring fast index modules

.iex.exs

@@ -5,7 +5,7 @@ alias Pinchflat.Tasks.Task
 alias Pinchflat.Sources.Source
 alias Pinchflat.Media.MediaItem
 alias Pinchflat.Tasks.SourceTasks
-alias Pinchflat.Media.MediaMetadata
+alias Pinchflat.Metadata.MediaMetadata
 alias Pinchflat.Profiles.MediaProfile
 
 alias Pinchflat.Tasks
@@ -18,7 +18,7 @@ alias Pinchflat.MediaClient.MediaDownloader
 alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
 alias Pinchflat.YtDlp.Backend.MediaCollection, as: YtDlpCollection
 
-alias Pinchflat.Api.YoutubeRss
+alias Pinchflat.FastIndexing.YoutubeRss
 alias Pinchflat.Metadata.MetadataFileHelpers
 
 alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer

lib/pinchflat/downloading/download_option_builder.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   @moduledoc """
   Builds the options for yt-dlp to download media based on the given media profile.
   """

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -0,0 +1,59 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.YoutubeRss
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+
+  @doc """
+  Starts tasks for running a fast indexing task for a source's media
+  regardless of the source's fast_index state. It's assumed the
+  caller will check for fast_index.
+
+  This is used for running fast index tasks on update. On creation, the
+  fast index is enqueued after the slow index is complete.
+
+  Returns {:ok, %Task{}}.
+  """
+  def kickoff_fast_indexing_task(%Source{} = source) do
+    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+
+    %{id: source.id}
+    # Schedule this one immediately, but future ones will be on an interval
+    |> FastIndexingWorker.new()
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
+  order of operations and how this fits into the indexing process.
+
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns :ok
+  """
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
+    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
+
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
+
+      %{id: source.id, media_url: url}
+      |> MediaIndexingWorker.new()
+      |> Tasks.create_job_with_task(source)
+    end)
+  end
+end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FastIndexingWorker do
+defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -10,7 +10,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -24,7 +24,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
       reschedule_indexing(source)
     else

lib/pinchflat/fast_indexing/youtube_rss.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Api.YoutubeRss do
+defmodule Pinchflat.FastIndexing.YoutubeRss do
   @moduledoc """
   Methods for interacting with YouTube RSS feeds
   """

lib/pinchflat/media.ex

@@ -9,7 +9,7 @@ defmodule Pinchflat.Media do
   alias Pinchflat.Tasks
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
 
   @doc """
   Returns the list of media_items.

lib/pinchflat/media/media_item.ex

@@ -8,7 +8,7 @@ defmodule Pinchflat.Media.MediaItem do
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
   alias Pinchflat.Media.MediaItemSearchIndex
 
   @allowed_fields [

lib/pinchflat/metadata/media_metadata.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaMetadata do
+defmodule Pinchflat.Metadata.MediaMetadata do
   @moduledoc """
   The MediaMetadata schema.
 

lib/pinchflat/sources.ex

@@ -12,6 +12,7 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
   alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -251,7 +252,7 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        SourceTasks.kickoff_fast_indexing_task(source)
+        FastIndexingHelpers.kickoff_fast_indexing_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/source_tasks.ex

@@ -12,9 +12,9 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Api.YoutubeRss
+  alias Pinchflat.FastIndexing.YoutubeRss
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.YtDlp.Backend.MediaCollection
@@ -40,46 +40,6 @@ defmodule Pinchflat.Tasks.SourceTasks do
     |> Tasks.create_job_with_task(source)
   end
 
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
-  for any new media items. See comments in `MediaIndexingWorker` for more info on the
-  order of operations and how this fits into the indexing process.
-
-  Returns :ok
-  """
-  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
-    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
-    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
-    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
-
-    Enum.each(new_media_ids, fn media_id ->
-      url = "https://www.youtube.com/watch?v=#{media_id}"
-
-      %{id: source.id, media_url: url}
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
-    end)
-  end
-
   @doc """
   Given a media source, creates (indexes) the media by creating media_items for each
   media ID in the source. Afterward, kicks off a download task for each pending media
@@ -134,7 +94,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
     end)
   end
 
-  def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
+  def enqueue_pending_media_tasks(%Source{download_media: false}) do
     :ok
   end
 

lib/pinchflat/workers/media_collection_indexing_worker.ex

@@ -11,7 +11,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
   alias Pinchflat.Tasks.SourceTasks
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
 
   @impl Oban.Worker
   @doc """

lib/pinchflat/workers/media_indexing_worker.ex

@@ -23,7 +23,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   and the media matches the profile's format preferences)
 
   Order of operations:
-    1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
        in its own worker) periodically checks the YouTube RSS feed for new media
     2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
        item

lib/pinchflat/yt_dlp/media_downloader.ex

@@ -10,7 +10,7 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
   alias Pinchflat.Media.MediaItem
 
   alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
   alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 

test/pinchflat/downloading/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -0,0 +1,74 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.TasksFixtures
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Tasks.Task
+  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+
+  setup :verify_on_exit!
+
+  describe "kickoff_fast_indexing_task/1" do
+    test "it schedules a job" do
+      source = source_fixture()
+
+      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "it creates and attaches a task" do
+      source = source_fixture()
+
+      assert {:ok, %Task{} = task} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert task.source_id == source.id
+    end
+
+    test "it deletes any fast indexing tasks for the source" do
+      source = source_fixture()
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+  end
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
+  end
+end

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.Workers.FastIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
 
   setup :verify_on_exit!
 

test/pinchflat/fast_indexing/youtube_rss_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Api.YoutubeRssTest do
+defmodule Pinchflat.FastIndexing.YoutubeRssTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.Api.YoutubeRss
+  alias Pinchflat.FastIndexing.YoutubeRss
 
   setup :verify_on_exit!
 

test/pinchflat/sources_test.exs

@@ -9,7 +9,7 @@ defmodule Pinchflat.SourcesTest do
   alias Pinchflat.Sources
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.Workers.MediaCollectionIndexingWorker

test/pinchflat/tasks/source_tasks_test.exs

@@ -11,7 +11,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.Workers.MediaCollectionIndexingWorker
@@ -66,59 +66,6 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
   end
 
-  describe "kickoff_fast_indexing_task/1" do
-    test "it schedules a job" do
-      source = source_fixture()
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
-    end
-
-    test "it creates and attaches a task" do
-      source = source_fixture()
-
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert task.source_id == source.id
-    end
-
-    test "it deletes any fast indexing tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-  end
-
-  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
-    setup do
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
-      assert worker.args["id"] == source.id
-      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
-    end
-
-    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      media_item_fixture(source_id: source.id, media_id: "test_1")
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      refute_enqueued(worker: MediaIndexingWorker)
-    end
-  end
-
   describe "index_and_enqueue_download_for_media_items/1" do
     setup do
       stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->

test/pinchflat/workers/media_collection_indexing_worker_test.exs

@@ -8,7 +8,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
   alias Pinchflat.Workers.MediaCollectionIndexingWorker