WIP - got everything hooked up for POC

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -79,21 +79,21 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
     case {source.index_frequency_minutes, source.last_indexed_at} do
       {index_freq, _} when index_freq > 0 ->
         # If the indexing is on a schedule simply run indexing and reschedule
-        perform_indexing_and_notification(source)
+        perform_indexing_and_notification(source, [force: args["force"]])
         maybe_enqueue_fast_indexing_task(source)
         reschedule_indexing(source)
 
       {_, nil} ->
         # If the source has never been indexed, index it once
         # even if it's not meant to reschedule
-        perform_indexing_and_notification(source)
+        perform_indexing_and_notification(source, [force: args["force"]])
         :ok
 
       _ ->
         # If the source HAS been indexed and is not meant to reschedule,
         # perform a no-op (unless forced)
         if args["force"] do
-          perform_indexing_and_notification(source)
+          perform_indexing_and_notification(source, [force: true])
         end
 
         :ok
@@ -103,11 +103,12 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
     Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
-  defp perform_indexing_and_notification(source) do
+  # TODO: test
+  defp perform_indexing_and_notification(source, indexing_opts) do
     apprise_server = Settings.get!(:apprise_server)
 
     SourceNotifications.wrap_new_media_notification(apprise_server, source, fn ->
-      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source, indexing_opts)
     end)
   end
 

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -31,7 +31,8 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   Returns {:ok, %Task{}}
   """
   def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
-    job_offset_seconds = calculate_job_offset_seconds(source)
+    # TODO: test
+    job_offset_seconds = if job_args[:force], do: 0, else: calculate_job_offset_seconds(source)
 
     Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true)
@@ -74,13 +75,15 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
 
   Returns [%MediaItem{} | %Ecto.Changeset{}]
   """
-  def index_and_enqueue_download_for_media_items(%Source{} = source) do
+  def index_and_enqueue_download_for_media_items(%Source{} = source, opts \\ []) do
+    # TODO: test
+    should_force = Keyword.get(opts, :force, false)
     # The media_profile is needed to determine the quality options to _then_ determine a more
     # accurate predicted filepath
     source = Repo.preload(source, [:media_profile])
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
-    {:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source)
+    {:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source, should_force)
     # Reload because the source may have been updated during the (long-running) indexing process
     # and important settings like `download_media` may have changed.
     source = Repo.reload!(source)
@@ -99,22 +102,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
     result
   end
 
-  # TODO: test
-  def create_download_archive_file(%Source{} = source) do
-    tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
-
-    archive_contents =
-      source
-      |> get_media_items_for_download_archive()
-      |> Enum.map(fn media_item -> "youtube #{media_item.media_id}" end)
-      |> Enum.join("\n")
-
-    case File.write(tmpfile, archive_contents) do
-      :ok -> tmpfile
-      err -> err
-    end
-  end
-
   # The file follower is a GenServer that watches a file for new lines and
   # processes them. This works well, but we have to be resilliant to partially-written
   # lines (ie: you should gracefully fail if you can't parse a line).
@@ -128,14 +115,15 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   # It attempts a graceful shutdown of the file follower after the indexing is done,
   # but the FileFollowerServer will also stop itself if it doesn't see any activity
   # for a sufficiently long time.
-  defp setup_file_watcher_and_kickoff_indexing(source) do
+  defp setup_file_watcher_and_kickoff_indexing(source, should_force) do
     {:ok, pid} = FileFollowerServer.start_link()
 
     handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end
 
     command_opts =
       [output: DownloadOptionBuilder.build_output_path_for(source)] ++
-        DownloadOptionBuilder.build_quality_options_for(source)
+        DownloadOptionBuilder.build_quality_options_for(source) ++
+        build_download_archive_options(source, should_force)
 
     runner_opts = [file_listener_handler: handler, use_cookies: source.use_cookies]
     result = MediaCollection.get_media_attributes_for_collection(source.original_url, command_opts, runner_opts)
@@ -186,6 +174,22 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
     max(0, index_frequency_seconds - offset_seconds)
   end
 
+  # TODO: test and doc
+  defp create_download_archive_file(source) do
+    tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
+
+    archive_contents =
+      source
+      |> get_media_items_for_download_archive()
+      |> Enum.map(fn media_item -> "youtube #{media_item.media_id}" end)
+      |> Enum.join("\n")
+
+    case File.write(tmpfile, archive_contents) do
+      :ok -> tmpfile
+      err -> err
+    end
+  end
+
   # TODO: document
   defp get_media_items_for_download_archive(source) do
     MediaQuery.new()
@@ -195,4 +199,14 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
     |> offset(20)
     |> Repo.all()
   end
+
+  # TODO: document
+  defp build_download_archive_options(%Source{collection_type: :playlist}, _should_force), do: []
+  defp build_download_archive_options(_source, true), do: []
+
+  defp build_download_archive_options(source, _should_force) do
+    archive_file = create_download_archive_file(source)
+
+    [:break_on_existing, download_archive: archive_file]
+  end
 end

lib/pinchflat/utils/filesystem_utils.ex

@@ -44,6 +44,7 @@ defmodule Pinchflat.Utils.FilesystemUtils do
 
   Returns binary()
   """
+  # TODO: consider namespacing these folders like they do in activestorage
   def generate_metadata_tmpfile(type) do
     tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
     filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])

lib/pinchflat/yt_dlp/command_runner.ex

@@ -39,7 +39,9 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts)
 
     case CliUtils.wrap_cmd(command, formatted_command_opts, stderr_to_stdout: true) do
-      {_, 0} ->
+      # 0 is normal exit, 101 is an intentional exit due to some
+      # break condition (like --break-on-existing)
+      {_, status} when status in [0, 101] ->
         # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
         # a huge deal, but it's still a good idea to clean up after ourselves.
         # (even on error? especially on error?)

lib/pinchflat_web/controllers/sources/source_html/actions_dropdown.html.heex

@@ -54,7 +54,7 @@
     <.link
       href={~p"/sources/#{@source}/force_index"}
       method="post"
-      data-confirm="Are you sure you want to force an index of this source? This isn't normally needed."
+      data-confirm="Are you sure you want index all content from this source? This isn't normally needed."
     >
       Force Index
     </.link>