diff --git a/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex b/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex index 5dbac3e..da78c75 100644 --- a/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex +++ b/lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex @@ -79,21 +79,21 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do case {source.index_frequency_minutes, source.last_indexed_at} do {index_freq, _} when index_freq > 0 -> # If the indexing is on a schedule simply run indexing and reschedule - perform_indexing_and_notification(source) + perform_indexing_and_notification(source, [force: args["force"]]) maybe_enqueue_fast_indexing_task(source) reschedule_indexing(source) {_, nil} -> # If the source has never been indexed, index it once # even if it's not meant to reschedule - perform_indexing_and_notification(source) + perform_indexing_and_notification(source, [force: args["force"]]) :ok _ -> # If the source HAS been indexed and is not meant to reschedule, # perform a no-op (unless forced) if args["force"] do - perform_indexing_and_notification(source) + perform_indexing_and_notification(source, [force: true]) end :ok @@ -103,11 +103,12 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale") end - defp perform_indexing_and_notification(source) do + # TODO: test + defp perform_indexing_and_notification(source, indexing_opts) do apprise_server = Settings.get!(:apprise_server) SourceNotifications.wrap_new_media_notification(apprise_server, source, fn -> - SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source) + SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source, indexing_opts) end) end diff --git a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex index ffefa70..4a9baf3 100644 --- a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex +++ b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex @@ -31,7 +31,8 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do Returns {:ok, %Task{}} """ def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do - job_offset_seconds = calculate_job_offset_seconds(source) + # TODO: test + job_offset_seconds = if job_args[:force], do: 0, else: calculate_job_offset_seconds(source) Tasks.delete_pending_tasks_for(source, "FastIndexingWorker") Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true) @@ -74,13 +75,15 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do Returns [%MediaItem{} | %Ecto.Changeset{}] """ - def index_and_enqueue_download_for_media_items(%Source{} = source) do + def index_and_enqueue_download_for_media_items(%Source{} = source, opts \\ []) do + # TODO: test + should_force = Keyword.get(opts, :force, false) # The media_profile is needed to determine the quality options to _then_ determine a more # accurate predicted filepath source = Repo.preload(source, [:media_profile]) # See the method definition below for more info on how file watchers work # (important reading if you're not familiar with it) - {:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source) + {:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source, should_force) # Reload because the source may have been updated during the (long-running) indexing process # and important settings like `download_media` may have changed. source = Repo.reload!(source) @@ -99,22 +102,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do result end - # TODO: test - def create_download_archive_file(%Source{} = source) do - tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt) - - archive_contents = - source - |> get_media_items_for_download_archive() - |> Enum.map(fn media_item -> "youtube #{media_item.media_id}" end) - |> Enum.join("\n") - - case File.write(tmpfile, archive_contents) do - :ok -> tmpfile - err -> err - end - end - # The file follower is a GenServer that watches a file for new lines and # processes them. This works well, but we have to be resilliant to partially-written # lines (ie: you should gracefully fail if you can't parse a line). @@ -128,14 +115,15 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do # It attempts a graceful shutdown of the file follower after the indexing is done, # but the FileFollowerServer will also stop itself if it doesn't see any activity # for a sufficiently long time. - defp setup_file_watcher_and_kickoff_indexing(source) do + defp setup_file_watcher_and_kickoff_indexing(source, should_force) do {:ok, pid} = FileFollowerServer.start_link() handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end command_opts = [output: DownloadOptionBuilder.build_output_path_for(source)] ++ - DownloadOptionBuilder.build_quality_options_for(source) + DownloadOptionBuilder.build_quality_options_for(source) ++ + build_download_archive_options(source, should_force) runner_opts = [file_listener_handler: handler, use_cookies: source.use_cookies] result = MediaCollection.get_media_attributes_for_collection(source.original_url, command_opts, runner_opts) @@ -186,6 +174,22 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do max(0, index_frequency_seconds - offset_seconds) end + # TODO: test and doc + defp create_download_archive_file(source) do + tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt) + + archive_contents = + source + |> get_media_items_for_download_archive() + |> Enum.map(fn media_item -> "youtube #{media_item.media_id}" end) + |> Enum.join("\n") + + case File.write(tmpfile, archive_contents) do + :ok -> tmpfile + err -> err + end + end + # TODO: document defp get_media_items_for_download_archive(source) do MediaQuery.new() @@ -195,4 +199,14 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do |> offset(20) |> Repo.all() end + + # TODO: document + defp build_download_archive_options(%Source{collection_type: :playlist}, _should_force), do: [] + defp build_download_archive_options(_source, true), do: [] + + defp build_download_archive_options(source, _should_force) do + archive_file = create_download_archive_file(source) + + [:break_on_existing, download_archive: archive_file] + end end diff --git a/lib/pinchflat/utils/filesystem_utils.ex b/lib/pinchflat/utils/filesystem_utils.ex index 355bb19..bac512e 100644 --- a/lib/pinchflat/utils/filesystem_utils.ex +++ b/lib/pinchflat/utils/filesystem_utils.ex @@ -44,6 +44,7 @@ defmodule Pinchflat.Utils.FilesystemUtils do Returns binary() """ + # TODO: consider namespacing these folders like they do in activestorage def generate_metadata_tmpfile(type) do tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory) filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"]) diff --git a/lib/pinchflat/yt_dlp/command_runner.ex b/lib/pinchflat/yt_dlp/command_runner.ex index 5d71661..db3f3f7 100644 --- a/lib/pinchflat/yt_dlp/command_runner.ex +++ b/lib/pinchflat/yt_dlp/command_runner.ex @@ -39,7 +39,9 @@ defmodule Pinchflat.YtDlp.CommandRunner do formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts) case CliUtils.wrap_cmd(command, formatted_command_opts, stderr_to_stdout: true) do - {_, 0} -> + # 0 is normal exit, 101 is an intentional exit due to some + # break condition (like --break-on-existing) + {_, status} when status in [0, 101] -> # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not # a huge deal, but it's still a good idea to clean up after ourselves. # (even on error? especially on error?) diff --git a/lib/pinchflat_web/controllers/sources/source_html/actions_dropdown.html.heex b/lib/pinchflat_web/controllers/sources/source_html/actions_dropdown.html.heex index 43d0e64..ac00877 100644 --- a/lib/pinchflat_web/controllers/sources/source_html/actions_dropdown.html.heex +++ b/lib/pinchflat_web/controllers/sources/source_html/actions_dropdown.html.heex @@ -54,7 +54,7 @@ <.link href={~p"/sources/#{@source}/force_index"} method="post" - data-confirm="Are you sure you want to force an index of this source? This isn't normally needed." + data-confirm="Are you sure you want index all content from this source? This isn't normally needed." > Force Index