d9c48370df
* [Unrelated] updated module name for existing liveview module * Updated toggle component and moved MP index table to a liveview * [WIP] reverted MP index table; added source count to MP index * Moved new live table to sources index * Added 'enabled' boolean to sources * Got 'enabled' logic working re: downloading pending media * Updated sources context to do the right thing when a source is updated * Docs and tests * Updated slow indexing to maintain its old schedule if re-enabled * Hooked up the enabled toggle to the sources page * [Unrelated] added direct links to various tabs on the sources table * More tests * Removed unneeded guard in * Removed outdated comment
101 lines
3.4 KiB
Elixir
101 lines
3.4 KiB
Elixir
defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
|
@moduledoc """
|
|
Methods for performing fast indexing tasks and managing the fast indexing process.
|
|
|
|
Many of these methods are made to be kickoff or be consumed by workers.
|
|
"""
|
|
|
|
require Logger
|
|
|
|
use Pinchflat.Media.MediaQuery
|
|
|
|
alias Pinchflat.Repo
|
|
alias Pinchflat.Media
|
|
alias Pinchflat.Tasks
|
|
alias Pinchflat.Sources.Source
|
|
alias Pinchflat.FastIndexing.YoutubeRss
|
|
alias Pinchflat.FastIndexing.YoutubeApi
|
|
alias Pinchflat.Downloading.DownloadingHelpers
|
|
alias Pinchflat.FastIndexing.FastIndexingWorker
|
|
alias Pinchflat.Downloading.DownloadOptionBuilder
|
|
|
|
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
|
|
|
|
@doc """
|
|
Kicks off a new fast indexing task for a source. This will delete any existing fast indexing
|
|
tasks for the source before starting a new one.
|
|
|
|
Returns {:ok, %Task{}}
|
|
"""
|
|
def kickoff_indexing_task(%Source{} = source) do
|
|
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
|
|
FastIndexingWorker.kickoff_with_task(source)
|
|
end
|
|
|
|
@doc """
|
|
Fetches new media IDs for a source from YT's API or RSS, indexes them, and kicks off downloading
|
|
tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the
|
|
order of operations and how this fits into the indexing process.
|
|
|
|
Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
|
|
downloaded_.
|
|
"""
|
|
def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
|
|
# The media_profile is needed to determine the quality options to _then_ determine a more
|
|
# accurate predicted filepath
|
|
source = Repo.preload(source, [:media_profile])
|
|
|
|
{:ok, media_ids} = get_recent_media_ids(source)
|
|
existing_media_items = list_media_items_by_media_id_for(source, media_ids)
|
|
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
|
|
|
|
maybe_new_media_items =
|
|
Enum.map(new_media_ids, fn media_id ->
|
|
case create_media_item_from_media_id(source, media_id) do
|
|
{:ok, media_item} ->
|
|
media_item
|
|
|
|
err ->
|
|
Logger.error("Error creating media item '#{media_id}' from URL: #{inspect(err)}")
|
|
nil
|
|
end
|
|
end)
|
|
|
|
DownloadingHelpers.enqueue_pending_download_tasks(source)
|
|
|
|
Enum.filter(maybe_new_media_items, & &1)
|
|
end
|
|
|
|
# If possible, use the YouTube API to fetch media IDs. If that fails, fall back to the RSS feed.
|
|
# If the YouTube API isn't set up, just use the RSS feed.
|
|
defp get_recent_media_ids(source) do
|
|
with true <- YoutubeApi.enabled?(),
|
|
{:ok, media_ids} <- YoutubeApi.get_recent_media_ids(source) do
|
|
{:ok, media_ids}
|
|
else
|
|
_ -> YoutubeRss.get_recent_media_ids(source)
|
|
end
|
|
end
|
|
|
|
defp list_media_items_by_media_id_for(source, media_ids) do
|
|
MediaQuery.new()
|
|
|> where(^dynamic([mi], ^MediaQuery.for_source(source) and mi.media_id in ^media_ids))
|
|
|> Repo.all()
|
|
end
|
|
|
|
defp create_media_item_from_media_id(source, media_id) do
|
|
url = "https://www.youtube.com/watch?v=#{media_id}"
|
|
|
|
command_opts =
|
|
[output: DownloadOptionBuilder.build_output_path_for(source)] ++
|
|
DownloadOptionBuilder.build_quality_options_for(source)
|
|
|
|
case YtDlpMedia.get_media_attributes(url, command_opts, use_cookies: source.use_cookies) do
|
|
{:ok, media_attrs} ->
|
|
Media.create_media_item_from_backend_attrs(source, media_attrs)
|
|
|
|
err ->
|
|
err
|
|
end
|
|
end
|
|
end
|