From 266f71cf75c7624dbe7d420885d4a0378e0f38ac Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Tue, 9 Apr 2024 16:10:41 -0700 Subject: [PATCH] Added methods for fetching re-downloadable media items --- .../downloading/media_redownload_worker.ex | 19 +++++ lib/pinchflat/media/media.ex | 26 +++++-- lib/pinchflat/media/media_item.ex | 4 +- lib/pinchflat/media/media_query.ex | 35 +++++++-- .../notifications/source_notifications.ex | 2 +- .../controllers/sources/source_controller.ex | 2 +- ...20240409224152_add_redownloaded_fields.exs | 2 +- test/pinchflat/media_test.exs | 77 +++++++++++++++++++ 8 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 lib/pinchflat/downloading/media_redownload_worker.ex diff --git a/lib/pinchflat/downloading/media_redownload_worker.ex b/lib/pinchflat/downloading/media_redownload_worker.ex new file mode 100644 index 0000000..4b38c58 --- /dev/null +++ b/lib/pinchflat/downloading/media_redownload_worker.ex @@ -0,0 +1,19 @@ +defmodule Pinchflat.Downloading.MediaRedownloadWorker do + @moduledoc false + + use Oban.Worker, + queue: :media_fetching, + unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]], + tags: ["media_item", "media_fetching"] + + require Logger + + alias Pinchflat.Media + + @doc """ + """ + # TODO + @impl Oban.Worker + def perform(%Oban.Job{}) do + end +end diff --git a/lib/pinchflat/media/media.ex b/lib/pinchflat/media/media.ex index ee8531d..a2c24e9 100644 --- a/lib/pinchflat/media/media.ex +++ b/lib/pinchflat/media/media.ex @@ -31,15 +31,31 @@ defmodule Pinchflat.Media do def list_cullable_media_items do MediaQuery.new() |> MediaQuery.with_media_filepath() - |> MediaQuery.with_passed_retention_period() - |> MediaQuery.with_no_culling_prevention() + |> MediaQuery.where_past_retention_period() + |> MediaQuery.where_culling_not_prevented() + |> Repo.all() + end + + @doc """ + Returns a list of media_items that are redownloadable based on the redownload delay + of the media_profile their source belongs to. + + Returns [%MediaItem{}, ...] + """ + def list_redownloadable_media_items do + MediaQuery.new() + |> MediaQuery.with_media_filepath() + |> MediaQuery.where_download_not_prevented() + |> MediaQuery.where_not_culled() + |> MediaQuery.where_media_not_redownloaded() + |> MediaQuery.where_past_redownload_delay() |> Repo.all() end @doc """ Returns a list of pending media_items for a given source, where pending means the `media_filepath` is `nil` AND the media_item - matches satisfies `MediaQuery.with_media_pending_download`. You + matches satisfies `MediaQuery.where_pending_download`. You should really check out that function if you need to know more because it has a lot going on. @@ -48,7 +64,7 @@ defmodule Pinchflat.Media do def list_pending_media_items_for(%Source{} = source) do MediaQuery.new() |> MediaQuery.for_source(source) - |> MediaQuery.with_media_pending_download() + |> MediaQuery.where_pending_download() |> Repo.all() end @@ -66,7 +82,7 @@ defmodule Pinchflat.Media do MediaQuery.new() |> MediaQuery.with_id(media_item.id) - |> MediaQuery.with_media_pending_download() + |> MediaQuery.where_pending_download() |> Repo.exists?() end diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index 3abcd48..a671d58 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -35,7 +35,7 @@ defmodule Pinchflat.Media.MediaItem do :prevent_download, :prevent_culling, :culled_at, - :redownloaded_at + :media_redownloaded_at ] # Pretty much all the fields captured at index are required. @required_fields ~w( @@ -62,6 +62,7 @@ defmodule Pinchflat.Media.MediaItem do field :livestream, :boolean, default: false field :short_form_content, :boolean, default: false field :media_downloaded_at, :utc_datetime + field :media_redownloaded_at, :utc_datetime field :upload_date, :date field :duration_seconds, :integer @@ -78,7 +79,6 @@ defmodule Pinchflat.Media.MediaItem do field :prevent_download, :boolean, default: false field :prevent_culling, :boolean, default: false field :culled_at, :utc_datetime - field :redownloaded_at, :utc_datetime field :matching_search_term, :string, virtual: true diff --git a/lib/pinchflat/media/media_query.ex b/lib/pinchflat/media/media_query.ex index 6751c28..d187aec 100644 --- a/lib/pinchflat/media/media_query.ex +++ b/lib/pinchflat/media/media_query.ex @@ -15,7 +15,7 @@ defmodule Pinchflat.Media.MediaQuery do # Prefixes: # - for_* - belonging to a certain record # - join_* - for joining on a certain record - # - with_* - for filtering based on full, concrete attributes + # - with_*, where_* - for filtering based on full, concrete attributes # - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search) # # Suffixes: @@ -33,7 +33,7 @@ defmodule Pinchflat.Media.MediaQuery do from(mi in query, join: s in assoc(mi, :source), as: :sources) end - def with_passed_retention_period(query) do + def where_past_retention_period(query) do query |> require_assoc(:source) |> where( @@ -47,10 +47,33 @@ defmodule Pinchflat.Media.MediaQuery do ) end - def with_no_culling_prevention(query) do + def where_past_redownload_delay(query) do + query + |> require_assoc(:source) + |> require_assoc(:media_profile) + |> where( + [mi, source, media_profile], + fragment( + "IFNULL(?, 0) > 0 AND DATETIME('now', '-' || ? || ' day') > ?", + media_profile.redownload_delay_days, + media_profile.redownload_delay_days, + mi.upload_date + ) + ) + end + + def where_culling_not_prevented(query) do where(query, [mi], mi.prevent_culling == false) end + def where_not_culled(query) do + where(query, [mi], is_nil(mi.culled_at)) + end + + def where_media_not_redownloaded(query) do + where(query, [mi], is_nil(mi.media_redownloaded_at)) + end + def with_id(query, id) do where(query, [mi], mi.id == ^id) end @@ -73,7 +96,7 @@ defmodule Pinchflat.Media.MediaQuery do |> where([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date) end - def with_no_prevented_download(query) do + def where_download_not_prevented(query) do where(query, [mi], mi.prevent_download == false) end @@ -129,9 +152,9 @@ defmodule Pinchflat.Media.MediaQuery do ) end - def with_media_pending_download(query) do + def where_pending_download(query) do query - |> with_no_prevented_download() + |> where_download_not_prevented() |> with_no_media_filepath() |> with_upload_date_after_source_cutoff() |> with_format_matching_profile_preference() diff --git a/lib/pinchflat/notifications/source_notifications.ex b/lib/pinchflat/notifications/source_notifications.ex index e39841d..1205858 100644 --- a/lib/pinchflat/notifications/source_notifications.ex +++ b/lib/pinchflat/notifications/source_notifications.ex @@ -59,7 +59,7 @@ defmodule Pinchflat.Notifications.SourceNotifications do defp pending_media_item_count(source) do MediaQuery.new() |> MediaQuery.for_source(source) - |> MediaQuery.with_media_pending_download() + |> MediaQuery.where_pending_download() |> Repo.aggregate(:count) end diff --git a/lib/pinchflat_web/controllers/sources/source_controller.ex b/lib/pinchflat_web/controllers/sources/source_controller.ex index d8e904d..ff12534 100644 --- a/lib/pinchflat_web/controllers/sources/source_controller.ex +++ b/lib/pinchflat_web/controllers/sources/source_controller.ex @@ -63,7 +63,7 @@ defmodule PinchflatWeb.Sources.SourceController do pending_media = MediaQuery.new() |> MediaQuery.for_source(source) - |> MediaQuery.with_media_pending_download() + |> MediaQuery.where_pending_download() |> order_by(desc: :id) |> limit(100) |> Repo.all() diff --git a/priv/repo/migrations/20240409224152_add_redownloaded_fields.exs b/priv/repo/migrations/20240409224152_add_redownloaded_fields.exs index d399110..11c528c 100644 --- a/priv/repo/migrations/20240409224152_add_redownloaded_fields.exs +++ b/priv/repo/migrations/20240409224152_add_redownloaded_fields.exs @@ -7,7 +7,7 @@ defmodule Pinchflat.Repo.Migrations.AddRedownloadedFields do end alter table(:media_items) do - add :redownloaded_at, :utc_datetime + add :media_redownloaded_at, :utc_datetime end end end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index 52eff4d..15d021d 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -130,6 +130,83 @@ defmodule Pinchflat.MediaTest do end end + describe "list_redownloadable_media_items/0" do + setup do + media_profile = media_profile_fixture(%{redownload_delay_days: 4}) + source = source_fixture(%{media_profile_id: media_profile.id}) + + {:ok, %{media_profile: media_profile, source: source}} + end + + test "returns media eligible for redownload", %{source: source} do + media_item = media_item_fixture(%{source_id: source.id, upload_date: now_minus(5, :days)}) + + assert Media.list_redownloadable_media_items() == [media_item] + end + + test "does not return media items without a media_filepath", %{source: source} do + _media_item = + media_item_fixture(%{ + source_id: source.id, + upload_date: now_minus(5, :days), + media_filepath: nil + }) + + assert Media.list_redownloadable_media_items() == [] + end + + test "does not return media items that are set to prevent download", %{source: source} do + _media_item = + media_item_fixture(%{ + source_id: source.id, + upload_date: now_minus(5, :days), + prevent_download: true + }) + + assert Media.list_redownloadable_media_items() == [] + end + + test "does not return media items that have been culled", %{source: source} do + _media_item = + media_item_fixture(%{ + source_id: source.id, + upload_date: now_minus(5, :days), + culled_at: now() + }) + + assert Media.list_redownloadable_media_items() == [] + end + + test "does not return media items before the download delay", %{source: source} do + _media_item = + media_item_fixture(%{ + source_id: source.id, + upload_date: now_minus(3, :days) + }) + + assert Media.list_redownloadable_media_items() == [] + end + + test "does not return media items that have already been redownloaded", %{source: source} do + _media_item = + media_item_fixture(%{ + source_id: source.id, + upload_date: now_minus(5, :days), + media_redownloaded_at: now() + }) + + assert Media.list_redownloadable_media_items() == [] + end + + test "does not return media items without a redownload delay" do + media_profile = media_profile_fixture(%{redownload_delay_days: nil}) + source = source_fixture(%{media_profile_id: media_profile.id}) + _media_item = media_item_fixture(%{source_id: source.id, upload_date: now_minus(5, :days)}) + + assert Media.list_redownloadable_media_items() == [] + end + end + describe "list_pending_media_items_for/1" do test "it returns pending without a filepath for a given source" do source = source_fixture()