diff --git a/config/config.exs b/config/config.exs index 9c60df4..49dde36 100644 --- a/config/config.exs +++ b/config/config.exs @@ -46,7 +46,13 @@ config :pinchflat, Oban, engine: Oban.Engines.Lite, repo: Pinchflat.Repo, # Keep old jobs for 30 days for display in the UI - plugins: [{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}], + plugins: [ + {Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}, + {Oban.Plugins.Cron, + crontab: [ + {"@midnight", Pinchflat.Downloading.MediaRetentionWorker} + ]} + ], # TODO: consider making this an env var or something? queues: [ default: 10, diff --git a/lib/pinchflat/downloading/media_retention_worker.ex b/lib/pinchflat/downloading/media_retention_worker.ex new file mode 100644 index 0000000..dd560d3 --- /dev/null +++ b/lib/pinchflat/downloading/media_retention_worker.ex @@ -0,0 +1,21 @@ +defmodule Pinchflat.Downloading.MediaRetentionWorker do + @moduledoc false + + use Oban.Worker, + queue: :local_metadata, + unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]], + tags: ["media_item", "local_metadata"] + + require Logger + + # TODO: docs + # TODO: test + # TODO: set to a 1 min interval to make sure that cron works + # TODO: update wiki after this is merged + # TODO: remove data backfill worker + # TODO: (other PR - not this one) add way to manually trigger an index of a source AND a pending media download + # of a source + @impl Oban.Worker + def perform(%Oban.Job{}) do + end +end diff --git a/lib/pinchflat/media/media.ex b/lib/pinchflat/media/media.ex index 4ab09d8..1caf266 100644 --- a/lib/pinchflat/media/media.ex +++ b/lib/pinchflat/media/media.ex @@ -22,6 +22,22 @@ defmodule Pinchflat.Media do Repo.all(MediaItem) end + # TODO: docs + # TODO: add attr for excluding media items from this list (ie: keep forever) + @doc """ + Returns a list of media_items that are cullable based on the retention period + of the source they belong to. + + Returns [%MediaItem{}, ...] + """ + def list_cullable_media_items do + MediaQuery.new() + |> MediaQuery.join_sources() + |> MediaQuery.with_media_filepath() + |> MediaQuery.with_passed_retention_period() + |> Repo.all() + end + @doc """ Returns a list of pending media_items for a given source, where pending means the `media_filepath` is `nil` AND the media_item diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index 04e7038..05ab537 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -44,7 +44,7 @@ defmodule Pinchflat.Media.MediaItem do source_id upload_date short_form_content - )a + )a schema "media_items" do # This is _not_ used as the primary key or internally in the database diff --git a/lib/pinchflat/media/media_query.ex b/lib/pinchflat/media/media_query.ex index 30960d6..da30300 100644 --- a/lib/pinchflat/media/media_query.ex +++ b/lib/pinchflat/media/media_query.ex @@ -17,6 +17,7 @@ defmodule Pinchflat.Media.MediaQuery do # Prefixes: # - for_* - belonging to a certain record + # - join_* - for joining on a certain record # - with_* - for filtering based on full, concrete attributes # - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search) # @@ -31,6 +32,23 @@ defmodule Pinchflat.Media.MediaQuery do where(query, [mi], mi.source_id == ^source.id) end + def join_sources(query) do + from(mi in query, join: s in assoc(mi, :source), as: :sources) + end + + def with_passed_retention_period(query) do + where( + query, + [mi, sources], + fragment( + "IFNULL(?, 0) > 0 AND DATETIME('now', '-' || ? || ' day') > ?", + sources.retention_period_days, + sources.retention_period_days, + mi.media_downloaded_at + ) + ) + end + def with_id(query, id) do where(query, [mi], mi.id == ^id) end diff --git a/lib/pinchflat/sources/source.ex b/lib/pinchflat/sources/source.ex index 2932ac9..c2ef216 100644 --- a/lib/pinchflat/sources/source.ex +++ b/lib/pinchflat/sources/source.ex @@ -108,6 +108,7 @@ defmodule Pinchflat.Sources.Source do |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end) |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end) |> validate_required(required_fields) + |> validate_number(:retention_period_days, greater_than_or_equal_to: 0) |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false) |> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url) end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index cc435b8..9a17ca9 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -38,6 +38,77 @@ defmodule Pinchflat.MediaTest do end end + describe "list_cullable_media_items/0" do + test "returns media items where the source has a retention period" do + source_one = source_fixture(%{retention_period_days: 2}) + source_two = source_fixture(%{retention_period_days: 0}) + source_three = source_fixture(%{retention_period_days: nil}) + + _media_item = + media_item_fixture(%{ + source_id: source_two.id, + media_filepath: "/video/#{Faker.File.file_name(:video)}", + media_downloaded_at: now_minus(3, :days) + }) + + _media_item = + media_item_fixture(%{ + source_id: source_three.id, + media_filepath: "/video/#{Faker.File.file_name(:video)}", + media_downloaded_at: now_minus(3, :days) + }) + + expected_media_item = + media_item_fixture(%{ + source_id: source_one.id, + media_filepath: "/video/#{Faker.File.file_name(:video)}", + media_downloaded_at: now_minus(3, :days) + }) + + assert Media.list_cullable_media_items() == [expected_media_item] + end + + test "returns media_items with a media_filepath" do + source = source_fixture(%{retention_period_days: 2}) + + _media_item = + media_item_fixture(%{ + source_id: source.id, + media_filepath: nil, + media_downloaded_at: now_minus(3, :days) + }) + + expected_media_item = + media_item_fixture(%{ + source_id: source.id, + media_filepath: "/video/#{Faker.File.file_name(:video)}", + media_downloaded_at: now_minus(3, :days) + }) + + assert Media.list_cullable_media_items() == [expected_media_item] + end + + test "returns items that have passed their retention period" do + source = source_fixture(%{retention_period_days: 2}) + + _media_item = + media_item_fixture(%{ + source_id: source.id, + media_filepath: "/video/#{Faker.File.file_name(:video)}", + media_downloaded_at: now_minus(2, :days) + }) + + expected_media_item = + media_item_fixture(%{ + source_id: source.id, + media_filepath: "/video/#{Faker.File.file_name(:video)}", + media_downloaded_at: now_minus(3, :days) + }) + + assert Media.list_cullable_media_items() == [expected_media_item] + end + end + describe "list_pending_media_items_for/1" do test "it returns pending without a filepath for a given source" do source = source_fixture()