[WIP] getting retention methods in place

config/config.exs

@@ -46,7 +46,13 @@ config :pinchflat, Oban,
   engine: Oban.Engines.Lite,
   repo: Pinchflat.Repo,
   # Keep old jobs for 30 days for display in the UI
-  plugins: [{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}],
+  plugins: [
+    {Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
+    {Oban.Plugins.Cron,
+     crontab: [
+       {"@midnight", Pinchflat.Downloading.MediaRetentionWorker}
+     ]}
+  ],
   # TODO: consider making this an env var or something?
   queues: [
     default: 10,

lib/pinchflat/downloading/media_retention_worker.ex

@@ -0,0 +1,21 @@
+defmodule Pinchflat.Downloading.MediaRetentionWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_metadata,
+    unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
+    tags: ["media_item", "local_metadata"]
+
+  require Logger
+
+  # TODO: docs
+  # TODO: test
+  # TODO: set to a 1 min interval to make sure that cron works
+  # TODO: update wiki after this is merged
+  # TODO: remove data backfill worker
+  # TODO: (other PR - not this one) add way to manually trigger an index of a source AND a pending media download
+  # of a source
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+  end
+end

lib/pinchflat/media/media.ex

@@ -22,6 +22,22 @@ defmodule Pinchflat.Media do
     Repo.all(MediaItem)
   end
 
+  # TODO: docs
+  # TODO: add attr for excluding media items from this list (ie: keep forever)
+  @doc """
+  Returns a list of media_items that are cullable based on the retention period
+  of the source they belong to.
+
+  Returns [%MediaItem{}, ...]
+  """
+  def list_cullable_media_items do
+    MediaQuery.new()
+    |> MediaQuery.join_sources()
+    |> MediaQuery.with_media_filepath()
+    |> MediaQuery.with_passed_retention_period()
+    |> Repo.all()
+  end
+
   @doc """
   Returns a list of pending media_items for a given source, where
   pending means the `media_filepath` is `nil` AND the media_item

lib/pinchflat/media/media_item.ex

@@ -44,7 +44,7 @@ defmodule Pinchflat.Media.MediaItem do
     source_id
     upload_date
     short_form_content
-    )a
+  )a
 
   schema "media_items" do
     # This is _not_ used as the primary key or internally in the database

lib/pinchflat/media/media_query.ex

@@ -17,6 +17,7 @@ defmodule Pinchflat.Media.MediaQuery do
 
   # Prefixes:
   # - for_* - belonging to a certain record
+  # - join_* - for joining on a certain record
   # - with_* - for filtering based on full, concrete attributes
   # - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search)
   #
@@ -31,6 +32,23 @@ defmodule Pinchflat.Media.MediaQuery do
     where(query, [mi], mi.source_id == ^source.id)
   end
 
+  def join_sources(query) do
+    from(mi in query, join: s in assoc(mi, :source), as: :sources)
+  end
+
+  def with_passed_retention_period(query) do
+    where(
+      query,
+      [mi, sources],
+      fragment(
+        "IFNULL(?, 0) > 0 AND DATETIME('now', '-' || ? || ' day') > ?",
+        sources.retention_period_days,
+        sources.retention_period_days,
+        mi.media_downloaded_at
+      )
+    )
+  end
+
   def with_id(query, id) do
     where(query, [mi], mi.id == ^id)
   end

lib/pinchflat/sources/source.ex

@@ -108,6 +108,7 @@ defmodule Pinchflat.Sources.Source do
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(required_fields)
+    |> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
   end

test/pinchflat/media_test.exs

@@ -38,6 +38,77 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "list_cullable_media_items/0" do
+    test "returns media items where the source has a retention period" do
+      source_one = source_fixture(%{retention_period_days: 2})
+      source_two = source_fixture(%{retention_period_days: 0})
+      source_three = source_fixture(%{retention_period_days: nil})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source_two.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source_three.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source_one.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+
+    test "returns media_items with a media_filepath" do
+      source = source_fixture(%{retention_period_days: 2})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: nil,
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+
+    test "returns items that have passed their retention period" do
+      source = source_fixture(%{retention_period_days: 2})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(2, :days)
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+  end
+
   describe "list_pending_media_items_for/1" do
     test "it returns pending without a filepath for a given source" do
       source = source_fixture()