From 3162ef492a3ba50c74340c8ff37baab202f98b4a Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Fri, 9 Feb 2024 20:38:03 -0800 Subject: [PATCH] adds lifecycle columns to source and media_item tables --- lib/pinchflat/media/media_item.ex | 5 ++++- lib/pinchflat/media_client/video_downloader.ex | 6 +++++- lib/pinchflat/media_source.ex | 4 ++++ lib/pinchflat/media_source/source.ex | 11 ++++++++++- lib/pinchflat/tasks/source_tasks.ex | 2 ++ ..._add_lifecycle_columns_to_sources_and_media.exs | 14 ++++++++++++++ .../media_client/video_downloader_test.exs | 6 ++++++ test/pinchflat/tasks/source_tasks_test.exs | 9 +++++++++ 8 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20240210005919_add_lifecycle_columns_to_sources_and_media.exs diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index d46ca83..da29691 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -15,11 +15,12 @@ defmodule Pinchflat.Media.MediaItem do media_id original_url livestream + media_downloaded_at media_filepath - source_id subtitle_filepaths thumbnail_filepath metadata_filepath + source_id )a @required_fields ~w(title original_url livestream media_id source_id)a @@ -28,6 +29,8 @@ defmodule Pinchflat.Media.MediaItem do field :media_id, :string field :original_url, :string field :livestream, :boolean, default: false + field :media_downloaded_at, :utc_datetime + field :media_filepath, :string field :thumbnail_filepath, :string field :metadata_filepath, :string diff --git a/lib/pinchflat/media_client/video_downloader.ex b/lib/pinchflat/media_client/video_downloader.ex index 93ab025..15c85b5 100644 --- a/lib/pinchflat/media_client/video_downloader.ex +++ b/lib/pinchflat/media_client/video_downloader.ex @@ -35,7 +35,11 @@ defmodule Pinchflat.MediaClient.VideoDownloader do case download_for_media_profile(media_item.media_id, media_profile, backend) do {:ok, parsed_json} -> parser = metadata_parser(backend) - parsed_attrs = parser.parse_for_media_item(parsed_json) + + parsed_attrs = + parsed_json + |> parser.parse_for_media_item() + |> Map.merge(%{media_downloaded_at: DateTime.utc_now()}) # Don't forgor to use preloaded associations or updates to # associations won't work! diff --git a/lib/pinchflat/media_source.ex b/lib/pinchflat/media_source.ex index f97be3b..e7c8abb 100644 --- a/lib/pinchflat/media_source.ex +++ b/lib/pinchflat/media_source.ex @@ -167,6 +167,10 @@ defmodule Pinchflat.MediaSource do {:ok, source} end + # IDEA: this uses a pattern where `kickoff_indexing_task` controls whether + # it should run based on the source, but `maybe_handle_media_tasks` handles that + # logic itself. Consider updating one or the other to be consistent (once I've + # decided which I like more) defp maybe_run_indexing_task(changeset, source) do case changeset.data do # If the changeset is new (not persisted), attempt indexing no matter what diff --git a/lib/pinchflat/media_source/source.ex b/lib/pinchflat/media_source/source.ex index 2ff79b5..572b35d 100644 --- a/lib/pinchflat/media_source/source.ex +++ b/lib/pinchflat/media_source/source.ex @@ -16,11 +16,19 @@ defmodule Pinchflat.MediaSource.Source do friendly_name index_frequency_minutes download_media + last_indexed_at original_url media_profile_id )a - @required_fields @allowed_fields -- ~w(index_frequency_minutes friendly_name)a + @required_fields ~w( + collection_name + collection_id + collection_type + download_media + original_url + media_profile_id + )a schema "sources" do field :friendly_name, :string @@ -29,6 +37,7 @@ defmodule Pinchflat.MediaSource.Source do field :collection_type, Ecto.Enum, values: [:channel, :playlist] field :index_frequency_minutes, :integer, default: 60 * 24 field :download_media, :boolean, default: true + field :last_indexed_at, :utc_datetime # This should only be used for user reference going forward # as the collection_id should be used for all API calls field :original_url, :string diff --git a/lib/pinchflat/tasks/source_tasks.ex b/lib/pinchflat/tasks/source_tasks.ex index af6757b..98d2c0d 100644 --- a/lib/pinchflat/tasks/source_tasks.ex +++ b/lib/pinchflat/tasks/source_tasks.ex @@ -5,6 +5,7 @@ defmodule Pinchflat.Tasks.SourceTasks do alias Pinchflat.Media alias Pinchflat.Tasks + alias Pinchflat.MediaSource alias Pinchflat.MediaSource.Source alias Pinchflat.MediaClient.SourceDetails alias Pinchflat.Workers.MediaIndexingWorker @@ -42,6 +43,7 @@ defmodule Pinchflat.Tasks.SourceTasks do """ def index_media_items(%Source{} = source) do {:ok, media_attributes} = SourceDetails.get_media_attributes(source.original_url) + MediaSource.update_source(source, %{last_indexed_at: DateTime.utc_now()}) media_attributes |> Enum.map(fn media_attrs -> diff --git a/priv/repo/migrations/20240210005919_add_lifecycle_columns_to_sources_and_media.exs b/priv/repo/migrations/20240210005919_add_lifecycle_columns_to_sources_and_media.exs new file mode 100644 index 0000000..0d08cc9 --- /dev/null +++ b/priv/repo/migrations/20240210005919_add_lifecycle_columns_to_sources_and_media.exs @@ -0,0 +1,14 @@ +defmodule Pinchflat.Repo.Migrations.AddLifecycleColumnsToSourcesAndMedia do + use Ecto.Migration + + def change do + alter table(:media_items) do + add :media_downloaded_at, :utc_datetime + add :details_updated_at, :utc_datetime + end + + alter table(:sources) do + add :last_indexed_at, :utc_datetime + end + end +end diff --git a/test/pinchflat/media_client/video_downloader_test.exs b/test/pinchflat/media_client/video_downloader_test.exs index b03ef8c..cfc362a 100644 --- a/test/pinchflat/media_client/video_downloader_test.exs +++ b/test/pinchflat/media_client/video_downloader_test.exs @@ -57,6 +57,12 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do :ok end + test "it sets the media_downloaded_at", %{media_item: media_item} do + assert media_item.media_downloaded_at == nil + assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item) + assert DateTime.diff(DateTime.utc_now(), updated_media_item.media_downloaded_at) < 1 + end + test "it extracts the title", %{media_item: media_item} do assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item) assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake" diff --git a/test/pinchflat/tasks/source_tasks_test.exs b/test/pinchflat/tasks/source_tasks_test.exs index 5d5ca11..6917401 100644 --- a/test/pinchflat/tasks/source_tasks_test.exs +++ b/test/pinchflat/tasks/source_tasks_test.exs @@ -103,6 +103,15 @@ defmodule Pinchflat.Tasks.SourceTasksTest do assert Enum.all?(first_run, fn %MediaItem{} -> true end) assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end) end + + test "it updates the source's last_indexed_at field", %{source: source} do + assert source.last_indexed_at == nil + + SourceTasks.index_media_items(source) + source = Repo.reload!(source) + + assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 1 + end end describe "enqueue_pending_media_tasks/1" do