From c9f3f585c0b314cadda12c6e468ea0056494babc Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Sun, 10 Mar 2024 16:49:08 -0700 Subject: [PATCH] Updated indexer to pull upload date --- lib/pinchflat/media/media_item.ex | 6 +++--- lib/pinchflat/yt_dlp/backend/media.ex | 17 +++++++++++++---- ...0230713_add_uploaded_at_to_media_items.exs | 5 +++-- test/pinchflat/media_test.exs | 3 ++- .../tasks/media_items_tasks_test.exs | 3 ++- test/pinchflat/tasks/source_tasks_test.exs | 3 ++- test/pinchflat/yt_dlp/backend/media_test.exs | 19 ++++++++++++------- test/support/fixtures/media_fixtures.ex | 6 ++++-- test/support/fixtures/sources_fixtures.ex | 9 ++++++--- 9 files changed, 47 insertions(+), 24 deletions(-) diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index 1c3e092..6b07cac 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -20,7 +20,7 @@ defmodule Pinchflat.Media.MediaItem do :livestream, :source_id, :short_form_content, - :uploaded_at, + :upload_date, # these fields are captured only on download :media_downloaded_at, :media_filepath, @@ -36,7 +36,7 @@ defmodule Pinchflat.Media.MediaItem do livestream media_id source_id - uploaded_at + upload_date short_form_content )a @@ -48,7 +48,7 @@ defmodule Pinchflat.Media.MediaItem do field :livestream, :boolean, default: false field :short_form_content, :boolean, default: false field :media_downloaded_at, :utc_datetime - field :uploaded_at, :utc_datetime + field :upload_date, :date field :media_filepath, :string field :media_size_bytes, :integer diff --git a/lib/pinchflat/yt_dlp/backend/media.ex b/lib/pinchflat/yt_dlp/backend/media.ex index 1c96237..09ad719 100644 --- a/lib/pinchflat/yt_dlp/backend/media.ex +++ b/lib/pinchflat/yt_dlp/backend/media.ex @@ -9,7 +9,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do :description, :original_url, :livestream, - :short_form_content + :short_form_content, + :upload_date ] defstruct [ @@ -18,7 +19,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do :description, :original_url, :livestream, - :short_form_content + :short_form_content, + :upload_date ] alias __MODULE__ @@ -67,7 +69,7 @@ defmodule Pinchflat.YtDlp.Backend.Media do Returns the output template for yt-dlp's indexing command. """ def indexing_output_template do - "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j" + "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" end @doc """ @@ -83,7 +85,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do description: response["description"], original_url: response["webpage_url"], livestream: response["was_live"], - short_form_content: short_form_content?(response) + short_form_content: short_form_content?(response), + upload_date: parse_upload_date(response["upload_date"]) } end @@ -100,6 +103,12 @@ defmodule Pinchflat.YtDlp.Backend.Media do end end + defp parse_upload_date(upload_date) do + <> <> <> <> <> = upload_date + + Date.from_iso8601!("#{year}-#{month}-#{day}") + end + defp backend_runner do # This approach lets us mock the command for testing Application.get_env(:pinchflat, :yt_dlp_runner) diff --git a/priv/repo/migrations/20240310230713_add_uploaded_at_to_media_items.exs b/priv/repo/migrations/20240310230713_add_uploaded_at_to_media_items.exs index 95a4297..f18c23e 100644 --- a/priv/repo/migrations/20240310230713_add_uploaded_at_to_media_items.exs +++ b/priv/repo/migrations/20240310230713_add_uploaded_at_to_media_items.exs @@ -4,8 +4,9 @@ defmodule Pinchflat.Repo.Migrations.AddUploadedAtToMediaItems do def change do alter table(:media_items) do # Setting default to unix epoch so I can enforce not null BUT also easily - # identify records that were created before this column was added - add :uploaded_at, :utc_datetime, default: "1970-01-01T00:00:00", null: false + # identify records that were created before this column was added. + # Not a DateTime because yt-dlp only returns the date + add :upload_date, :date, default: "1970-01-01", null: false end end end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index 870cef7..8979108 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -373,7 +373,8 @@ defmodule Pinchflat.MediaTest do title: Faker.Commerce.product_name(), media_filepath: "/video/#{Faker.File.file_name(:video)}", source_id: source_fixture().id, - original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}" + original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}", + upload_date: Date.utc_today() } assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs) diff --git a/test/pinchflat/tasks/media_items_tasks_test.exs b/test/pinchflat/tasks/media_items_tasks_test.exs index 7cf18e8..36bc91f 100644 --- a/test/pinchflat/tasks/media_items_tasks_test.exs +++ b/test/pinchflat/tasks/media_items_tasks_test.exs @@ -88,7 +88,8 @@ defmodule Pinchflat.Tasks.MediaItemTasksTest do was_live: true, description: "desc2", aspect_ratio: 1.67, - duration: 345.67 + duration: 345.67, + upload_date: "20210101" }) {:ok, output} diff --git a/test/pinchflat/tasks/source_tasks_test.exs b/test/pinchflat/tasks/source_tasks_test.exs index 55df00c..b3a8274 100644 --- a/test/pinchflat/tasks/source_tasks_test.exs +++ b/test/pinchflat/tasks/source_tasks_test.exs @@ -282,7 +282,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do was_live: true, description: "desc2", aspect_ratio: 1.67, - duration: 345.67 + duration: 345.67, + upload_date: "20210101" }) File.write(filepath, contents) diff --git a/test/pinchflat/yt_dlp/backend/media_test.exs b/test/pinchflat/yt_dlp/backend/media_test.exs index 94c031d..e69ce2c 100644 --- a/test/pinchflat/yt_dlp/backend/media_test.exs +++ b/test/pinchflat/yt_dlp/backend/media_test.exs @@ -79,7 +79,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do describe "indexing_output_template/0" do test "contains all the greatest hits" do - assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j" == + assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" == Media.indexing_output_template() end end @@ -93,7 +93,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "was_live" => false, "aspect_ratio" => 1.0, - "duration" => 60 + "duration" => 60, + "upload_date" => "20210101" } assert %Media{ @@ -102,15 +103,17 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do description: "I'm not sure what I expected.", original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk", livestream: false, - short_form_content: false - } = Media.response_to_struct(response) + short_form_content: false, + upload_date: Date.from_iso8601!("2021-01-01") + } == Media.response_to_struct(response) end test "sets short_form_content to true if the URL contains /shorts/" do response = %{ "webpage_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk", "aspect_ratio" => 1.0, - "duration" => 61 + "duration" => 61, + "upload_date" => "20210101" } assert %Media{short_form_content: true} = Media.response_to_struct(response) @@ -120,7 +123,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do response = %{ "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "aspect_ratio" => 0.5, - "duration" => 59 + "duration" => 59, + "upload_date" => "20210101" } assert %Media{short_form_content: true} = Media.response_to_struct(response) @@ -130,7 +134,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do response = %{ "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "aspect_ratio" => 1.0, - "duration" => 61 + "duration" => 61, + "upload_date" => "20210101" } assert %Media{short_form_content: false} = Media.response_to_struct(response) diff --git a/test/support/fixtures/media_fixtures.ex b/test/support/fixtures/media_fixtures.ex index e4f3ffe..ce0a3a7 100644 --- a/test/support/fixtures/media_fixtures.ex +++ b/test/support/fixtures/media_fixtures.ex @@ -21,7 +21,8 @@ defmodule Pinchflat.MediaFixtures do livestream: false, short_form_content: false, media_filepath: "/video/#{Faker.File.file_name(:video)}", - source_id: SourcesFixtures.source_fixture().id + source_id: SourcesFixtures.source_fixture().id, + upload_date: DateTime.utc_now() }) |> Pinchflat.Media.create_media_item() @@ -75,7 +76,8 @@ defmodule Pinchflat.MediaFixtures do was_live: false, description: "desc1", aspect_ratio: 1.67, - duration: 123.45 + duration: 123.45, + upload_date: "20210101" } Phoenix.json_library().encode!(media_attributes) diff --git a/test/support/fixtures/sources_fixtures.ex b/test/support/fixtures/sources_fixtures.ex index 89c113c..9f4b076 100644 --- a/test/support/fixtures/sources_fixtures.ex +++ b/test/support/fixtures/sources_fixtures.ex @@ -39,7 +39,8 @@ defmodule Pinchflat.SourcesFixtures do was_live: false, description: "desc1", aspect_ratio: 1.67, - duration: 12.34 + duration: 12.34, + upload_date: "20210101" }, %{ id: "video2", @@ -48,7 +49,8 @@ defmodule Pinchflat.SourcesFixtures do was_live: true, description: "desc2", aspect_ratio: 1.67, - duration: 345.67 + duration: 345.67, + upload_date: "20220202" }, %{ id: "video3", @@ -57,7 +59,8 @@ defmodule Pinchflat.SourcesFixtures do was_live: false, description: "desc3", aspect_ratio: 1.0, - duration: 678.90 + duration: 678.90, + upload_date: "20230303" } ]