Updated indexer to pull upload date

lib/pinchflat/media/media_item.ex

@@ -20,7 +20,7 @@ defmodule Pinchflat.Media.MediaItem do
     :livestream,
     :source_id,
     :short_form_content,
-    :uploaded_at,
+    :upload_date,
     # these fields are captured only on download
     :media_downloaded_at,
     :media_filepath,
@@ -36,7 +36,7 @@ defmodule Pinchflat.Media.MediaItem do
     livestream
     media_id
     source_id
-    uploaded_at
+    upload_date
     short_form_content
     )a
 
@@ -48,7 +48,7 @@ defmodule Pinchflat.Media.MediaItem do
     field :livestream, :boolean, default: false
     field :short_form_content, :boolean, default: false
     field :media_downloaded_at, :utc_datetime
-    field :uploaded_at, :utc_datetime
+    field :upload_date, :date
 
     field :media_filepath, :string
     field :media_size_bytes, :integer

lib/pinchflat/yt_dlp/backend/media.ex

@@ -9,7 +9,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
     :description,
     :original_url,
     :livestream,
-    :short_form_content
+    :short_form_content,
+    :upload_date
   ]
 
   defstruct [
@@ -18,7 +19,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
     :description,
     :original_url,
     :livestream,
-    :short_form_content
+    :short_form_content,
+    :upload_date
   ]
 
   alias __MODULE__
@@ -67,7 +69,7 @@ defmodule Pinchflat.YtDlp.Backend.Media do
   Returns the output template for yt-dlp's indexing command.
   """
   def indexing_output_template do
-    "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j"
+    "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j"
   end
 
   @doc """
@@ -83,7 +85,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       description: response["description"],
       original_url: response["webpage_url"],
       livestream: response["was_live"],
-      short_form_content: short_form_content?(response)
+      short_form_content: short_form_content?(response),
+      upload_date: parse_upload_date(response["upload_date"])
     }
   end
 
@@ -100,6 +103,12 @@ defmodule Pinchflat.YtDlp.Backend.Media do
     end
   end
 
+  defp parse_upload_date(upload_date) do
+    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
+
+    Date.from_iso8601!("#{year}-#{month}-#{day}")
+  end
+
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

priv/repo/migrations/20240310230713_add_uploaded_at_to_media_items.exs

@@ -4,8 +4,9 @@ defmodule Pinchflat.Repo.Migrations.AddUploadedAtToMediaItems do
   def change do
     alter table(:media_items) do
       # Setting default to unix epoch so I can enforce not null BUT also easily
-      # identify records that were created before this column was added
+      # identify records that were created before this column was added.
-      add :uploaded_at, :utc_datetime, default: "1970-01-01T00:00:00", null: false
+      # Not a DateTime because yt-dlp only returns the date
+      add :upload_date, :date, default: "1970-01-01", null: false
     end
   end
 end

test/pinchflat/media_test.exs

@@ -373,7 +373,8 @@ defmodule Pinchflat.MediaTest do
         title: Faker.Commerce.product_name(),
         media_filepath: "/video/#{Faker.File.file_name(:video)}",
         source_id: source_fixture().id,
-        original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}"
+        original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
+        upload_date: Date.utc_today()
       }
 
       assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)

test/pinchflat/tasks/media_items_tasks_test.exs

@@ -88,7 +88,8 @@ defmodule Pinchflat.Tasks.MediaItemTasksTest do
             was_live: true,
             description: "desc2",
             aspect_ratio: 1.67,
-            duration: 345.67
+            duration: 345.67,
+            upload_date: "20210101"
           })
 
         {:ok, output}

test/pinchflat/tasks/source_tasks_test.exs

@@ -282,7 +282,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
             was_live: true,
             description: "desc2",
             aspect_ratio: 1.67,
-            duration: 345.67
+            duration: 345.67,
+            upload_date: "20210101"
           })
 
         File.write(filepath, contents)

test/pinchflat/yt_dlp/backend/media_test.exs

@@ -79,7 +79,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
 
   describe "indexing_output_template/0" do
     test "contains all the greatest hits" do
-      assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j" ==
+      assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" ==
                Media.indexing_output_template()
     end
   end
@@ -93,7 +93,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
         "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
         "was_live" => false,
         "aspect_ratio" => 1.0,
-        "duration" => 60
+        "duration" => 60,
+        "upload_date" => "20210101"
       }
 
       assert %Media{
@@ -102,15 +103,17 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
                description: "I'm not sure what I expected.",
                original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk",
                livestream: false,
-               short_form_content: false
+               short_form_content: false,
-             } = Media.response_to_struct(response)
+               upload_date: Date.from_iso8601!("2021-01-01")
+             } == Media.response_to_struct(response)
     end
 
     test "sets short_form_content to true if the URL contains /shorts/" do
       response = %{
         "webpage_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk",
         "aspect_ratio" => 1.0,
-        "duration" => 61
+        "duration" => 61,
+        "upload_date" => "20210101"
       }
 
       assert %Media{short_form_content: true} = Media.response_to_struct(response)
@@ -120,7 +123,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
       response = %{
         "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
         "aspect_ratio" => 0.5,
-        "duration" => 59
+        "duration" => 59,
+        "upload_date" => "20210101"
       }
 
       assert %Media{short_form_content: true} = Media.response_to_struct(response)
@@ -130,7 +134,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
       response = %{
         "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
         "aspect_ratio" => 1.0,
-        "duration" => 61
+        "duration" => 61,
+        "upload_date" => "20210101"
       }
 
       assert %Media{short_form_content: false} = Media.response_to_struct(response)

test/support/fixtures/media_fixtures.ex

@@ -21,7 +21,8 @@ defmodule Pinchflat.MediaFixtures do
         livestream: false,
         short_form_content: false,
         media_filepath: "/video/#{Faker.File.file_name(:video)}",
-        source_id: SourcesFixtures.source_fixture().id
+        source_id: SourcesFixtures.source_fixture().id,
+        upload_date: DateTime.utc_now()
       })
       |> Pinchflat.Media.create_media_item()
 
@@ -75,7 +76,8 @@ defmodule Pinchflat.MediaFixtures do
       was_live: false,
       description: "desc1",
       aspect_ratio: 1.67,
-      duration: 123.45
+      duration: 123.45,
+      upload_date: "20210101"
     }
 
     Phoenix.json_library().encode!(media_attributes)

test/support/fixtures/sources_fixtures.ex

@@ -39,7 +39,8 @@ defmodule Pinchflat.SourcesFixtures do
         was_live: false,
         description: "desc1",
         aspect_ratio: 1.67,
-        duration: 12.34
+        duration: 12.34,
+        upload_date: "20210101"
       },
       %{
         id: "video2",
@@ -48,7 +49,8 @@ defmodule Pinchflat.SourcesFixtures do
         was_live: true,
         description: "desc2",
         aspect_ratio: 1.67,
-        duration: 345.67
+        duration: 345.67,
+        upload_date: "20220202"
       },
       %{
         id: "video3",
@@ -57,7 +59,8 @@ defmodule Pinchflat.SourcesFixtures do
         was_live: false,
         description: "desc3",
         aspect_ratio: 1.0,
-        duration: 678.90
+        duration: 678.90,
+        upload_date: "20230303"
       }
     ]