Refactored yt-dlp media module

lib/pinchflat/media/media_item.ex

@@ -51,7 +51,7 @@ defmodule Pinchflat.Media.MediaItem do
     livestream
     media_id
     source_id
-    upload_date
+    uploaded_at
     short_form_content
   )a
 
@@ -130,6 +130,7 @@ defmodule Pinchflat.Media.MediaItem do
     ~w(__meta__ __struct__ metadata tasks media_items_search_index)a
   end
 
+  # TODO: refactor
   defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :upload_date) do
     source_id = get_field(changeset, :source_id)
     source = Sources.get_source!(source_id)

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -95,7 +95,11 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
   def parse_upload_date(upload_date) do
     <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
 
-    Date.from_iso8601!("#{year}-#{month}-#{day}")
+    # TODO: test new
+    case DateTime.from_iso8601("#{year}-#{month}-#{day}T00:00:00Z") do
+      {:ok, datetime, _} -> datetime
+      _ -> raise "Invalid upload date: #{upload_date}"
+    end
   end
 
   @doc """

lib/pinchflat/metadata/nfo_builder.ex

@@ -38,6 +38,7 @@ defmodule Pinchflat.Metadata.NfoBuilder do
   end
 
   defp build_for_media_item(metadata) do
+    # TODO: see how this works post-change
     upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"])
     # Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub.
     # WHO NEEDS A FANCY XML PARSER ANYWAY?!

lib/pinchflat/yt_dlp/media.ex

@@ -10,7 +10,7 @@ defmodule Pinchflat.YtDlp.Media do
     :original_url,
     :livestream,
     :short_form_content,
-    :upload_date,
+    :uploaded_at,
     :duration_seconds
   ]
 
@@ -21,7 +21,7 @@ defmodule Pinchflat.YtDlp.Media do
     :original_url,
     :livestream,
     :short_form_content,
-    :upload_date,
+    :uploaded_at,
     :duration_seconds
   ]
 
@@ -72,7 +72,7 @@ defmodule Pinchflat.YtDlp.Media do
   Returns the output template for yt-dlp's indexing command.
   """
   def indexing_output_template do
-    "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j"
+    "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date,timestamp})j"
   end
 
   @doc """
@@ -90,7 +90,7 @@ defmodule Pinchflat.YtDlp.Media do
       livestream: !!response["was_live"],
       duration_seconds: response["duration"] && round(response["duration"]),
       short_form_content: response["webpage_url"] && short_form_content?(response),
-      upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
+      uploaded_at: parse_uploaded_at(response)
     }
   end
 
@@ -110,6 +110,18 @@ defmodule Pinchflat.YtDlp.Media do
     end
   end
 
+  defp parse_uploaded_at(%{"timestamp" => ts} = response) when is_number(ts) do
+    case DateTime.from_unix(ts) do
+      {:ok, datetime} -> datetime
+      _ -> MetadataFileHelpers.parse_upload_date(response["upload_date"])
+    end
+  end
+
+  # This field is needed before inserting into the database, but absence
+  # of this field should fail at insert-time rather than here
+  defp parse_uploaded_at(%{"upload_date" => nil}), do: nil
+  defp parse_uploaded_at(response), do: MetadataFileHelpers.parse_upload_date(response["upload_date"])
+
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

test/pinchflat/yt_dlp/media_test.exs

@@ -79,7 +79,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
 
   describe "indexing_output_template/0" do
     test "contains all the greatest hits" do
-      assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" ==
+      assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date,timestamp})j" ==
                Media.indexing_output_template()
     end
   end
@@ -94,7 +94,8 @@ defmodule Pinchflat.YtDlp.MediaTest do
         "was_live" => false,
         "aspect_ratio" => 1.0,
         "duration" => 60,
-        "upload_date" => "20210101"
+        "upload_date" => "20210101",
+        "timestamp" => 1_600_000_000
       }
 
       assert %Media{
@@ -104,7 +105,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
                original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk",
                livestream: false,
                short_form_content: false,
-               upload_date: Date.from_iso8601!("2021-01-01"),
+               uploaded_at: ~U[2020-09-13 12:26:40Z],
                duration_seconds: 60
              } == Media.response_to_struct(response)
     end
@@ -146,34 +147,11 @@ defmodule Pinchflat.YtDlp.MediaTest do
       response = %{
         "webpage_url" => nil,
         "aspect_ratio" => nil,
-        "duration" => nil
-      }
-
-      assert %Media{short_form_content: nil} = Media.response_to_struct(response)
-    end
-
-    test "parses the upload date" do
-      response = %{
-        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
-        "aspect_ratio" => 1.0,
-        "duration" => 61,
+        "duration" => nil,
         "upload_date" => "20210101"
       }
 
-      expected_date = Date.from_iso8601!("2021-01-01")
-
-      assert %Media{upload_date: ^expected_date} = Media.response_to_struct(response)
-    end
-
-    test "doesn't blow up if upload date is missing" do
-      response = %{
-        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
-        "aspect_ratio" => 1.0,
-        "duration" => 61,
-        "upload_date" => nil
-      }
-
-      assert %Media{upload_date: nil} = Media.response_to_struct(response)
+      assert %Media{short_form_content: nil} = Media.response_to_struct(response)
     end
 
     test "parses the duration" do
@@ -209,4 +187,58 @@ defmodule Pinchflat.YtDlp.MediaTest do
       assert %Media{livestream: false} = Media.response_to_struct(response)
     end
   end
+
+  describe "response_to_struct/1 when testing uploaded_at" do
+    test "parses the upload date from the timestamp if present" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61,
+        "upload_date" => "20210101",
+        "timestamp" => 1_600_000_000
+      }
+
+      expected_date = ~U[2020-09-13 12:26:40Z]
+
+      assert %Media{uploaded_at: ^expected_date} = Media.response_to_struct(response)
+    end
+
+    test "parses the upload date from the uploaded_at if timestamp is present but nil" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61,
+        "upload_date" => "20210101",
+        "timestamp" => nil
+      }
+
+      expected_date = ~U[2021-01-01 00:00:00Z]
+
+      assert %Media{uploaded_at: ^expected_date} = Media.response_to_struct(response)
+    end
+
+    test "parses the upload date from the uploaded_at if timestamp absent" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61,
+        "upload_date" => "20210101"
+      }
+
+      expected_date = ~U[2021-01-01 00:00:00Z]
+
+      assert %Media{uploaded_at: ^expected_date} = Media.response_to_struct(response)
+    end
+
+    test "doesn't blow up if upload date is missing" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61,
+        "upload_date" => nil
+      }
+
+      assert %Media{uploaded_at: nil} = Media.response_to_struct(response)
+    end
+  end
 end