Improved robustness of file downloads (#225)

lib/pinchflat/downloading/download_option_builder.ex

@@ -124,9 +124,18 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
       [format_sort: "res:#{res},+codec:avc:m4a", remux_video: "mp4"]
     end
 
+    audio_format_precedence = [
+      "bestaudio[ext=m4a]",
+      "bestaudio[ext=mp3]",
+      "bestaudio",
+      "best[ext=m4a]",
+      "best[ext=mp3]",
+      "best"
+    ]
+
     case media_profile.preferred_resolution do
       # Also be aware that :audio disabled all embedding options for subtitles
-      :audio -> [:extract_audio, format: "bestaudio[ext=m4a]"]
+      :audio -> [:extract_audio, format: Enum.join(audio_format_precedence, "/")]
       :"360p" -> video_codec_option.("360")
       :"480p" -> video_codec_option.("480")
       :"720p" -> video_codec_option.("720")

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -54,24 +54,36 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
 
   @doc """
   Downloads and stores a thumbnail for a media item, returning the filepath.
-  Chooses the highest quality jpg thumbnail available.
+  Chooses the highest quality thumbnail available (preferring jpg). Returns
+  nil if no thumbnails are available.
 
-  Returns binary()
+  Returns binary() | nil
   """
   def download_and_store_thumbnail_for(database_record, metadata_map) do
-    thumbnail_url =
-      metadata_map["thumbnails"]
-      |> Enum.filter(&(&1["preference"] && String.ends_with?(&1["url"], ".jpg")))
-      |> Enum.sort(&(&1["preference"] >= &2["preference"]))
-      |> List.first()
-      |> Map.get("url")
+    thumbnails =
+      (metadata_map["thumbnails"] || [])
+      # Give it a low preference if the `preference` key doesn't exist
+      |> Enum.map(&Map.put_new(&1, "preference", -1000))
+      # Give it a low preference if image isn't a jpg
+      |> Enum.map(fn t ->
+        preference_weight = if String.ends_with?(t["url"], ".jpg"), do: t["preference"], else: t["preference"] - 1000
 
-    filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
-    thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
+        Map.put(t, "preference", preference_weight)
+      end)
 
-    :ok = FilesystemUtils.write_p!(filepath, thumbnail_blob)
+    case Enum.sort_by(thumbnails, & &1["preference"], :desc) do
+      [thumbnail_map | _] ->
+        thumbnail_url = thumbnail_map["url"]
+        filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
+        thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
 
-    filepath
+        :ok = FilesystemUtils.write_p!(filepath, thumbnail_blob)
+
+        filepath
+
+      _ ->
+        nil
+    end
   end
 
   @doc """

lib/pinchflat/metadata/metadata_parser.ex

@@ -30,7 +30,7 @@ defmodule Pinchflat.Metadata.MetadataParser do
       original_url: metadata["original_url"],
       description: metadata["description"],
       media_filepath: metadata["filepath"],
-      livestream: metadata["was_live"],
+      livestream: !!metadata["was_live"],
       duration_seconds: metadata["duration"] && round(metadata["duration"])
     }
   end

lib/pinchflat/yt_dlp/media.ex

@@ -87,7 +87,7 @@ defmodule Pinchflat.YtDlp.Media do
       title: response["title"],
       description: response["description"],
       original_url: response["webpage_url"],
-      livestream: response["was_live"],
+      livestream: !!response["was_live"],
       duration_seconds: response["duration"] && round(response["duration"]),
       short_form_content: response["webpage_url"] && short_form_content?(response),
       upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -32,7 +32,7 @@
               </div>
               <aside class="mt-4 xl:mt-0">
                 <div>Uploaded: <%= @media_item.upload_date %></div>
-                <div>
+                <div :if={URI.parse(@media_item.original_url).scheme =~ "http"}>
                   <.subtle_link href={@media_item.original_url} target="_blank">Open Original</.subtle_link>
                 </div>
                 <div class="mt-4 text-bodydark">

test/pinchflat/downloading/download_option_builder_test.exs

@@ -259,7 +259,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
       assert {:ok, res} = DownloadOptionBuilder.build(media_item)
 
       assert :extract_audio in res
-      assert {:format, "bestaudio[ext=m4a]"} in res
+      assert {:format, "bestaudio[ext=m4a]/bestaudio[ext=mp3]/bestaudio/best[ext=m4a]/best[ext=mp3]/best"} in res
 
       refute {:remux_video, "mp4"} in res
     end

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -89,6 +89,48 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
 
       assert filepath =~ ~r{/media_items/#{media_item.id}/img_2.jpg}
     end
+
+    test "will fall back to a non-jpg if it has to", %{media_item: media_item} do
+      metadata = %{
+        "thumbnails" => [
+          %{"url" => "https://i.ytimg.com/vi/ABC123/img_1.webp", "preference" => -1}
+        ]
+      }
+
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert filepath =~ ~r{/media_items/#{media_item.id}/img_1.webp}
+    end
+
+    test "does not require a preference field", %{media_item: media_item} do
+      metadata = %{
+        "thumbnails" => [
+          %{"url" => "https://i.ytimg.com/vi/ABC123/img_1.webp"}
+        ]
+      }
+
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert filepath =~ ~r{/media_items/#{media_item.id}/img_1.webp}
+    end
+  end
+
+  describe "download_and_store_thumbnail_for/2 when not downloading thumbnails" do
+    test "returns nil if there are no thumbnails", %{media_item: media_item} do
+      metadata = %{"thumbnails" => []}
+
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert filepath == nil
+    end
+
+    test "returns nil if there is no thumbnail field", %{media_item: media_item} do
+      metadata = %{}
+
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert filepath == nil
+    end
   end
 
   describe "parse_upload_date/1" do

test/pinchflat/metadata/metadata_parser_test.exs

@@ -46,6 +46,14 @@ defmodule Pinchflat.Metadata.MetadataParserTest do
       assert result.livestream == metadata["was_live"]
     end
 
+    test "the livestream flag defaults to false", %{metadata: metadata} do
+      metadata = Map.put(metadata, "was_live", nil)
+
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.livestream == false
+    end
+
     test "it extracts the duration in seconds", %{metadata: metadata} do
       result = Parser.parse_for_media_item(metadata)
 

test/pinchflat/yt_dlp/media_test.exs

@@ -199,5 +199,16 @@ defmodule Pinchflat.YtDlp.MediaTest do
 
       assert %Media{duration_seconds: nil} = Media.response_to_struct(response)
     end
+
+    test "sets livestream to false if the was_live field isn't present" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 60,
+        "upload_date" => "20210101"
+      }
+
+      assert %Media{livestream: false} = Media.response_to_struct(response)
+    end
   end
 end