Add media streaming (#108)

* [WIP] set up streaming route

* Added UUID to sources and media items

* Added media preview to MI show page

.iex.exs

@@ -21,31 +21,3 @@ alias Pinchflat.FastIndexing.YoutubeRss
 alias Pinchflat.Metadata.MetadataFileHelpers
 
 alias Pinchflat.SlowIndexing.FileFollowerServer
-
-defmodule IexHelpers do
-  def last_media_item do
-    Repo.one(from m in MediaItem, limit: 1)
-  end
-
-  def details(type) do
-    source =
-      case type do
-        :playlist -> playlist_url()
-        :channel -> channel_url()
-      end
-
-    YtDlpCollection.get_source_details(source)
-  end
-
-  def ids(type) do
-    source =
-      case type do
-        :playlist -> playlist_url()
-        :channel -> channel_url()
-      end
-
-    YtDlpCollection.get_media_attributes_for_collection(source)
-  end
-end
-
-import IexHelpers

.sobelow-conf

@@ -9,7 +9,15 @@
   threshold: :low,
   # All of these are ignorable because this app is intended to be single-user and self-hosted.
   # There is an expectation that the user won't intentionally run a FS Traversal on themselves
-  ignore: ["CI.System", "Traversal.FileModule", "Config.HTTPS", "Config.CSP"],
+  ignore:
+    [
+      "CI.System",
+      "Traversal.FileModule",
+      "Config.HTTPS",
+      "Config.CSP",
+      "XSS.ContentType",
+      "Traversal.SendFile"
+    ],
   ignore_files: [],
   version: false
 ]

README.md

@@ -85,6 +85,10 @@ NOTE: it's recommended to not run the container as root. Doing so can create per
 
 HTTP basic authentication is optionally supported. To use it, set the `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD` environment variables when starting the container. No authentication will be required unless you set _both_ of these.
 
+### Important note:
+
+The media streaming endpoint is not protected by basic auth. To help protect your media, these endpoints work with UUIDs instead of sequential IDs but this is still essentially security through obscurity. If you're concerned about the security of your media, consider using a reverse proxy with authentication or a VPN.
+
 ## EFF donations
 
 A portion of all donations to Pinchflat will be donated to the [Electronic Frontier Foundation](https://www.eff.org/). The EFF defends your online liberties and [backed](https://github.com/github/dmca/blob/9a85e0f021f7967af80e186b890776a50443f06c/2020/11/2020-11-16-RIAA-reversal-effletter.pdf) `youtube-dl` when Google took them down. [See here](https://github.com/kieraneglin/pinchflat/wiki/EFF-Donation-Receipts) for a list of donation receipts.

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -14,6 +14,8 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
 
   alias Pinchflat.Repo
   alias Pinchflat.Settings
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Filesystem.FilesystemHelpers
 
   def start_link(opts \\ []) do
@@ -32,6 +34,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
   @impl true
   def init(state) do
     apply_default_settings()
+    backfill_uuids()
     ensure_directories_are_writeable()
     rename_old_job_workers()
 
@@ -43,6 +46,19 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
     Settings.fetch!(:pro_enabled, false)
   end
 
+  defp backfill_uuids do
+    # This is a one-time backfill to ensure that all media items have a UUID
+    # This is important for the RSS feed and the streaming endpoint
+    source_query = from(m in Source, where: is_nil(m.uuid), update: [set: [uuid: fragment("gen_random_uuid()")]])
+    media_item_query = from(m in MediaItem, where: is_nil(m.uuid), update: [set: [uuid: fragment("gen_random_uuid()")]])
+
+    {source_count, _} = Repo.update_all(source_query, [])
+    {media_item_count, _} = Repo.update_all(media_item_query, [])
+
+    Logger.info("Backfilled UUIDs for #{source_count} sources.")
+    Logger.info("Backfilled UUIDs for #{media_item_count} media items.")
+  end
+
   defp ensure_directories_are_writeable do
     directories = [
       Application.get_env(:pinchflat, :media_directory),

lib/pinchflat/media/media_item.ex

@@ -5,6 +5,7 @@ defmodule Pinchflat.Media.MediaItem do
 
   use Ecto.Schema
   import Ecto.Changeset
+  import Pinchflat.Utils.ChangesetUtils
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
@@ -32,6 +33,7 @@ defmodule Pinchflat.Media.MediaItem do
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
+    uuid
     title
     original_url
     livestream
@@ -42,6 +44,11 @@ defmodule Pinchflat.Media.MediaItem do
     )a
 
   schema "media_items" do
+    # This is _not_ used as the primary key or internally in the database
+    # relations. This is only used to prevent an enumeration attack on the streaming
+    # and RSS feed endpoints since those _must_ be public (ie: no basic auth)
+    field :uuid, Ecto.UUID
+
     field :title, :string
     field :media_id, :string
     field :description, :string
@@ -78,6 +85,7 @@ defmodule Pinchflat.Media.MediaItem do
     media_item
     |> cast(attrs, @allowed_fields)
     |> cast_assoc(:metadata, with: &MediaMetadata.changeset/2, required: false)
+    |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(@required_fields)
     |> unique_constraint([:media_id, :source_id])
   end

lib/pinchflat/sources/source.ex

@@ -47,6 +47,7 @@ defmodule Pinchflat.Sources.Source do
 
   @pre_insert_required_fields @initially_required_fields ++
                                 ~w(
+                                  uuid
                                   custom_name
                                   collection_name
                                   collection_id
@@ -54,6 +55,11 @@ defmodule Pinchflat.Sources.Source do
                                 )a
 
   schema "sources" do
+    # This is _not_ used as the primary key or internally in the database
+    # relations. This is only used to prevent an enumeration attack on the streaming
+    # and RSS feed endpoints since those _must_ be public (ie: no basic auth)
+    field :uuid, Ecto.UUID
+
     field :custom_name, :string
     field :collection_name, :string
     field :collection_id, :string
@@ -96,6 +102,7 @@ defmodule Pinchflat.Sources.Source do
     source
     |> cast(attrs, @allowed_fields)
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
+    |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(required_fields)
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
   end

lib/pinchflat_web/controllers/media_items/media_item_controller.ex

@@ -3,6 +3,7 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do
 
   alias Pinchflat.Repo
   alias Pinchflat.Media
+  alias Pinchflat.Media.MediaItem
 
   def show(conn, %{"id" => id}) do
     media_item =
@@ -29,4 +30,66 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do
     |> put_flash(:info, flash_message)
     |> redirect(to: ~p"/sources/#{media_item.source_id}")
   end
+
+  # See here for details on streaming files and range requests:
+  # https://www.zeng.dev/post/2023-http-range-and-play-mp4-in-browser/
+  #
+  # Uses the UUID instead of the ID to avoid enumeration attacks
+  # since streaming is a public endpoint (ie: no auth required)
+  def stream(conn, %{"id" => uuid}) do
+    media_item = Repo.get_by!(MediaItem, uuid: uuid)
+
+    if File.exists?(media_item.media_filepath) do
+      file_size = File.stat!(media_item.media_filepath).size
+      mime_type = MIME.from_path(media_item.media_filepath)
+
+      case parse_range(conn, file_size) do
+        {:ok, {start_pos, end_pos}} ->
+          length = end_pos - start_pos + 1
+
+          conn
+          |> put_resp_content_type(mime_type)
+          |> put_resp_header("accept-ranges", "bytes")
+          |> put_resp_header("content-range", "bytes #{start_pos}-#{end_pos}/#{file_size}")
+          |> put_resp_header("content-length", to_string(length))
+          |> send_file(206, media_item.media_filepath, start_pos, length)
+
+        {:error, :invalid_range} ->
+          conn
+          |> put_resp_content_type(mime_type)
+          |> put_resp_header("content-length", to_string(file_size))
+          |> put_resp_header("accept-ranges", "bytes")
+          |> send_file(200, media_item.media_filepath)
+      end
+    else
+      send_resp(conn, 404, "File not found")
+    end
+  end
+
+  defp parse_range(conn, file_size) do
+    with [range_header | _] <- get_req_header(conn, "range"),
+         ["bytes", range] <- String.split(range_header, "="),
+         [start_pos, end_pos] <- String.split(range, "-") do
+      validate_range(start_pos, end_pos, file_size)
+    else
+      _ -> {:error, :invalid_range}
+    end
+  end
+
+  defp validate_range(start_pos, end_pos, file_size) do
+    case {Integer.parse(start_pos), Integer.parse(end_pos)} do
+      {:error, :error} ->
+        {:error, :invalid_range}
+
+      {{start_pos, _}, :error} ->
+        {:ok, {start_pos, file_size - 1}}
+
+      # See RFC7233
+      {{start_pos, _}, {end_pos, _}} when end_pos >= file_size ->
+        {:ok, {start_pos, file_size - 1}}
+
+      {{start_pos, _}, {end_pos, _}} ->
+        {:ok, {start_pos, end_pos}}
+    end
+  end
 end

lib/pinchflat_web/controllers/media_items/media_item_html.ex

@@ -2,4 +2,16 @@ defmodule PinchflatWeb.MediaItems.MediaItemHTML do
   use PinchflatWeb, :html
 
   embed_templates "media_item_html/*"
+
+  def media_file_exists?(media_item) do
+    !!media_item.media_filepath and File.exists?(media_item.media_filepath)
+  end
+
+  def media_type(media_item) do
+    case Path.extname(media_item.media_filepath) do
+      ext when ext in [".mp4", ".webm", ".mkv"] -> :video
+      ext when ext in [".mp3", ".m4a"] -> :audio
+      _ -> :unknown
+    end
+  end
 end

lib/pinchflat_web/controllers/media_items/media_item_html/media_preview.heex

@@ -0,0 +1,13 @@
+<%= if media_type(@media_item) == :video do %>
+  <video controls class="max-h-96 w-full lg:w-2/3 xl:w-1/2">
+    <source src={~p"/media/#{@media_item.uuid}/stream"} type="video/mp4" />
+    Your browser does not support the video element.
+  </video>
+<% end %>
+
+<%= if media_type(@media_item) == :audio do %>
+  <audio controls class="w-full lg:w-2/3 xl:w-1/2">
+    <source src={~p"/media/#{@media_item.uuid}/stream"} type="audio/mpeg" />
+    Your browser does not support the audio element.
+  </audio>
+<% end %>

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -8,11 +8,16 @@
     </h2>
   </div>
 </div>
-<div class="rounded-sm border border-stroke bg-white py-5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5">
+<div class="rounded-sm border border-stroke bg-white py-5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark px-7.5">
   <div class="max-w-full overflow-x-auto">
     <.tabbed_layout>
       <:tab title="Attributes">
         <div class="flex flex-col gap-10 dark:text-white">
+          <%= if media_file_exists?(@media_item) do %>
+            <h3 class="font-bold text-xl">Preview</h3>
+            <.media_preview media_item={@media_item} />
+          <% end %>
+
           <h3 class="font-bold text-xl">Attributes</h3>
           <section>
             <strong>Source:</strong>

lib/pinchflat_web/router.ex

@@ -28,10 +28,14 @@ defmodule PinchflatWeb.Router do
     end
   end
 
-  # Other scopes may use custom stacks.
-  # scope "/api", PinchflatWeb do
-  #   pipe_through :api
-  # end
+  # Routes in here are NOT protected by basic auth. This is necessary for
+  # media streaming to work for RSS podcast feeds.
+  #
+  # TODO: consider putting the basic auth here behind a config flag
+  # so people that want RSS feeds to work can enable it.
+  scope "/", PinchflatWeb do
+    get "/media/:id/stream", MediaItems.MediaItemController, :stream
+  end
 
   # Enable LiveDashboard and Swoosh mailbox preview in development
   if Application.compile_env(:pinchflat, :dev_routes) do

priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs

@@ -0,0 +1,13 @@
+defmodule Pinchflat.Repo.Migrations.AddUuidToSourceAndMedia do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :uuid, :uuid
+    end
+
+    alter table(:media_items) do
+      add :uuid, :uuid
+    end
+  end
+end

test/pinchflat/media_test.exs

@@ -398,6 +398,37 @@ defmodule Pinchflat.MediaTest do
       assert media_item.media_filepath == valid_attrs.media_filepath
     end
 
+    test "automatically sets the UUID" do
+      valid_attrs = %{
+        media_id: Faker.String.base64(12),
+        title: Faker.Commerce.product_name(),
+        media_filepath: "/video/#{Faker.File.file_name(:video)}",
+        source_id: source_fixture().id,
+        original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
+        upload_date: Date.utc_today()
+      }
+
+      assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
+
+      assert String.length(media_item.uuid) == 36
+    end
+
+    test "UUID is not writable by the user" do
+      valid_attrs = %{
+        media_id: Faker.String.base64(12),
+        title: Faker.Commerce.product_name(),
+        media_filepath: "/video/#{Faker.File.file_name(:video)}",
+        source_id: source_fixture().id,
+        original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
+        upload_date: Date.utc_today(),
+        uuid: "some-uuid"
+      }
+
+      assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
+
+      assert String.length(media_item.uuid) == 36
+    end
+
     test "creating with invalid data returns error changeset" do
       assert {:error, %Ecto.Changeset{}} = Media.create_media_item(@invalid_attrs)
     end

test/pinchflat/sources_test.exs

@@ -59,6 +59,31 @@ defmodule Pinchflat.SourcesTest do
   end
 
   describe "create_source/2" do
+    test "automatically sets the UUID" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/3)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123"
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+      assert String.length(source.uuid) == 36
+    end
+
+    test "UUID is not writable by the user" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/3)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        uuid: "some_uuid"
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+      assert String.length(source.uuid) == 36
+    end
+
     test "creates a source and adds name + ID from runner response for channels" do
       expect(YtDlpRunnerMock, :run, &channel_mock/3)
 

test/pinchflat_web/controllers/media_item_controller_test.exs

@@ -66,6 +66,147 @@ defmodule PinchflatWeb.MediaItemControllerTest do
     end
   end
 
+  describe "streaming media" do
+    test "returns 404 if the media isn't found", %{conn: conn} do
+      media_item = media_item_fixture()
+      conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.status == 404
+    end
+
+    test "automatically sets the content type", %{conn: conn} do
+      media_item = media_item_with_attachments()
+      conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
+
+      assert {"content-type", "video/mp4; charset=utf-8"} in conn.resp_headers
+    end
+
+    test "sets the content length", %{conn: conn} do
+      media_item = media_item_with_attachments()
+      filesize = File.stat!(media_item.media_filepath).size
+
+      conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
+
+      assert {"content-length", to_string(filesize)} in conn.resp_headers
+    end
+  end
+
+  describe "streaming media when range is valid" do
+    setup do
+      media_item = media_item_with_attachments()
+
+      %{media_item: media_item}
+    end
+
+    test "sets the correct status and headers", %{conn: conn, media_item: media_item} do
+      filesize = File.stat!(media_item.media_filepath).size
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=0-100")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.status == 206
+      assert {"content-range", "bytes 0-100/#{filesize}"} in conn.resp_headers
+      assert {"content-length", "101"} in conn.resp_headers
+    end
+
+    test "streams the specified range", %{conn: conn, media_item: media_item} do
+      conn =
+        conn
+        |> put_req_header("range", "bytes=0-100")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert byte_size(conn.resp_body) == 101
+    end
+
+    test "supports range offsets", %{conn: conn, media_item: media_item} do
+      contents = File.read!(media_item.media_filepath)
+      expected = String.slice(contents, 100..200)
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=100-200")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.resp_body == expected
+    end
+
+    test "returns as expected if the requested range is larger than the file", %{conn: conn, media_item: media_item} do
+      contents = File.read!(media_item.media_filepath)
+      filesize = File.stat!(media_item.media_filepath).size
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=0-#{filesize * 10}")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.resp_body == contents
+      assert {"content-range", "bytes 0-#{filesize - 1}/#{filesize}"} in conn.resp_headers
+      assert {"content-length", to_string(filesize)} in conn.resp_headers
+    end
+
+    test "supports endless ranges", %{conn: conn, media_item: media_item} do
+      contents = File.read!(media_item.media_filepath)
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=0-")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.resp_body == contents
+    end
+
+    test "supports endless ranges with offsets", %{conn: conn, media_item: media_item} do
+      contents = File.read!(media_item.media_filepath)
+      {_, expected} = String.split_at(contents, 100)
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=100-")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.resp_body == expected
+    end
+  end
+
+  describe "streaming media when range is invalid or not present" do
+    setup do
+      media_item = media_item_with_attachments()
+
+      %{media_item: media_item}
+    end
+
+    test "sets the correct status and headers", %{conn: conn, media_item: media_item} do
+      filesize = File.stat!(media_item.media_filepath).size
+
+      conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.status == 200
+      assert {"content-length", to_string(filesize)} in conn.resp_headers
+    end
+
+    test "streams the entire file", %{conn: conn, media_item: media_item} do
+      contents = File.read!(media_item.media_filepath)
+
+      conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.resp_body == contents
+    end
+
+    test "doesn't blow up if the range header is invalid", %{conn: conn, media_item: media_item} do
+      contents = File.read!(media_item.media_filepath)
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=-")
+        |> get(~p"/media/#{media_item.uuid}/stream")
+
+      assert conn.status == 200
+      assert conn.resp_body == contents
+    end
+  end
+
   defp create_media_item(_) do
     media_item = media_item_fixture()
     %{media_item: media_item}

test/support/fixtures/media_fixtures.ex

@@ -49,7 +49,7 @@ defmodule Pinchflat.MediaFixtures do
       Path.join([
         Application.get_env(:pinchflat, :media_directory),
         "#{:rand.uniform(1_000_000)}",
-        "#{:rand.uniform(1_000_000)}_media.mkv"
+        "#{:rand.uniform(1_000_000)}_media.mp4"
       ])
 
     fixture_media_filepath =