From 566941228c9e61e0a21af77cfca5676ab13ab39a Mon Sep 17 00:00:00 2001 From: Kieran Date: Sat, 23 Mar 2024 12:38:06 -0700 Subject: [PATCH] Add media streaming (#108) * [WIP] set up streaming route * Added UUID to sources and media items * Added media preview to MI show page --- .iex.exs | 28 ---- .sobelow-conf | 10 +- README.md | 4 + lib/pinchflat/boot/pre_job_startup_tasks.ex | 16 ++ lib/pinchflat/media/media_item.ex | 8 + lib/pinchflat/sources/source.ex | 7 + .../media_items/media_item_controller.ex | 63 ++++++++ .../media_items/media_item_html.ex | 12 ++ .../media_item_html/media_preview.heex | 13 ++ .../media_item_html/show.html.heex | 7 +- lib/pinchflat_web/router.ex | 12 +- ...323165649_add_uuid_to_source_and_media.exs | 13 ++ test/pinchflat/media_test.exs | 31 ++++ test/pinchflat/sources_test.exs | 25 ++++ .../media_item_controller_test.exs | 141 ++++++++++++++++++ test/support/fixtures/media_fixtures.ex | 2 +- 16 files changed, 357 insertions(+), 35 deletions(-) create mode 100644 lib/pinchflat_web/controllers/media_items/media_item_html/media_preview.heex create mode 100644 priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs diff --git a/.iex.exs b/.iex.exs index 19b65cb..2d68cf5 100644 --- a/.iex.exs +++ b/.iex.exs @@ -21,31 +21,3 @@ alias Pinchflat.FastIndexing.YoutubeRss alias Pinchflat.Metadata.MetadataFileHelpers alias Pinchflat.SlowIndexing.FileFollowerServer - -defmodule IexHelpers do - def last_media_item do - Repo.one(from m in MediaItem, limit: 1) - end - - def details(type) do - source = - case type do - :playlist -> playlist_url() - :channel -> channel_url() - end - - YtDlpCollection.get_source_details(source) - end - - def ids(type) do - source = - case type do - :playlist -> playlist_url() - :channel -> channel_url() - end - - YtDlpCollection.get_media_attributes_for_collection(source) - end -end - -import IexHelpers diff --git a/.sobelow-conf b/.sobelow-conf index ed5bb1f..0c4088d 100644 --- a/.sobelow-conf +++ b/.sobelow-conf @@ -9,7 +9,15 @@ threshold: :low, # All of these are ignorable because this app is intended to be single-user and self-hosted. # There is an expectation that the user won't intentionally run a FS Traversal on themselves - ignore: ["CI.System", "Traversal.FileModule", "Config.HTTPS", "Config.CSP"], + ignore: + [ + "CI.System", + "Traversal.FileModule", + "Config.HTTPS", + "Config.CSP", + "XSS.ContentType", + "Traversal.SendFile" + ], ignore_files: [], version: false ] diff --git a/README.md b/README.md index f57c059..c78bd08 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,10 @@ NOTE: it's recommended to not run the container as root. Doing so can create per HTTP basic authentication is optionally supported. To use it, set the `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD` environment variables when starting the container. No authentication will be required unless you set _both_ of these. +### Important note: + +The media streaming endpoint is not protected by basic auth. To help protect your media, these endpoints work with UUIDs instead of sequential IDs but this is still essentially security through obscurity. If you're concerned about the security of your media, consider using a reverse proxy with authentication or a VPN. + ## EFF donations A portion of all donations to Pinchflat will be donated to the [Electronic Frontier Foundation](https://www.eff.org/). The EFF defends your online liberties and [backed](https://github.com/github/dmca/blob/9a85e0f021f7967af80e186b890776a50443f06c/2020/11/2020-11-16-RIAA-reversal-effletter.pdf) `youtube-dl` when Google took them down. [See here](https://github.com/kieraneglin/pinchflat/wiki/EFF-Donation-Receipts) for a list of donation receipts. diff --git a/lib/pinchflat/boot/pre_job_startup_tasks.ex b/lib/pinchflat/boot/pre_job_startup_tasks.ex index c3e5e9b..a16935c 100644 --- a/lib/pinchflat/boot/pre_job_startup_tasks.ex +++ b/lib/pinchflat/boot/pre_job_startup_tasks.ex @@ -14,6 +14,8 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do alias Pinchflat.Repo alias Pinchflat.Settings + alias Pinchflat.Sources.Source + alias Pinchflat.Media.MediaItem alias Pinchflat.Filesystem.FilesystemHelpers def start_link(opts \\ []) do @@ -32,6 +34,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do @impl true def init(state) do apply_default_settings() + backfill_uuids() ensure_directories_are_writeable() rename_old_job_workers() @@ -43,6 +46,19 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do Settings.fetch!(:pro_enabled, false) end + defp backfill_uuids do + # This is a one-time backfill to ensure that all media items have a UUID + # This is important for the RSS feed and the streaming endpoint + source_query = from(m in Source, where: is_nil(m.uuid), update: [set: [uuid: fragment("gen_random_uuid()")]]) + media_item_query = from(m in MediaItem, where: is_nil(m.uuid), update: [set: [uuid: fragment("gen_random_uuid()")]]) + + {source_count, _} = Repo.update_all(source_query, []) + {media_item_count, _} = Repo.update_all(media_item_query, []) + + Logger.info("Backfilled UUIDs for #{source_count} sources.") + Logger.info("Backfilled UUIDs for #{media_item_count} media items.") + end + defp ensure_directories_are_writeable do directories = [ Application.get_env(:pinchflat, :media_directory), diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index a87e919..a11b5c0 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -5,6 +5,7 @@ defmodule Pinchflat.Media.MediaItem do use Ecto.Schema import Ecto.Changeset + import Pinchflat.Utils.ChangesetUtils alias Pinchflat.Tasks.Task alias Pinchflat.Sources.Source @@ -32,6 +33,7 @@ defmodule Pinchflat.Media.MediaItem do ] # Pretty much all the fields captured at index are required. @required_fields ~w( + uuid title original_url livestream @@ -42,6 +44,11 @@ defmodule Pinchflat.Media.MediaItem do )a schema "media_items" do + # This is _not_ used as the primary key or internally in the database + # relations. This is only used to prevent an enumeration attack on the streaming + # and RSS feed endpoints since those _must_ be public (ie: no basic auth) + field :uuid, Ecto.UUID + field :title, :string field :media_id, :string field :description, :string @@ -78,6 +85,7 @@ defmodule Pinchflat.Media.MediaItem do media_item |> cast(attrs, @allowed_fields) |> cast_assoc(:metadata, with: &MediaMetadata.changeset/2, required: false) + |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end) |> validate_required(@required_fields) |> unique_constraint([:media_id, :source_id]) end diff --git a/lib/pinchflat/sources/source.ex b/lib/pinchflat/sources/source.ex index 1de247c..8664f23 100644 --- a/lib/pinchflat/sources/source.ex +++ b/lib/pinchflat/sources/source.ex @@ -47,6 +47,7 @@ defmodule Pinchflat.Sources.Source do @pre_insert_required_fields @initially_required_fields ++ ~w( + uuid custom_name collection_name collection_id @@ -54,6 +55,11 @@ defmodule Pinchflat.Sources.Source do )a schema "sources" do + # This is _not_ used as the primary key or internally in the database + # relations. This is only used to prevent an enumeration attack on the streaming + # and RSS feed endpoints since those _must_ be public (ie: no basic auth) + field :uuid, Ecto.UUID + field :custom_name, :string field :collection_name, :string field :collection_id, :string @@ -96,6 +102,7 @@ defmodule Pinchflat.Sources.Source do source |> cast(attrs, @allowed_fields) |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end) + |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end) |> validate_required(required_fields) |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false) end diff --git a/lib/pinchflat_web/controllers/media_items/media_item_controller.ex b/lib/pinchflat_web/controllers/media_items/media_item_controller.ex index 95847dd..bf44c77 100644 --- a/lib/pinchflat_web/controllers/media_items/media_item_controller.ex +++ b/lib/pinchflat_web/controllers/media_items/media_item_controller.ex @@ -3,6 +3,7 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do alias Pinchflat.Repo alias Pinchflat.Media + alias Pinchflat.Media.MediaItem def show(conn, %{"id" => id}) do media_item = @@ -29,4 +30,66 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do |> put_flash(:info, flash_message) |> redirect(to: ~p"/sources/#{media_item.source_id}") end + + # See here for details on streaming files and range requests: + # https://www.zeng.dev/post/2023-http-range-and-play-mp4-in-browser/ + # + # Uses the UUID instead of the ID to avoid enumeration attacks + # since streaming is a public endpoint (ie: no auth required) + def stream(conn, %{"id" => uuid}) do + media_item = Repo.get_by!(MediaItem, uuid: uuid) + + if File.exists?(media_item.media_filepath) do + file_size = File.stat!(media_item.media_filepath).size + mime_type = MIME.from_path(media_item.media_filepath) + + case parse_range(conn, file_size) do + {:ok, {start_pos, end_pos}} -> + length = end_pos - start_pos + 1 + + conn + |> put_resp_content_type(mime_type) + |> put_resp_header("accept-ranges", "bytes") + |> put_resp_header("content-range", "bytes #{start_pos}-#{end_pos}/#{file_size}") + |> put_resp_header("content-length", to_string(length)) + |> send_file(206, media_item.media_filepath, start_pos, length) + + {:error, :invalid_range} -> + conn + |> put_resp_content_type(mime_type) + |> put_resp_header("content-length", to_string(file_size)) + |> put_resp_header("accept-ranges", "bytes") + |> send_file(200, media_item.media_filepath) + end + else + send_resp(conn, 404, "File not found") + end + end + + defp parse_range(conn, file_size) do + with [range_header | _] <- get_req_header(conn, "range"), + ["bytes", range] <- String.split(range_header, "="), + [start_pos, end_pos] <- String.split(range, "-") do + validate_range(start_pos, end_pos, file_size) + else + _ -> {:error, :invalid_range} + end + end + + defp validate_range(start_pos, end_pos, file_size) do + case {Integer.parse(start_pos), Integer.parse(end_pos)} do + {:error, :error} -> + {:error, :invalid_range} + + {{start_pos, _}, :error} -> + {:ok, {start_pos, file_size - 1}} + + # See RFC7233 + {{start_pos, _}, {end_pos, _}} when end_pos >= file_size -> + {:ok, {start_pos, file_size - 1}} + + {{start_pos, _}, {end_pos, _}} -> + {:ok, {start_pos, end_pos}} + end + end end diff --git a/lib/pinchflat_web/controllers/media_items/media_item_html.ex b/lib/pinchflat_web/controllers/media_items/media_item_html.ex index ffae16a..12d7ffa 100644 --- a/lib/pinchflat_web/controllers/media_items/media_item_html.ex +++ b/lib/pinchflat_web/controllers/media_items/media_item_html.ex @@ -2,4 +2,16 @@ defmodule PinchflatWeb.MediaItems.MediaItemHTML do use PinchflatWeb, :html embed_templates "media_item_html/*" + + def media_file_exists?(media_item) do + !!media_item.media_filepath and File.exists?(media_item.media_filepath) + end + + def media_type(media_item) do + case Path.extname(media_item.media_filepath) do + ext when ext in [".mp4", ".webm", ".mkv"] -> :video + ext when ext in [".mp3", ".m4a"] -> :audio + _ -> :unknown + end + end end diff --git a/lib/pinchflat_web/controllers/media_items/media_item_html/media_preview.heex b/lib/pinchflat_web/controllers/media_items/media_item_html/media_preview.heex new file mode 100644 index 0000000..2146a98 --- /dev/null +++ b/lib/pinchflat_web/controllers/media_items/media_item_html/media_preview.heex @@ -0,0 +1,13 @@ +<%= if media_type(@media_item) == :video do %> + +<% end %> + +<%= if media_type(@media_item) == :audio do %> + +<% end %> diff --git a/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex b/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex index 660c179..7d56c51 100644 --- a/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex +++ b/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex @@ -8,11 +8,16 @@ -
+
<.tabbed_layout> <:tab title="Attributes">
+ <%= if media_file_exists?(@media_item) do %> +

Preview

+ <.media_preview media_item={@media_item} /> + <% end %> +

Attributes

Source: diff --git a/lib/pinchflat_web/router.ex b/lib/pinchflat_web/router.ex index e68e799..eae7109 100644 --- a/lib/pinchflat_web/router.ex +++ b/lib/pinchflat_web/router.ex @@ -28,10 +28,14 @@ defmodule PinchflatWeb.Router do end end - # Other scopes may use custom stacks. - # scope "/api", PinchflatWeb do - # pipe_through :api - # end + # Routes in here are NOT protected by basic auth. This is necessary for + # media streaming to work for RSS podcast feeds. + # + # TODO: consider putting the basic auth here behind a config flag + # so people that want RSS feeds to work can enable it. + scope "/", PinchflatWeb do + get "/media/:id/stream", MediaItems.MediaItemController, :stream + end # Enable LiveDashboard and Swoosh mailbox preview in development if Application.compile_env(:pinchflat, :dev_routes) do diff --git a/priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs b/priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs new file mode 100644 index 0000000..40b69db --- /dev/null +++ b/priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs @@ -0,0 +1,13 @@ +defmodule Pinchflat.Repo.Migrations.AddUuidToSourceAndMedia do + use Ecto.Migration + + def change do + alter table(:sources) do + add :uuid, :uuid + end + + alter table(:media_items) do + add :uuid, :uuid + end + end +end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index 21cdf2c..40e338c 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -398,6 +398,37 @@ defmodule Pinchflat.MediaTest do assert media_item.media_filepath == valid_attrs.media_filepath end + test "automatically sets the UUID" do + valid_attrs = %{ + media_id: Faker.String.base64(12), + title: Faker.Commerce.product_name(), + media_filepath: "/video/#{Faker.File.file_name(:video)}", + source_id: source_fixture().id, + original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}", + upload_date: Date.utc_today() + } + + assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs) + + assert String.length(media_item.uuid) == 36 + end + + test "UUID is not writable by the user" do + valid_attrs = %{ + media_id: Faker.String.base64(12), + title: Faker.Commerce.product_name(), + media_filepath: "/video/#{Faker.File.file_name(:video)}", + source_id: source_fixture().id, + original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}", + upload_date: Date.utc_today(), + uuid: "some-uuid" + } + + assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs) + + assert String.length(media_item.uuid) == 36 + end + test "creating with invalid data returns error changeset" do assert {:error, %Ecto.Changeset{}} = Media.create_media_item(@invalid_attrs) end diff --git a/test/pinchflat/sources_test.exs b/test/pinchflat/sources_test.exs index ce247a2..392cc2f 100644 --- a/test/pinchflat/sources_test.exs +++ b/test/pinchflat/sources_test.exs @@ -59,6 +59,31 @@ defmodule Pinchflat.SourcesTest do end describe "create_source/2" do + test "automatically sets the UUID" do + expect(YtDlpRunnerMock, :run, &channel_mock/3) + + valid_attrs = %{ + media_profile_id: media_profile_fixture().id, + original_url: "https://www.youtube.com/channel/abc123" + } + + assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs) + assert String.length(source.uuid) == 36 + end + + test "UUID is not writable by the user" do + expect(YtDlpRunnerMock, :run, &channel_mock/3) + + valid_attrs = %{ + media_profile_id: media_profile_fixture().id, + original_url: "https://www.youtube.com/channel/abc123", + uuid: "some_uuid" + } + + assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs) + assert String.length(source.uuid) == 36 + end + test "creates a source and adds name + ID from runner response for channels" do expect(YtDlpRunnerMock, :run, &channel_mock/3) diff --git a/test/pinchflat_web/controllers/media_item_controller_test.exs b/test/pinchflat_web/controllers/media_item_controller_test.exs index 1f50462..428d337 100644 --- a/test/pinchflat_web/controllers/media_item_controller_test.exs +++ b/test/pinchflat_web/controllers/media_item_controller_test.exs @@ -66,6 +66,147 @@ defmodule PinchflatWeb.MediaItemControllerTest do end end + describe "streaming media" do + test "returns 404 if the media isn't found", %{conn: conn} do + media_item = media_item_fixture() + conn = get(conn, ~p"/media/#{media_item.uuid}/stream") + + assert conn.status == 404 + end + + test "automatically sets the content type", %{conn: conn} do + media_item = media_item_with_attachments() + conn = get(conn, ~p"/media/#{media_item.uuid}/stream") + + assert {"content-type", "video/mp4; charset=utf-8"} in conn.resp_headers + end + + test "sets the content length", %{conn: conn} do + media_item = media_item_with_attachments() + filesize = File.stat!(media_item.media_filepath).size + + conn = get(conn, ~p"/media/#{media_item.uuid}/stream") + + assert {"content-length", to_string(filesize)} in conn.resp_headers + end + end + + describe "streaming media when range is valid" do + setup do + media_item = media_item_with_attachments() + + %{media_item: media_item} + end + + test "sets the correct status and headers", %{conn: conn, media_item: media_item} do + filesize = File.stat!(media_item.media_filepath).size + + conn = + conn + |> put_req_header("range", "bytes=0-100") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert conn.status == 206 + assert {"content-range", "bytes 0-100/#{filesize}"} in conn.resp_headers + assert {"content-length", "101"} in conn.resp_headers + end + + test "streams the specified range", %{conn: conn, media_item: media_item} do + conn = + conn + |> put_req_header("range", "bytes=0-100") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert byte_size(conn.resp_body) == 101 + end + + test "supports range offsets", %{conn: conn, media_item: media_item} do + contents = File.read!(media_item.media_filepath) + expected = String.slice(contents, 100..200) + + conn = + conn + |> put_req_header("range", "bytes=100-200") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert conn.resp_body == expected + end + + test "returns as expected if the requested range is larger than the file", %{conn: conn, media_item: media_item} do + contents = File.read!(media_item.media_filepath) + filesize = File.stat!(media_item.media_filepath).size + + conn = + conn + |> put_req_header("range", "bytes=0-#{filesize * 10}") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert conn.resp_body == contents + assert {"content-range", "bytes 0-#{filesize - 1}/#{filesize}"} in conn.resp_headers + assert {"content-length", to_string(filesize)} in conn.resp_headers + end + + test "supports endless ranges", %{conn: conn, media_item: media_item} do + contents = File.read!(media_item.media_filepath) + + conn = + conn + |> put_req_header("range", "bytes=0-") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert conn.resp_body == contents + end + + test "supports endless ranges with offsets", %{conn: conn, media_item: media_item} do + contents = File.read!(media_item.media_filepath) + {_, expected} = String.split_at(contents, 100) + + conn = + conn + |> put_req_header("range", "bytes=100-") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert conn.resp_body == expected + end + end + + describe "streaming media when range is invalid or not present" do + setup do + media_item = media_item_with_attachments() + + %{media_item: media_item} + end + + test "sets the correct status and headers", %{conn: conn, media_item: media_item} do + filesize = File.stat!(media_item.media_filepath).size + + conn = get(conn, ~p"/media/#{media_item.uuid}/stream") + + assert conn.status == 200 + assert {"content-length", to_string(filesize)} in conn.resp_headers + end + + test "streams the entire file", %{conn: conn, media_item: media_item} do + contents = File.read!(media_item.media_filepath) + + conn = get(conn, ~p"/media/#{media_item.uuid}/stream") + + assert conn.resp_body == contents + end + + test "doesn't blow up if the range header is invalid", %{conn: conn, media_item: media_item} do + contents = File.read!(media_item.media_filepath) + + conn = + conn + |> put_req_header("range", "bytes=-") + |> get(~p"/media/#{media_item.uuid}/stream") + + assert conn.status == 200 + assert conn.resp_body == contents + end + end + defp create_media_item(_) do media_item = media_item_fixture() %{media_item: media_item} diff --git a/test/support/fixtures/media_fixtures.ex b/test/support/fixtures/media_fixtures.ex index ce0a3a7..a9cf75d 100644 --- a/test/support/fixtures/media_fixtures.ex +++ b/test/support/fixtures/media_fixtures.ex @@ -49,7 +49,7 @@ defmodule Pinchflat.MediaFixtures do Path.join([ Application.get_env(:pinchflat, :media_directory), "#{:rand.uniform(1_000_000)}", - "#{:rand.uniform(1_000_000)}_media.mkv" + "#{:rand.uniform(1_000_000)}_media.mp4" ]) fixture_media_filepath =