diff --git a/.iex.exs b/.iex.exs index 19b65cb..2d68cf5 100644 --- a/.iex.exs +++ b/.iex.exs @@ -21,31 +21,3 @@ alias Pinchflat.FastIndexing.YoutubeRss alias Pinchflat.Metadata.MetadataFileHelpers alias Pinchflat.SlowIndexing.FileFollowerServer - -defmodule IexHelpers do - def last_media_item do - Repo.one(from m in MediaItem, limit: 1) - end - - def details(type) do - source = - case type do - :playlist -> playlist_url() - :channel -> channel_url() - end - - YtDlpCollection.get_source_details(source) - end - - def ids(type) do - source = - case type do - :playlist -> playlist_url() - :channel -> channel_url() - end - - YtDlpCollection.get_media_attributes_for_collection(source) - end -end - -import IexHelpers diff --git a/lib/pinchflat/boot/pre_job_startup_tasks.ex b/lib/pinchflat/boot/pre_job_startup_tasks.ex index c3e5e9b..a16935c 100644 --- a/lib/pinchflat/boot/pre_job_startup_tasks.ex +++ b/lib/pinchflat/boot/pre_job_startup_tasks.ex @@ -14,6 +14,8 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do alias Pinchflat.Repo alias Pinchflat.Settings + alias Pinchflat.Sources.Source + alias Pinchflat.Media.MediaItem alias Pinchflat.Filesystem.FilesystemHelpers def start_link(opts \\ []) do @@ -32,6 +34,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do @impl true def init(state) do apply_default_settings() + backfill_uuids() ensure_directories_are_writeable() rename_old_job_workers() @@ -43,6 +46,19 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do Settings.fetch!(:pro_enabled, false) end + defp backfill_uuids do + # This is a one-time backfill to ensure that all media items have a UUID + # This is important for the RSS feed and the streaming endpoint + source_query = from(m in Source, where: is_nil(m.uuid), update: [set: [uuid: fragment("gen_random_uuid()")]]) + media_item_query = from(m in MediaItem, where: is_nil(m.uuid), update: [set: [uuid: fragment("gen_random_uuid()")]]) + + {source_count, _} = Repo.update_all(source_query, []) + {media_item_count, _} = Repo.update_all(media_item_query, []) + + Logger.info("Backfilled UUIDs for #{source_count} sources.") + Logger.info("Backfilled UUIDs for #{media_item_count} media items.") + end + defp ensure_directories_are_writeable do directories = [ Application.get_env(:pinchflat, :media_directory), diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index a87e919..a11b5c0 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -5,6 +5,7 @@ defmodule Pinchflat.Media.MediaItem do use Ecto.Schema import Ecto.Changeset + import Pinchflat.Utils.ChangesetUtils alias Pinchflat.Tasks.Task alias Pinchflat.Sources.Source @@ -32,6 +33,7 @@ defmodule Pinchflat.Media.MediaItem do ] # Pretty much all the fields captured at index are required. @required_fields ~w( + uuid title original_url livestream @@ -42,6 +44,11 @@ defmodule Pinchflat.Media.MediaItem do )a schema "media_items" do + # This is _not_ used as the primary key or internally in the database + # relations. This is only used to prevent an enumeration attack on the streaming + # and RSS feed endpoints since those _must_ be public (ie: no basic auth) + field :uuid, Ecto.UUID + field :title, :string field :media_id, :string field :description, :string @@ -78,6 +85,7 @@ defmodule Pinchflat.Media.MediaItem do media_item |> cast(attrs, @allowed_fields) |> cast_assoc(:metadata, with: &MediaMetadata.changeset/2, required: false) + |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end) |> validate_required(@required_fields) |> unique_constraint([:media_id, :source_id]) end diff --git a/lib/pinchflat/sources/source.ex b/lib/pinchflat/sources/source.ex index 1de247c..8664f23 100644 --- a/lib/pinchflat/sources/source.ex +++ b/lib/pinchflat/sources/source.ex @@ -47,6 +47,7 @@ defmodule Pinchflat.Sources.Source do @pre_insert_required_fields @initially_required_fields ++ ~w( + uuid custom_name collection_name collection_id @@ -54,6 +55,11 @@ defmodule Pinchflat.Sources.Source do )a schema "sources" do + # This is _not_ used as the primary key or internally in the database + # relations. This is only used to prevent an enumeration attack on the streaming + # and RSS feed endpoints since those _must_ be public (ie: no basic auth) + field :uuid, Ecto.UUID + field :custom_name, :string field :collection_name, :string field :collection_id, :string @@ -96,6 +102,7 @@ defmodule Pinchflat.Sources.Source do source |> cast(attrs, @allowed_fields) |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end) + |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end) |> validate_required(required_fields) |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false) end diff --git a/priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs b/priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs new file mode 100644 index 0000000..40b69db --- /dev/null +++ b/priv/repo/migrations/20240323165649_add_uuid_to_source_and_media.exs @@ -0,0 +1,13 @@ +defmodule Pinchflat.Repo.Migrations.AddUuidToSourceAndMedia do + use Ecto.Migration + + def change do + alter table(:sources) do + add :uuid, :uuid + end + + alter table(:media_items) do + add :uuid, :uuid + end + end +end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index 21cdf2c..40e338c 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -398,6 +398,37 @@ defmodule Pinchflat.MediaTest do assert media_item.media_filepath == valid_attrs.media_filepath end + test "automatically sets the UUID" do + valid_attrs = %{ + media_id: Faker.String.base64(12), + title: Faker.Commerce.product_name(), + media_filepath: "/video/#{Faker.File.file_name(:video)}", + source_id: source_fixture().id, + original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}", + upload_date: Date.utc_today() + } + + assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs) + + assert String.length(media_item.uuid) == 36 + end + + test "UUID is not writable by the user" do + valid_attrs = %{ + media_id: Faker.String.base64(12), + title: Faker.Commerce.product_name(), + media_filepath: "/video/#{Faker.File.file_name(:video)}", + source_id: source_fixture().id, + original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}", + upload_date: Date.utc_today(), + uuid: "some-uuid" + } + + assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs) + + assert String.length(media_item.uuid) == 36 + end + test "creating with invalid data returns error changeset" do assert {:error, %Ecto.Changeset{}} = Media.create_media_item(@invalid_attrs) end diff --git a/test/pinchflat/sources_test.exs b/test/pinchflat/sources_test.exs index ce247a2..392cc2f 100644 --- a/test/pinchflat/sources_test.exs +++ b/test/pinchflat/sources_test.exs @@ -59,6 +59,31 @@ defmodule Pinchflat.SourcesTest do end describe "create_source/2" do + test "automatically sets the UUID" do + expect(YtDlpRunnerMock, :run, &channel_mock/3) + + valid_attrs = %{ + media_profile_id: media_profile_fixture().id, + original_url: "https://www.youtube.com/channel/abc123" + } + + assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs) + assert String.length(source.uuid) == 36 + end + + test "UUID is not writable by the user" do + expect(YtDlpRunnerMock, :run, &channel_mock/3) + + valid_attrs = %{ + media_profile_id: media_profile_fixture().id, + original_url: "https://www.youtube.com/channel/abc123", + uuid: "some_uuid" + } + + assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs) + assert String.length(source.uuid) == 36 + end + test "creates a source and adds name + ID from runner response for channels" do expect(YtDlpRunnerMock, :run, &channel_mock/3)