diff --git a/.iex.exs b/.iex.exs index 52a238b..b2a95ca 100644 --- a/.iex.exs +++ b/.iex.exs @@ -14,7 +14,7 @@ alias Pinchflat.Sources alias Pinchflat.Settings alias Pinchflat.MediaClient.{SourceDetails, MediaDownloader} -alias Pinchflat.Metadata.{Zipper, ThumbnailFetcher} +alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer @@ -48,7 +48,7 @@ defmodule IexHelpers do :channel -> channel_url() end - SourceDetails.get_media_attributes(source) + SourceDetails.get_media_attributes_for_collection(source) end end diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index 0023e93..99c9fbc 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -12,14 +12,14 @@ defmodule Pinchflat.Media.MediaItem do alias Pinchflat.Media.MediaItemSearchIndex @allowed_fields [ - # these fields are captured on indexing + # these fields are captured on indexing (and again on download) :title, :media_id, :description, :original_url, :livestream, :source_id, - # these fields are captured on download + # these fields are captured only on download :media_downloaded_at, :media_filepath, :media_size_bytes, diff --git a/lib/pinchflat/media_client/backends/yt_dlp/media.ex b/lib/pinchflat/media_client/backends/yt_dlp/media.ex index a1e8c03..2b73db1 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/media.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/media.ex @@ -20,6 +20,24 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.Media do end end + # TODO: test + def get_media_attributes(url) do + runner = Application.get_env(:pinchflat, :yt_dlp_runner) + command_opts = [:simulate, :skip_download] + output_template = indexing_output_template() + + case runner.run(url, command_opts, output_template) do + {:ok, output} -> Phoenix.json_library().decode!(output) + res -> res + end + end + + # TODO: test + # TODO: test that media_collection consumes this maybe? + def indexing_output_template do + "%(.{id,title,was_live,original_url,description})j" + end + defp backend_runner do Application.get_env(:pinchflat, :yt_dlp_runner) end diff --git a/lib/pinchflat/media_client/backends/yt_dlp/media_collection.ex b/lib/pinchflat/media_client/backends/yt_dlp/media_collection.ex index 45a2a17..85225c4 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/media_collection.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/media_collection.ex @@ -8,6 +8,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollection do alias Pinchflat.Utils.FunctionUtils alias Pinchflat.Utils.FilesystemUtils + alias Pinchflat.MediaClient.Backends.YtDlp.Media, as: YtDlpMedia @doc """ Returns a list of maps representing the media in the collection. @@ -19,10 +20,10 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollection do Returns {:ok, [map()]} | {:error, any, ...}. """ - def get_media_attributes(url, addl_opts \\ []) do + def get_media_attributes_for_collection(url, addl_opts \\ []) do runner = Application.get_env(:pinchflat, :yt_dlp_runner) command_opts = [:simulate, :skip_download] - output_template = "%(.{id,title,was_live,original_url,description})j" + output_template = YtDlpMedia.indexing_output_template() output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json) file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false) diff --git a/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex b/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex index a32ea73..aab92a9 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex @@ -23,11 +23,15 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do |> Map.merge(parse_infojson_metadata(metadata)) end + # TODO: test new defp parse_media_metadata(metadata) do %{ + media_id: metadata["id"], title: metadata["title"], + original_url: metadata["original_url"], description: metadata["description"], - media_filepath: metadata["filepath"] + media_filepath: metadata["filepath"], + livestream: metadata["was_live"] } end diff --git a/lib/pinchflat/media_client/media_downloader.ex b/lib/pinchflat/media_client/media_downloader.ex index b952d8f..dbe724c 100644 --- a/lib/pinchflat/media_client/media_downloader.ex +++ b/lib/pinchflat/media_client/media_downloader.ex @@ -55,6 +55,11 @@ defmodule Pinchflat.MediaClient.MediaDownloader do end end + # def download_for_source(source, url, backend \\ :yt_dlp) do + # # Create MI from source and URL + # media_item = nil + # end + defp download_with_options(url, item_with_preloads, backend) do option_builder = option_builder(backend) media_backend = media_backend(backend) diff --git a/lib/pinchflat/media_client/source_details.ex b/lib/pinchflat/media_client/source_details.ex index d3cd2a1..1aa5a76 100644 --- a/lib/pinchflat/media_client/source_details.ex +++ b/lib/pinchflat/media_client/source_details.ex @@ -29,14 +29,14 @@ defmodule Pinchflat.MediaClient.SourceDetails do Returns {:ok, [map()]} | {:error, any, ...}. """ - def get_media_attributes(sourceable, opts \\ [], backend \\ :yt_dlp) + def get_media_attributes_for_collection(sourceable, opts \\ [], backend \\ :yt_dlp) - def get_media_attributes(%Source{} = source, opts, backend) do - get_media_attributes(source.collection_id, opts, backend) + def get_media_attributes_for_collection(%Source{} = source, opts, backend) do + get_media_attributes_for_collection(source.collection_id, opts, backend) end - def get_media_attributes(source_url, opts, backend) when is_binary(source_url) do - source_module(backend).get_media_attributes(source_url, opts) + def get_media_attributes_for_collection(source_url, opts, backend) when is_binary(source_url) do + source_module(backend).get_media_attributes_for_collection(source_url, opts) end defp source_module(backend) do diff --git a/lib/pinchflat/tasks/source_tasks.ex b/lib/pinchflat/tasks/source_tasks.ex index ddec433..2b62e3a 100644 --- a/lib/pinchflat/tasks/source_tasks.ex +++ b/lib/pinchflat/tasks/source_tasks.ex @@ -65,7 +65,7 @@ defmodule Pinchflat.Tasks.SourceTasks do def index_and_enqueue_download_for_media_items(%Source{} = source) do # See the method definition below for more info on how file watchers work # (important reading if you're not familiar with it) - {:ok, media_attributes} = get_media_attributes_and_setup_file_watcher(source) + {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source) result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end) Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()}) @@ -116,7 +116,7 @@ defmodule Pinchflat.Tasks.SourceTasks do # lines (ie: you should gracefully fail if you can't parse a line). # # This works in-tandem with the normal (blocking) media indexing behaviour. When - # the `get_media_attributes` method completes it'll return the FULL result to + # the `get_media_attributes_for_collection` method completes it'll return the FULL result to # the caller for parsing. Ideally, every item in the list will have already # been processed by the file follower, but if not, the caller handles creation # of any media items that were missed/initially failed. @@ -124,11 +124,11 @@ defmodule Pinchflat.Tasks.SourceTasks do # It attempts a graceful shutdown of the file follower after the indexing is done, # but the FileFollowerServer will also stop itself if it doesn't see any activity # for a sufficiently long time. - defp get_media_attributes_and_setup_file_watcher(source) do + defp get_media_attributes_for_collection_and_setup_file_watcher(source) do {:ok, pid} = FileFollowerServer.start_link() handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end - result = SourceDetails.get_media_attributes(source.original_url, file_listener_handler: handler) + result = SourceDetails.get_media_attributes_for_collection(source.original_url, file_listener_handler: handler) FileFollowerServer.stop(pid) diff --git a/test/pinchflat/media_client/backends/yt_dlp/media_collection_test.exs b/test/pinchflat/media_client/backends/yt_dlp/media_collection_test.exs index e82a188..db22d40 100644 --- a/test/pinchflat/media_client/backends/yt_dlp/media_collection_test.exs +++ b/test/pinchflat/media_client/backends/yt_dlp/media_collection_test.exs @@ -9,14 +9,14 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do setup :verify_on_exit! - describe "get_media_attributes/2" do + describe "get_media_attributes_for_collection/2" do test "returns a list of video attributes with no blank elements" do expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, source_attributes_return_fixture() <> "\n\n"} end) assert {:ok, [%{"id" => "video1"}, %{"id" => "video2"}, %{"id" => "video3"}]} = - MediaCollection.get_media_attributes(@channel_url) + MediaCollection.get_media_attributes_for_collection(@channel_url) end test "it passes the expected default args" do @@ -27,13 +27,13 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do {:ok, ""} end) - assert {:ok, _} = MediaCollection.get_media_attributes(@channel_url) + assert {:ok, _} = MediaCollection.get_media_attributes_for_collection(@channel_url) end test "returns the error straight through when the command fails" do expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:error, "Big issue", 1} end) - assert {:error, "Big issue", 1} = MediaCollection.get_media_attributes(@channel_url) + assert {:error, "Big issue", 1} = MediaCollection.get_media_attributes_for_collection(@channel_url) end test "passes the explict tmpfile path to runner" do @@ -44,7 +44,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do {:ok, ""} end) - assert {:ok, _} = MediaCollection.get_media_attributes(@channel_url) + assert {:ok, _} = MediaCollection.get_media_attributes_for_collection(@channel_url) end test "supports an optional file_listener_handler that gets passed a filename" do @@ -55,7 +55,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do send(current_self, {:handler, filename}) end - assert {:ok, _} = MediaCollection.get_media_attributes(@channel_url, file_listener_handler: handler) + assert {:ok, _} = + MediaCollection.get_media_attributes_for_collection(@channel_url, file_listener_handler: handler) assert_receive {:handler, filename} assert String.ends_with?(filename, ".json") diff --git a/test/pinchflat/media_client/source_details_test.exs b/test/pinchflat/media_client/source_details_test.exs index 482c08e..c99af2b 100644 --- a/test/pinchflat/media_client/source_details_test.exs +++ b/test/pinchflat/media_client/source_details_test.exs @@ -43,7 +43,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do end end - describe "get_media_attributes/2 when passed a string" do + describe "get_media_attributes_for_collection/2 when passed a string" do test "it passes the expected arguments to the backend" do expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot, _addl_opts -> assert opts == [:simulate, :skip_download] @@ -52,7 +52,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do {:ok, ""} end) - assert {:ok, _} = SourceDetails.get_media_attributes(@channel_url) + assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(@channel_url) end test "it returns a list of maps" do @@ -60,11 +60,11 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do {:ok, source_attributes_return_fixture()} end) - assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes(@channel_url) + assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes_for_collection(@channel_url) end end - describe "get_media_attributes/2 when passed a Source record" do + describe "get_media_attributes_for_collection/2 when passed a Source record" do test "it calls the backend with the source's collection ID" do source = source_fixture() @@ -73,7 +73,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do {:ok, source_attributes_return_fixture()} end) - assert {:ok, _} = SourceDetails.get_media_attributes(source) + assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(source) end test "it builds options based on the source's media profile" do @@ -89,7 +89,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do ) source = source_fixture(media_profile_id: media_profile.id) - assert {:ok, _} = SourceDetails.get_media_attributes(source) + assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(source) end test "lets you pass through an optional file_listener_handler" do @@ -104,7 +104,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do send(current_self, {:handler, filename}) end - assert {:ok, _} = SourceDetails.get_media_attributes(source, file_listener_handler: handler) + assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(source, file_listener_handler: handler) assert_receive {:handler, _} end