From c77047951f756656f2c60d922bb520a19d122bd6 Mon Sep 17 00:00:00 2001 From: Kieran Date: Sat, 24 Feb 2024 13:40:05 -0800 Subject: [PATCH] Improve metadata storage (#37) * Updated config path to specify metadata storage path * Removed metadata column from DB, replacing it with filepath columns * Updated app to store compressed metadata; automatically download thumbnails * Ensured metadata is deleted when other files are deleted * Updated docs * Added inets to application start so http calls work in prod --- .iex.exs | 1 + config/config.exs | 4 +- config/dev.exs | 3 +- config/runtime.exs | 21 +++-- config/test.exs | 3 +- lib/pinchflat/http/http_behaviour.ex | 8 ++ lib/pinchflat/http/http_client.ex | 35 ++++++++ lib/pinchflat/media.ex | 37 +++++++- lib/pinchflat/media/media_metadata.ex | 15 +++- .../backends/yt_dlp/command_runner.ex | 4 +- .../backends/yt_dlp/metadata_file_helpers.ex | 72 +++++++++++++++ .../backends/yt_dlp/metadata_parser.ex | 9 +- .../media_client/video_downloader.ex | 15 +++- mix.exs | 2 +- ...dd_metadata_filepath_to_media_metadata.exs | 12 +++ selfhosted.Dockerfile | 3 +- .../yt_dlp/metadata_file_helpers_test.exs | 87 +++++++++++++++++++ .../backends/yt_dlp/metadata_parser_test.exs | 8 +- .../yt_dlp/output_path_builder_test.exs | 2 +- .../backends/yt_dlp/video_collection_test.exs | 2 +- .../media_client/video_downloader_test.exs | 11 ++- test/pinchflat/media_test.exs | 49 ++++++++++- .../workers/video_download_worker_test.exs | 4 + test/support/fixtures/media_fixtures.ex | 19 ++-- test/test_helper.exs | 5 ++ 25 files changed, 370 insertions(+), 61 deletions(-) create mode 100644 lib/pinchflat/http/http_behaviour.ex create mode 100644 lib/pinchflat/http/http_client.ex create mode 100644 lib/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers.ex create mode 100644 priv/repo/migrations/20240223184259_add_metadata_filepath_to_media_metadata.exs create mode 100644 test/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers_test.exs diff --git a/.iex.exs b/.iex.exs index f9b879b..b15931d 100644 --- a/.iex.exs +++ b/.iex.exs @@ -13,6 +13,7 @@ alias Pinchflat.Profiles alias Pinchflat.Sources alias Pinchflat.MediaClient.{SourceDetails, VideoDownloader} +alias Pinchflat.Metadata.{Zipper, ThumbnailFetcher} defmodule IexHelpers do def playlist_url do diff --git a/config/config.exs b/config/config.exs index a8957b3..2551df6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,7 +14,9 @@ config :pinchflat, yt_dlp_executable: System.find_executable("yt-dlp"), yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, media_directory: "/downloads", - metadata_directory: Path.join([System.tmp_dir!(), "pinchflat", "metadata"]), + # The user may or may not store metadata for their needs, but the app will always store its copy + metadata_directory: "/config/metadata", + tmpfile_directory: Path.join([System.tmp_dir!(), "pinchflat", "data"]), # Setting AUTH_USERNAME and AUTH_PASSWORD implies you want to use basic auth. # If either is unset, basic auth will not be used. basic_auth_username: System.get_env("AUTH_USERNAME"), diff --git a/config/dev.exs b/config/dev.exs index 2dda9d9..2e1baae 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -2,7 +2,8 @@ import Config config :pinchflat, media_directory: Path.join([File.cwd!(), "tmp", "videos"]), - metadata_directory: Path.join([File.cwd!(), "tmp", "metadata"]) + metadata_directory: Path.join([File.cwd!(), "tmp", "metadata"]), + tmpfile_directory: Path.join([File.cwd!(), "tmp", "tmpfiles"]) # Configure your database config :pinchflat, Pinchflat.Repo, diff --git a/config/runtime.exs b/config/runtime.exs index 4086fc0..8fb0033 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -22,19 +22,24 @@ if System.get_env("PHX_SERVER") do end if config_env() == :prod do - database_path = - System.get_env("DATABASE_PATH") || + config_path = + System.get_env("CONFIG_PATH") || raise """ - environment variable DATABASE_PATH is missing. - For example: /etc/pinchflat/pinchflat.db + environment variable CONFIG_PATH is missing. + For example: /etc/pinchflat/config """ - log_path = System.get_env("LOG_PATH", "log/pinchflat.log") + db_path = System.get_env("DATABASE_PATH", Path.join([config_path, "db", "pinchflat.db"])) + log_path = System.get_env("LOG_PATH", Path.join([config_path, "logs", "pinchflat.log"])) + metadata_path = System.get_env("METADATA_PATH", Path.join([config_path, "metadata"])) - config :pinchflat, yt_dlp_executable: System.find_executable("yt-dlp") + config :pinchflat, + yt_dlp_executable: System.find_executable("yt-dlp"), + metadata_directory: metadata_path, + dns_cluster_query: System.get_env("DNS_CLUSTER_QUERY") config :pinchflat, Pinchflat.Repo, - database: database_path, + database: db_path, pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5") # The secret key base is used to sign/encrypt cookies and other secrets. @@ -63,8 +68,6 @@ if config_env() == :prod do end end - config :pinchflat, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") - config :pinchflat, PinchflatWeb.Endpoint, http: [ # Enable IPv6 and bind on all interfaces. diff --git a/config/test.exs b/config/test.exs index e51fb69..005d72f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -4,7 +4,8 @@ config :pinchflat, # Specifying backend data here makes mocking and local testing SUPER easy yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"]), media_directory: Path.join([System.tmp_dir!(), "test", "videos"]), - metadata_directory: Path.join([System.tmp_dir!(), "test", "metadata"]) + metadata_directory: Path.join([System.tmp_dir!(), "test", "metadata"]), + tmpfile_directory: Path.join([System.tmp_dir!(), "test", "tmpfiles"]) config :pinchflat, Oban, testing: :manual diff --git a/lib/pinchflat/http/http_behaviour.ex b/lib/pinchflat/http/http_behaviour.ex new file mode 100644 index 0000000..2fe639e --- /dev/null +++ b/lib/pinchflat/http/http_behaviour.ex @@ -0,0 +1,8 @@ +defmodule Pinchflat.HTTP.HTTPBehaviour do + @moduledoc """ + This module defines the behaviour for HTTP clients. Literally just + so I can use Mox to create an HTTP mock + """ + + @callback get(String.t(), Keyword.t(), Keyword.t()) :: {:ok, String.t()} | {:error, String.t()} +end diff --git a/lib/pinchflat/http/http_client.ex b/lib/pinchflat/http/http_client.ex new file mode 100644 index 0000000..45abda4 --- /dev/null +++ b/lib/pinchflat/http/http_client.ex @@ -0,0 +1,35 @@ +defmodule Pinchflat.HTTP.HTTPClient do + @moduledoc """ + This module provides a simple interface for making HTTP requests. + + Made to be easily swappable with other HTTP clients. If you need more complexity + or security, check out HTTPoison or Mint. + """ + + alias Pinchflat.HTTP.HTTPBehaviour + + @behaviour HTTPBehaviour + + @doc """ + Makes a GET request to the given URL and returns the response. + + NOTE: I can't really test this with Mox and I can't think of a way to test this + that isn't ultimately redundant. I'm just going to leave it untested for now and + focus more on testing the consumers of this module. + + Returns {:ok, String.t()} | {:error, String.t()} + """ + @impl HTTPBehaviour + def get(url, headers \\ [], opts \\ []) do + case :httpc.request(:get, {url, headers}, [], opts) do + {:ok, {{_version, 200, _reason_phrase}, _headers, body}} -> + {:ok, body} + + {:ok, {{_version, status_code, reason_phrase}, _headers, _body}} -> + {:error, "HTTP request failed with status code #{status_code}: #{reason_phrase}"} + + {:error, reason} -> + {:error, "HTTP request failed: #{reason}"} + end + end +end diff --git a/lib/pinchflat/media.ex b/lib/pinchflat/media.ex index 05931a9..ccd6e48 100644 --- a/lib/pinchflat/media.ex +++ b/lib/pinchflat/media.ex @@ -9,6 +9,7 @@ defmodule Pinchflat.Media do alias Pinchflat.Tasks alias Pinchflat.Media.MediaItem alias Pinchflat.Sources.Source + alias Pinchflat.Media.MediaMetadata @doc """ Returns the list of media_items. Returns [%MediaItem{}, ...]. @@ -103,6 +104,21 @@ defmodule Pinchflat.Media do |> Enum.filter(&is_binary/1) end + @doc """ + Produces a flat list of the filesystem paths for a media_item's metadata files. + Returns an empty list if the media_item has no metadata. + + Returns [binary()] | [] + """ + def metadata_filepaths(media_item) do + metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{} + mapped_struct = Map.from_struct(metadata) + + MediaMetadata.filepath_attributes() + |> Enum.map(fn field -> mapped_struct[field] end) + |> Enum.filter(&is_binary/1) + end + @doc """ Creates a media_item. Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}. """ @@ -134,18 +150,31 @@ defmodule Pinchflat.Media do @doc """ Deletes the media_item's associated files. Will leave the media_item in the database. + NOTE: this deletes the metadata files as well, but maybe it shouldn't? I'm wondering if + the metadata is more a concern of the DB record itself and should be lumped in with those + delete operations. But the metadata does come from the download operation of the file. + Food for thought but not a priority at the moment. + Returns {:ok, %MediaItem{}} """ def delete_attachments(media_item) do + media_item = Repo.preload(media_item, :metadata) + media_item |> media_filepaths() + |> Enum.concat(metadata_filepaths(media_item)) |> Enum.each(&File.rm/1) - # Fails if the directory is not empty - case File.rmdir(Path.dirname(media_item.media_filepath)) do - :ok -> {:ok, media_item} - {:error, :eexist} -> {:ok, media_item} + # rmdir will attempt to delete the directory, but only if it is empty + if media_item.media_filepath do + File.rmdir(Path.dirname(media_item.media_filepath)) end + + if media_item.metadata && media_item.metadata.metadata_filepath do + File.rmdir(Path.dirname(media_item.metadata.metadata_filepath)) + end + + {:ok, media_item} end @doc """ diff --git a/lib/pinchflat/media/media_metadata.ex b/lib/pinchflat/media/media_metadata.ex index e41db27..18c3ba2 100644 --- a/lib/pinchflat/media/media_metadata.ex +++ b/lib/pinchflat/media/media_metadata.ex @@ -10,8 +10,12 @@ defmodule Pinchflat.Media.MediaMetadata do alias Pinchflat.Media.MediaItem + @allowed_fields ~w(metadata_filepath thumbnail_filepath)a + @required_fields ~w(metadata_filepath thumbnail_filepath)a + schema "media_metadata" do - field :client_response, :map + field :metadata_filepath, :string + field :thumbnail_filepath, :string belongs_to :media_item, MediaItem @@ -21,8 +25,13 @@ defmodule Pinchflat.Media.MediaMetadata do @doc false def changeset(media_metadata, attrs) do media_metadata - |> cast(attrs, [:client_response]) - |> validate_required([:client_response]) + |> cast(attrs, @allowed_fields) + |> validate_required(@required_fields) |> unique_constraint([:media_item_id]) end + + @doc false + def filepath_attributes do + ~w(metadata_filepath thumbnail_filepath)a + end end diff --git a/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex b/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex index bb98b20..9abe6cb 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex @@ -44,8 +44,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do end defp generate_json_output_path do - metadata_directory = Application.get_env(:pinchflat, :metadata_directory) - filepath = Path.join([metadata_directory, "#{StringUtils.random_string(64)}.json"]) + tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory) + filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.json"]) # Ensure the file can be created and written to BEFORE we run the `yt-dlp` command :ok = File.mkdir_p!(Path.dirname(filepath)) diff --git a/lib/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers.ex b/lib/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers.ex new file mode 100644 index 0000000..d737e69 --- /dev/null +++ b/lib/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers.ex @@ -0,0 +1,72 @@ +defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers do + @moduledoc """ + Provides methods for creating/downloading/storing related metadata + out-of-band of the normal yt-dlp backend process. + + The idea is that I don't want to craft a complicated yt-dlp command, + instead focusing on downloading the video as the user wants it then + I can use the result of that here to grab the additional information + needed + """ + + @doc """ + Compresses and stores metadata for a media item, returning the filepath. + + Returns binary() + """ + def compress_and_store_metadata_for(database_record, metadata_map) do + filepath = generate_filepath_for(database_record, "metadata.json.gz") + {:ok, json} = Phoenix.json_library().encode(metadata_map) + + File.mkdir_p!(Path.dirname(filepath)) + :ok = File.write(filepath, json, [:compressed]) + + filepath + end + + @doc """ + Reads and decodes compressed metadata from a filepath. + + Returns {:ok, map()} | {:error, any} + """ + def read_compressed_metadata(filepath) do + {:ok, json} = File.open(filepath, [:read, :compressed], &IO.read(&1, :all)) + + Phoenix.json_library().decode(json) + end + + @doc """ + Downloads and stores a thumbnail for a media item, returning the filepath. + + Returns binary() + """ + def download_and_store_thumbnail_for(database_record, metadata_map) do + thumbnail_url = metadata_map["thumbnail"] + filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url)) + thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url) + + File.mkdir_p!(Path.dirname(filepath)) + :ok = File.write(filepath, thumbnail_blob) + + filepath + end + + defp fetch_thumbnail_from_url(url) do + http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient) + {:ok, body} = http_client.get(url, [], body_format: :binary) + + body + end + + defp generate_filepath_for(database_record, filename) do + metadata_directory = Application.get_env(:pinchflat, :metadata_directory) + record_table_name = database_record.__meta__.source + + Path.join([ + metadata_directory, + record_table_name, + to_string(database_record.id), + filename + ]) + end +end diff --git a/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex b/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex index 757829b..a32ea73 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex @@ -16,13 +16,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do Returns map() """ def parse_for_media_item(metadata) do - metadata_attrs = %{ - metadata: %{ - client_response: metadata - } - } - - metadata_attrs + Map.new() |> Map.merge(parse_media_metadata(metadata)) |> Map.merge(parse_subtitle_metadata(metadata)) |> Map.merge(parse_thumbnail_metadata(metadata)) @@ -38,7 +32,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do end defp parse_subtitle_metadata(metadata) do - # IDEA: if needed, consider filtering out subtitles that don't exist on-disk subtitle_filepaths = (metadata["requested_subtitles"] || %{}) |> Enum.map(fn {lang, attrs} -> [lang, attrs["filepath"]] end) diff --git a/lib/pinchflat/media_client/video_downloader.ex b/lib/pinchflat/media_client/video_downloader.ex index 9643ef1..4ab205f 100644 --- a/lib/pinchflat/media_client/video_downloader.ex +++ b/lib/pinchflat/media_client/video_downloader.ex @@ -16,6 +16,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser + alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: YtDlpMetadataHelpers @doc """ Downloads a video for a media item, updating the media item based on the metadata @@ -34,12 +35,18 @@ defmodule Pinchflat.MediaClient.VideoDownloader do case download_for_media_profile(media_item.original_url, media_profile, backend) do {:ok, parsed_json} -> - parser = metadata_parser(backend) + {parser, helpers} = metadata_parsers(backend) parsed_attrs = parsed_json |> parser.parse_for_media_item() - |> Map.merge(%{media_downloaded_at: DateTime.utc_now()}) + |> Map.merge(%{ + media_downloaded_at: DateTime.utc_now(), + metadata: %{ + metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json), + thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json) + } + }) # Don't forgor to use preloaded associations or updates to # associations won't work! @@ -70,9 +77,9 @@ defmodule Pinchflat.MediaClient.VideoDownloader do end end - defp metadata_parser(backend) do + defp metadata_parsers(backend) do case backend do - :yt_dlp -> YtDlpMetadataParser + :yt_dlp -> {YtDlpMetadataParser, YtDlpMetadataHelpers} end end end diff --git a/mix.exs b/mix.exs index 7f3e61b..9a108b3 100644 --- a/mix.exs +++ b/mix.exs @@ -23,7 +23,7 @@ defmodule Pinchflat.MixProject do def application do [ mod: {Pinchflat.Application, []}, - extra_applications: [:logger, :runtime_tools] + extra_applications: [:logger, :runtime_tools, :inets] ] end diff --git a/priv/repo/migrations/20240223184259_add_metadata_filepath_to_media_metadata.exs b/priv/repo/migrations/20240223184259_add_metadata_filepath_to_media_metadata.exs new file mode 100644 index 0000000..07a3ace --- /dev/null +++ b/priv/repo/migrations/20240223184259_add_metadata_filepath_to_media_metadata.exs @@ -0,0 +1,12 @@ +defmodule Pinchflat.Repo.Migrations.AddMetadataFilepathToMediaMetadata do + use Ecto.Migration + + def change do + alter table(:media_metadata) do + add :metadata_filepath, :string, null: false + add :thumbnail_filepath, :string, null: false + + remove :client_response, :json + end + end +end diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile index ded2309..79ffaff 100644 --- a/selfhosted.Dockerfile +++ b/selfhosted.Dockerfile @@ -99,8 +99,7 @@ RUN chown nobody /config /downloads # set runner ENV ENV MIX_ENV="prod" -ENV DATABASE_PATH="/config/db/pinchflat.db" -ENV LOG_PATH="/config/logs/pinchflat.log" +ENV CONFIG_PATH="/config" ENV PORT=8945 ENV RUN_CONTEXT="selfhosted" diff --git a/test/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers_test.exs b/test/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers_test.exs new file mode 100644 index 0000000..ffbfff9 --- /dev/null +++ b/test/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers_test.exs @@ -0,0 +1,87 @@ +defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpersTest do + use Pinchflat.DataCase + import Mox + import Pinchflat.MediaFixtures + + alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: Helpers + + setup do + media_item = media_item_fixture() + + {:ok, %{media_item: media_item}} + end + + setup :verify_on_exit! + + describe "compress_and_store_metadata_for/2" do + test "returns the filepath", %{media_item: media_item} do + metadata_map = %{"foo" => "bar"} + + filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map) + + assert filepath =~ ~r{/media_items/#{media_item.id}/metadata.json.gz} + end + + test "creates folder structure based on passed record", %{media_item: media_item} do + metadata_map = %{"foo" => "bar"} + + filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map) + + assert File.exists?(Path.dirname(filepath)) + end + + test "stores it as compressed JSON", %{media_item: media_item} do + metadata_map = %{"foo" => "bar"} + + filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map) + {:ok, json} = File.open(filepath, [:read, :compressed], &IO.read(&1, :all)) + + assert json == Phoenix.json_library().encode!(metadata_map) + end + end + + describe "read_compressed_metadata/1" do + test "returns the compressed and decoded metadata", %{media_item: media_item} do + metadata_map = %{"foo" => "bar"} + + filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map) + {:ok, decoded_json} = Helpers.read_compressed_metadata(filepath) + + assert decoded_json == metadata_map + end + end + + describe "download_and_store_thumbnail_for/2" do + setup do + # This tests that the HTTP endpoint is being called with every test + expect(HTTPClientMock, :get, fn url, _headers, _opts -> + assert url =~ "example.com" + + {:ok, "thumbnail data"} + end) + + metadata = %{"thumbnail" => "example.com/thumbnail.jpg"} + + {:ok, %{metadata: metadata}} + end + + test "returns the filepath", %{media_item: media_item, metadata: metadata} do + filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata) + + assert filepath =~ ~r{/media_items/#{media_item.id}/thumbnail.jpg} + end + + test "creates folder structure based on passed record", %{media_item: media_item, metadata: metadata} do + filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata) + + assert File.exists?(Path.dirname(filepath)) + end + + test "the filename and extension is based on the URL", %{media_item: media_item} do + metadata = %{"thumbnail" => "example.com/maxres.webp"} + filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata) + + assert Path.basename(filepath) == "maxres.webp" + end + end +end diff --git a/test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs b/test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs index c29675d..b40fc70 100644 --- a/test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs +++ b/test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs @@ -1,5 +1,5 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do - use ExUnit.Case, async: true + use Pinchflat.DataCase alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: Parser @@ -41,12 +41,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do assert is_binary(result.description) end - - test "it returns the metadata as a map", %{metadata: metadata} do - result = Parser.parse_for_media_item(metadata) - - assert result.metadata.client_response == metadata - end end describe "parse_for_media_item/1 when testing subtitle metadata" do diff --git a/test/pinchflat/media_client/backends/yt_dlp/output_path_builder_test.exs b/test/pinchflat/media_client/backends/yt_dlp/output_path_builder_test.exs index 4c67a44..8915639 100644 --- a/test/pinchflat/media_client/backends/yt_dlp/output_path_builder_test.exs +++ b/test/pinchflat/media_client/backends/yt_dlp/output_path_builder_test.exs @@ -1,5 +1,5 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.OutputPathBuilderTest do - use ExUnit.Case, async: true + use Pinchflat.DataCase alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder diff --git a/test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs b/test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs index b23bfcf..1eae92a 100644 --- a/test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs +++ b/test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs @@ -1,5 +1,5 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do - use ExUnit.Case, async: true + use Pinchflat.DataCase import Mox import Pinchflat.SourcesFixtures diff --git a/test/pinchflat/media_client/video_downloader_test.exs b/test/pinchflat/media_client/video_downloader_test.exs index 65631bf..27013d4 100644 --- a/test/pinchflat/media_client/video_downloader_test.exs +++ b/test/pinchflat/media_client/video_downloader_test.exs @@ -14,6 +14,10 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do [:metadata, source: :media_profile] ) + stub(HTTPClientMock, :get, fn _url, _headers, _opts -> + {:ok, ""} + end) + {:ok, %{media_item: media_item}} end @@ -29,15 +33,16 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do assert {:ok, _} = VideoDownloader.download_for_media_item(media_item) end - test "it saves the metadata to the database", %{media_item: media_item} do + test "it saves the metadata filepatha to the database", %{media_item: media_item} do expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end) assert is_nil(media_item.metadata) assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item) - assert updated_media_item.metadata - assert is_map(updated_media_item.metadata.client_response) + + assert updated_media_item.metadata.metadata_filepath =~ "media_items/#{media_item.id}/metadata.json.gz" + assert updated_media_item.metadata.thumbnail_filepath =~ "media_items/#{media_item.id}/maxresdefault.jpg" end test "errors are passed through", %{media_item: media_item} do diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index cb9ab77..a02fe29 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -1,6 +1,7 @@ defmodule Pinchflat.MediaTest do use Pinchflat.DataCase + import Mox import Pinchflat.TasksFixtures import Pinchflat.MediaFixtures import Pinchflat.ProfilesFixtures @@ -8,12 +9,17 @@ defmodule Pinchflat.MediaTest do alias Pinchflat.Media alias Pinchflat.Media.MediaItem + alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers + + setup :verify_on_exit! @invalid_attrs %{title: nil, media_id: nil, media_filepath: nil} describe "schema" do test "media_metadata is deleted when media_item is deleted" do - media_item = media_item_fixture(%{metadata: %{client_response: %{foo: "bar"}}}) + media_item = + media_item_fixture(%{metadata: %{metadata_filepath: "/metadata.json.gz", thumbnail_filepath: "/thumbnail.jpg"}}) + metadata = media_item.metadata assert {:ok, %MediaItem{}} = Media.delete_media_item(media_item) @@ -259,6 +265,27 @@ defmodule Pinchflat.MediaTest do end end + describe "metadata_filepaths" do + test "returns filepaths in a flat list" do + filepaths = %{ + metadata_filepath: "/metadata.json.gz", + thumbnail_filepath: "/thumbnail.jpg" + } + + media_item = media_item_fixture(%{metadata: filepaths}) + + assert Media.metadata_filepaths(media_item) == [ + "/metadata.json.gz", + "/thumbnail.jpg" + ] + end + + test "returns an empty list when there is no metadata" do + media_item = media_item_fixture() + assert Media.metadata_filepaths(media_item) == [] + end + end + describe "create_media_item/1" do test "creating with valid data creates a media_item" do valid_attrs = %{ @@ -328,6 +355,26 @@ defmodule Pinchflat.MediaTest do refute File.exists?(media_item.media_filepath) end + test "deletes the media item's metadata files" do + stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end) + media_item = Repo.preload(media_item_with_attachments(), :metadata) + + update_attrs = %{ + metadata: %{ + metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}), + thumbnail_filepath: + MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{ + "thumbnail" => "https://example.com/thumbnail.jpg" + }) + } + } + + {:ok, updated_media_item} = Media.update_media_item(media_item, update_attrs) + + assert {:ok, _} = Media.delete_attachments(updated_media_item) + refute File.exists?(updated_media_item.metadata.metadata_filepath) + end + test "does not delete the media item" do media_item = media_item_with_attachments() diff --git a/test/pinchflat/workers/video_download_worker_test.exs b/test/pinchflat/workers/video_download_worker_test.exs index 376b14b..0e043f1 100644 --- a/test/pinchflat/workers/video_download_worker_test.exs +++ b/test/pinchflat/workers/video_download_worker_test.exs @@ -16,6 +16,10 @@ defmodule Pinchflat.Workers.VideoDownloadWorkerTest do [:metadata, source: :media_profile] ) + stub(HTTPClientMock, :get, fn _url, _headers, _opts -> + {:ok, ""} + end) + {:ok, %{media_item: media_item}} end diff --git a/test/support/fixtures/media_fixtures.ex b/test/support/fixtures/media_fixtures.ex index 29cbec0..c327c46 100644 --- a/test/support/fixtures/media_fixtures.ex +++ b/test/support/fixtures/media_fixtures.ex @@ -31,18 +31,13 @@ defmodule Pinchflat.MediaFixtures do Generate a media_item with metadata. """ def media_item_with_metadata(attrs \\ %{}) do - json_filepath = - Path.join([ - File.cwd!(), - "test", - "support", - "files", - "media_metadata.json" - ]) - - {:ok, file_body} = File.read(json_filepath) - {:ok, parsed_json} = Phoenix.json_library().decode(file_body) - merged_attrs = Map.merge(attrs, %{metadata: %{client_respinse: parsed_json}}) + merged_attrs = + Map.merge(attrs, %{ + metadata: %{ + metadata_filepath: Application.get_env(:pinchflat, :metadata_directory) <> "/metadata.json.gz", + thumbnail_filepath: Application.get_env(:pinchflat, :metadata_directory) <> "/thumbnail.jpg" + } + }) media_item_fixture(merged_attrs) end diff --git a/test/test_helper.exs b/test/test_helper.exs index b0bcf23..00db531 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,6 +1,9 @@ Mox.defmock(YtDlpRunnerMock, for: Pinchflat.MediaClient.Backends.BackendCommandRunner) Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock) +Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour) +Application.put_env(:pinchflat, :http_client, HTTPClientMock) + ExUnit.start() Ecto.Adapters.SQL.Sandbox.mode(Pinchflat.Repo, :manual) Faker.start() @@ -8,7 +11,9 @@ Faker.start() ExUnit.after_suite(fn _ -> File.rm_rf!(Application.get_env(:pinchflat, :media_directory)) File.rm_rf!(Application.get_env(:pinchflat, :metadata_directory)) + File.rm_rf!(Application.get_env(:pinchflat, :tmpfile_directory)) File.mkdir_p!(Application.get_env(:pinchflat, :media_directory)) File.mkdir_p!(Application.get_env(:pinchflat, :metadata_directory)) + File.mkdir_p!(Application.get_env(:pinchflat, :tmpfile_directory)) end)