Takes a fun and flirty digression to remove abstractions around yt-dlp since I'm 100% committed to using it exclusively

.iex.exs

@@ -14,7 +14,7 @@ alias Pinchflat.Sources
 alias Pinchflat.Settings
 
 alias Pinchflat.MediaClient.{SourceDetails, MediaDownloader}
-alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers
+alias Pinchflat.Metadata.MetadataFileHelpers
 
 alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
 

config/config.exs

@@ -12,7 +12,7 @@ config :pinchflat,
   generators: [timestamp_type: :utc_datetime],
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
-  yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner,
+  yt_dlp_runner: Pinchflat.YtDlp.Backend.CommandRunner,
   media_directory: "/downloads",
   # The user may or may not store metadata for their needs, but the app will always store its copy
   metadata_directory: "/config/metadata",

lib/pinchflat/media_client/source_details.ex

@@ -1,47 +0,0 @@
-defmodule Pinchflat.MediaClient.SourceDetails do
-  @moduledoc """
-  This is the integration layer for actually working with sources.
-
-  Technically hardcodes the yt-dlp backend for now, but should leave
-  it open-ish for future expansion (just in case).
-  """
-
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.MediaClient.Backends.YtDlp.MediaCollection, as: YtDlpSource
-
-  @doc """
-  Gets a source's ID and name from its URL using the given backend.
-
-  Returns {:ok, map()} | {:error, any, ...}.
-  """
-  def get_source_details(source_url, backend \\ :yt_dlp) do
-    source_module(backend).get_source_details(source_url)
-  end
-
-  @doc """
-  Returns a list of basic media data maps for the given source URL OR
-  source record using the given backend.
-
-  Options:
-    - :file_listener_handler - a function that will be called with the path to the
-      file that will be written to by yt-dlp. This is useful for
-      setting up a file watcher to read the file as it gets written to.
-
-  Returns {:ok, [map()]} | {:error, any, ...}.
-  """
-  def get_media_attributes_for_collection(sourceable, opts \\ [], backend \\ :yt_dlp)
-
-  def get_media_attributes_for_collection(%Source{} = source, opts, backend) do
-    get_media_attributes_for_collection(source.collection_id, opts, backend)
-  end
-
-  def get_media_attributes_for_collection(source_url, opts, backend) when is_binary(source_url) do
-    source_module(backend).get_media_attributes_for_collection(source_url, opts)
-  end
-
-  defp source_module(backend) do
-    case backend do
-      :yt_dlp -> YtDlpSource
-    end
-  end
-end

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers do
+defmodule Pinchflat.Metadata.MetadataFileHelpers do
   @moduledoc """
   Provides methods for creating/downloading/storing related metadata
   out-of-band of the normal yt-dlp backend process.

lib/pinchflat/metadata/metadata_parser.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
+defmodule Pinchflat.Metadata.MetadataParser do
   @moduledoc """
   yt-dlp offers a LOT of metadata in its JSON response, some of which
   needs to be extracted and included in various models.

lib/pinchflat/profiles/output_path_builder.ex

@@ -1,8 +1,6 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
+defmodule Pinchflat.Profiles.OutputPathBuilder do
   @moduledoc """
   Builds yt-dlp-friendly output paths for downloaded media
-
-  IDEA: consider making this a behaviour so I can add other backends later
   """
 
   alias Pinchflat.RenderedString.Parser, as: TemplateParser

lib/pinchflat/sources.ex

@@ -11,7 +11,7 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Sources.Source
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.MediaClient.SourceDetails
+  alias Pinchflat.YtDlp.Backend.MediaCollection
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -116,7 +116,7 @@ defmodule Pinchflat.Sources do
   defp add_source_details_to_changeset(source, changeset) do
     %Ecto.Changeset{changes: changes} = changeset
 
-    case SourceDetails.get_source_details(changes.original_url) do
+    case MediaCollection.get_source_details(changes.original_url) do
       {:ok, source_details} ->
         add_source_details_by_collection_type(source, changeset, source_details)
 

lib/pinchflat/tasks/source_tasks.ex

@@ -13,9 +13,9 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.MediaClient.SourceDetails
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.YtDlp.Backend.MediaCollection
   alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
 
   @doc """
@@ -128,7 +128,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
     {:ok, pid} = FileFollowerServer.start_link()
 
     handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end
-    result = SourceDetails.get_media_attributes_for_collection(source.original_url, file_listener_handler: handler)
+    result = MediaCollection.get_media_attributes_for_collection(source.original_url, file_listener_handler: handler)
 
     FileFollowerServer.stop(pid)
 

lib/pinchflat/yt_dlp/backend/backend_command_runner.ex

@@ -1,6 +1,9 @@
-defmodule Pinchflat.MediaClient.Backends.BackendCommandRunner do
+defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
   @moduledoc """
-  A behaviour for running CLI commands against a downloader backend
+  A behaviour for running CLI commands against a downloader backend (yt-dlp).
+
+  Used so we can implement Mox for testing without actually running the
+  yt-dlp command.
   """
 
   @callback run(binary(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}

lib/pinchflat/yt_dlp/backend/command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do
+defmodule Pinchflat.YtDlp.Backend.CommandRunner do
   @moduledoc """
   Runs yt-dlp commands using the `System.cmd/3` function
   """
@@ -7,7 +7,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do
 
   alias Pinchflat.Utils.StringUtils
   alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
-  alias Pinchflat.MediaClient.Backends.BackendCommandRunner
+  alias Pinchflat.YtDlp.Backend.BackendCommandRunner
 
   @behaviour BackendCommandRunner
 
@@ -25,6 +25,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do
   """
   @impl BackendCommandRunner
   def run(url, command_opts, output_template, addl_opts \\ []) do
+    # This approach lets us mock the command for testing
     command = backend_executable()
     # These must stay in exactly this order, hence why I'm giving it its own variable.
     # Also, can't use RAM file since yt-dlp needs a concrete filepath.

lib/pinchflat/yt_dlp/backend/media.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.Media do
+defmodule Pinchflat.YtDlp.Backend.Media do
   @moduledoc """
   Contains utilities for working with singular pieces of media
   """
@@ -39,6 +39,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.Media do
   end
 
   defp backend_runner do
+    # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)
   end
 end

lib/pinchflat/yt_dlp/backend/media_collection.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollection do
+defmodule Pinchflat.YtDlp.Backend.MediaCollection do
   @moduledoc """
   Contains utilities for working with collections of
   media (aka: a source [ie: channels, playlists]).
@@ -8,7 +8,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollection do
 
   alias Pinchflat.Utils.FunctionUtils
   alias Pinchflat.Utils.FilesystemUtils
-  alias Pinchflat.MediaClient.Backends.YtDlp.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
 
   @doc """
   Returns a list of maps representing the media in the collection.
@@ -74,6 +74,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollection do
   end
 
   defp backend_runner do
+    # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)
   end
 end

lib/pinchflat/yt_dlp/download_option_builder.ex

@@ -1,12 +1,10 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder do
+defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
   @moduledoc """
   Builds the options for yt-dlp to download media based on the given media profile.
-
-  IDEA: consider making this a behaviour so I can add other backends later
   """
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
+  alias Pinchflat.Profiles.OutputPathBuilder
 
   @doc """
   Builds the options for yt-dlp to download media based on the given media's profile.

lib/pinchflat/yt_dlp/media_downloader.ex

@@ -1,25 +1,22 @@
 defmodule Pinchflat.MediaClient.MediaDownloader do
   @moduledoc """
-  This is the integration layer for actually downloading medias.
+  This is the integration layer for actually downloading media.
   It takes into account the media profile's settings in order
   to download the media with the desired options.
-
-  Technically hardcodes the yt-dlp backend for now, but should leave
-  it open-ish for future expansion (just in case).
   """
 
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.Media, as: YtDlpMedia
-  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: YtDlpMetadataHelpers
+  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
+  alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
+  alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 
   @doc """
   Downloads media for a media item, updating the media item based on the metadata
-  returned by the backend. Also saves the entire metadata response to the associated
+  returned by yt-dlp. Also saves the entire metadata response to the associated
   media_metadata record.
 
   NOTE: related methods (like the download worker) won't download if the media item's source
@@ -28,12 +25,12 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
 
   Returns {:ok, %MediaItem{}} | {:error, any, ...any}
   """
-  def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do
+  def download_for_media_item(%MediaItem{} = media_item) do
     item_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
 
-    case download_with_options(media_item.original_url, item_with_preloads, backend) do
+    case download_with_options(media_item.original_url, item_with_preloads) do
       {:ok, parsed_json} ->
-        {parser, helpers} = metadata_parsers(backend)
+        {parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
 
         parsed_attrs =
           parsed_json
@@ -55,34 +52,14 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
     end
   end
 
-  # def download_for_source(source, url, backend \\ :yt_dlp) do
+  # def download_for_source(source, url) do
   #   # Create MI from source and URL
   #   media_item = nil
   # end
 
-  defp download_with_options(url, item_with_preloads, backend) do
-    option_builder = option_builder(backend)
-    media_backend = media_backend(backend)
-    {:ok, options} = option_builder.build(item_with_preloads)
+  defp download_with_options(url, item_with_preloads) do
+    {:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
 
-    media_backend.download(url, options)
-  end
-
-  defp option_builder(backend) do
-    case backend do
-      :yt_dlp -> YtDlpDownloadOptionBuilder
-    end
-  end
-
-  defp media_backend(backend) do
-    case backend do
-      :yt_dlp -> YtDlpMedia
-    end
-  end
-
-  defp metadata_parsers(backend) do
-    case backend do
-      :yt_dlp -> {YtDlpMetadataParser, YtDlpMetadataHelpers}
-    end
+    YtDlpMedia.download(url, options)
   end
 end

test/pinchflat/media_client/source_details_test.exs

@@ -1,112 +0,0 @@
-defmodule Pinchflat.MediaClient.SourceDetailsTest do
-  use Pinchflat.DataCase
-  import Mox
-  import Pinchflat.ProfilesFixtures
-  import Pinchflat.SourcesFixtures
-
-  alias Pinchflat.MediaClient.SourceDetails
-
-  @channel_url "https://www.youtube.com/c/TheUselessTrials"
-
-  setup :verify_on_exit!
-
-  describe "get_source_details/2" do
-    test "it passes the expected arguments to the backend" do
-      expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
-        assert opts == [:simulate, :skip_download, playlist_end: 1]
-        assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j"
-
-        {:ok, "{}"}
-      end)
-
-      assert {:ok, _} = SourceDetails.get_source_details(@channel_url)
-    end
-
-    test "it returns a map composed of the returned data" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        Phoenix.json_library().encode(%{
-          channel: "TheUselessTrials",
-          channel_id: "UCQH2",
-          playlist_id: "PLQH2",
-          playlist_title: "TheUselessTrials - Videos"
-        })
-      end)
-
-      assert {:ok, res} = SourceDetails.get_source_details(@channel_url)
-
-      assert %{
-               channel_id: "UCQH2",
-               channel_name: "TheUselessTrials",
-               playlist_id: "PLQH2",
-               playlist_name: "TheUselessTrials - Videos"
-             } = res
-    end
-  end
-
-  describe "get_media_attributes_for_collection/2 when passed a string" do
-    test "it passes the expected arguments to the backend" do
-      expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot, _addl_opts ->
-        assert opts == [:simulate, :skip_download]
-        assert ot == "%(.{id,title,was_live,original_url,description})j"
-
-        {:ok, ""}
-      end)
-
-      assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(@channel_url)
-    end
-
-    test "it returns a list of maps" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes_for_collection(@channel_url)
-    end
-  end
-
-  describe "get_media_attributes_for_collection/2 when passed a Source record" do
-    test "it calls the backend with the source's collection ID" do
-      source = source_fixture()
-
-      expect(YtDlpRunnerMock, :run, fn url, _opts, _ot, _addl_opts ->
-        assert source.collection_id == url
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(source)
-    end
-
-    test "it builds options based on the source's media profile" do
-      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, _addl_opts ->
-        assert opts == [:simulate, :skip_download]
-        {:ok, ""}
-      end)
-
-      media_profile =
-        media_profile_fixture(
-          shorts_behaviour: :include,
-          livestream_behaviour: :exclude
-        )
-
-      source = source_fixture(media_profile_id: media_profile.id)
-      assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(source)
-    end
-
-    test "lets you pass through an optional file_listener_handler" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      source = source_fixture()
-      current_self = self()
-
-      handler = fn filename ->
-        send(current_self, {:handler, filename})
-      end
-
-      assert {:ok, _} = SourceDetails.get_media_attributes_for_collection(source, file_listener_handler: handler)
-
-      assert_receive {:handler, _}
-    end
-  end
-end

test/pinchflat/media_test.exs

@@ -9,7 +9,7 @@ defmodule Pinchflat.MediaTest do
 
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers
+  alias Pinchflat.Metadata.MetadataFileHelpers
 
   setup :verify_on_exit!
 

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpersTest do
+defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: Helpers
+  alias Pinchflat.Metadata.MetadataFileHelpers, as: Helpers
 
   setup do
     media_item = media_item_fixture()

test/pinchflat/metadata/metadata_parser_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
+defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: Parser
+  alias Pinchflat.Metadata.MetadataParser, as: Parser
 
   setup do
     json_filepath =

test/pinchflat/yt_dlp/backend/command_runner_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunnerTest do
+defmodule Pinchflat.YtDlp.Backend.CommandRunnerTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, as: Runner
+  alias Pinchflat.YtDlp.Backend.CommandRunner, as: Runner
 
   @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
   @media_url "https://www.youtube.com/watch?v=-LHXuyzpex0"

test/pinchflat/yt_dlp/backend/media_collection_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do
+defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.MediaCollection
+  alias Pinchflat.YtDlp.Backend.MediaCollection
 
   @channel_url "https://www.youtube.com/c/TheUselessTrials"
 

test/pinchflat/yt_dlp/backend/media_test.exs

@@ -1,8 +1,8 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaTest do
+defmodule Pinchflat.YtDlp.Backend.MediaTest do
   use Pinchflat.DataCase
   import Mox
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.Media
+  alias Pinchflat.YtDlp.Backend.Media
 
   @media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
 

test/pinchflat/yt_dlp/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.YtDlp.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})

test/profiles/output_path_builder_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.OutputPathBuilderTest do
+defmodule Pinchflat.Profiles.OutputPathBuilderTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
+  alias Pinchflat.Profiles.OutputPathBuilder
 
   describe "build/2" do
     test "it expands 'standard' curly brace variables in the template" do

test/test_helper.exs

@@ -1,4 +1,4 @@
-Mox.defmock(YtDlpRunnerMock, for: Pinchflat.MediaClient.Backends.BackendCommandRunner)
+Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.Backend.BackendCommandRunner)
 Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
 
 Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour)