Reworked yt-dlp runner; added proper test mocks

config/config.exs

@@ -10,9 +10,9 @@ import Config
 config :pinchflat,
   ecto_repos: [Pinchflat.Repo],
   generators: [timestamp_type: :utc_datetime],
-  backend_executables: %{
+  # Specifying backend data here makes mocking and local testing SUPER easy
-    yt_dlp: "false"
+  yt_dlp_executable: System.find_executable("yt-dlp"),
-  }
+  yt_dlp_runner: Pinchflat.DownloaderBackends.YtDlp.CommandRunner
 
 # Configures the endpoint
 config :pinchflat, PinchflatWeb.Endpoint,

config/test.exs

@@ -1,5 +1,9 @@
 import Config
 
+config :pinchflat,
+  # Specifying backend data here makes mocking and local testing SUPER easy
+  yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"])
+
 # Configure your database
 #
 # The MIX_TEST_PARTITION environment variable can be used

lib/pinchflat/downloader_backends/backend_command_runner.ex

@@ -0,0 +1,7 @@
+defmodule Pinchflat.DownloaderBackends.BackendCommandRunner do
+  @moduledoc """
+  A behaviour for running CLI commands against a downloader backend
+  """
+
+  @callback run(binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
+end

lib/pinchflat/downloader_backends/yt_dlp/command_runner.ex

@@ -4,40 +4,32 @@ defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunner do
   """
 
   alias Pinchflat.Utils.StringUtils
+  alias Pinchflat.DownloaderBackends.BackendCommandRunner
+
+  @behaviour BackendCommandRunner
 
   @doc """
-  Runs a yt-dlp command and returns the output and status
+  Runs a yt-dlp command and returns the string output
-
-  TODO: look into using a behavior for this (if I ever add other backends)
   """
-  def run(url, command_options) do
+  @impl BackendCommandRunner
-    command = Application.get_env(:pinchflat, :backend_executables)[:yt_dlp]
+  def run(url, command_opts) do
-    formatted_command_options = parse_options(command_options) ++ [url]
+    command = backend_executable()
+    formatted_command_opts = parse_options(command_opts) ++ [url]
 
-    case System.cmd(command, formatted_command_options, stderr_to_stdout: true) do
+    case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {output, 0} -> {:ok, output}
       {output, status} -> {:error, output, status}
     end
   end
 
-  @doc """
-  Runs a yt-dlp command and returns the output as a JSON object
-  """
-  def run_json(url, command_options) do
-    case run(url, command_options ++ [:dump_json]) do
-      {:ok, output} -> {:ok, Phoenix.json_library().decode!(output)}
-      res -> res
-    end
-  end
-
   # We want to satisfy the following behaviours:
   #
   # 1. If the key is an atom, convert it to a string and convert it to kebab case (for convenience)
   # 2. If the key is a string, assume we want it as-is and don't convert it
   # 3. If the key is accompanied by a value, append the value to the list
   # 4. If the key is not accompanied by a value, assume it's a flag and PREpend it to the list
-  defp parse_options(command_options) do
+  defp parse_options(command_opts) do
-    Enum.reduce(command_options, [], &parse_option/2)
+    Enum.reduce(command_opts, [], &parse_option/2)
   end
 
   defp parse_option({k, v}, acc) when is_atom(k) do
@@ -59,4 +51,8 @@ defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunner do
   defp parse_option(arg, acc) when is_binary(arg) do
     [arg | acc]
   end
+
+  defp backend_executable do
+    Application.get_env(:pinchflat, :yt_dlp_executable)
+  end
 end

lib/pinchflat/downloader_backends/yt_dlp/video_collection.ex

@@ -0,0 +1,21 @@
+defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollection do
+  @moduledoc """
+  Contains utilities for working with collections of videos (ie: channels, playlists)
+  """
+
+  @doc """
+  Returns a list of strings representing the video ids in the collection
+  """
+  def get_video_ids(url, command_opts \\ []) do
+    opts = command_opts ++ [:simulate, :skip_download, :get_id]
+
+    case backend_runner().run(url, opts) do
+      {:ok, output} -> {:ok, String.split(output, "\n", trim: true)}
+      res -> res
+    end
+  end
+
+  defp backend_runner do
+    Application.get_env(:pinchflat, :yt_dlp_runner)
+  end
+end

mix.exs

@@ -50,7 +50,8 @@ defmodule Pinchflat.MixProject do
       {:gettext, "~> 0.20"},
       {:jason, "~> 1.2"},
       {:dns_cluster, "~> 0.1.1"},
-      {:plug_cowboy, "~> 2.5"}
+      {:plug_cowboy, "~> 2.5"},
+      {:mox, "~> 1.0", only: :test}
     ]
   end
 

mix.lock

@@ -18,6 +18,7 @@
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"},
+  "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
   "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
   "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
   "phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"},

test/pinchflat/downloader_backends/yt_dlp/command_runner_test.exs

@@ -3,16 +3,11 @@ defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunnerTest do
 
   alias Pinchflat.DownloaderBackends.YtDlp.CommandRunner, as: Runner
 
-  @cmd Path.join([File.cwd!(), "/test/support/scripts/mock-yt-dlp-repeater.sh"])
+  @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
   @video_url "https://www.youtube.com/watch?v=9bZkp7q19f0"
-  @original_executables Application.compile_env(:pinchflat, :backend_executables)
 
   setup do
-    Application.put_env(:pinchflat, :backend_executables, %{@original_executables | yt_dlp: @cmd})
+    on_exit(&reset_executable/0)
-
-    on_exit(fn ->
-      Application.put_env(:pinchflat, :backend_executables, @original_executables)
-    end)
   end
 
   describe "run/2" do
@@ -59,35 +54,19 @@ defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunnerTest do
     end
 
     test "it returns the output and status when the command fails" do
-      Application.put_env(:pinchflat, :backend_executables, %{
+      wrap_executable("/bin/false", fn ->
-        @original_executables
+        assert {:error, "", 1} = Runner.run(@video_url, [])
-        | yt_dlp: "/bin/false"
+      end)
-      })
-
-      assert {:error, "", 1} = Runner.run(@video_url, [])
     end
   end
 
-  describe "run_json/2" do
+  defp wrap_executable(new_executable, fun) do
-    test "it returns decoded JSON when the command succeeds" do
+    Application.put_env(:pinchflat, :yt_dlp_executable, new_executable)
-      assert {:ok, output} = Runner.run_json(@video_url, [])
+    fun.()
+    reset_executable()
+  end
 
-      assert is_map(output)
+  def reset_executable do
-    end
+    Application.put_env(:pinchflat, :yt_dlp_executable, @original_executable)
-
-    test "it adds the --dump-json flag automatically" do
-      assert {:ok, %{"args" => output}} = Runner.run_json(@video_url, [])
-
-      assert String.contains?(output, "--dump-json")
-    end
-
-    test "it returns errors when the command fails" do
-      Application.put_env(:pinchflat, :backend_executables, %{
-        @original_executables
-        | yt_dlp: "/bin/false"
-      })
-
-      assert {:error, "", 1} = Runner.run_json(@video_url, [])
-    end
   end
 end

test/pinchflat/downloader_backends/yt_dlp/video_collection_test.exs

@@ -0,0 +1,44 @@
+defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollectionTest do
+  use ExUnit.Case, async: true
+  import Mox
+
+  alias Pinchflat.DownloaderBackends.YtDlp.VideoCollection, as: VideoCollection
+
+  @channel_url "https://www.youtube.com/@TheUselessTrials"
+
+  setup :verify_on_exit!
+
+  describe "get_video_ids/2" do
+    test "returns a list of video ids with no blank elements" do
+      expect(CommandRunnerMock, :run, fn _url, _opts -> {:ok, "id1\nid2\n\nid3\n"} end)
+
+      assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url)
+    end
+
+    test "it passes the expected default args" do
+      expect(CommandRunnerMock, :run, fn _url, opts ->
+        assert opts == [:simulate, :skip_download, :get_id]
+
+        {:ok, ""}
+      end)
+
+      assert {:ok, _} = VideoCollection.get_video_ids(@channel_url)
+    end
+
+    test "it passes the expected custom args" do
+      expect(CommandRunnerMock, :run, fn _url, opts ->
+        assert opts == [:custom_arg, :simulate, :skip_download, :get_id]
+
+        {:ok, ""}
+      end)
+
+      assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg])
+    end
+
+    test "returns the error straight through when the command fails" do
+      expect(CommandRunnerMock, :run, fn _url, _opts -> {:error, "Big issue", 1} end)
+
+      assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url)
+    end
+  end
+end

test/test_helper.exs

@@ -1,2 +1,5 @@
+Mox.defmock(CommandRunnerMock, for: Pinchflat.DownloaderBackends.BackendCommandRunner)
+Application.put_env(:pinchflat, :yt_dlp_runner, CommandRunnerMock)
+
 ExUnit.start()
 Ecto.Adapters.SQL.Sandbox.mode(Pinchflat.Repo, :manual)