Initial yt-dlp implementation (#3)

* Added basic runner for yt-dlp commands

* Updated formatting rules

* Added JSON runner; Added test script for CLI interactions

* Reworked yt-dlp runner; added proper test mocks

.formatter.exs

@@ -2,5 +2,6 @@
   import_deps: [:ecto, :ecto_sql, :phoenix],
   subdirectories: ["priv/*/migrations"],
   plugins: [Phoenix.LiveView.HTMLFormatter],
-  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"]
+  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
+  line_length: 100
 ]

config/config.exs

@@ -9,7 +9,10 @@ import Config
 
 config :pinchflat,
   ecto_repos: [Pinchflat.Repo],
-  generators: [timestamp_type: :utc_datetime]
+  generators: [timestamp_type: :utc_datetime],
+  # Specifying backend data here makes mocking and local testing SUPER easy
+  yt_dlp_executable: System.find_executable("yt-dlp"),
+  yt_dlp_runner: Pinchflat.DownloaderBackends.YtDlp.CommandRunner
 
 # Configures the endpoint
 config :pinchflat, PinchflatWeb.Endpoint,

config/test.exs

@@ -1,14 +1,18 @@
 import Config
 
+config :pinchflat,
+  # Specifying backend data here makes mocking and local testing SUPER easy
+  yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"])
+
 # Configure your database
 #
 # The MIX_TEST_PARTITION environment variable can be used
 # to provide built-in test partitioning in CI environment.
 # Run `mix help test` for more information.
 config :pinchflat, Pinchflat.Repo,
-  username: "postgres",
-  password: "postgres",
-  hostname: "localhost",
+  username: System.get_env("POSTGRES_USER"),
+  password: System.get_env("POSTGRES_PASSWORD"),
+  hostname: System.get_env("POSTGRES_HOST"),
   database: "pinchflat_test#{System.get_env("MIX_TEST_PARTITION")}",
   pool: Ecto.Adapters.SQL.Sandbox,
   pool_size: 10

lib/pinchflat/downloader_backends/backend_command_runner.ex

@@ -0,0 +1,7 @@
+defmodule Pinchflat.DownloaderBackends.BackendCommandRunner do
+  @moduledoc """
+  A behaviour for running CLI commands against a downloader backend
+  """
+
+  @callback run(binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
+end

lib/pinchflat/downloader_backends/yt_dlp/command_runner.ex

@@ -0,0 +1,58 @@
+defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunner do
+  @moduledoc """
+  Runs yt-dlp commands using the `System.cmd/3` function
+  """
+
+  alias Pinchflat.Utils.StringUtils
+  alias Pinchflat.DownloaderBackends.BackendCommandRunner
+
+  @behaviour BackendCommandRunner
+
+  @doc """
+  Runs a yt-dlp command and returns the string output
+  """
+  @impl BackendCommandRunner
+  def run(url, command_opts) do
+    command = backend_executable()
+    formatted_command_opts = parse_options(command_opts) ++ [url]
+
+    case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
+      {output, 0} -> {:ok, output}
+      {output, status} -> {:error, output, status}
+    end
+  end
+
+  # We want to satisfy the following behaviours:
+  #
+  # 1. If the key is an atom, convert it to a string and convert it to kebab case (for convenience)
+  # 2. If the key is a string, assume we want it as-is and don't convert it
+  # 3. If the key is accompanied by a value, append the value to the list
+  # 4. If the key is not accompanied by a value, assume it's a flag and PREpend it to the list
+  defp parse_options(command_opts) do
+    Enum.reduce(command_opts, [], &parse_option/2)
+  end
+
+  defp parse_option({k, v}, acc) when is_atom(k) do
+    stringified_key = StringUtils.to_kebab_case(Atom.to_string(k))
+
+    parse_option({"--#{stringified_key}", v}, acc)
+  end
+
+  defp parse_option({k, v}, acc) when is_binary(k) do
+    acc ++ [k, to_string(v)]
+  end
+
+  defp parse_option(arg, acc) when is_atom(arg) do
+    stringified_arg = StringUtils.to_kebab_case(Atom.to_string(arg))
+
+    parse_option("--#{stringified_arg}", acc)
+  end
+
+  defp parse_option(arg, acc) when is_binary(arg) do
+    [arg | acc]
+  end
+
+  defp backend_executable do
+    Application.get_env(:pinchflat, :yt_dlp_executable)
+  end
+end

lib/pinchflat/downloader_backends/yt_dlp/video_collection.ex

@@ -0,0 +1,21 @@
+defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollection do
+  @moduledoc """
+  Contains utilities for working with collections of videos (ie: channels, playlists)
+  """
+
+  @doc """
+  Returns a list of strings representing the video ids in the collection
+  """
+  def get_video_ids(url, command_opts \\ []) do
+    opts = command_opts ++ [:simulate, :skip_download, :get_id]
+
+    case backend_runner().run(url, opts) do
+      {:ok, output} -> {:ok, String.split(output, "\n", trim: true)}
+      res -> res
+    end
+  end
+
+  defp backend_runner do
+    Application.get_env(:pinchflat, :yt_dlp_runner)
+  end
+end

lib/pinchflat_web/components/core_components.ex

@@ -614,8 +614,7 @@ defmodule PinchflatWeb.CoreComponents do
       to: selector,
       time: 200,
       transition:
-        {"transition-all transform ease-in duration-200",
-         "opacity-100 translate-y-0 sm:scale-100",
+        {"transition-all transform ease-in duration-200", "opacity-100 translate-y-0 sm:scale-100",
          "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95"}
     )
   end

lib/pinchflat_web/telemetry.ex

@@ -70,8 +70,7 @@ defmodule PinchflatWeb.Telemetry do
       ),
       summary("pinchflat.repo.query.idle_time",
         unit: {:native, :millisecond},
-        description:
-          "The time the connection spent waiting before being checked out for the query"
+        description: "The time the connection spent waiting before being checked out for the query"
       ),
 
       # VM Metrics

lib/utils/string_utils.ex

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Utils.StringUtils do
+  @moduledoc """
+  Utility functions for working with strings
+  """
+
+  @doc """
+  Converts a string to kebab-case (ie: `hello world` -> `hello-world`)
+  """
+  def to_kebab_case(string) do
+    string
+    |> String.replace(~r/[\s_]/, "-")
+    |> String.downcase()
+  end
+end

mix.exs

@@ -50,7 +50,8 @@ defmodule Pinchflat.MixProject do
       {:gettext, "~> 0.20"},
       {:jason, "~> 1.2"},
       {:dns_cluster, "~> 0.1.1"},
-      {:plug_cowboy, "~> 2.5"}
+      {:plug_cowboy, "~> 2.5"},
+      {:mox, "~> 1.0", only: :test}
     ]
   end
 

mix.lock

@@ -18,6 +18,7 @@
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"},
+  "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
   "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
   "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
   "phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"},

test/pinchflat/downloader_backends/yt_dlp/command_runner_test.exs

@@ -0,0 +1,72 @@
+defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunnerTest do
+  use ExUnit.Case, async: true
+
+  alias Pinchflat.DownloaderBackends.YtDlp.CommandRunner, as: Runner
+
+  @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
+  @video_url "https://www.youtube.com/watch?v=9bZkp7q19f0"
+
+  setup do
+    on_exit(&reset_executable/0)
+  end
+
+  describe "run/2" do
+    test "it returns the output and status when the command succeeds" do
+      assert {:ok, _output} = Runner.run(@video_url, [])
+    end
+
+    test "it converts symbol k-v arg keys to kebab case" do
+      assert {:ok, output} = Runner.run(@video_url, buffer_size: 1024)
+
+      assert String.contains?(output, "--buffer-size 1024")
+    end
+
+    test "it keeps string k-v arg keys untouched" do
+      assert {:ok, output} = Runner.run(@video_url, [{"--under_score", 1024}])
+
+      assert String.contains?(output, "--under_score 1024")
+    end
+
+    test "it converts symbol arg keys to kebab case" do
+      assert {:ok, output} = Runner.run(@video_url, [:ignore_errors])
+
+      assert String.contains?(output, "--ignore-errors")
+    end
+
+    test "it keeps string arg keys untouched" do
+      assert {:ok, output} = Runner.run(@video_url, ["-v"])
+
+      assert String.contains?(output, "-v")
+      refute String.contains?(output, "--v")
+    end
+
+    test "it places arg keys (flags) at the beginning of the command" do
+      assert {:ok, output} =
+               Runner.run(@video_url, [{"--under_score", 1024}, :ignore_errors])
+
+      assert String.contains?(output, "--ignore-errors --under_score 1024")
+    end
+
+    test "it includes the video url as the last argument" do
+      assert {:ok, output} = Runner.run(@video_url, [:ignore_errors])
+
+      assert String.contains?(output, "--ignore-errors #{@video_url}\n")
+    end
+
+    test "it returns the output and status when the command fails" do
+      wrap_executable("/bin/false", fn ->
+        assert {:error, "", 1} = Runner.run(@video_url, [])
+      end)
+    end
+  end
+
+  defp wrap_executable(new_executable, fun) do
+    Application.put_env(:pinchflat, :yt_dlp_executable, new_executable)
+    fun.()
+    reset_executable()
+  end
+
+  def reset_executable do
+    Application.put_env(:pinchflat, :yt_dlp_executable, @original_executable)
+  end
+end

test/pinchflat/downloader_backends/yt_dlp/video_collection_test.exs

@@ -0,0 +1,44 @@
+defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollectionTest do
+  use ExUnit.Case, async: true
+  import Mox
+
+  alias Pinchflat.DownloaderBackends.YtDlp.VideoCollection, as: VideoCollection
+
+  @channel_url "https://www.youtube.com/@TheUselessTrials"
+
+  setup :verify_on_exit!
+
+  describe "get_video_ids/2" do
+    test "returns a list of video ids with no blank elements" do
+      expect(CommandRunnerMock, :run, fn _url, _opts -> {:ok, "id1\nid2\n\nid3\n"} end)
+
+      assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url)
+    end
+
+    test "it passes the expected default args" do
+      expect(CommandRunnerMock, :run, fn _url, opts ->
+        assert opts == [:simulate, :skip_download, :get_id]
+
+        {:ok, ""}
+      end)
+
+      assert {:ok, _} = VideoCollection.get_video_ids(@channel_url)
+    end
+
+    test "it passes the expected custom args" do
+      expect(CommandRunnerMock, :run, fn _url, opts ->
+        assert opts == [:custom_arg, :simulate, :skip_download, :get_id]
+
+        {:ok, ""}
+      end)
+
+      assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg])
+    end
+
+    test "returns the error straight through when the command fails" do
+      expect(CommandRunnerMock, :run, fn _url, _opts -> {:error, "Big issue", 1} end)
+
+      assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url)
+    end
+  end
+end

test/pinchflat/utils/string_utils_test.exs

@@ -0,0 +1,15 @@
+defmodule Pinchflat.Utils.StringUtilsTest do
+  use ExUnit.Case, async: true
+
+  alias Pinchflat.Utils.StringUtils, as: StringUtils
+
+  describe "to_kebab_case/1" do
+    test "converts a space-delimited string to kebab-case" do
+      assert StringUtils.to_kebab_case("hello world") == "hello-world"
+    end
+
+    test "converts an underscore-delimited string to kebab-case" do
+      assert StringUtils.to_kebab_case("hello_world") == "hello-world"
+    end
+  end
+end

test/support/scripts/yt-dlp-mocks/repeater.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ "$@" == *"--dump-json"* ]]; then
+  echo '{ "args": "'$@'"}'
+else
+  echo $@
+fi

test/test_helper.exs

@@ -1,2 +1,5 @@
+Mox.defmock(CommandRunnerMock, for: Pinchflat.DownloaderBackends.BackendCommandRunner)
+Application.put_env(:pinchflat, :yt_dlp_runner, CommandRunnerMock)
+
 ExUnit.start()
 Ecto.Adapters.SQL.Sandbox.mode(Pinchflat.Repo, :manual)