diff --git a/.formatter.exs b/.formatter.exs index ef8840c..602260a 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -2,5 +2,6 @@ import_deps: [:ecto, :ecto_sql, :phoenix], subdirectories: ["priv/*/migrations"], plugins: [Phoenix.LiveView.HTMLFormatter], - inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"] + inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"], + line_length: 100 ] diff --git a/config/config.exs b/config/config.exs index 5a7d4e9..f920c3c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -9,7 +9,10 @@ import Config config :pinchflat, ecto_repos: [Pinchflat.Repo], - generators: [timestamp_type: :utc_datetime] + generators: [timestamp_type: :utc_datetime], + # Specifying backend data here makes mocking and local testing SUPER easy + yt_dlp_executable: System.find_executable("yt-dlp"), + yt_dlp_runner: Pinchflat.DownloaderBackends.YtDlp.CommandRunner # Configures the endpoint config :pinchflat, PinchflatWeb.Endpoint, diff --git a/config/test.exs b/config/test.exs index 957c6d3..20d03ff 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,14 +1,18 @@ import Config +config :pinchflat, + # Specifying backend data here makes mocking and local testing SUPER easy + yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"]) + # Configure your database # # The MIX_TEST_PARTITION environment variable can be used # to provide built-in test partitioning in CI environment. # Run `mix help test` for more information. config :pinchflat, Pinchflat.Repo, - username: "postgres", - password: "postgres", - hostname: "localhost", + username: System.get_env("POSTGRES_USER"), + password: System.get_env("POSTGRES_PASSWORD"), + hostname: System.get_env("POSTGRES_HOST"), database: "pinchflat_test#{System.get_env("MIX_TEST_PARTITION")}", pool: Ecto.Adapters.SQL.Sandbox, pool_size: 10 diff --git a/lib/pinchflat/downloader_backends/backend_command_runner.ex b/lib/pinchflat/downloader_backends/backend_command_runner.ex new file mode 100644 index 0000000..6e1bcb7 --- /dev/null +++ b/lib/pinchflat/downloader_backends/backend_command_runner.ex @@ -0,0 +1,7 @@ +defmodule Pinchflat.DownloaderBackends.BackendCommandRunner do + @moduledoc """ + A behaviour for running CLI commands against a downloader backend + """ + + @callback run(binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()} +end diff --git a/lib/pinchflat/downloader_backends/yt_dlp/command_runner.ex b/lib/pinchflat/downloader_backends/yt_dlp/command_runner.ex new file mode 100644 index 0000000..014ae0e --- /dev/null +++ b/lib/pinchflat/downloader_backends/yt_dlp/command_runner.ex @@ -0,0 +1,58 @@ +defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunner do + @moduledoc """ + Runs yt-dlp commands using the `System.cmd/3` function + """ + + alias Pinchflat.Utils.StringUtils + alias Pinchflat.DownloaderBackends.BackendCommandRunner + + @behaviour BackendCommandRunner + + @doc """ + Runs a yt-dlp command and returns the string output + """ + @impl BackendCommandRunner + def run(url, command_opts) do + command = backend_executable() + formatted_command_opts = parse_options(command_opts) ++ [url] + + case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do + {output, 0} -> {:ok, output} + {output, status} -> {:error, output, status} + end + end + + # We want to satisfy the following behaviours: + # + # 1. If the key is an atom, convert it to a string and convert it to kebab case (for convenience) + # 2. If the key is a string, assume we want it as-is and don't convert it + # 3. If the key is accompanied by a value, append the value to the list + # 4. If the key is not accompanied by a value, assume it's a flag and PREpend it to the list + defp parse_options(command_opts) do + Enum.reduce(command_opts, [], &parse_option/2) + end + + defp parse_option({k, v}, acc) when is_atom(k) do + stringified_key = StringUtils.to_kebab_case(Atom.to_string(k)) + + parse_option({"--#{stringified_key}", v}, acc) + end + + defp parse_option({k, v}, acc) when is_binary(k) do + acc ++ [k, to_string(v)] + end + + defp parse_option(arg, acc) when is_atom(arg) do + stringified_arg = StringUtils.to_kebab_case(Atom.to_string(arg)) + + parse_option("--#{stringified_arg}", acc) + end + + defp parse_option(arg, acc) when is_binary(arg) do + [arg | acc] + end + + defp backend_executable do + Application.get_env(:pinchflat, :yt_dlp_executable) + end +end diff --git a/lib/pinchflat/downloader_backends/yt_dlp/video_collection.ex b/lib/pinchflat/downloader_backends/yt_dlp/video_collection.ex new file mode 100644 index 0000000..324895b --- /dev/null +++ b/lib/pinchflat/downloader_backends/yt_dlp/video_collection.ex @@ -0,0 +1,21 @@ +defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollection do + @moduledoc """ + Contains utilities for working with collections of videos (ie: channels, playlists) + """ + + @doc """ + Returns a list of strings representing the video ids in the collection + """ + def get_video_ids(url, command_opts \\ []) do + opts = command_opts ++ [:simulate, :skip_download, :get_id] + + case backend_runner().run(url, opts) do + {:ok, output} -> {:ok, String.split(output, "\n", trim: true)} + res -> res + end + end + + defp backend_runner do + Application.get_env(:pinchflat, :yt_dlp_runner) + end +end diff --git a/lib/pinchflat_web/components/core_components.ex b/lib/pinchflat_web/components/core_components.ex index 2d5ae2f..e715b97 100644 --- a/lib/pinchflat_web/components/core_components.ex +++ b/lib/pinchflat_web/components/core_components.ex @@ -614,8 +614,7 @@ defmodule PinchflatWeb.CoreComponents do to: selector, time: 200, transition: - {"transition-all transform ease-in duration-200", - "opacity-100 translate-y-0 sm:scale-100", + {"transition-all transform ease-in duration-200", "opacity-100 translate-y-0 sm:scale-100", "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95"} ) end diff --git a/lib/pinchflat_web/telemetry.ex b/lib/pinchflat_web/telemetry.ex index d5ba1f1..172e290 100644 --- a/lib/pinchflat_web/telemetry.ex +++ b/lib/pinchflat_web/telemetry.ex @@ -70,8 +70,7 @@ defmodule PinchflatWeb.Telemetry do ), summary("pinchflat.repo.query.idle_time", unit: {:native, :millisecond}, - description: - "The time the connection spent waiting before being checked out for the query" + description: "The time the connection spent waiting before being checked out for the query" ), # VM Metrics diff --git a/lib/utils/string_utils.ex b/lib/utils/string_utils.ex new file mode 100644 index 0000000..f52bcfb --- /dev/null +++ b/lib/utils/string_utils.ex @@ -0,0 +1,14 @@ +defmodule Pinchflat.Utils.StringUtils do + @moduledoc """ + Utility functions for working with strings + """ + + @doc """ + Converts a string to kebab-case (ie: `hello world` -> `hello-world`) + """ + def to_kebab_case(string) do + string + |> String.replace(~r/[\s_]/, "-") + |> String.downcase() + end +end diff --git a/mix.exs b/mix.exs index 577a9aa..726bc74 100644 --- a/mix.exs +++ b/mix.exs @@ -50,7 +50,8 @@ defmodule Pinchflat.MixProject do {:gettext, "~> 0.20"}, {:jason, "~> 1.2"}, {:dns_cluster, "~> 0.1.1"}, - {:plug_cowboy, "~> 2.5"} + {:plug_cowboy, "~> 2.5"}, + {:mox, "~> 1.0", only: :test} ] end diff --git a/mix.lock b/mix.lock index 5ed3226..3c8ee07 100644 --- a/mix.lock +++ b/mix.lock @@ -18,6 +18,7 @@ "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, "phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"}, diff --git a/test/pinchflat/downloader_backends/yt_dlp/command_runner_test.exs b/test/pinchflat/downloader_backends/yt_dlp/command_runner_test.exs new file mode 100644 index 0000000..02e18c2 --- /dev/null +++ b/test/pinchflat/downloader_backends/yt_dlp/command_runner_test.exs @@ -0,0 +1,72 @@ +defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunnerTest do + use ExUnit.Case, async: true + + alias Pinchflat.DownloaderBackends.YtDlp.CommandRunner, as: Runner + + @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable) + @video_url "https://www.youtube.com/watch?v=9bZkp7q19f0" + + setup do + on_exit(&reset_executable/0) + end + + describe "run/2" do + test "it returns the output and status when the command succeeds" do + assert {:ok, _output} = Runner.run(@video_url, []) + end + + test "it converts symbol k-v arg keys to kebab case" do + assert {:ok, output} = Runner.run(@video_url, buffer_size: 1024) + + assert String.contains?(output, "--buffer-size 1024") + end + + test "it keeps string k-v arg keys untouched" do + assert {:ok, output} = Runner.run(@video_url, [{"--under_score", 1024}]) + + assert String.contains?(output, "--under_score 1024") + end + + test "it converts symbol arg keys to kebab case" do + assert {:ok, output} = Runner.run(@video_url, [:ignore_errors]) + + assert String.contains?(output, "--ignore-errors") + end + + test "it keeps string arg keys untouched" do + assert {:ok, output} = Runner.run(@video_url, ["-v"]) + + assert String.contains?(output, "-v") + refute String.contains?(output, "--v") + end + + test "it places arg keys (flags) at the beginning of the command" do + assert {:ok, output} = + Runner.run(@video_url, [{"--under_score", 1024}, :ignore_errors]) + + assert String.contains?(output, "--ignore-errors --under_score 1024") + end + + test "it includes the video url as the last argument" do + assert {:ok, output} = Runner.run(@video_url, [:ignore_errors]) + + assert String.contains?(output, "--ignore-errors #{@video_url}\n") + end + + test "it returns the output and status when the command fails" do + wrap_executable("/bin/false", fn -> + assert {:error, "", 1} = Runner.run(@video_url, []) + end) + end + end + + defp wrap_executable(new_executable, fun) do + Application.put_env(:pinchflat, :yt_dlp_executable, new_executable) + fun.() + reset_executable() + end + + def reset_executable do + Application.put_env(:pinchflat, :yt_dlp_executable, @original_executable) + end +end diff --git a/test/pinchflat/downloader_backends/yt_dlp/video_collection_test.exs b/test/pinchflat/downloader_backends/yt_dlp/video_collection_test.exs new file mode 100644 index 0000000..08296fc --- /dev/null +++ b/test/pinchflat/downloader_backends/yt_dlp/video_collection_test.exs @@ -0,0 +1,44 @@ +defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollectionTest do + use ExUnit.Case, async: true + import Mox + + alias Pinchflat.DownloaderBackends.YtDlp.VideoCollection, as: VideoCollection + + @channel_url "https://www.youtube.com/@TheUselessTrials" + + setup :verify_on_exit! + + describe "get_video_ids/2" do + test "returns a list of video ids with no blank elements" do + expect(CommandRunnerMock, :run, fn _url, _opts -> {:ok, "id1\nid2\n\nid3\n"} end) + + assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url) + end + + test "it passes the expected default args" do + expect(CommandRunnerMock, :run, fn _url, opts -> + assert opts == [:simulate, :skip_download, :get_id] + + {:ok, ""} + end) + + assert {:ok, _} = VideoCollection.get_video_ids(@channel_url) + end + + test "it passes the expected custom args" do + expect(CommandRunnerMock, :run, fn _url, opts -> + assert opts == [:custom_arg, :simulate, :skip_download, :get_id] + + {:ok, ""} + end) + + assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg]) + end + + test "returns the error straight through when the command fails" do + expect(CommandRunnerMock, :run, fn _url, _opts -> {:error, "Big issue", 1} end) + + assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url) + end + end +end diff --git a/test/pinchflat/utils/string_utils_test.exs b/test/pinchflat/utils/string_utils_test.exs new file mode 100644 index 0000000..59ebbfc --- /dev/null +++ b/test/pinchflat/utils/string_utils_test.exs @@ -0,0 +1,15 @@ +defmodule Pinchflat.Utils.StringUtilsTest do + use ExUnit.Case, async: true + + alias Pinchflat.Utils.StringUtils, as: StringUtils + + describe "to_kebab_case/1" do + test "converts a space-delimited string to kebab-case" do + assert StringUtils.to_kebab_case("hello world") == "hello-world" + end + + test "converts an underscore-delimited string to kebab-case" do + assert StringUtils.to_kebab_case("hello_world") == "hello-world" + end + end +end diff --git a/test/support/scripts/yt-dlp-mocks/repeater.sh b/test/support/scripts/yt-dlp-mocks/repeater.sh new file mode 100755 index 0000000..169303d --- /dev/null +++ b/test/support/scripts/yt-dlp-mocks/repeater.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ "$@" == *"--dump-json"* ]]; then + echo '{ "args": "'$@'"}' +else + echo $@ +fi diff --git a/test/test_helper.exs b/test/test_helper.exs index d5d35e7..2511d46 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,5 @@ +Mox.defmock(CommandRunnerMock, for: Pinchflat.DownloaderBackends.BackendCommandRunner) +Application.put_env(:pinchflat, :yt_dlp_runner, CommandRunnerMock) + ExUnit.start() Ecto.Adapters.SQL.Sandbox.mode(Pinchflat.Repo, :manual)