From d8e9f8ce576ff8e0acbc1d788326c5d3dd40a29b Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Sun, 28 Jan 2024 17:52:23 -0800 Subject: [PATCH] [WIP] hooked up basic video downloading; starting work on metadata --- config/config.exs | 3 +- config/dev.exs | 3 +- config/test.exs | 3 +- .../backends/yt_dlp/command_runner.ex | 8 +++-- .../backends/yt_dlp/metadata_parser.ex | 7 ++++ .../media_client/backends/yt_dlp/video.ex | 24 +++++++++++-- .../media_client/video_downloader.ex | 35 ++++++++++++++++++- .../profiles/options/yt_dlp/option_builder.ex | 5 --- lib/pinchflat/utils/string_utils.ex | 9 +++++ .../workers/video_download_worker.ex | 27 ++++++++++++++ 10 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex create mode 100644 lib/pinchflat/workers/video_download_worker.ex diff --git a/config/config.exs b/config/config.exs index 30edd54..91b3920 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,7 +14,8 @@ config :pinchflat, yt_dlp_executable: System.find_executable("yt-dlp"), yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, # TODO: figure this out - media_directory: :not_implemented + media_directory: :not_implemented, + metadata_directory: Path.join([System.tmp_dir!(), "pinchflat", "metadata"]) # Configures the endpoint config :pinchflat, PinchflatWeb.Endpoint, diff --git a/config/dev.exs b/config/dev.exs index 847ef3c..ccf1d70 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,7 +1,8 @@ import Config config :pinchflat, - media_directory: Path.join([System.tmp_dir!(), "yt-dlp"]) + media_directory: Path.join([File.cwd!(), "tmp", "videos"]), + metadata_directory: Path.join([File.cwd!(), "tmp", "metadata"]) # Configure your database config :pinchflat, Pinchflat.Repo, diff --git a/config/test.exs b/config/test.exs index 2a2bdce..cd4c199 100644 --- a/config/test.exs +++ b/config/test.exs @@ -3,7 +3,8 @@ import Config config :pinchflat, # Specifying backend data here makes mocking and local testing SUPER easy yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"]), - media_directory: Path.join([System.tmp_dir!(), "yt-dlp"]) + media_directory: Path.join([File.cwd!(), "tmp", "videos"]), + metadata_directory: Path.join([File.cwd!(), "tmp", "metadata"]) config :pinchflat, Oban, testing: :manual diff --git a/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex b/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex index 9f24390..de0bbbf 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex @@ -3,6 +3,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do Runs yt-dlp commands using the `System.cmd/3` function """ + require Logger + alias Pinchflat.Utils.StringUtils alias Pinchflat.MediaClient.Backends.BackendCommandRunner @@ -20,7 +22,9 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do @impl BackendCommandRunner def run(url, command_opts) do command = backend_executable() - formatted_command_opts = parse_options(command_opts) ++ [url] + formatted_command_opts = [url] ++ parse_options(command_opts) + + Logger.debug("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}") case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do {output, 0} -> {:ok, output} @@ -55,7 +59,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do end defp parse_option(arg, acc) when is_binary(arg) do - [arg | acc] + acc ++ [arg] end defp backend_executable do diff --git a/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex b/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex new file mode 100644 index 0000000..2a93744 --- /dev/null +++ b/lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex @@ -0,0 +1,7 @@ +defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do + def parse_for_media_item(metadata) do + %{ + video_filepath: metadata["filepath"] + } + end +end diff --git a/lib/pinchflat/media_client/backends/yt_dlp/video.ex b/lib/pinchflat/media_client/backends/yt_dlp/video.ex index bea5b3b..cce5c7e 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/video.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/video.ex @@ -3,17 +3,31 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.Video do Contains utilities for working with singular videos """ + alias Pinchflat.Utils.StringUtils + @doc """ Downloads a single video (and possible metadata) directly to its final destination. Returns the parsed JSON output from yt-dlp. + It writes to file and then immediately reads from it since printing + to stdout also contains any warnings/errors that may have occurred, + even if the command is otherwise successful (which creates invalid + JSON). + Returns {:ok, map()} | {:error, any, ...}. + + TODO: test changes (writing to then reading from a file) """ def download(url, command_opts \\ []) do - opts = [:no_simulate, print: "%()j"] ++ command_opts + json_output_path = Path.join([metadata_directory(), "#{StringUtils.random_string()}.json"]) + # These must stay in exactly this order, hence why I'm giving it its own variable. + # Also, can't use RAM file since yt-dlp needs a concrete filepath. + print_to_file_opts = [{:print_to_file, "after_move:%()j"}, json_output_path] + opts = [:no_simulate] ++ print_to_file_opts ++ command_opts - with {:ok, output} <- backend_runner().run(url, opts), - {:ok, parsed_json} <- Phoenix.json_library().decode(output) do + with {:ok, _} <- backend_runner().run(url, opts), + {:ok, file_body} <- File.read(json_output_path), + {:ok, parsed_json} <- Phoenix.json_library().decode(file_body) do {:ok, parsed_json} else err -> err @@ -23,4 +37,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.Video do defp backend_runner do Application.get_env(:pinchflat, :yt_dlp_runner) end + + defp metadata_directory do + Application.get_env(:pinchflat, :metadata_directory) + end end diff --git a/lib/pinchflat/media_client/video_downloader.ex b/lib/pinchflat/media_client/video_downloader.ex index 8d526d7..fe7f061 100644 --- a/lib/pinchflat/media_client/video_downloader.ex +++ b/lib/pinchflat/media_client/video_downloader.ex @@ -8,15 +8,19 @@ defmodule Pinchflat.MediaClient.VideoDownloader do it open-ish for future expansion (just in case). """ + alias Pinchflat.Repo + alias Pinchflat.Media + alias Pinchflat.Media.MediaItem alias Pinchflat.Profiles.MediaProfile alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder, as: YtDlpOptionBuilder + alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser @doc """ Downloads a single video based on the settings in the given media profile. - Returns {:ok, %ChannelDetails{}} | {:error, any, ...}. + Returns {:ok, map()} | {:error, any, ...}. """ def download_for_media_profile(url, %MediaProfile{} = media_profile, backend \\ :yt_dlp) do option_builder = option_builder(backend) @@ -26,6 +30,29 @@ defmodule Pinchflat.MediaClient.VideoDownloader do video_backend.download(url, options) end + @doc """ + TODO: test + TODO: consider removing the above function? I don't know if it's actually useful + TODO: save metadata filepath to media item record + TODO: consider saving the output JSON to the database instead of the filesystem. + reason: would make updating metadata easier (no orphans). Also queryable. + """ + def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do + item_with_preloads = Repo.preload(media_item, channel: :media_profile) + media_profile = item_with_preloads.channel.media_profile + + case download_for_media_profile(media_item.media_id, media_profile, backend) do + {:ok, parsed_json} -> + parser = metadata_parser(backend) + parsed_attrs = parser.parse_for_media_item(parsed_json) + + Media.update_media_item(media_item, parsed_attrs) + + err -> + err + end + end + defp option_builder(backend) do case backend do :yt_dlp -> YtDlpOptionBuilder @@ -37,4 +64,10 @@ defmodule Pinchflat.MediaClient.VideoDownloader do :yt_dlp -> YtDlpVideo end end + + defp metadata_parser(backend) do + case backend do + :yt_dlp -> YtDlpMetadataParser + end + end end diff --git a/lib/pinchflat/profiles/options/yt_dlp/option_builder.ex b/lib/pinchflat/profiles/options/yt_dlp/option_builder.ex index 783be1e..ce99e1f 100644 --- a/lib/pinchflat/profiles/options/yt_dlp/option_builder.ex +++ b/lib/pinchflat/profiles/options/yt_dlp/option_builder.ex @@ -24,15 +24,10 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do # see: https://github.com/yt-dlp/yt-dlp#output-template {:ok, [ - :write_thumbnail, - :write_subs, :embed_metadata, :embed_thumbnail, :embed_subs, - :write_info_json, - :write_auto_subs, :no_progress, - convert_thumbnails: "jpg", sub_langs: "en.*", output: Path.join(base_directory(), output_path) ]} diff --git a/lib/pinchflat/utils/string_utils.ex b/lib/pinchflat/utils/string_utils.ex index f52bcfb..48baa94 100644 --- a/lib/pinchflat/utils/string_utils.ex +++ b/lib/pinchflat/utils/string_utils.ex @@ -11,4 +11,13 @@ defmodule Pinchflat.Utils.StringUtils do |> String.replace(~r/[\s_]/, "-") |> String.downcase() end + + @doc """ + TODO: test + """ + def random_string(length \\ 32) do + :crypto.strong_rand_bytes(length) + |> Base.encode16(case: :lower) + |> String.slice(0..(length - 1)) + end end diff --git a/lib/pinchflat/workers/video_download_worker.ex b/lib/pinchflat/workers/video_download_worker.ex new file mode 100644 index 0000000..830b64d --- /dev/null +++ b/lib/pinchflat/workers/video_download_worker.ex @@ -0,0 +1,27 @@ +defmodule Pinchflat.Workers.VideoDownloadWorker do + @moduledoc false + + use Oban.Worker, + queue: :media_fetching, + unique: [period: :infinity, states: [:available, :scheduled, :retryable]], + tags: ["media_itwm", "media_fetching"] + + alias Pinchflat.Media + alias Pinchflat.MediaClient.VideoDownloader + + @impl Oban.Worker + @doc """ + TODO: test + """ + def perform(%Oban.Job{args: %{"id" => media_item_id}}) do + media_item = Media.get_media_item!(media_item_id) + + case VideoDownloader.download_for_media_item(media_item) do + {:ok, _} -> + {:ok, media_item} + + err -> + err + end + end +end