From cf0133bae4841c93603cc4d2b345df198c9402ec Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Wed, 14 Aug 2024 09:24:24 -0700 Subject: [PATCH] Updated every command to optionally include cookies --- lib/pinchflat/downloading/media_downloader.ex | 4 +++- .../fast_indexing/fast_indexing_helpers.ex | 3 ++- .../metadata/metadata_file_helpers.ex | 11 ++++++---- .../source_metadata_storage_worker.ex | 8 +++++-- .../slow_indexing/slow_indexing_helpers.ex | 4 +++- lib/pinchflat/sources/sources.ex | 4 +++- lib/pinchflat/yt_dlp/command_runner.ex | 13 ++++++++++-- lib/pinchflat/yt_dlp/media.ex | 12 ++++++----- lib/pinchflat/yt_dlp/media_collection.ex | 21 ++++++++++++------- 9 files changed, 55 insertions(+), 25 deletions(-) diff --git a/lib/pinchflat/downloading/media_downloader.ex b/lib/pinchflat/downloading/media_downloader.ex index a10e6b9..bcb2929 100644 --- a/lib/pinchflat/downloading/media_downloader.ex +++ b/lib/pinchflat/downloading/media_downloader.ex @@ -106,8 +106,10 @@ defmodule Pinchflat.Downloading.MediaDownloader do defp download_with_options(url, item_with_preloads, output_filepath, override_opts) do {:ok, options} = DownloadOptionBuilder.build(item_with_preloads, override_opts) + # TODO: test + runner_opts = [output_filepath: output_filepath, use_cookies: item_with_preloads.source.use_cookies] - YtDlpMedia.download(url, options, output_filepath: output_filepath) + YtDlpMedia.download(url, options, runner_opts) end defp recoverable_errors do diff --git a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex index 02fc00f..9fbd453 100644 --- a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex +++ b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex @@ -68,7 +68,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do defp create_media_item_from_media_id(source, media_id) do url = "https://www.youtube.com/watch?v=#{media_id}" - case YtDlpMedia.get_media_attributes(url) do + # TODO: test + case YtDlpMedia.get_media_attributes(url, use_cookies: source.use_cookies) do {:ok, media_attrs} -> Media.create_media_item_from_backend_attrs(source, media_attrs) diff --git a/lib/pinchflat/metadata/metadata_file_helpers.ex b/lib/pinchflat/metadata/metadata_file_helpers.ex index b728f47..75c9ed3 100644 --- a/lib/pinchflat/metadata/metadata_file_helpers.ex +++ b/lib/pinchflat/metadata/metadata_file_helpers.ex @@ -62,11 +62,14 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do Returns binary() | nil """ - def download_and_store_thumbnail_for(database_record) do - yt_dlp_filepath = generate_filepath_for(database_record, "thumbnail.%(ext)s") - real_filepath = generate_filepath_for(database_record, "thumbnail.jpg") + def download_and_store_thumbnail_for(media_item_with_preloads) do + yt_dlp_filepath = generate_filepath_for(media_item_with_preloads, "thumbnail.%(ext)s") + real_filepath = generate_filepath_for(media_item_with_preloads, "thumbnail.jpg") + command_opts = [output: yt_dlp_filepath] + addl_opts = [use_cookies: media_item_with_preloads.source.use_cookies] - case YtDlpMedia.download_thumbnail(database_record.original_url, output: yt_dlp_filepath) do + # TODO: test + case YtDlpMedia.download_thumbnail(media_item_with_preloads.original_url, command_opts, addl_opts) do {:ok, _} -> real_filepath _ -> nil end diff --git a/lib/pinchflat/metadata/source_metadata_storage_worker.ex b/lib/pinchflat/metadata/source_metadata_storage_worker.ex index 8ccfdf9..deef087 100644 --- a/lib/pinchflat/metadata/source_metadata_storage_worker.ex +++ b/lib/pinchflat/metadata/source_metadata_storage_worker.ex @@ -92,7 +92,10 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do defp determine_series_directory(source) do output_path = DownloadOptionBuilder.build_output_path_for(source) - {:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, output: output_path) + # TODO: test + runner_opts = [output: output_path] + addl_opts = [use_cookies: source.use_cookies] + {:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, runner_opts, addl_opts) case MetadataFileHelpers.series_directory_from_media_filepath(filepath) do {:ok, series_directory} -> series_directory @@ -119,7 +122,8 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do base_opts ++ [:write_thumbnail, playlist_items: 1] end - MediaCollection.get_source_metadata(source.original_url, opts) + # TODO: test + MediaCollection.get_source_metadata(source.original_url, opts, use_cookies: source.use_cookies) end defp tmp_directory do diff --git a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex index 2434b92..a9d067c 100644 --- a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex +++ b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex @@ -94,7 +94,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do {:ok, pid} = FileFollowerServer.start_link() handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end - result = MediaCollection.get_media_attributes_for_collection(source.original_url, file_listener_handler: handler) + runner_opts = [file_listener_handler: handler, use_cookies: source.use_cookies] + # TODO: test + result = MediaCollection.get_media_attributes_for_collection(source.original_url, runner_opts) FileFollowerServer.stop(pid) diff --git a/lib/pinchflat/sources/sources.ex b/lib/pinchflat/sources/sources.ex index 6cd45c1..6e2d2f3 100644 --- a/lib/pinchflat/sources/sources.ex +++ b/lib/pinchflat/sources/sources.ex @@ -180,7 +180,9 @@ defmodule Pinchflat.Sources do end defp add_source_details_to_changeset(source, changeset) do - case MediaCollection.get_source_details(changeset.changes.original_url) do + use_cookies = Ecto.Changeset.get_field(changeset, :use_cookies) + + case MediaCollection.get_source_details(changeset.changes.original_url, [], use_cookies: use_cookies) do {:ok, source_details} -> add_source_details_by_collection_type(source, changeset, source_details) diff --git a/lib/pinchflat/yt_dlp/command_runner.ex b/lib/pinchflat/yt_dlp/command_runner.ex index 30b2404..7cd21dd 100644 --- a/lib/pinchflat/yt_dlp/command_runner.ex +++ b/lib/pinchflat/yt_dlp/command_runner.ex @@ -18,6 +18,8 @@ defmodule Pinchflat.YtDlp.CommandRunner do - :output_filepath - the path to save the output to. If not provided, a temporary file will be created and used. Useful for if you need a reference to the file for a file watcher. + - :use_cookies - if true, will add a cookie file to the command options. Will not + attach a cookie file if the user hasn't set one up. Returns {:ok, binary()} | {:error, output, status}. """ @@ -28,7 +30,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do output_filepath = generate_output_filepath(addl_opts) print_to_file_opts = [{:print_to_file, output_template}, output_filepath] - user_configured_opts = cookie_file_options() + user_configured_opts = cookie_file_options(addl_opts) # These must stay in exactly this order, hence why I'm giving it its own variable. all_opts = command_opts ++ print_to_file_opts ++ user_configured_opts ++ global_options() formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts) @@ -78,7 +80,14 @@ defmodule Pinchflat.YtDlp.CommandRunner do ] end - defp cookie_file_options do + defp cookie_file_options(addl_opts) do + case Keyword.get(addl_opts, :use_cookies) do + true -> add_cookie_file() + _ -> [] + end + end + + defp add_cookie_file do base_dir = Application.get_env(:pinchflat, :extras_directory) filename_options_map = %{cookies: "cookies.txt"} diff --git a/lib/pinchflat/yt_dlp/media.ex b/lib/pinchflat/yt_dlp/media.ex index cecf08a..a90aaa9 100644 --- a/lib/pinchflat/yt_dlp/media.ex +++ b/lib/pinchflat/yt_dlp/media.ex @@ -53,12 +53,13 @@ defmodule Pinchflat.YtDlp.Media do Returns {:ok, ""} | {:error, any, ...}. """ - def download_thumbnail(url, command_opts \\ []) do - opts = [:no_simulate, :skip_download, :write_thumbnail, convert_thumbnail: "jpg"] ++ command_opts + def download_thumbnail(url, command_opts \\ [], addl_opts \\ []) do + all_command_opts = [:no_simulate, :skip_download, :write_thumbnail, convert_thumbnail: "jpg"] ++ command_opts # NOTE: it doesn't seem like this command actually returns anything in `after_move` since # we aren't downloading the main media file - backend_runner().run(url, opts, "after_move:%()j") + # TODO: test + backend_runner().run(url, all_command_opts, "after_move:%()j", addl_opts) end @doc """ @@ -66,12 +67,13 @@ defmodule Pinchflat.YtDlp.Media do Returns {:ok, %Media{}} | {:error, any, ...}. """ - def get_media_attributes(url) do + def get_media_attributes(url, addl_opts \\ []) do runner = Application.get_env(:pinchflat, :yt_dlp_runner) command_opts = [:simulate, :skip_download] output_template = indexing_output_template() - case runner.run(url, command_opts, output_template) do + # TODO: test + case runner.run(url, command_opts, output_template, addl_opts) do {:ok, output} -> output |> Phoenix.json_library().decode!() diff --git a/lib/pinchflat/yt_dlp/media_collection.ex b/lib/pinchflat/yt_dlp/media_collection.ex index f657195..100079d 100644 --- a/lib/pinchflat/yt_dlp/media_collection.ex +++ b/lib/pinchflat/yt_dlp/media_collection.ex @@ -25,15 +25,18 @@ defmodule Pinchflat.YtDlp.MediaCollection do # the first video has not released yet (ie: is a premier). We don't care about # available formats since we're just getting the media details command_opts = [:simulate, :skip_download, :ignore_no_formats_error, :no_warnings] + use_cookies = Keyword.get(addl_opts, :use_cookies, false) output_template = YtDlpMedia.indexing_output_template() output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json) file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false) + # TODO: test + runner_opts = [output_filepath: output_filepath, use_cookies: use_cookies] if file_listener_handler do file_listener_handler.(output_filepath) end - case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do + case runner.run(url, command_opts, output_template, runner_opts) do {:ok, output} -> parsed_lines = output @@ -64,7 +67,7 @@ defmodule Pinchflat.YtDlp.MediaCollection do Returns {:ok, map()} | {:error, any, ...}. """ - def get_source_details(source_url, addl_opts \\ []) do + def get_source_details(source_url, command_opts \\ [], addl_opts \\ []) do # `ignore_no_formats_error` is necessary because yt-dlp will error out if # the first video has not released yet (ie: is a premier). We don't care about # available formats since we're just getting the source details @@ -75,10 +78,11 @@ defmodule Pinchflat.YtDlp.MediaCollection do playlist_end: 1 ] - command_opts = default_opts ++ addl_opts + all_command_opts = default_opts ++ command_opts output_template = "%(.{channel,channel_id,playlist_id,playlist_title,filename})j" - with {:ok, output} <- backend_runner().run(source_url, command_opts, output_template), + # TODO: test + with {:ok, output} <- backend_runner().run(source_url, all_command_opts, output_template, addl_opts), {:ok, parsed_json} <- Phoenix.json_library().decode(output) do {:ok, format_source_details(parsed_json)} else @@ -109,14 +113,15 @@ defmodule Pinchflat.YtDlp.MediaCollection do Returns {:ok, map()} | {:error, any, ...}. """ - def get_source_metadata(source_url, addl_opts \\ [playlist_items: 0]) do + def get_source_metadata(source_url, command_opts \\ [playlist_items: 0], addl_opts \\ []) do # This only validates that the `playlist_items` key is present. It's otherwise unused - _playlist_items = Keyword.fetch!(addl_opts, :playlist_items) + _playlist_items = Keyword.fetch!(command_opts, :playlist_items) - opts = [:skip_download] ++ addl_opts + all_command_opts = [:skip_download] ++ command_opts output_template = "playlist:%()j" - with {:ok, output} <- backend_runner().run(source_url, opts, output_template), + # TODO: test + with {:ok, output} <- backend_runner().run(source_url, all_command_opts, output_template, addl_opts), {:ok, parsed_json} <- Phoenix.json_library().decode(output) do {:ok, parsed_json} else