diff --git a/docker-compose.yml b/docker-compose.yml index 23cde70..31e897e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,3 +12,5 @@ services: - ./docker-run.dev.sh stdin_open: true tty: true + env_file: + - .env diff --git a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex index 684ef49..62c2f99 100644 --- a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex +++ b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex @@ -26,7 +26,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do downloaded_. """ def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do - {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source) + {:ok, media_ids} = YoutubeRss.get_recent_media_ids(source) existing_media_items = list_media_items_by_media_id_for(source, media_ids) new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id) diff --git a/lib/pinchflat/fast_indexing/youtube_api.ex b/lib/pinchflat/fast_indexing/youtube_api.ex new file mode 100644 index 0000000..2e2a45a --- /dev/null +++ b/lib/pinchflat/fast_indexing/youtube_api.ex @@ -0,0 +1,66 @@ +defmodule Pinchflat.FastIndexing.YoutubeApi do + @moduledoc """ + TODO + """ + + require Logger + + alias Pinchflat.Sources.Source + alias Pinchflat.Utils.FunctionUtils + + # TODO: test + # TODO: make this a behaviour + def get_recent_media_ids(%Source{} = source) do + api_response = + source + |> determine_playlist_id() + |> do_api_request() + + case api_response do + {:ok, parsed_json} -> get_media_ids_from_response(parsed_json) + {:error, reason} -> {:error, reason} + end + end + + # The UC prefix is for channels which won't work with this API endpoint. Swapping + # the prefix to UU will get us the playlist that represents the channel's uploads + defp determine_playlist_id(%{collection_id: c_id}) do + String.replace_prefix(c_id, "UC", "UU") + end + + defp do_api_request(playlist_id) do + Logger.debug("Fetching recent media IDs from YouTube API for playlist: #{playlist_id}") + + api_base = "https://youtube.googleapis.com/youtube/v3/playlistItems" + request_url = "#{api_base}?part=contentDetails&maxResults=50&playlistId=#{playlist_id}&key=#{api_key()}" + + case http_client().get(request_url, accept: "application/json") do + {:ok, response} -> + Phoenix.json_library().decode(response) + + {:error, reason} -> + {:error, reason} + end + end + + defp get_media_ids_from_response(parsed_json) do + parsed_json + |> Map.get("items", []) + |> Enum.map(fn item -> + item + |> Map.get("contentDetails", %{}) + |> Map.get("videoId", nil) + end) + |> Enum.reject(&is_nil/1) + |> FunctionUtils.wrap_ok() + end + + # TODO: replace this with a user setting + defp api_key do + System.get_env("YOUTUBE_API_KEY") + end + + defp http_client do + Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient) + end +end diff --git a/lib/pinchflat/fast_indexing/youtube_behaviour.ex b/lib/pinchflat/fast_indexing/youtube_behaviour.ex new file mode 100644 index 0000000..ea474de --- /dev/null +++ b/lib/pinchflat/fast_indexing/youtube_behaviour.ex @@ -0,0 +1,3 @@ +defmodule Pinchflat.FastIndexing.YoutubeBehaviour do + # TODO +end diff --git a/lib/pinchflat/fast_indexing/youtube_rss.ex b/lib/pinchflat/fast_indexing/youtube_rss.ex index 5caaab8..8aa0e8d 100644 --- a/lib/pinchflat/fast_indexing/youtube_rss.ex +++ b/lib/pinchflat/fast_indexing/youtube_rss.ex @@ -12,7 +12,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRss do Returns {:ok, [binary()]} | {:error, binary()} """ - def get_recent_media_ids_from_rss(%Source{} = source) do + def get_recent_media_ids(%Source{} = source) do Logger.debug("Fetching recent media IDs from YouTube RSS feed for source: #{source.collection_id}") case http_client().get(rss_url_for_source(source)) do diff --git a/lib/pinchflat/http/http_client.ex b/lib/pinchflat/http/http_client.ex index 45abda4..c7d4550 100644 --- a/lib/pinchflat/http/http_client.ex +++ b/lib/pinchflat/http/http_client.ex @@ -21,9 +21,11 @@ defmodule Pinchflat.HTTP.HTTPClient do """ @impl HTTPBehaviour def get(url, headers \\ [], opts \\ []) do + headers = parse_headers(headers) + case :httpc.request(:get, {url, headers}, [], opts) do {:ok, {{_version, 200, _reason_phrase}, _headers, body}} -> - {:ok, body} + {:ok, to_string(body)} {:ok, {{_version, status_code, reason_phrase}, _headers, _body}} -> {:error, "HTTP request failed with status code #{status_code}: #{reason_phrase}"} @@ -32,4 +34,8 @@ defmodule Pinchflat.HTTP.HTTPClient do {:error, "HTTP request failed: #{reason}"} end end + + defp parse_headers(headers) do + Enum.map(headers, fn {k, v} -> {to_charlist(k), to_charlist(v)} end) + end end diff --git a/test/pinchflat/fast_indexing/youtube_rss_test.exs b/test/pinchflat/fast_indexing/youtube_rss_test.exs index a769498..a9374a9 100644 --- a/test/pinchflat/fast_indexing/youtube_rss_test.exs +++ b/test/pinchflat/fast_indexing/youtube_rss_test.exs @@ -11,7 +11,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, source: source} end - describe "get_recent_media_ids_from_rss/1" do + describe "get_recent_media_ids/1" do test "calls the expected URL for channel sources" do source = source_fixture(collection_type: :channel, collection_id: "channel_id") @@ -21,7 +21,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, ""} end) - assert {:ok, _} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:ok, _} = YoutubeRss.get_recent_media_ids(source) end test "calls the expected URL for playlist sources" do @@ -33,13 +33,13 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, ""} end) - assert {:ok, _} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:ok, _} = YoutubeRss.get_recent_media_ids(source) end test "returns an error if the HTTP request fails", %{source: source} do expect(HTTPClientMock, :get, fn _url -> {:error, ""} end) - assert {:error, "Failed to fetch RSS feed"} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:error, "Failed to fetch RSS feed"} = YoutubeRss.get_recent_media_ids(source) end test "returns the media IDs from the RSS feed", %{source: source} do @@ -47,7 +47,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, "test_1test_2"} end) - assert {:ok, ["test_1", "test_2"]} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:ok, ["test_1", "test_2"]} = YoutubeRss.get_recent_media_ids(source) end test "strips whitespace from media IDs", %{source: source} do @@ -55,7 +55,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, " test_1 test_2 "} end) - assert {:ok, ["test_1", "test_2"]} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:ok, ["test_1", "test_2"]} = YoutubeRss.get_recent_media_ids(source) end test "removes empty media IDs", %{source: source} do @@ -63,7 +63,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, "test_1"} end) - assert {:ok, ["test_1"]} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:ok, ["test_1"]} = YoutubeRss.get_recent_media_ids(source) end test "removes duplicate media IDs", %{source: source} do @@ -71,7 +71,7 @@ defmodule Pinchflat.FastIndexing.YoutubeRssTest do {:ok, "test_1test_1"} end) - assert {:ok, ["test_1"]} = YoutubeRss.get_recent_media_ids_from_rss(source) + assert {:ok, ["test_1"]} = YoutubeRss.get_recent_media_ids(source) end end end