Start integrating playlist support (#13)

* [WIP] updated the output of VideoCollection to include playlists

* Updated source's name to collection_name; supported playlist ID/name fetching

* Hooked up collection_type to form; refactored enqueue_pending_media_downloads

* Added friendly_name to form

* Added media profile link to source view

* Updates comment
This commit is contained in:
Kieran 2024-02-05 11:38:19 -08:00 committed by GitHub
parent bdef6c75bb
commit ca01f17b58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 291 additions and 100 deletions

View file

@ -2,6 +2,7 @@ alias Pinchflat.Repo
alias Pinchflat.Tasks.Task
alias Pinchflat.Media.MediaItem
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Media.MediaMetadata
alias Pinchflat.MediaSource.Source
alias Pinchflat.Profiles.MediaProfile
@ -12,3 +13,39 @@ alias Pinchflat.Profiles
alias Pinchflat.MediaSource
alias Pinchflat.MediaClient.{SourceDetails, VideoDownloader}
defmodule IexHelpers do
def playlist_url do
"https://www.youtube.com/playlist?list=PLmqC3wPkeL8kSlTCcSMDD63gmSi7evcXS"
end
def channel_url do
"https://www.youtube.com/c/TheUselessTrials"
end
def video_url do
"https://www.youtube.com/watch?v=bR52O78ZIUw"
end
def details(type) do
source =
case type do
:playlist -> playlist_url()
:channel -> channel_url()
end
SourceDetails.get_source_details(source)
end
def ids(type) do
source =
case type do
:playlist -> playlist_url()
:channel -> channel_url()
end
SourceDetails.get_video_ids(source)
end
end
import IexHelpers

View file

@ -4,8 +4,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
videos (aka: a source [ie: channels, playlists]).
"""
alias Pinchflat.MediaClient.SourceDetails
@doc """
Returns a list of strings representing the video ids in the collection.
@ -28,19 +26,29 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
instead we're fetching just the first video (using playlist_end: 1)
and parsing the source ID and name from _its_ metadata
Returns {:ok, %SourceDetails{}} | {:error, any, ...}.
Returns {:ok, map()} | {:error, any, ...}.
"""
def get_source_details(source_url) do
opts = [:skip_download, playlist_end: 1]
opts = [:simulate, :skip_download, playlist_end: 1]
output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
with {:ok, output} <- backend_runner().run(source_url, opts, "%(.{channel,channel_id})j"),
with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
{:ok, SourceDetails.new(parsed_json["channel_id"], parsed_json["channel"])}
{:ok, format_source_details(parsed_json)}
else
err -> err
end
end
defp format_source_details(response) do
%{
channel_id: response["channel_id"],
channel_name: response["channel"],
playlist_id: response["playlist_id"],
playlist_name: response["playlist_title"]
}
end
defp backend_runner do
Application.get_env(:pinchflat, :yt_dlp_runner)
end

View file

@ -5,16 +5,9 @@ defmodule Pinchflat.MediaClient.SourceDetails do
Technically hardcodes the yt-dlp backend for now, but should leave
it open-ish for future expansion (just in case).
"""
@enforce_keys [:id, :name]
defstruct [:id, :name]
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
@doc false
def new(id, name) do
%__MODULE__{id: id, name: name}
end
@doc """
Gets a source's ID and name from its URL, using the given backend.

View file

@ -100,6 +100,10 @@ defmodule Pinchflat.MediaSource do
Note that this fetches source details as long as the `original_url` is present.
This means that it'll go for it even if a changeset is otherwise invalid. This
is pretty easy to change, but for MVP I'm not concerned.
IDEA: Maybe I could discern `collection_type` based on the original URL?
It also seems like it's a channel when the returned yt-dlp channel_id is the
same as the playlist_id - maybe could use that?
"""
def change_source_from_url(%Source{} = source, attrs) do
case change_source(source, attrs) do
@ -115,14 +119,8 @@ defmodule Pinchflat.MediaSource do
%Ecto.Changeset{changes: changes} = changeset
case SourceDetails.get_source_details(changes.original_url) do
{:ok, %SourceDetails{} = source_details} ->
change_source(
source,
Map.merge(changes, %{
name: source_details.name,
collection_id: source_details.id
})
)
{:ok, source_details} ->
add_source_details_by_collection_type(source, changeset, source_details)
{:error, runner_error, _status_code} ->
Ecto.Changeset.add_error(
@ -134,15 +132,35 @@ defmodule Pinchflat.MediaSource do
end
end
defp add_source_details_by_collection_type(source, changeset, source_details) do
%Ecto.Changeset{changes: changes} = changeset
collection_type = source.collection_type || changes[:collection_type]
collection_changes =
case collection_type do
:channel ->
%{
collection_id: source_details.channel_id,
collection_name: source_details.channel_name
}
:playlist ->
%{
collection_id: source_details.playlist_id,
collection_name: source_details.playlist_name
}
_ ->
%{}
end
change_source(source, Map.merge(changes, collection_changes))
end
defp commit_and_start_indexing(changeset) do
case Repo.insert_or_update(changeset) do
{:ok, %Source{} = source} ->
maybe_run_indexing_task(changeset, source)
{:ok, source}
err ->
err
{:ok, %Source{} = source} -> maybe_run_indexing_task(changeset, source)
err -> err
end
end
@ -159,5 +177,7 @@ defmodule Pinchflat.MediaSource do
SourceTasks.kickoff_indexing_task(source)
end
end
{:ok, source}
end
end

View file

@ -9,14 +9,24 @@ defmodule Pinchflat.MediaSource.Source do
alias Pinchflat.Media.MediaItem
alias Pinchflat.Profiles.MediaProfile
@allowed_fields ~w(name collection_id collection_type index_frequency_minutes original_url media_profile_id)a
@required_fields @allowed_fields -- ~w(index_frequency_minutes)a
@allowed_fields ~w(
collection_name
collection_id
collection_type
friendly_name
index_frequency_minutes
original_url
media_profile_id
)a
@required_fields @allowed_fields -- ~w(index_frequency_minutes friendly_name)a
schema "sources" do
field :name, :string
field :friendly_name, :string
field :collection_name, :string
field :collection_id, :string
field :collection_type, Ecto.Enum, values: [:channel, :playlist]
field :index_frequency_minutes, :integer
field :index_frequency_minutes, :integer, default: 60 * 24
# This should only be used for user reference going forward
# as the collection_id should be used for all API calls
field :original_url, :string

View file

@ -3,9 +3,11 @@ defmodule Pinchflat.Tasks.SourceTasks do
This module contains methods for managing tasks (workers) related to sources.
"""
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.MediaSource.Source
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.VideoDownloadWorker
@doc """
Starts tasks for indexing a source's media.
@ -30,4 +32,27 @@ defmodule Pinchflat.Tasks.SourceTasks do
end
end
end
@doc """
Starts tasks for downloading videos for any of a sources _pending_ media items.
NOTE: this starts a download for each media item that is pending,
not just the ones that were indexed in this job run. This should ensure
that any stragglers are caught if, for some reason, they weren't enqueued
or somehow got de-queued.
I'm not sure of a case where this would happen, but it's cheap insurance.
Returns :ok
"""
def enqueue_pending_media_downloads(%Source{} = source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
media_item
|> Map.take([:id])
|> VideoDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end)
end
end

View file

@ -7,10 +7,9 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
tags: ["media_source", "media_indexing"]
alias __MODULE__
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.MediaSource
alias Pinchflat.Workers.VideoDownloadWorker
alias Pinchflat.Tasks.SourceTasks
@impl Oban.Worker
@doc """
@ -49,7 +48,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
defp index_media_and_reschedule(source) do
MediaSource.index_media_items(source)
enqueue_video_downloads(source)
SourceTasks.enqueue_pending_media_downloads(source)
source
|> Map.take([:id])
@ -60,21 +59,4 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
{:error, :duplicate_job} -> {:ok, :job_exists}
end
end
# NOTE: this starts a download for each media item that is pending,
# not just the ones that were indexed in this job run. This should ensure
# that any stragglers are caught if, for some reason, they weren't enqueued
# or somehow got de-queued.
#
# I'm not sure of a case where this would happen, but it's cheap insurance.
defp enqueue_video_downloads(source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
media_item
|> Map.take([:id])
|> VideoDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end)
end
end

View file

@ -1,6 +1,7 @@
defmodule PinchflatWeb.MediaSources.SourceController do
use PinchflatWeb, :controller
alias Pinchflat.Repo
alias Pinchflat.Profiles
alias Pinchflat.MediaSource
alias Pinchflat.MediaSource.Source
@ -30,7 +31,10 @@ defmodule PinchflatWeb.MediaSources.SourceController do
end
def show(conn, %{"id" => id}) do
source = MediaSource.get_source!(id)
source =
id
|> MediaSource.get_source!()
|> Repo.preload(:media_profile)
render(conn, :show, source: source)
end

View file

@ -24,4 +24,11 @@ defmodule PinchflatWeb.MediaSources.SourceHTML do
{"Monthly", 30 * 24 * 60}
]
end
def friendly_collection_types do
[
{"Channel", "channel"},
{"Playlist", "playlist"}
]
end
end

View file

@ -8,8 +8,8 @@
</.header>
<.table id="sources" rows={@sources} row_click={&JS.navigate(~p"/media_sources/sources/#{&1}")}>
<:col :let={source} label="Name"><%= source.name %></:col>
<:col :let={source} label="Source"><%= source.collection_id %></:col>
<:col :let={source} label="Collection Name"><%= source.collection_name %></:col>
<:col :let={source} label="Collection ID"><%= source.collection_id %></:col>
<:action :let={source}>
<div class="sr-only">
<.link navigate={~p"/media_sources/sources/#{source}"}>Show</.link>

View file

@ -9,9 +9,18 @@
</.header>
<.list>
<:item title="Source Name"><%= @source.name %></:item>
<:item title="Source ID"><%= @source.collection_id %></:item>
<:item title="Original URL"><%= @source.original_url %></:item>
<:item title="media_profile">
<.link href={~p"/media_profiles/#{@source.media_profile}"}>
<%= @source.media_profile.name %>
</.link>
</:item>
<:item
:for={attr <- ~w(collection_type collection_name collection_id original_url friendly_name)a}
title={attr}
>
<%= Map.get(@source, attr) %>
</:item>
</.list>
<.back navigate={~p"/media_sources/sources"}>Back to sources</.back>

View file

@ -3,6 +3,8 @@
Oops, something went wrong! Please check the errors below.
</.error>
<.input field={f[:friendly_name]} type="text" label="Friendly Name" />
<.input
field={f[:media_profile_id]}
options={Enum.map(@media_profiles, &{&1.name, &1.id})}
@ -10,7 +12,13 @@
label="Media Profile"
/>
<.input field={f[:collection_type]} type="text" label="Collection Type" />
<.input
field={f[:collection_type]}
options={friendly_collection_types()}
type="select"
label="Collection Type"
/>
<.input field={f[:original_url]} type="text" label="Source URL" />
<.input

View file

@ -0,0 +1,11 @@
defmodule Pinchflat.Repo.Migrations.RenameSourceNameToCollectionName do
use Ecto.Migration
def change do
rename table(:sources), :name, to: :collection_name
alter table(:sources) do
add :friendly_name, :string
end
end
end

View file

@ -2,7 +2,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
use ExUnit.Case, async: true
import Mox
alias Pinchflat.MediaClient.SourceDetails
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection
@channel_url "https://www.youtube.com/c/TheUselessTrials"
@ -45,19 +44,30 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
end
describe "get_source_details/1" do
test "it returns a %SourceDetails{} with data on success" do
test "it returns a map with data on success" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, "{\"channel\": \"TheUselessTrials\", \"channel_id\": \"UCQH2\"}"}
Phoenix.json_library().encode(%{
channel: "TheUselessTrials",
channel_id: "UCQH2",
playlist_id: "PLQH2",
playlist_title: "TheUselessTrials - Videos"
})
end)
assert {:ok, res} = VideoCollection.get_source_details(@channel_url)
assert %SourceDetails{id: "UCQH2", name: "TheUselessTrials"} = res
assert %{
channel_id: "UCQH2",
channel_name: "TheUselessTrials",
playlist_id: "PLQH2",
playlist_name: "TheUselessTrials - Videos"
} = res
end
test "it passes the expected args to the backend runner" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [:skip_download, playlist_end: 1]
assert ot == "%(.{channel,channel_id})j"
assert opts == [:simulate, :skip_download, playlist_end: 1]
assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j"
{:ok, "{}"}
end)

View file

@ -8,20 +8,13 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
setup :verify_on_exit!
describe "new/2" do
test "it returns a struct with the given values" do
assert %SourceDetails{id: "UCQH2", name: "TheUselessTrials"} =
SourceDetails.new("UCQH2", "TheUselessTrials")
end
end
describe "get_source_details/2" do
test "it passes the expected arguments to the backend" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [:skip_download, playlist_end: 1]
assert ot == "%(.{channel,channel_id})j"
assert opts == [:simulate, :skip_download, playlist_end: 1]
assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j"
{:ok, "{\"channel\": \"TheUselessTrials\", \"channel_id\": \"UCQH2\"}"}
{:ok, "{}"}
end)
assert {:ok, _} = SourceDetails.get_source_details(@channel_url)
@ -29,11 +22,22 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
test "it returns a struct composed of the returned data" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, "{\"channel\": \"TheUselessTrials\", \"channel_id\": \"UCQH2\"}"}
Phoenix.json_library().encode(%{
channel: "TheUselessTrials",
channel_id: "UCQH2",
playlist_id: "PLQH2",
playlist_title: "TheUselessTrials - Videos"
})
end)
assert {:ok, res} = SourceDetails.get_source_details(@channel_url)
assert %SourceDetails{id: "UCQH2", name: "TheUselessTrials"} = res
assert %{
channel_id: "UCQH2",
channel_name: "TheUselessTrials",
playlist_id: "PLQH2",
playlist_name: "TheUselessTrials - Videos"
} = res
end
end

View file

@ -29,7 +29,7 @@ defmodule Pinchflat.MediaSourceTest do
end
describe "create_source/1" do
test "creates a source and adds name + ID from runner response" do
test "creates a source and adds name + ID from runner response for channels" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
valid_attrs = %{
@ -39,20 +39,34 @@ defmodule Pinchflat.MediaSourceTest do
}
assert {:ok, %Source{} = source} = MediaSource.create_source(valid_attrs)
assert source.name == "some name"
assert String.starts_with?(source.collection_id, "some_source_id_")
assert source.collection_name == "some channel name"
assert String.starts_with?(source.collection_id, "some_channel_id_")
end
test "creates a source and adds name + ID for playlists" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
valid_attrs = %{
media_profile_id: media_profile_fixture().id,
original_url: "https://www.youtube.com/playlist?list=abc123",
collection_type: "playlist"
}
assert {:ok, %Source{} = source} = MediaSource.create_source(valid_attrs)
assert source.collection_name == "some playlist name"
assert String.starts_with?(source.collection_id, "some_playlist_id_")
end
test "creation with invalid data returns error changeset" do
assert {:error, %Ecto.Changeset{}} = MediaSource.create_source(@invalid_source_attrs)
end
test "creation enforces uniqueness of source_id scoped to the media_profile" do
test "creation enforces uniqueness of collection_id scoped to the media_profile" do
expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
{:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_source_id_12345678"
channel: "some channel name",
channel_id: "some_channel_id_12345678"
})}
end)
@ -70,8 +84,8 @@ defmodule Pinchflat.MediaSourceTest do
expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
{:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_source_id_12345678"
channel: "some channel name",
channel_id: "some_channel_id_12345678"
})}
end)
@ -159,21 +173,32 @@ defmodule Pinchflat.MediaSourceTest do
describe "update_source/2" do
test "updates with valid data updates the source" do
source = source_fixture()
update_attrs = %{name: "some updated name"}
update_attrs = %{collection_name: "some updated name"}
assert {:ok, %Source{} = source} = MediaSource.update_source(source, update_attrs)
assert source.name == "some updated name"
assert source.collection_name == "some updated name"
end
test "updating the original_url will re-fetch the source details" do
test "updating the original_url will re-fetch the source details for channels" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
source = source_fixture()
update_attrs = %{original_url: "https://www.youtube.com/channel/abc123"}
assert {:ok, %Source{} = source} = MediaSource.update_source(source, update_attrs)
assert source.name == "some name"
assert String.starts_with?(source.collection_id, "some_source_id_")
assert source.collection_name == "some channel name"
assert String.starts_with?(source.collection_id, "some_channel_id_")
end
test "updating the original_url will re-fetch the source details for playlists" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
source = source_fixture(collection_type: "playlist")
update_attrs = %{original_url: "https://www.youtube.com/playlist?list=abc123"}
assert {:ok, %Source{} = source} = MediaSource.update_source(source, update_attrs)
assert source.collection_name == "some playlist name"
assert String.starts_with?(source.collection_id, "some_playlist_id_")
end
test "not updating the original_url will not re-fetch the source details" do
@ -275,16 +300,16 @@ defmodule Pinchflat.MediaSourceTest do
media_profile_id = media_profile.id
changeset =
MediaSource.change_source_from_url(%Source{}, %{
MediaSource.change_source_from_url(%Source{collection_type: :channel}, %{
original_url: "https://www.youtube.com/channel/abc123",
media_profile_id: media_profile.id
})
assert %Ecto.Changeset{} = changeset
assert String.starts_with?(changeset.changes.collection_id, "some_source_id_")
assert String.starts_with?(changeset.changes.collection_id, "some_channel_id_")
assert %{
name: "some name",
collection_name: "some channel name",
media_profile_id: ^media_profile_id,
original_url: "https://www.youtube.com/channel/abc123"
} = changeset.changes
@ -309,8 +334,10 @@ defmodule Pinchflat.MediaSourceTest do
{
:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_source_id_#{:rand.uniform(1_000_000)}"
channel: "some channel name",
channel_id: "some_channel_id_#{:rand.uniform(1_000_000)}",
playlist_id: "some_playlist_id_#{:rand.uniform(1_000_000)}",
playlist_title: "some playlist name"
})
}
end

View file

@ -2,11 +2,14 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
use Pinchflat.DataCase
import Pinchflat.TasksFixtures
import Pinchflat.MediaFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.Tasks
alias Pinchflat.Tasks.Task
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.VideoDownloadWorker
describe "kickoff_indexing_task/1" do
test "it does not schedule a job if the interval is <= 0" do
@ -42,4 +45,35 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
describe "enqueue_pending_media_downloads/1" do
test "it enqueues a job for each pending media item" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
assert_enqueued(worker: VideoDownloadWorker, args: %{"id" => media_item.id})
end
test "it does not enqueue a job for media items with a filepath" do
source = source_fixture()
_media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
refute_enqueued(worker: VideoDownloadWorker)
end
test "it attaches a task to each enqueued job" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
end
end

View file

@ -104,16 +104,17 @@ defmodule PinchflatWeb.SourceControllerTest do
end
defp create_source(_) do
source = source_fixture()
%{source: source}
%{source: source_fixture()}
end
defp runner_function_mock(_url, _opts, _ot) do
{
:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_source_id_#{:rand.uniform(1_000_000)}"
channel: "some channel name",
channel_id: "some_channel_id_#{:rand.uniform(1_000_000)}",
playlist_id: "some_playlist_id_#{:rand.uniform(1_000_000)}",
playlist_title: "some playlist name"
})
}
end

View file

@ -16,9 +16,10 @@ defmodule Pinchflat.MediaSourceFixtures do
%Source{}
|> Source.changeset(
Enum.into(attrs, %{
name: "Source ##{:rand.uniform(1_000_000)}",
collection_name: "Source ##{:rand.uniform(1_000_000)}",
collection_id: Base.encode16(:crypto.hash(:md5, "#{:rand.uniform(1_000_000)}")),
collection_type: "channel",
friendly_name: "Cool and good internal name!",
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
media_profile_id: ProfilesFixtures.media_profile_fixture().id,
index_frequency_minutes: 60