danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Media downloading indexing options (round 2) (#14)

Browse source 
* Adds options + option builder + metadata parsing for media thumbnails

* Added release-type options to media profile; built option parser for indexing operations

* Added new media_profile options to creation form; made show helper for rendering database items

* Added options for downloading/embedding metadata

* Adds option on sources to not download media (index only)

* reformatted docs
This commit is contained in:
Kieran 2024-02-06 18:36:41 -08:00 committed by GitHub
parent b19c01e3ed
commit 9e4fbfa35d
31 changed files with 2317 additions and 1731 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -39,3 +39,4 @@ npm-debug.log
/.elixir_ls
.env
.DS_Store
scratchpad.md

12
lib/pinchflat/media/media_item.ex
View file

@ -10,13 +10,23 @@ defmodule Pinchflat.Media.MediaItem do
alias Pinchflat.MediaSource.Source
alias Pinchflat.Media.MediaMetadata
@allowed_fields ~w(
title
media_id
media_filepath
source_id
subtitle_filepaths
thumbnail_filepath
metadata_filepath
)a
@required_fields ~w(media_id source_id)a
@allowed_fields ~w(title media_id media_filepath source_id subtitle_filepaths)a
schema "media_items" do
field :title, :string
field :media_id, :string
field :media_filepath, :string
field :thumbnail_filepath, :string
field :metadata_filepath, :string
# This is an array of [iso-2 language, filepath] pairs. Probably could
# be an associated record, but I don't see the benefit right now.
# Will very likely revisit because I can't leave well-enough alone.

25
lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex
View file

@ -25,6 +25,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
metadata_attrs
|> Map.merge(parse_media_metadata(metadata))
|> Map.merge(parse_subtitle_metadata(metadata))
|> Map.merge(parse_thumbnail_metadata(metadata))
|> Map.merge(parse_infojson_metadata(metadata))
end
defp parse_media_metadata(metadata) do
@ -35,10 +37,9 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
end
defp parse_subtitle_metadata(metadata) do
subtitle_map = metadata["requested_subtitles"] || %{}
# IDEA: if needed, consider filtering out subtitles that don't exist on-disk
subtitle_filepaths =
subtitle_map
(metadata["requested_subtitles"] || %{})
|> Enum.map(fn {lang, attrs} -> [lang, attrs["filepath"]] end)
|> Enum.sort(fn [lang_a, _], [lang_b, _] -> lang_a < lang_b end)
@ -46,4 +47,24 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
subtitle_filepaths: subtitle_filepaths
}
end
defp parse_thumbnail_metadata(metadata) do
thumbnail_filepath =
(metadata["thumbnails"] || %{})
# Reverse so that higher resolution thumbnails come first.
# This _shouldn't_ matter yet, but I'd rather default to the best
# in case I'm wrong.
|> Enum.reverse()
|> Enum.find_value(fn attrs -> attrs["filepath"] end)
%{
thumbnail_filepath: thumbnail_filepath
}
end
defp parse_infojson_metadata(metadata) do
%{
metadata_filepath: metadata["infojson_filename"]
}
end
end

28
lib/pinchflat/media_client/source_details.ex
View file

@ -6,10 +6,14 @@ defmodule Pinchflat.MediaClient.SourceDetails do
it open-ish for future expansion (just in case).
"""
alias Pinchflat.Repo
alias Pinchflat.MediaSource.Source
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder, as: YtDlpIndexOptionBuilder
@doc """
Gets a source's ID and name from its URL, using the given backend.
Gets a source's ID and name from its URL using the given backend.
Returns {:ok, map()} | {:error, any, ...}.
"""
@ -18,11 +22,23 @@ defmodule Pinchflat.MediaClient.SourceDetails do
end
@doc """
Returns a list of video IDs for the given source URL, using the given backend.
Returns a list of video IDs for the given source URL OR source record using the given backend.
If passing a source record, the call to the backend may have custom options applied based on
the `option_builder`.
Returns {:ok, list(binary())} | {:error, any, ...}.
"""
def get_video_ids(source_url, backend \\ :yt_dlp) do
def get_video_ids(sourceable, backend \\ :yt_dlp)
def get_video_ids(%Source{} = source, backend) do
media_profile = Repo.preload(source, :media_profile).media_profile
{:ok, options} = option_builder(backend).build(media_profile)
source_module(backend).get_video_ids(source.collection_id, options)
end
def get_video_ids(source_url, backend) when is_binary(source_url) do
source_module(backend).get_video_ids(source_url)
end
@ -31,4 +47,10 @@ defmodule Pinchflat.MediaClient.SourceDetails do
:yt_dlp -> YtDlpSource
end
end
defp option_builder(backend) do
case backend do
:yt_dlp -> YtDlpIndexOptionBuilder
end
end
end

8
lib/pinchflat/media_client/video_downloader.ex
View file

@ -14,7 +14,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo
alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder, as: YtDlpOptionBuilder
alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
@doc """
@ -22,6 +22,10 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
returned by the backend. Also saves the entire metadata response to the associated
media_metadata record.
NOTE: related methods (like the download worker) won't download if the media item's source
is set to not download media. However, I'm not enforcing that here since I need this for testing.
This may change in the future but I'm not stressed.
Returns {:ok, %MediaItem{}} | {:error, any, ...any}
"""
def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do
@ -52,7 +56,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
defp option_builder(backend) do
case backend do
:yt_dlp -> YtDlpOptionBuilder
:yt_dlp -> YtDlpDownloadOptionBuilder
end
end

2
lib/pinchflat/media_source/source.ex
View file

@ -15,6 +15,7 @@ defmodule Pinchflat.MediaSource.Source do
collection_type
friendly_name
index_frequency_minutes
download_media
original_url
media_profile_id
)a
@ -27,6 +28,7 @@ defmodule Pinchflat.MediaSource.Source do
field :collection_id, :string
field :collection_type, Ecto.Enum, values: [:channel, :playlist]
field :index_frequency_minutes, :integer, default: 60 * 24
field :download_media, :boolean, default: true
# This should only be used for user reference going forward
# as the collection_id should be used for all API calls
field :original_url, :string

21
lib/pinchflat/profiles/media_profile.ex
View file

@ -15,6 +15,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
download_auto_subs
embed_subs
sub_langs
download_thumbnail
embed_thumbnail
download_metadata
embed_metadata
shorts_behaviour
livestream_behaviour
)a
@required_fields ~w(name output_path_template)a
@ -22,11 +28,26 @@ defmodule Pinchflat.Profiles.MediaProfile do
schema "media_profiles" do
field :name, :string
field :output_path_template, :string
field :download_subs, :boolean, default: true
field :download_auto_subs, :boolean, default: true
field :embed_subs, :boolean, default: true
field :sub_langs, :string, default: "en"
field :download_thumbnail, :boolean, default: true
field :embed_thumbnail, :boolean, default: true
field :download_metadata, :boolean, default: true
field :embed_metadata, :boolean, default: true
# NOTE: these do NOT speed up indexing - the indexer still has to go
# through the entire collection to determine if a video is a short or
# a livestream.
# NOTE: these can BOTH be set to :only which will download shorts and
# livestreams _only_ and ignore regular videos.
field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
has_many :sources, Source
timestamps(type: :utc_datetime)

38
lib/pinchflat/profiles/options/yt_dlp/option_builder.ex → lib/pinchflat/profiles/options/yt_dlp/download_option_builder.ex
View file

@ -1,6 +1,6 @@
defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder do
@moduledoc """
Builds the options for yt-dlp based on the given media profile.
Builds the options for yt-dlp to download media based on the given media profile.
IDEA: consider making this a behaviour so I can add other backends later
"""
@ -9,7 +9,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
@doc """
Builds the options for yt-dlp based on the given media profile.
Builds the options for yt-dlp to download media based on the given media profile.
IDEA: consider adding the ability to pass in a second argument to override
these options
@ -24,6 +24,8 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
built_options =
default_options() ++
subtitle_options(media_profile) ++
thumbnail_options(media_profile) ++
metadata_options(media_profile) ++
output_options(media_profile)
{:ok, built_options}
@ -31,11 +33,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
# This will be updated a lot as I add new options to profiles
defp default_options do
[
:embed_metadata,
:embed_thumbnail,
:no_progress
]
[:no_progress]
end
defp subtitle_options(media_profile) do
@ -65,6 +63,30 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
end)
end
defp thumbnail_options(media_profile) do
mapped_struct = Map.from_struct(media_profile)
Enum.reduce(mapped_struct, [], fn attr, acc ->
case attr do
{:download_thumbnail, true} -> acc ++ [:write_thumbnail]
{:embed_thumbnail, true} -> acc ++ [:embed_thumbnail]
_ -> acc
end
end)
end
defp metadata_options(media_profile) do
mapped_struct = Map.from_struct(media_profile)
Enum.reduce(mapped_struct, [], fn attr, acc ->
case attr do
{:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
{:embed_metadata, true} -> acc ++ [:embed_metadata]
_ -> acc
end
end)
end
defp output_options(media_profile) do
{:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template)

52
lib/pinchflat/profiles/options/yt_dlp/index_option_builder.ex Normal file
View file

@ -0,0 +1,52 @@
defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
@moduledoc """
Builds the options for yt-dlp to index a media source based on the given media profile.
"""
alias Pinchflat.Profiles.MediaProfile
@doc """
Builds the options for yt-dlp to index a media source based on the given media profile.
"""
def build(%MediaProfile{} = media_profile) do
built_options = release_type_options(media_profile)
{:ok, built_options}
end
defp release_type_options(media_profile) do
mapped_struct = Map.from_struct(media_profile)
# Appending multiple match filters treats them as an OR condition,
# so we have to be careful around combining `only` and `exclude` options.
# eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
# which will return all shorts AND normal videos.
Enum.reduce(mapped_struct, [], fn attr, acc ->
case {attr, media_profile} do
{{:shorts_behaviour, :only}, _} ->
acc ++ [match_filter: "original_url*=/shorts/"]
{{:livestream_behaviour, :only}, _} ->
acc ++ [match_filter: "was_live"]
# Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
# other type is set to `only`. There is also special behaviour if they're both excludes,
# hence why these check against `:include` alone.
{{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
acc ++ [match_filter: "original_url!*=/shorts/"]
{{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
acc ++ [match_filter: "!was_live"]
# Again, since it's an OR, there's a special syntax if they're both excluded
# to make it an AND. Note that I'm not checking for the other permutation of
# both excluding since this MUST get hit so adding the other version would double up.
{{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
_ ->
acc
end
end)
end
end

5
lib/pinchflat/profiles/options/yt_dlp/output_path_builder.ex
View file

@ -12,6 +12,11 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
Translates liquid-style templates into yt-dlp-style templates,
leaving yt-dlp syntax intact.
IDEA: apart from any custom options I've defined, I can support any yt-dlp
option by assuming `{{ identifier }}` should transform to `%(identifier)S`.
It's not doing anything huge, but it's nicer to type and more approachable IMO.
IDEA: set a default for `MediaProfile`'s `output_path_template` field
"""
def build(template_string) do
TemplateParser.parse(template_string, full_yt_dlp_options_map())

7
lib/pinchflat/tasks/source_tasks.ex
View file

@ -35,6 +35,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
@doc """
Starts tasks for downloading videos for any of a sources _pending_ media items.
Jobs are not enqueued if the source is set to not download media. This will return :ok.
NOTE: this starts a download for each media item that is pending,
not just the ones that were indexed in this job run. This should ensure
@ -45,7 +46,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
Returns :ok
"""
def enqueue_pending_media_downloads(%Source{} = source) do
def enqueue_pending_media_downloads(%Source{download_media: true} = source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
@ -55,4 +56,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
|> Tasks.create_job_with_task(media_item)
end)
end
def enqueue_pending_media_downloads(%Source{download_media: false} = _source) do
:ok
end
end

20
lib/pinchflat/workers/video_download_worker.ex
View file

@ -6,18 +6,32 @@ defmodule Pinchflat.Workers.VideoDownloadWorker do
unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
tags: ["media_item", "media_fetching"]
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.MediaClient.VideoDownloader
@impl Oban.Worker
@doc """
For a given media item, download the video and save the metadata.
For a given media item, download the media alongside any options.
Does not download media if its source is set to not download media.
Returns {:ok, %MediaItem{}} | {:error, any, ...any}
Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
"""
def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
media_item = Media.get_media_item!(media_item_id)
media_item =
media_item_id
|> Media.get_media_item!()
|> Repo.preload(:source)
# If the source is set to not download media, perform a no-op
if media_item.source.download_media do
download_media(media_item)
else
:ok
end
end
defp download_media(media_item) do
case VideoDownloader.download_for_media_item(media_item) do
{:ok, _} -> {:ok, media_item}
err -> err

26
lib/pinchflat_web/components/core_components.ex
View file

@ -535,7 +535,7 @@ defmodule PinchflatWeb.CoreComponents do
def list(assigns) do
~H"""
<div class="mt-14">
<div class="mt-2 mb-14">
<dl class="-my-4 divide-y divide-zinc-100">
<div :for={item <- @item} class="flex gap-4 py-4 text-sm leading-6 sm:gap-8">
<dt class="w-1/4 flex-none text-zinc-500"><%= item.title %></dt>
@ -546,6 +546,30 @@ defmodule PinchflatWeb.CoreComponents do
"""
end
@doc """
Renders a data list from a given map. Used in development to
quickly show the attributes of a database record.
"""
attr :map, :map, required: true
def list_items_from_map(assigns) do
attrs =
Enum.filter(assigns.map, fn
{_, %{__struct__: _}} -> false
_ -> true
end)
assigns = assign(assigns, iterable_attributes: attrs)
~H"""
<.list>
<:item :for={{k, v} <- @iterable_attributes} title={k}>
<%= v %>
</:item>
</.list>
"""
end
@doc """
Renders a back navigation link.

8
lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex
View file

@ -10,4 +10,12 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
attr :action, :string, required: true
def media_profile_form(assigns)
def friendly_format_type_options do
[
{"Include (default)", :include},
{"Exclude", :exclude},
{"Only", :only}
]
end
end

25
lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex
View file

@ -4,10 +4,35 @@
</.error>
<.input field={f[:name]} type="text" label="Name" />
<.input field={f[:output_path_template]} type="text" label="Output path template" />
<h3>Subtitle Options</h3>
<.input field={f[:download_subs]} type="checkbox" label="Download Subs" />
<.input field={f[:download_auto_subs]} type="checkbox" label="Download Autogenerated Subs" />
<.input field={f[:embed_subs]} type="checkbox" label="Embed Subs" />
<.input field={f[:sub_langs]} type="text" label="Sub Langs" />
<h3>Thumbnail Options</h3>
<.input field={f[:download_thumbnail]} type="checkbox" label="Download Thumbnail" />
<.input field={f[:embed_thumbnail]} type="checkbox" label="Embed Thumbnail" />
<h3>Metadata Options</h3>
<.input field={f[:download_metadata]} type="checkbox" label="Download Metadata" />
<.input field={f[:embed_metadata]} type="checkbox" label="Embed Metadata" />
<h3>Release Format Options</h3>
<.input
field={f[:shorts_behaviour]}
options={friendly_format_type_options()}
type="select"
label="Include Shorts?"
/>
<.input
field={f[:livestream_behaviour]}
options={friendly_format_type_options()}
type="select"
label="Include Livestreams?"
/>
<:actions>
<.button>Save Media profile</.button>
</:actions>

11
lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex
View file

@ -8,15 +8,6 @@
</:actions>
</.header>
<.list>
<:item
:for={
attr <- ~w(name output_path_template download_subs download_auto_subs embed_subs sub_langs)a
}
title={attr}
>
<%= Map.get(@media_profile, attr) %>
</:item>
</.list>
<.list_items_from_map map={Map.from_struct(@media_profile)} />
<.back navigate={~p"/media_profiles"}>Back to media_profiles</.back>

11
lib/pinchflat_web/controllers/media_sources/source_html/show.html.heex
View file

@ -8,19 +8,16 @@
</:actions>
</.header>
<h3 class="mt-14">Relationships</h3>
<.list>
<:item title="media_profile">
<.link href={~p"/media_profiles/#{@source.media_profile}"}>
<%= @source.media_profile.name %>
</.link>
</:item>
<:item
:for={attr <- ~w(collection_type collection_name collection_id original_url friendly_name)a}
title={attr}
>
<%= Map.get(@source, attr) %>
</:item>
</.list>
<h3>Attributes</h3>
<.list_items_from_map map={Map.from_struct(@source)} />
<.back navigate={~p"/media_sources/sources"}>Back to sources</.back>

2
lib/pinchflat_web/controllers/media_sources/source_html/source_form.html.heex
View file

@ -28,6 +28,8 @@
label="Index Frequency"
/>
<.input field={f[:download_media]} type="checkbox" label="Download Media?" />
<:actions>
<.button>Save Source</.button>
</:actions>

14
priv/repo/migrations/20240205194558_add_thumbnail_options_to_media_profiles.exs Normal file
View file

@ -0,0 +1,14 @@
defmodule Pinchflat.Repo.Migrations.AddThumbnailOptionsToMediaProfiles do
use Ecto.Migration
def change do
alter table(:media_profiles) do
add :download_thumbnail, :boolean, default: true, null: false
add :embed_thumbnail, :boolean, default: true, null: false
end
alter table(:media_items) do
add :thumbnail_filepath, :string
end
end
end

10
priv/repo/migrations/20240205215634_add_release_type_behaviour_to_media_profiles.exs Normal file
View file

@ -0,0 +1,10 @@
defmodule Pinchflat.Repo.Migrations.AddReleaseTypeBehaviourToMediaProfiles do
use Ecto.Migration
def change do
alter table(:media_profiles) do
add :shorts_behaviour, :string, null: false, default: "include"
add :livestream_behaviour, :string, null: false, default: "include"
end
end
end

14
priv/repo/migrations/20240206201218_add_metadata_options_to_media_profiles.exs Normal file
View file

@ -0,0 +1,14 @@
defmodule Pinchflat.Repo.Migrations.AddMetadataOptionsToMediaProfiles do
use Ecto.Migration
def change do
alter table(:media_profiles) do
add :download_metadata, :boolean, default: true, null: false
add :embed_metadata, :boolean, default: true, null: false
end
alter table(:media_items) do
add :metadata_filepath, :string
end
end
end

9
priv/repo/migrations/20240206204740_add_download_media_to_sources.exs Normal file
View file

@ -0,0 +1,9 @@
defmodule Pinchflat.Repo.Migrations.AddDownloadMediaToSources do
use Ecto.Migration
def change do
alter table(:sources) do
add :download_media, :boolean, default: true, null: false
end
end
end

42
test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs
View file

@ -47,7 +47,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
test "extracts the subtitle filepaths", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert [["de", german_filepath], ["en", english_filepath]] = result.subtitle_filepaths
assert [["de", german_filepath], ["en", english_filepath] | _rest] = result.subtitle_filepaths
assert String.ends_with?(english_filepath, ".en.srt")
assert String.ends_with?(german_filepath, ".de.srt")
@ -83,4 +83,44 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
assert result.subtitle_filepaths == []
end
end
describe "parse_for_media_item/1 when testing thumbnail metadata" do
test "extracts the thumbnail filepath", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert String.ends_with?(result.thumbnail_filepath, ".webp")
end
test "doesn't freak out if the video has no thumbnails", %{metadata: metadata} do
metadata = Map.put(metadata, "thumbnails", %{})
result = Parser.parse_for_media_item(metadata)
assert result.thumbnail_filepath == nil
end
test "doesn't freak out if the thumbnails key is missing", %{metadata: metadata} do
metadata = Map.delete(metadata, "thumbnails")
result = Parser.parse_for_media_item(metadata)
assert result.thumbnail_filepath == nil
end
end
describe "parse_for_media_item/1 when testing infojson metadata" do
test "extracts the metadata filepath", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert String.ends_with?(result.metadata_filepath, ".info.json")
end
test "doesn't freak out if the video has no infojson", %{metadata: metadata} do
metadata = Map.put(metadata, "infojson_filename", nil)
result = Parser.parse_for_media_item(metadata)
assert result.metadata_filepath == nil
end
end
end

35
test/pinchflat/media_client/source_details_test.exs
View file

@ -1,6 +1,8 @@
defmodule Pinchflat.MediaClient.SourceDetailsTest do
use ExUnit.Case, async: true
use Pinchflat.DataCase
import Mox
import Pinchflat.ProfilesFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.MediaClient.SourceDetails
@ -41,7 +43,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
end
end
describe "get_video_ids/2" do
describe "get_video_ids/2 when passed a string" do
test "it passes the expected arguments to the backend" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [:simulate, :skip_download]
@ -61,4 +63,33 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
assert {:ok, ["video1", "video2", "video3"]} = SourceDetails.get_video_ids(@channel_url)
end
end
describe "get_video_ids/2 when passed a Source record" do
test "it calls the backend with the source's collection ID" do
source = source_fixture()
expect(YtDlpRunnerMock, :run, fn url, _opts, _ot ->
assert source.collection_id == url
{:ok, "video1\nvideo2\nvideo3"}
end)
assert {:ok, _} = SourceDetails.get_video_ids(source)
end
test "it builds options based on the source's media profile" do
expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
assert opts == [{:match_filter, "!was_live"}, :simulate, :skip_download]
{:ok, ""}
end)
media_profile =
media_profile_fixture(
shorts_behaviour: :include,
livestream_behaviour: :exclude
)
source = source_fixture(media_profile_id: media_profile.id)
assert {:ok, _} = SourceDetails.get_video_ids(source)
end
end
end

52
test/pinchflat/media_client/video_downloader_test.exs
View file

@ -28,18 +28,6 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
assert {:ok, _} = VideoDownloader.download_for_media_item(media_item)
end
test "it writes attributes to the media item", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
assert %{media_filepath: nil, title: nil, subtitle_filepaths: []} = media_item
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert updated_media_item.media_filepath
assert updated_media_item.title
assert length(updated_media_item.subtitle_filepaths) > 0
end
test "it saves the metadata to the database", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
@ -59,4 +47,44 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
assert {:error, :some_error} = VideoDownloader.download_for_media_item(media_item)
end
end
describe "download_for_media_item/3 when testing media_item attributes" do
setup do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
:ok
end
test "it extracts the title", %{media_item: media_item} do
assert media_item.title == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake"
end
test "it extracts the media_filepath", %{media_item: media_item} do
assert media_item.media_filepath == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert String.ends_with?(updated_media_item.media_filepath, ".mkv")
end
test "it extracts the subtitle_filepaths", %{media_item: media_item} do
assert media_item.subtitle_filepaths == []
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert [["de", _], ["en", _] | _rest] = updated_media_item.subtitle_filepaths
end
test "it extracts the thumbnail_filepath", %{media_item: media_item} do
assert media_item.thumbnail_filepath == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert String.ends_with?(updated_media_item.thumbnail_filepath, ".webp")
end
test "it extracts the metadata_filepath", %{media_item: media_item} do
assert media_item.metadata_filepath == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert String.ends_with?(updated_media_item.metadata_filepath, ".info.json")
end
end
end

161
test/pinchflat/profiles/options/yt_dlp/download_option_builder_test.exs Normal file
View file

@ -0,0 +1,161 @@
defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilderTest do
use ExUnit.Case, async: true
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder
@media_profile %MediaProfile{
output_path_template: "{{ title }}.%(ext)s"
}
describe "build/1" do
test "it generates an expanded output path based on the given template" do
assert {:ok, res} = DownloadOptionBuilder.build(@media_profile)
assert {:output, "/tmp/videos/%(title)S.%(ext)s"} in res
end
end
describe "build/1 when testing subtitle options" do
test "includes :write_subs option when specified" do
media_profile = %MediaProfile{@media_profile | download_subs: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :write_subs in res
end
test "forces SRT format when download_subs is true" do
media_profile = %MediaProfile{@media_profile | download_subs: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert {:convert_subs, "srt"} in res
end
test "includes :write_auto_subs option when specified" do
media_profile = %MediaProfile{@media_profile | download_subs: true, download_auto_subs: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :write_auto_subs in res
end
test "doesn't include :write_auto_subs option when download_subs is false" do
media_profile = %MediaProfile{@media_profile | download_subs: false, download_auto_subs: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
refute :write_auto_subs in res
end
test "includes :embed_subs option when specified" do
media_profile = %MediaProfile{@media_profile | embed_subs: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :embed_subs in res
end
test "includes sub_langs option when download_subs is true" do
media_profile = %MediaProfile{@media_profile | download_subs: true, sub_langs: "en"}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert {:sub_langs, "en"} in res
end
test "includes sub_langs option when embed_subs is true" do
media_profile = %MediaProfile{@media_profile | embed_subs: true, sub_langs: "en"}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert {:sub_langs, "en"} in res
end
test "doesn't include sub_langs option when neither downloading nor embedding" do
media_profile = %MediaProfile{
@media_profile
| embed_subs: false,
download_subs: false,
sub_langs: "en"
}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
refute {:sub_langs, "en"} in res
end
test "other struct attributes are ignored" do
media_profile = %MediaProfile{@media_profile | id: -1}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
refute {:id, -1} in res
end
end
describe "build/1 when testing thumbnail options" do
test "includes :write_thumbnail option when specified" do
media_profile = %MediaProfile{@media_profile | download_thumbnail: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :write_thumbnail in res
end
test "includes :embed_thumbnail option when specified" do
media_profile = %MediaProfile{@media_profile | embed_thumbnail: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :embed_thumbnail in res
end
test "doesn't include these options when not specified" do
media_profile = %MediaProfile{
@media_profile
| embed_thumbnail: false,
download_thumbnail: false
}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
refute :write_thumbnail in res
refute :embed_thumbnail in res
end
end
describe "build/1 when testing metadata options" do
test "includes :write_info_json option when specified" do
media_profile = %MediaProfile{@media_profile | download_metadata: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :write_info_json in res
assert :clean_info_json in res
end
test "includes :embed_metadata option when specified" do
media_profile = %MediaProfile{@media_profile | embed_metadata: true}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
assert :embed_metadata in res
end
test "doesn't include these options when not specified" do
media_profile = %MediaProfile{
@media_profile
| embed_metadata: false,
download_metadata: false
}
assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
refute :write_info_json in res
refute :clean_info_json in res
refute :embed_metadata in res
end
end
end

104
test/pinchflat/profiles/options/yt_dlp/index_option_builder_test.exs Normal file
View file

@ -0,0 +1,104 @@
defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilderTest do
use ExUnit.Case, async: true
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder
@media_profile %MediaProfile{
output_path_template: "{{ title }}.%(ext)s",
shorts_behaviour: :include,
livestream_behaviour: :include
}
describe "build/1 when testing release type options" do
test "adds correct filter when shorts_behaviour is :only" do
media_profile = %MediaProfile{@media_profile | shorts_behaviour: :only}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
refute {:match_filter, "was_live"} in res
end
test "adds correct filter when livestream_behaviour is :only" do
media_profile = %MediaProfile{@media_profile | livestream_behaviour: :only}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "was_live"} in res
refute {:match_filter, "!was_live"} in res
refute {:match_filter, "!original_url*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
end
test "adds correct filter when both livestreams and shorts are :only" do
media_profile = %MediaProfile{
@media_profile
| shorts_behaviour: :only,
livestream_behaviour: :only
}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "original_url*=/shorts/"} in res
assert {:match_filter, "was_live"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
end
test "adds correct filter when shorts_behaviour is :exclude" do
media_profile = %MediaProfile{@media_profile | shorts_behaviour: :exclude}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "was_live"} in res
refute {:match_filter, "!was_live"} in res
end
test "adds correct filter when livestream_behaviour is :exclude" do
media_profile = %MediaProfile{@media_profile | livestream_behaviour: :exclude}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "!was_live"} in res
refute {:match_filter, "was_live"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
end
test "adds correct filter when shorts and livestreams are both exclude" do
media_profile = %MediaProfile{
@media_profile
| shorts_behaviour: :exclude,
livestream_behaviour: :exclude
}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "!was_live & original_url!*=/shorts/"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "was_live"} in res
end
test "does not add exclusion filter if one is excluded and the other is only" do
media_profile = %MediaProfile{
@media_profile
| shorts_behaviour: :exclude,
livestream_behaviour: :only
}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "was_live"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
end
end
end

97
test/pinchflat/profiles/options/yt_dlp/option_builder_test.exs
View file

@ -1,97 +0,0 @@
defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilderTest do
use ExUnit.Case, async: true
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder
@media_profile %MediaProfile{
output_path_template: "{{ title }}.%(ext)s"
}
describe "build/1" do
test "it generates an expanded output path based on the given template" do
assert {:ok, res} = OptionBuilder.build(@media_profile)
assert {:output, "/tmp/videos/%(title)S.%(ext)s"} in res
end
end
describe "build/1 when testing subtitle options" do
test "includes :write_subs option when specified" do
media_profile = %MediaProfile{@media_profile | download_subs: true}
assert {:ok, res} = OptionBuilder.build(media_profile)
assert :write_subs in res
end
test "forces SRT format when download_subs is true" do
media_profile = %MediaProfile{@media_profile | download_subs: true}
assert {:ok, res} = OptionBuilder.build(media_profile)
assert {:convert_subs, "srt"} in res
end
test "includes :write_auto_subs option when specified" do
media_profile = %MediaProfile{@media_profile | download_subs: true, download_auto_subs: true}
assert {:ok, res} = OptionBuilder.build(media_profile)
assert :write_auto_subs in res
end
test "doesn't include :write_auto_subs option when download_subs is false" do
media_profile = %MediaProfile{@media_profile | download_subs: false, download_auto_subs: true}
assert {:ok, res} = OptionBuilder.build(media_profile)
refute :write_auto_subs in res
end
test "includes :embed_subs option when specified" do
media_profile = %MediaProfile{@media_profile | embed_subs: true}
assert {:ok, res} = OptionBuilder.build(media_profile)
assert :embed_subs in res
end
test "includes sub_langs option when download_subs is true" do
media_profile = %MediaProfile{@media_profile | download_subs: true, sub_langs: "en"}
assert {:ok, res} = OptionBuilder.build(media_profile)
assert {:sub_langs, "en"} in res
end
test "includes sub_langs option when embed_subs is true" do
media_profile = %MediaProfile{@media_profile | embed_subs: true, sub_langs: "en"}
assert {:ok, res} = OptionBuilder.build(media_profile)
assert {:sub_langs, "en"} in res
end
test "doesn't include sub_langs option when neither downloading nor embedding" do
media_profile = %MediaProfile{
@media_profile
| embed_subs: false,
download_subs: false,
sub_langs: "en"
}
assert {:ok, res} = OptionBuilder.build(media_profile)
refute {:sub_langs, "en"} in res
end
test "other struct attributes are ignored" do
media_profile = %MediaProfile{@media_profile | id: -1}
assert {:ok, res} = OptionBuilder.build(media_profile)
refute {:id, -1} in res
end
end
end

16
test/pinchflat/tasks/source_tasks_test.exs
View file

@ -75,5 +75,21 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
test "it does not create a job if the source is set to not download" do
source = source_fixture(download_media: false)
assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
refute_enqueued(worker: VideoDownloadWorker)
end
test "it does not attach tasks if the source is set to not download" do
source = source_fixture(download_media: false)
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
end
end

9
test/pinchflat/workers/video_download_worker_test.exs
View file

@ -4,6 +4,7 @@ defmodule Pinchflat.Workers.VideoDownloadWorkerTest do
import Mox
import Pinchflat.MediaFixtures
alias Pinchflat.MediaSource
alias Pinchflat.Workers.VideoDownloadWorker
setup :verify_on_exit!
@ -55,5 +56,13 @@ defmodule Pinchflat.Workers.VideoDownloadWorkerTest do
assert job.state == "retryable"
end)
end
test "it does not download if the source is set to not download", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot -> :ok end)
MediaSource.update_source(media_item.source, %{download_media: false})
perform_job(VideoDownloadWorker, %{id: media_item.id})
end
end
end

3183
test/support/files/media_metadata.json
View file

File diff suppressed because one or more lines are too long