danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Download video(s) (first iteration) (#5)

Browse source 
* Updated package.json (and made an excuse to make a branch)

* Video filepath parser (#6)

* Restructured files; Added parser placeholder

* More restructuring

* Added basic parser for hydrating template strings

* Improved docs

* More docs

* Initial implementation of media profiles (#7)

* [WIP] Added basic video download method

* [WIP] Very-WIP first steps at parsing options and downloading

* Made my options safe by default and removed special safe versions

* Ran html generator for mediaprofile model - leaving as-is for now

* Addressed a bunch of TODO comments

* Add "channel" type Media Source (#8)

* [WIP] Working on fetching channel metadata in yt-dlp backend

* Finished first draft of methods to do with querying channels

* Renamed CommandRunnerMock to have a more descriptive name

* Ran the phx generator for the channel model

* Renamed Downloader namespace to MediaClient

* [WIP] saving before attempting LiveView

* LiveView did not work out but here's a working controller how about

* Index a channel (#9)

* Ran a MediaItem generator; Reformatted to my liking

* [WIP] added basic index function

* setup oban

* Added basic Oban job for indexing

* Added in workers for indexing; hooked them into record creation flow

* Added a task model with a phx generator

* Tied together tasks with jobs and channels

* Download indexed videos (#10)

* Clarified documentation

* more comments

* [WIP] hooked up basic video downloading; starting work on metadata

* Added metadata model and parsing

Adding the metadata model made me realize that, in many cases, yt-dlp
returns undesired input in stdout, breaking parsing. In order to get
the metadata model working, I had to change the way in which the app
interacts with yt-dlp. Now, output is written as a file to disk which
is immediately re-read and returned.

* Added tests for video download worker

* Hooked up video downloading to the channel indexing pipeline

* Adds tasks for media items

* Updated video metadata parser to extract the title

* Ran linting
This commit is contained in:
Kieran 2024-01-30 20:15:59 -08:00 committed by GitHub
parent 44bec0a127
commit 4dd9d837a3
96 changed files with 9358 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -38,3 +38,4 @@ npm-debug.log
/.elixir_ls
.env
.DS_Store

2
Dockerfile
View file

@ -29,5 +29,7 @@ RUN chmod +x ./docker-run.sh
# Install Elixir deps
RUN mix deps.get
# Gives us iex shell history
ENV ERL_AFLAGS="-kernel shell_history enabled"
EXPOSE 4008

4
assets/yarn.lock Normal file
View file

@ -0,0 +1,4 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1

12
config/config.exs
View file

@ -12,7 +12,10 @@ config :pinchflat,
generators: [timestamp_type: :utc_datetime],
# Specifying backend data here makes mocking and local testing SUPER easy
yt_dlp_executable: System.find_executable("yt-dlp"),
yt_dlp_runner: Pinchflat.DownloaderBackends.YtDlp.CommandRunner
yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner,
# TODO: figure this out
media_directory: :not_implemented,
metadata_directory: Path.join([System.tmp_dir!(), "pinchflat", "metadata"])
# Configures the endpoint
config :pinchflat, PinchflatWeb.Endpoint,
@ -25,6 +28,13 @@ config :pinchflat, PinchflatWeb.Endpoint,
pubsub_server: Pinchflat.PubSub,
live_view: [signing_salt: "/t5878kO"]
config :pinchflat, Oban,
repo: Pinchflat.Repo,
# Keep old jobs for 30 days for display in the UI
plugins: [{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}],
# TODO: consider making this an env var or something?
queues: [default: 10, media_indexing: 2, media_fetching: 2]
# Configures the mailer
#
# By default it uses the "Local" adapter which stores the emails

4
config/dev.exs
View file

@ -1,5 +1,9 @@
import Config
config :pinchflat,
media_directory: Path.join([File.cwd!(), "tmp", "videos"]),
metadata_directory: Path.join([File.cwd!(), "tmp", "metadata"])
# Configure your database
config :pinchflat, Pinchflat.Repo,
username: System.get_env("POSTGRES_USER"),

6
config/test.exs
View file

@ -2,7 +2,11 @@ import Config
config :pinchflat,
# Specifying backend data here makes mocking and local testing SUPER easy
yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"])
yt_dlp_executable: Path.join([File.cwd!(), "/test/support/scripts/yt-dlp-mocks/repeater.sh"]),
media_directory: Path.join([System.tmp_dir!(), "videos"]),
metadata_directory: Path.join([System.tmp_dir!(), "metadata"])
config :pinchflat, Oban, testing: :manual
# Configure your database
#

3
ideas.md Normal file
View file

@ -0,0 +1,3 @@
- Write media datbase ID as metadata/to file/whatever so it gives us an option to retroactively match media to the DB down the line. Useful if someone moves the media without informing the UI
- Use a UUID for the media database ID (or at least alongside it)
- Look into this and its recommended plugins https://hexdocs.pm/ex_check/readme.html

1
lib/pinchflat/application.ex
View file

@ -10,6 +10,7 @@ defmodule Pinchflat.Application do
children = [
PinchflatWeb.Telemetry,
Pinchflat.Repo,
{Oban, Application.fetch_env!(:pinchflat, Oban)},
{DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
{Phoenix.PubSub, name: Pinchflat.PubSub},
# Start the Finch HTTP client for sending emails

7
lib/pinchflat/downloader_backends/backend_command_runner.ex
View file

@ -1,7 +0,0 @@
defmodule Pinchflat.DownloaderBackends.BackendCommandRunner do
@moduledoc """
A behaviour for running CLI commands against a downloader backend
"""
@callback run(binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
end

58
lib/pinchflat/downloader_backends/yt_dlp/command_runner.ex
View file

@ -1,58 +0,0 @@
defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunner do
@moduledoc """
Runs yt-dlp commands using the `System.cmd/3` function
"""
alias Pinchflat.Utils.StringUtils
alias Pinchflat.DownloaderBackends.BackendCommandRunner
@behaviour BackendCommandRunner
@doc """
Runs a yt-dlp command and returns the string output
"""
@impl BackendCommandRunner
def run(url, command_opts) do
command = backend_executable()
formatted_command_opts = parse_options(command_opts) ++ [url]
case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
{output, 0} -> {:ok, output}
{output, status} -> {:error, output, status}
end
end
# We want to satisfy the following behaviours:
#
# 1. If the key is an atom, convert it to a string and convert it to kebab case (for convenience)
# 2. If the key is a string, assume we want it as-is and don't convert it
# 3. If the key is accompanied by a value, append the value to the list
# 4. If the key is not accompanied by a value, assume it's a flag and PREpend it to the list
defp parse_options(command_opts) do
Enum.reduce(command_opts, [], &parse_option/2)
end
defp parse_option({k, v}, acc) when is_atom(k) do
stringified_key = StringUtils.to_kebab_case(Atom.to_string(k))
parse_option({"--#{stringified_key}", v}, acc)
end
defp parse_option({k, v}, acc) when is_binary(k) do
acc ++ [k, to_string(v)]
end
defp parse_option(arg, acc) when is_atom(arg) do
stringified_arg = StringUtils.to_kebab_case(Atom.to_string(arg))
parse_option("--#{stringified_arg}", acc)
end
defp parse_option(arg, acc) when is_binary(arg) do
[arg | acc]
end
defp backend_executable do
Application.get_env(:pinchflat, :yt_dlp_executable)
end
end

21
lib/pinchflat/downloader_backends/yt_dlp/video_collection.ex
View file

@ -1,21 +0,0 @@
defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollection do
@moduledoc """
Contains utilities for working with collections of videos (ie: channels, playlists)
"""
@doc """
Returns a list of strings representing the video ids in the collection
"""
def get_video_ids(url, command_opts \\ []) do
opts = command_opts ++ [:simulate, :skip_download, :get_id]
case backend_runner().run(url, opts) do
{:ok, output} -> {:ok, String.split(output, "\n", trim: true)}
res -> res
end
end
defp backend_runner do
Application.get_env(:pinchflat, :yt_dlp_runner)
end
end

75
lib/pinchflat/media.ex Normal file
View file

@ -0,0 +1,75 @@
defmodule Pinchflat.Media do
@moduledoc """
The Media context.
"""
import Ecto.Query, warn: false
alias Pinchflat.Repo
alias Pinchflat.Tasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.MediaSource.Channel
@doc """
Returns the list of media_items. Returns [%MediaItem{}, ...].
"""
def list_media_items do
Repo.all(MediaItem)
end
@doc """
Returns a list of pending media_items for a given channel, where
pending means the `video_filepath` is `nil`.
Returns [%MediaItem{}, ...].
"""
def list_pending_media_items_for(%Channel{} = channel) do
from(
m in MediaItem,
where: m.channel_id == ^channel.id and is_nil(m.video_filepath)
)
|> Repo.all()
end
@doc """
Gets a single media_item.
Returns %MediaItem{}. Raises `Ecto.NoResultsError` if the Media item does not exist.
"""
def get_media_item!(id), do: Repo.get!(MediaItem, id)
@doc """
Creates a media_item. Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}.
"""
def create_media_item(attrs) do
%MediaItem{}
|> MediaItem.changeset(attrs)
|> Repo.insert()
end
@doc """
Updates a media_item. Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}.
"""
def update_media_item(%MediaItem{} = media_item, attrs) do
media_item
|> MediaItem.changeset(attrs)
|> Repo.update()
end
@doc """
Deletes a media_item and its associated tasks.
Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}.
"""
def delete_media_item(%MediaItem{} = media_item) do
Tasks.delete_tasks_for(media_item)
Repo.delete(media_item)
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking media_item changes.
"""
def change_media_item(%MediaItem{} = media_item, attrs \\ %{}) do
MediaItem.changeset(media_item, attrs)
end
end

38
lib/pinchflat/media/media_item.ex Normal file
View file

@ -0,0 +1,38 @@
defmodule Pinchflat.Media.MediaItem do
@moduledoc """
The MediaItem schema.
"""
use Ecto.Schema
import Ecto.Changeset
alias Pinchflat.Tasks.Task
alias Pinchflat.MediaSource.Channel
alias Pinchflat.Media.MediaMetadata
@required_fields ~w(media_id channel_id)a
@allowed_fields ~w(title media_id video_filepath channel_id)a
schema "media_items" do
field :title, :string
field :media_id, :string
field :video_filepath, :string
belongs_to :channel, Channel
has_one :metadata, MediaMetadata, on_replace: :update
has_many :tasks, Task
timestamps(type: :utc_datetime)
end
@doc false
def changeset(media_item, attrs) do
media_item
|> cast(attrs, @allowed_fields)
|> cast_assoc(:metadata, with: &MediaMetadata.changeset/2, required: false)
|> validate_required(@required_fields)
|> unique_constraint([:media_id, :channel_id])
end
end

28
lib/pinchflat/media/media_metadata.ex Normal file
View file

@ -0,0 +1,28 @@
defmodule Pinchflat.Media.MediaMetadata do
@moduledoc """
The MediaMetadata schema.
Look. Don't @ me about Metadata vs. Metadatum. I'm very sensitive.
"""
use Ecto.Schema
import Ecto.Changeset
alias Pinchflat.Media.MediaItem
schema "media_metadata" do
field :client_response, :map
belongs_to :media_item, MediaItem
timestamps(type: :utc_datetime)
end
@doc false
def changeset(media_metadata, attrs) do
media_metadata
|> cast(attrs, [:client_response])
|> validate_required([:client_response])
|> unique_constraint([:media_item_id])
end
end

7
lib/pinchflat/media_client/backends/backend_command_runner.ex Normal file
View file

@ -0,0 +1,7 @@
defmodule Pinchflat.MediaClient.Backends.BackendCommandRunner do
@moduledoc """
A behaviour for running CLI commands against a downloader backend
"""
@callback run(binary(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
end

32
lib/pinchflat/media_client/backends/yt_dlp/channel.ex Normal file
View file

@ -0,0 +1,32 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.Channel do
@moduledoc """
Contains utilities for working with a channel's videos
"""
use Pinchflat.MediaClient.Backends.YtDlp.VideoCollection
alias Pinchflat.MediaClient.ChannelDetails
@doc """
Gets a channel's ID and name from its URL.
yt-dlp does not _really_ have channel-specific functions, so
instead we're fetching just the first video (using playlist_end: 1)
and parsing the channel ID and name from _its_ metadata
Returns {:ok, %ChannelDetails{}} | {:error, any, ...}.
"""
def get_channel_details(channel_url) do
opts = [playlist_end: 1]
with {:ok, output} <- backend_runner().run(channel_url, opts, "%(.{channel,channel_id})j"),
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
{:ok, ChannelDetails.new(parsed_json["channel_id"], parsed_json["channel"])}
else
err -> err
end
end
defp backend_runner do
Application.get_env(:pinchflat, :yt_dlp_runner)
end
end

90
lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex Normal file
View file

@ -0,0 +1,90 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do
@moduledoc """
Runs yt-dlp commands using the `System.cmd/3` function
"""
require Logger
alias Pinchflat.Utils.StringUtils
alias Pinchflat.MediaClient.Backends.BackendCommandRunner
@behaviour BackendCommandRunner
@doc """
Runs a yt-dlp command and returns the string output. Saves the output to
a file and then returns its contents because yt-dlp will return warnings
to stdout even if the command is successful, but these will break JSON parsing.
Returns {:ok, binary()} | {:error, output, status}.
IDEA: Indexing takes a long time, but the output is actually streamed to stdout.
Maybe we could listen to that stream instead so we can index videos as they're discovered.
See: https://stackoverflow.com/a/49061086/5665799
"""
@impl BackendCommandRunner
def run(url, command_opts, output_template) do
command = backend_executable()
# These must stay in exactly this order, hence why I'm giving it its own variable.
# Also, can't use RAM file since yt-dlp needs a concrete filepath.
json_output_path = generate_json_output_path()
print_to_file_opts = [{:print_to_file, output_template}, json_output_path]
formatted_command_opts = [url] ++ parse_options(command_opts ++ print_to_file_opts)
Logger.debug("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}")
case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
{_, 0} ->
# IDEA: consider deleting the file after reading it
# (even on error? especially on error?)
File.read(json_output_path)
{output, status} ->
{:error, output, status}
end
end
defp generate_json_output_path do
metadata_directory = Application.get_env(:pinchflat, :metadata_directory)
filepath = Path.join([metadata_directory, "#{StringUtils.random_string(64)}.json"])
# Ensure the file can be created and written to BEFORE we run the `yt-dlp` command
:ok = File.mkdir_p!(Path.dirname(filepath))
:ok = File.write(filepath, "")
filepath
end
# We want to satisfy the following behaviours:
#
# 1. If the key is an atom, convert it to a string and convert it to kebab case (for convenience)
# 2. If the key is a string, assume we want it as-is and don't convert it
# 3. If the key is accompanied by a value, append the value to the list
# 4. If the key is not accompanied by a value, assume it's a flag and PREpend it to the list
defp parse_options(command_opts) do
Enum.reduce(command_opts, [], &parse_option/2)
end
defp parse_option({k, v}, acc) when is_atom(k) do
stringified_key = StringUtils.to_kebab_case(Atom.to_string(k))
parse_option({"--#{stringified_key}", v}, acc)
end
defp parse_option({k, v}, acc) when is_binary(k) do
acc ++ [k, to_string(v)]
end
defp parse_option(arg, acc) when is_atom(arg) do
stringified_arg = StringUtils.to_kebab_case(Atom.to_string(arg))
parse_option("--#{stringified_arg}", acc)
end
defp parse_option(arg, acc) when is_binary(arg) do
acc ++ [arg]
end
defp backend_executable do
Application.get_env(:pinchflat, :yt_dlp_executable)
end
end

27
lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex Normal file
View file

@ -0,0 +1,27 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
@moduledoc """
yt-dlp offers a LOT of metadata in its JSON response, some of which
needs to be extracted and included in various models.
For now, also squirrel all of it away in the `media_metadata` table.
I might revisit this or pare it down later, but I'd rather need it
and not have it, ya know?
"""
@doc """
Parses the given JSON response from yt-dlp and returns a map of
the needful media_item attributes, along with anything needed for
its associations.
Returns map()
"""
def parse_for_media_item(metadata) do
%{
title: metadata["title"],
video_filepath: metadata["filepath"],
metadata: %{
client_response: metadata
}
}
end
end

26
lib/pinchflat/media_client/backends/yt_dlp/video.ex Normal file
View file

@ -0,0 +1,26 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.Video do
@moduledoc """
Contains utilities for working with singular videos
"""
@doc """
Downloads a single video (and possibly its metadata) directly to its
final destination. Returns the parsed JSON output from yt-dlp.
Returns {:ok, map()} | {:error, any, ...}.
"""
def download(url, command_opts \\ []) do
opts = [:no_simulate] ++ command_opts
with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j"),
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
{:ok, parsed_json}
else
err -> err
end
end
defp backend_runner do
Application.get_env(:pinchflat, :yt_dlp_runner)
end
end

28
lib/pinchflat/media_client/backends/yt_dlp/video_collection.ex Normal file
View file

@ -0,0 +1,28 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
@moduledoc """
Contains utilities for working with collections of videos (ie: channels, playlists).
Meant to be included in other modules but can be used on its own. Channels and playlists
will have many of their own methods, but also share a lot of methods. This module is for
those shared methods.
"""
defmacro __using__(_) do
quote do
@doc """
Returns a list of strings representing the video ids in the collection.
Returns {:ok, [binary()]} | {:error, any, ...}.
"""
def get_video_ids(url, command_opts \\ []) do
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
opts = command_opts ++ [:simulate, :skip_download]
case runner.run(url, opts, "%(id)s") do
{:ok, output} -> {:ok, String.split(output, "\n", trim: true)}
res -> res
end
end
end
end
end

41
lib/pinchflat/media_client/channel_details.ex Normal file
View file

@ -0,0 +1,41 @@
defmodule Pinchflat.MediaClient.ChannelDetails do
@moduledoc """
This is the integration layer for actually working with channels.
Technically hardcodes the yt-dlp backend for now, but should leave
it open-ish for future expansion (just in case).
"""
@enforce_keys [:id, :name]
defstruct [:id, :name]
alias Pinchflat.MediaClient.Backends.YtDlp.Channel, as: YtDlpChannel
@doc false
def new(id, name) do
%__MODULE__{id: id, name: name}
end
@doc """
Gets a channel's ID and name from its URL, using the given backend.
Returns {:ok, map()} | {:error, any, ...}.
"""
def get_channel_details(channel_url, backend \\ :yt_dlp) do
channel_module(backend).get_channel_details(channel_url)
end
@doc """
Returns a list of video IDs for the given channel URL, using the given backend.
Returns {:ok, list(binary())} | {:error, any, ...}.
"""
def get_video_ids(channel_url, backend \\ :yt_dlp) do
channel_module(backend).get_video_ids(channel_url)
end
defp channel_module(backend) do
case backend do
:yt_dlp -> YtDlpChannel
end
end
end

70
lib/pinchflat/media_client/video_downloader.ex Normal file
View file

@ -0,0 +1,70 @@
defmodule Pinchflat.MediaClient.VideoDownloader do
@moduledoc """
This is the integration layer for actually downloading videos.
It takes into account the media profile's settings in order
to download the video with the desired options.
Technically hardcodes the yt-dlp backend for now, but should leave
it open-ish for future expansion (just in case).
"""
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Media.MediaItem
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo
alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder, as: YtDlpOptionBuilder
alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
@doc """
Downloads a video for a media item, updating the media item based on the metadata
returned by the backend. Also saves the entire metadata response to the associated
media_metadata record.
Returns {:ok, %MediaItem{}} | {:error, any, ...any}
"""
def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do
item_with_preloads = Repo.preload(media_item, [:metadata, channel: :media_profile])
media_profile = item_with_preloads.channel.media_profile
case download_for_media_profile(media_item.media_id, media_profile, backend) do
{:ok, parsed_json} ->
parser = metadata_parser(backend)
parsed_attrs = parser.parse_for_media_item(parsed_json)
# Don't forgor to use preloaded associations or updates to
# associations won't work!
Media.update_media_item(item_with_preloads, parsed_attrs)
err ->
err
end
end
defp download_for_media_profile(url, %MediaProfile{} = media_profile, backend) do
option_builder = option_builder(backend)
video_backend = video_backend(backend)
{:ok, options} = option_builder.build(media_profile)
video_backend.download(url, options)
end
defp option_builder(backend) do
case backend do
:yt_dlp -> YtDlpOptionBuilder
end
end
defp video_backend(backend) do
case backend do
:yt_dlp -> YtDlpVideo
end
end
defp metadata_parser(backend) do
case backend do
:yt_dlp -> YtDlpMetadataParser
end
end
end

163
lib/pinchflat/media_source.ex Normal file
View file

@ -0,0 +1,163 @@
defmodule Pinchflat.MediaSource do
@moduledoc """
The MediaSource context.
"""
import Ecto.Query, warn: false
alias Pinchflat.Repo
alias Pinchflat.Tasks
alias Pinchflat.Media
alias Pinchflat.Tasks.ChannelTasks
alias Pinchflat.MediaSource.Channel
alias Pinchflat.MediaClient.ChannelDetails
@doc """
Returns the list of channels. Returns [%Channel{}, ...]
"""
def list_channels do
Repo.all(Channel)
end
@doc """
Gets a single channel.
Returns %Channel{}. Raises `Ecto.NoResultsError` if the Channel does not exist.
"""
def get_channel!(id), do: Repo.get!(Channel, id)
@doc """
Creates a channel. May attempt to pull additional channel details from the
original_url (if provided). Will attempt to start indexing the channel's
media if successfully inserted.
Returns {:ok, %Channel{}} | {:error, %Ecto.Changeset{}}
"""
def create_channel(attrs) do
%Channel{}
|> change_channel_from_url(attrs)
|> commit_and_start_indexing()
end
@doc """
Given a media source, creates (indexes) the media by creating media_items for each
media ID in the source.
Returns [%MediaItem{}, ...] | [%Ecto.Changeset{}, ...]
"""
def index_media_items(%Channel{} = channel) do
{:ok, media_ids} = ChannelDetails.get_video_ids(channel.original_url)
media_ids
|> Enum.map(fn media_id ->
attrs = %{channel_id: channel.id, media_id: media_id}
case Media.create_media_item(attrs) do
{:ok, media_item} -> media_item
{:error, changeset} -> changeset
end
end)
end
@doc """
Updates a channel. May attempt to pull additional channel details from the
original_url (if changed). May attempt to start indexing the channel's
media if the indexing frequency has been changed.
Existing indexing tasks will be cancelled if the indexing frequency has been
changed (logic in `ChannelTasks.kickoff_indexing_task`)
Returns {:ok, %Channel{}} | {:error, %Ecto.Changeset{}}
"""
def update_channel(%Channel{} = channel, attrs) do
channel
|> change_channel_from_url(attrs)
|> commit_and_start_indexing()
end
@doc """
Deletes a channel and it's associated tasks (of any state).
Returns {:ok, %Channel{}} | {:error, %Ecto.Changeset{}}
"""
def delete_channel(%Channel{} = channel) do
Tasks.delete_tasks_for(channel)
Repo.delete(channel)
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking channel changes.
"""
def change_channel(%Channel{} = channel, attrs \\ %{}) do
Channel.changeset(channel, attrs)
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking channel changes and additionally
fetches channel details from the original_url (if provided). If the channel
details cannot be fetched, an error is added to the changeset.
Note that this fetches channel details as long as the `original_url` is present.
This means that it'll go for it even if a changeset is otherwise invalid. This
is pretty easy to change, but for MVP I'm not concerned.
"""
def change_channel_from_url(%Channel{} = channel, attrs) do
case change_channel(channel, attrs) do
%Ecto.Changeset{changes: %{original_url: _}} = changeset ->
add_channel_details_to_changeset(channel, changeset)
changeset ->
changeset
end
end
defp add_channel_details_to_changeset(channel, changeset) do
%Ecto.Changeset{changes: changes} = changeset
case ChannelDetails.get_channel_details(changes.original_url) do
{:ok, %ChannelDetails{} = channel_details} ->
change_channel(
channel,
Map.merge(changes, %{
name: channel_details.name,
channel_id: channel_details.id
})
)
{:error, runner_error, _status_code} ->
Ecto.Changeset.add_error(
changeset,
:original_url,
"could not fetch channel details from URL",
error: runner_error
)
end
end
defp commit_and_start_indexing(changeset) do
case Repo.insert_or_update(changeset) do
{:ok, %Channel{} = channel} ->
maybe_run_indexing_task(changeset, channel)
{:ok, channel}
err ->
err
end
end
defp maybe_run_indexing_task(changeset, channel) do
case changeset.data do
# If the changeset is new (not persisted), attempt indexing no matter what
%{__meta__: %{state: :built}} ->
ChannelTasks.kickoff_indexing_task(channel)
# If the record has been persisted, only attempt indexing if the
# indexing frequency has been changed
%{__meta__: %{state: :loaded}} ->
if Map.has_key?(changeset.changes, :index_frequency_minutes) do
ChannelTasks.kickoff_indexing_task(channel)
end
end
end
end

37
lib/pinchflat/media_source/channel.ex Normal file
View file

@ -0,0 +1,37 @@
defmodule Pinchflat.MediaSource.Channel do
@moduledoc """
The Channel schema.
"""
use Ecto.Schema
import Ecto.Changeset
alias Pinchflat.Media.MediaItem
alias Pinchflat.Profiles.MediaProfile
@allowed_fields ~w(name channel_id index_frequency_minutes original_url media_profile_id)a
@required_fields @allowed_fields -- ~w(index_frequency_minutes)a
schema "channels" do
field :name, :string
field :channel_id, :string
field :index_frequency_minutes, :integer
# This should only be used for user reference going forward
# as the channel_id should be used for all API calls
field :original_url, :string
belongs_to :media_profile, MediaProfile
has_many :media_items, MediaItem
timestamps(type: :utc_datetime)
end
@doc false
def changeset(channel, attrs) do
channel
|> cast(attrs, @allowed_fields)
|> validate_required(@required_fields)
|> unique_constraint([:channel_id, :media_profile_id])
end
end

56
lib/pinchflat/profiles.ex Normal file
View file

@ -0,0 +1,56 @@
defmodule Pinchflat.Profiles do
@moduledoc """
The Profiles context.
"""
import Ecto.Query, warn: false
alias Pinchflat.Repo
alias Pinchflat.Profiles.MediaProfile
@doc """
Returns the list of media_profiles. Returns [%MediaProfile{}, ...]
"""
def list_media_profiles do
Repo.all(MediaProfile)
end
@doc """
Gets a single media_profile.
Returns %MediaProfile{}. Raises `Ecto.NoResultsError` if the Media profile does not exist.
"""
def get_media_profile!(id), do: Repo.get!(MediaProfile, id)
@doc """
Creates a media_profile. Returns {:ok, %MediaProfile{}} | {:error, %Ecto.Changeset{}}
"""
def create_media_profile(attrs) do
%MediaProfile{}
|> MediaProfile.changeset(attrs)
|> Repo.insert()
end
@doc """
Updates a media_profile. Returns {:ok, %MediaProfile{}} | {:error, %Ecto.Changeset{}}
"""
def update_media_profile(%MediaProfile{} = media_profile, attrs) do
media_profile
|> MediaProfile.changeset(attrs)
|> Repo.update()
end
@doc """
Deletes a media_profile. Returns {:ok, %MediaProfile{}} | {:error, %Ecto.Changeset{}}
"""
def delete_media_profile(%MediaProfile{} = media_profile) do
Repo.delete(media_profile)
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking media_profile changes.
"""
def change_media_profile(%MediaProfile{} = media_profile, attrs \\ %{}) do
MediaProfile.changeset(media_profile, attrs)
end
end

27
lib/pinchflat/profiles/media_profile.ex Normal file
View file

@ -0,0 +1,27 @@
defmodule Pinchflat.Profiles.MediaProfile do
@moduledoc """
A media profile is a set of settings that can be applied to many media sources
"""
use Ecto.Schema
import Ecto.Changeset
alias Pinchflat.MediaSource.Channel
schema "media_profiles" do
field :name, :string
field :output_path_template, :string
has_many :channels, Channel
timestamps(type: :utc_datetime)
end
@doc false
def changeset(media_profile, attrs) do
media_profile
|> cast(attrs, [:name, :output_path_template])
|> validate_required([:name, :output_path_template])
|> unique_constraint(:name)
end
end

39
lib/pinchflat/profiles/options/yt_dlp/option_builder.ex Normal file
View file

@ -0,0 +1,39 @@
defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
@moduledoc """
Builds the options for yt-dlp based on the given media profile.
IDEA: consider making this a behaviour so I can add other backends later
"""
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
@doc """
Builds the options for yt-dlp based on the given media profile.
IDEA: consider adding the ability to pass in a second argument to override
these options
"""
def build(%MediaProfile{} = media_profile) do
{:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template)
# NOTE: I'll be hardcoding most things for now (esp. options to help me test) -
# add more configuration later as I build out the models. Walk before you can run!
# NOTE: Looks like you can put different media types in different directories.
# see: https://github.com/yt-dlp/yt-dlp#output-template
{:ok,
[
:embed_metadata,
:embed_thumbnail,
:embed_subs,
:no_progress,
sub_langs: "en.*",
output: Path.join(base_directory(), output_path)
]}
end
defp base_directory do
Application.get_env(:pinchflat, :media_directory)
end
end

64
lib/pinchflat/profiles/options/yt_dlp/output_path_builder.ex Normal file
View file

@ -0,0 +1,64 @@
defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
@moduledoc """
Builds yt-dlp-friendly output paths for downloaded media
IDEA: consider making this a behaviour so I can add other backends later
"""
alias Pinchflat.RenderedString.Parser, as: TemplateParser
@doc """
Builds the actual final filepath from a given template.
Translates liquid-style templates into yt-dlp-style templates,
leaving yt-dlp syntax intact.
"""
def build(template_string) do
TemplateParser.parse(template_string, full_yt_dlp_options_map())
end
defp full_yt_dlp_options_map do
Map.merge(
standard_yt_dlp_option_map(),
custom_yt_dlp_option_map()
)
end
defp standard_yt_dlp_option_map do
%{
# Uppercase "S" means "safe" - ie: filepath-safe
"id" => "%(id)S",
"ext" => "%(ext)S",
"title" => "%(title)S",
"fulltitle" => "%(fulltitle)S",
"uploader" => "%(uploader)S",
"creator" => "%(creator)S",
"upload_date" => "%(upload_date)S",
"release_date" => "%(release_date)S",
"duration" => "%(duration)S",
# For videos classified as an episode of a series:
"series" => "%(series)S",
"season" => "%(season)S",
"season_number" => "%(season_number)S",
"episode" => "%(episode)S",
"episode_number" => "%(episode_number)S",
"episode_id" => "%(episode_id)S",
# For videos classified as music:
"track" => "%(track)S",
"track_number" => "%(track_number)S",
"artist" => "%(artist)S",
"album" => "%(album)S",
"album_type" => "%(album_type)S",
"genre" => "%(genre)S"
}
end
defp custom_yt_dlp_option_map do
%{
# Individual parts of the upload date
"upload_year" => "%(upload_date>%Y)S",
"upload_month" => "%(upload_date>%m)S",
"upload_day" => "%(upload_date>%d)S"
}
end
end

52
lib/pinchflat/rendered_string/base.ex Normal file
View file

@ -0,0 +1,52 @@
defmodule Pinchflat.RenderedString.Base do
@moduledoc """
A base module for parsing rendered strings, designed as a macro to be used
in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
NOTE: if the needs here get any more complicated, look into using a Liquid
template parser. No need to reinvent the wheel any more than I already have.
NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
"""
defmacro __using__(_opts) do
quote location: :keep do
import NimbleParsec
opening_tag = string("{{")
closing_tag = string("}}")
optional_whitespaces = ascii_string(~c[ \t\n\r], min: 0)
# Capture everything up to the opening object
text =
lookahead_not(opening_tag)
# ... as long as it's a character
|> utf8_char([])
# ... and there's at least one character
|> times(min: 1)
# ... and then convert it to a string
|> reduce({List, :to_string, []})
# ... finally bag it and tag it
|> unwrap_and_tag(:text)
identifier =
utf8_string([?a..?z, ?A..?Z, ?_, ?0..?9], min: 1)
|> reduce({Enum, :join, []})
|> unwrap_and_tag(:identifier)
defparsecp(:expression, identifier)
# when spotting interpolation, ignore the opening tag and any whitespace
interpolation =
ignore(concat(opening_tag, optional_whitespaces))
# ... then parse the expression (identifier)
|> parsec(:expression)
# ... then ignore any whitespace and the closing tags after the expression
|> ignore(concat(optional_whitespaces, closing_tag))
# ... once again we bag it and tag it
|> unwrap_and_tag(:interpolation)
defparsec(:do_parse, choice([interpolation, text]) |> repeat() |> eos())
end
end
end

35
lib/pinchflat/rendered_string/parser.ex Normal file
View file

@ -0,0 +1,35 @@
defmodule Pinchflat.RenderedString.Parser do
@moduledoc """
Parses liquid-ish-style strings into a rendered string
Used for turning filepath templates into real filepaths
"""
use Pinchflat.RenderedString.Base
@doc """
Parses a string into a rendered string, using the provided variables.
Variable identifiers are surrounded by {{ and }}. The variable keys MUST be strings.
If an identifier is not found in the provided variables, it will be removed from the string.
"""
def parse(string, variables) do
# `do_parse` comes from `RenderedString.Base`
case do_parse(string) do
{:ok, parsed, _, _, _, _} ->
{:ok, build_string(parsed, variables)}
{:error, message, _, _, _, _} ->
{:error, message}
end
end
defp build_string(parsed, variables) do
Enum.reduce(parsed, "", fn element, acc ->
case element do
{:text, text} -> acc <> text
{:interpolation, {:identifier, identifier}} -> acc <> to_string(variables[identifier])
end
end)
end
end

14
lib/pinchflat/repo.ex
View file

@ -2,4 +2,18 @@ defmodule Pinchflat.Repo do
use Ecto.Repo,
otp_app: :pinchflat,
adapter: Ecto.Adapters.Postgres
@doc """
It's not immediately obvious if an Oban job qualifies as unique, so this method
attempts creating a job and checks for the `conflict?` field in the returned job.
Returns {:ok, %Oban.Job{}} | {:duplicate, %Oban.Job{}} | {:error, any()}.
"""
def insert_unique_job(job_struct) do
case Oban.insert(job_struct) do
{:ok, %Oban.Job{conflict?: false} = job} -> {:ok, job}
{:ok, %Oban.Job{conflict?: true} = job} -> {:duplicate, job}
err -> err
end
end
end

148
lib/pinchflat/tasks.ex Normal file
View file

@ -0,0 +1,148 @@
defmodule Pinchflat.Tasks do
@moduledoc """
The Tasks context.
"""
import Ecto.Query, warn: false
alias Pinchflat.Repo
alias Pinchflat.Tasks.Task
alias Pinchflat.Media.MediaItem
alias Pinchflat.MediaSource.Channel
@doc """
Returns the list of tasks. Returns [%Task{}, ...]
"""
def list_tasks do
Repo.all(Task)
end
@doc """
Returns the list of tasks for a given record type and ID. Optionally allows you to specify
which job states to include.
Returns [%Task{}, ...]
"""
def list_tasks_for(attached_record_type, attached_record_id, job_states \\ Oban.Job.states()) do
stringified_states = Enum.map(job_states, &to_string/1)
Repo.all(
from t in Task,
join: j in assoc(t, :job),
where: field(t, ^attached_record_type) == ^attached_record_id,
where: j.state in ^stringified_states
)
end
@doc """
Returns the list of pending tasks for a given record type and ID.
Returns [%Task{}, ...]
"""
def list_pending_tasks_for(attached_record_type, attached_record_id) do
list_tasks_for(
attached_record_type,
attached_record_id,
[:available, :scheduled, :retryable]
)
end
@doc """
Gets a single task.
Returns %Task{}. Raises `Ecto.NoResultsError` if the Task does not exist.
"""
def get_task!(id), do: Repo.get!(Task, id)
@doc """
Creates a task.
Accepts map() | %Oban.Job{}, %Channel{} | %Oban.Job{}, %MediaItem{}.
Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}.
"""
def create_task(attrs) do
%Task{}
|> Task.changeset(attrs)
|> Repo.insert()
end
# This function's signature is designed to help simplify
# usage of `create_job_with_task/2`
def create_task(%Oban.Job{} = job, attached_record) do
attached_record_attr =
case attached_record do
%Channel{} = channel -> %{channel_id: channel.id}
%MediaItem{} = media_item -> %{media_item_id: media_item.id}
end
%Task{}
|> Task.changeset(Map.merge(%{job_id: job.id}, attached_record_attr))
|> Repo.insert()
end
@doc """
Creates a job from given attrs, creating a task with an attached record
if successful. Returns an error if the job already exists.
Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}.
"""
def create_job_with_task(job_attrs, task_attached_record) do
case Repo.insert_unique_job(job_attrs) do
{:ok, job} -> create_task(job, task_attached_record)
{:duplicate, _} -> {:error, :duplicate_job}
err -> err
end
end
@doc """
Deletes a task. Also cancels any attached job.
Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}.
"""
def delete_task(%Task{} = task) do
:ok = Oban.cancel_job(task.job_id)
Repo.delete(task)
end
@doc """
Deletes all tasks attached to a given record, cancelling any attached jobs.
Returns :ok
"""
def delete_tasks_for(attached_record) do
tasks =
case attached_record do
%Channel{} = channel -> list_tasks_for(:channel_id, channel.id)
%MediaItem{} = media_item -> list_tasks_for(:media_item_id, media_item.id)
end
Enum.each(tasks, fn task ->
delete_task(task)
end)
end
@doc """
Deletes all _pending_ tasks attached to a given record, cancelling any attached jobs.
Returns :ok
"""
def delete_pending_tasks_for(attached_record) do
tasks =
case attached_record do
%Channel{} = channel -> list_pending_tasks_for(:channel_id, channel.id)
%MediaItem{} = media_item -> list_pending_tasks_for(:media_item_id, media_item.id)
end
Enum.each(tasks, fn task ->
delete_task(task)
end)
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking task changes.
"""
def change_task(%Task{} = task, attrs \\ %{}) do
Task.changeset(task, attrs)
end
end

33
lib/pinchflat/tasks/channel_tasks.ex Normal file
View file

@ -0,0 +1,33 @@
defmodule Pinchflat.Tasks.ChannelTasks do
@moduledoc """
This module contains methods for managing tasks (workers) related to channels.
"""
alias Pinchflat.Tasks
alias Pinchflat.MediaSource.Channel
alias Pinchflat.Workers.MediaIndexingWorker
@doc """
Starts tasks for indexing a channel's media.
Returns {:ok, :should_not_index} | {:ok, %Task{}}.
"""
def kickoff_indexing_task(%Channel{} = channel) do
Tasks.delete_pending_tasks_for(channel)
if channel.index_frequency_minutes <= 0 do
{:ok, :should_not_index}
else
channel
|> Map.take([:id])
# Schedule this one immediately, but future ones will be on an interval
|> MediaIndexingWorker.new()
|> Tasks.create_job_with_task(channel)
|> case do
# This should never return {:error, :duplicate_job} since we just deleted
# any pending tasks. I'm being assertive about it so it's obvious if I'm wrong
{:ok, task} -> {:ok, task}
end
end
end
end

26
lib/pinchflat/tasks/task.ex Normal file
View file

@ -0,0 +1,26 @@
defmodule Pinchflat.Tasks.Task do
@moduledoc """
The Task schema.
"""
use Ecto.Schema
import Ecto.Changeset
alias Pinchflat.Media.MediaItem
alias Pinchflat.MediaSource.Channel
schema "tasks" do
belongs_to :job, Oban.Job
belongs_to :channel, Channel
belongs_to :media_item, MediaItem
timestamps(type: :utc_datetime)
end
@doc false
def changeset(task, attrs) do
task
|> cast(attrs, [:job_id, :channel_id, :media_item_id])
|> validate_required([:job_id])
end
end

13
lib/utils/string_utils.ex → lib/pinchflat/utils/string_utils.ex
View file

@ -5,10 +5,23 @@ defmodule Pinchflat.Utils.StringUtils do
@doc """
Converts a string to kebab-case (ie: `hello world` -> `hello-world`)
Returns binary()
"""
def to_kebab_case(string) do
string
|> String.replace(~r/[\s_]/, "-")
|> String.downcase()
end
@doc """
Returns a random string of the given length. Base 16 encoded, lower case.
Returns binary()
"""
def random_string(length \\ 32) do
:crypto.strong_rand_bytes(length)
|> Base.encode16(case: :lower)
|> String.slice(0..(length - 1))
end
end

80
lib/pinchflat/workers/media_indexing_worker.ex Normal file
View file

@ -0,0 +1,80 @@
defmodule Pinchflat.Workers.MediaIndexingWorker do
@moduledoc false
use Oban.Worker,
queue: :media_indexing,
unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
tags: ["media_source", "media_indexing"]
alias __MODULE__
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.MediaSource
alias Pinchflat.Workers.VideoDownloadWorker
@impl Oban.Worker
@doc """
The ID is that of a channel _record_, not a YouTube channel ID. Indexes
the provided channel, kicks off downloads for each new MediaItem, and
reschedules the job to run again in the future (as determined by the
channel's `index_frequency_minutes` field).
README: Re-scheduling here works a little different than you may expect.
The reschedule time is relative to the time the job has actually _completed_.
This has some benefits but also side effects to be aware of:
- Benefit: No chance for jobs to overlap if a job takes longer than the
scheduled interval. Less likely to hit API rate limits.
- Side effect: Intervals are "soft" and _always_ walk forward. This may cause
user confusion since a 30-minute job scheduled for every hour will
actually run every 1 hour and 30 minutes. The tradeoff of not inundating
the API with requests and also not overlapping jobs is worth it, IMO.
NOTE: Since indexing can take a LONG time, I should check what happens if an
application restart occurs while a job is running. Will the job be lost?
IDEA: Should I use paging and do indexing in chunks? Is that even faster?
Returns :ok | {:ok, %Task{}}
"""
def perform(%Oban.Job{args: %{"id" => channel_id}}) do
channel = MediaSource.get_channel!(channel_id)
if channel.index_frequency_minutes <= 0 do
:ok
else
index_media_and_reschedule(channel)
end
end
defp index_media_and_reschedule(channel) do
MediaSource.index_media_items(channel)
enqueue_video_downloads(channel)
channel
|> Map.take([:id])
|> MediaIndexingWorker.new(schedule_in: channel.index_frequency_minutes * 60)
|> Tasks.create_job_with_task(channel)
|> case do
{:ok, task} -> {:ok, task}
{:error, :duplicate_job} -> {:ok, :job_exists}
end
end
# NOTE: this starts a download for each media item that is pending,
# not just the ones that were indexed in this job run. This should ensure
# that any stragglers are caught if, for some reason, they weren't enqueued
# or somehow got de-queued.
#
# I'm not sure of a case where this would happen, but it's cheap insurance.
defp enqueue_video_downloads(channel) do
channel
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
media_item
|> Map.take([:id])
|> VideoDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end)
end
end

26
lib/pinchflat/workers/video_download_worker.ex Normal file
View file

@ -0,0 +1,26 @@
defmodule Pinchflat.Workers.VideoDownloadWorker do
@moduledoc false
use Oban.Worker,
queue: :media_fetching,
unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
tags: ["media_item", "media_fetching"]
alias Pinchflat.Media
alias Pinchflat.MediaClient.VideoDownloader
@impl Oban.Worker
@doc """
For a given media item, download the video and save the metadata.
Returns {:ok, %MediaItem{}} | {:error, any, ...any}
"""
def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
media_item = Media.get_media_item!(media_item_id)
case VideoDownloader.download_for_media_item(media_item) do
{:ok, _} -> {:ok, media_item}
err -> err
end
end
end

62
lib/pinchflat_web/controllers/media_profiles/media_profile_controller.ex Normal file
View file

@ -0,0 +1,62 @@
defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
use PinchflatWeb, :controller
alias Pinchflat.Profiles
alias Pinchflat.Profiles.MediaProfile
def index(conn, _params) do
media_profiles = Profiles.list_media_profiles()
render(conn, :index, media_profiles: media_profiles)
end
def new(conn, _params) do
changeset = Profiles.change_media_profile(%MediaProfile{})
render(conn, :new, changeset: changeset)
end
def create(conn, %{"media_profile" => media_profile_params}) do
case Profiles.create_media_profile(media_profile_params) do
{:ok, media_profile} ->
conn
|> put_flash(:info, "Media profile created successfully.")
|> redirect(to: ~p"/media_profiles/#{media_profile}")
{:error, %Ecto.Changeset{} = changeset} ->
render(conn, :new, changeset: changeset)
end
end
def show(conn, %{"id" => id}) do
media_profile = Profiles.get_media_profile!(id)
render(conn, :show, media_profile: media_profile)
end
def edit(conn, %{"id" => id}) do
media_profile = Profiles.get_media_profile!(id)
changeset = Profiles.change_media_profile(media_profile)
render(conn, :edit, media_profile: media_profile, changeset: changeset)
end
def update(conn, %{"id" => id, "media_profile" => media_profile_params}) do
media_profile = Profiles.get_media_profile!(id)
case Profiles.update_media_profile(media_profile, media_profile_params) do
{:ok, media_profile} ->
conn
|> put_flash(:info, "Media profile updated successfully.")
|> redirect(to: ~p"/media_profiles/#{media_profile}")
{:error, %Ecto.Changeset{} = changeset} ->
render(conn, :edit, media_profile: media_profile, changeset: changeset)
end
end
def delete(conn, %{"id" => id}) do
media_profile = Profiles.get_media_profile!(id)
{:ok, _media_profile} = Profiles.delete_media_profile(media_profile)
conn
|> put_flash(:info, "Media profile deleted successfully.")
|> redirect(to: ~p"/media_profiles")
end
end

13
lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex Normal file
View file

@ -0,0 +1,13 @@
defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
use PinchflatWeb, :html
embed_templates "media_profile_html/*"
@doc """
Renders a media_profile form.
"""
attr :changeset, Ecto.Changeset, required: true
attr :action, :string, required: true
def media_profile_form(assigns)
end

8
lib/pinchflat_web/controllers/media_profiles/media_profile_html/edit.html.heex Normal file
View file

@ -0,0 +1,8 @@
<.header>
Edit Media profile <%= @media_profile.id %>
<:subtitle>Use this form to manage media_profile records in your database.</:subtitle>
</.header>
<.media_profile_form changeset={@changeset} action={~p"/media_profiles/#{@media_profile}"} />
<.back navigate={~p"/media_profiles"}>Back to media_profiles</.back>

30
lib/pinchflat_web/controllers/media_profiles/media_profile_html/index.html.heex Normal file
View file

@ -0,0 +1,30 @@
<.header>
Listing Media profiles
<:actions>
<.link href={~p"/media_profiles/new"}>
<.button>New Media profile</.button>
</.link>
</:actions>
</.header>
<.table
id="media_profiles"
rows={@media_profiles}
row_click={&JS.navigate(~p"/media_profiles/#{&1}")}
>
<:col :let={media_profile} label="Name"><%= media_profile.name %></:col>
<:col :let={media_profile} label="Output path template">
<%= media_profile.output_path_template %>
</:col>
<:action :let={media_profile}>
<div class="sr-only">
<.link navigate={~p"/media_profiles/#{media_profile}"}>Show</.link>
</div>
<.link navigate={~p"/media_profiles/#{media_profile}/edit"}>Edit</.link>
</:action>
<:action :let={media_profile}>
<.link href={~p"/media_profiles/#{media_profile}"} method="delete" data-confirm="Are you sure?">
Delete
</.link>
</:action>
</.table>

10
lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex Normal file
View file

@ -0,0 +1,10 @@
<.simple_form :let={f} for={@changeset} action={@action}>
<.error :if={@changeset.action}>
Oops, something went wrong! Please check the errors below.
</.error>
<.input field={f[:name]} type="text" label="Name" />
<.input field={f[:output_path_template]} type="text" label="Output path template" />
<:actions>
<.button>Save Media profile</.button>
</:actions>
</.simple_form>

8
lib/pinchflat_web/controllers/media_profiles/media_profile_html/new.html.heex Normal file
View file

@ -0,0 +1,8 @@
<.header>
New Media profile
<:subtitle>Use this form to manage media_profile records in your database.</:subtitle>
</.header>
<.media_profile_form changeset={@changeset} action={~p"/media_profiles"} />
<.back navigate={~p"/media_profiles"}>Back to media_profiles</.back>

16
lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex Normal file
View file

@ -0,0 +1,16 @@
<.header>
Media profile <%= @media_profile.id %>
<:subtitle>This is a media_profile record from your database.</:subtitle>
<:actions>
<.link href={~p"/media_profiles/#{@media_profile}/edit"}>
<.button>Edit media_profile</.button>
</.link>
</:actions>
</.header>
<.list>
<:item title="Name"><%= @media_profile.name %></:item>
<:item title="Output path template"><%= @media_profile.output_path_template %></:item>
</.list>
<.back navigate={~p"/media_profiles"}>Back to media_profiles</.back>

75
lib/pinchflat_web/controllers/media_sources/channel_controller.ex Normal file
View file

@ -0,0 +1,75 @@
defmodule PinchflatWeb.MediaSources.ChannelController do
use PinchflatWeb, :controller
alias Pinchflat.Profiles
alias Pinchflat.MediaSource
alias Pinchflat.MediaSource.Channel
def index(conn, _params) do
channels = MediaSource.list_channels()
render(conn, :index, channels: channels)
end
def new(conn, _params) do
changeset = MediaSource.change_channel(%Channel{})
render(conn, :new, changeset: changeset, media_profiles: media_profiles())
end
def create(conn, %{"channel" => channel_params}) do
case MediaSource.create_channel(channel_params) do
{:ok, channel} ->
conn
|> put_flash(:info, "Channel created successfully.")
|> redirect(to: ~p"/media_sources/channels/#{channel}")
{:error, %Ecto.Changeset{} = changeset} ->
render(conn, :new, changeset: changeset, media_profiles: media_profiles())
end
end
def show(conn, %{"id" => id}) do
channel = MediaSource.get_channel!(id)
render(conn, :show, channel: channel)
end
def edit(conn, %{"id" => id}) do
channel = MediaSource.get_channel!(id)
changeset = MediaSource.change_channel(channel)
render(conn, :edit, channel: channel, changeset: changeset, media_profiles: media_profiles())
end
def update(conn, %{"id" => id, "channel" => channel_params}) do
channel = MediaSource.get_channel!(id)
case MediaSource.update_channel(channel, channel_params) do
{:ok, channel} ->
conn
|> put_flash(:info, "Channel updated successfully.")
|> redirect(to: ~p"/media_sources/channels/#{channel}")
{:error, %Ecto.Changeset{} = changeset} ->
render(conn, :edit,
channel: channel,
changeset: changeset,
media_profiles: media_profiles()
)
end
end
def delete(conn, %{"id" => id}) do
channel = MediaSource.get_channel!(id)
{:ok, _channel} = MediaSource.delete_channel(channel)
conn
|> put_flash(:info, "Channel deleted successfully.")
|> redirect(to: ~p"/media_sources/channels")
end
defp media_profiles do
Profiles.list_media_profiles()
end
end

27
lib/pinchflat_web/controllers/media_sources/channel_html.ex Normal file
View file

@ -0,0 +1,27 @@
defmodule PinchflatWeb.MediaSources.ChannelHTML do
use PinchflatWeb, :html
embed_templates "channel_html/*"
@doc """
Renders a channel form.
"""
attr :changeset, Ecto.Changeset, required: true
attr :action, :string, required: true
attr :media_profiles, :list, required: true
def channel_form(assigns)
def friendly_index_frequencies do
[
{"Never", -1},
{"1 Hour", 60},
{"3 Hours", 3 * 60},
{"6 Hours", 6 * 60},
{"12 Hours", 12 * 60},
{"Daily (recommended)", 24 * 60},
{"Weekly", 7 * 24 * 60},
{"Monthly", 30 * 24 * 60}
]
end
end

25
lib/pinchflat_web/controllers/media_sources/channel_html/channel_form.html.heex Normal file
View file

@ -0,0 +1,25 @@
<.simple_form :let={f} for={@changeset} action={@action}>
<.error :if={@changeset.action}>
Oops, something went wrong! Please check the errors below.
</.error>
<.input
field={f[:media_profile_id]}
options={Enum.map(@media_profiles, &{&1.name, &1.id})}
type="select"
label="Media Profile"
/>
<.input field={f[:original_url]} type="text" label="Channel URL" />
<.input
field={f[:index_frequency_minutes]}
options={friendly_index_frequencies()}
type="select"
label="Index Frequency"
/>
<:actions>
<.button>Save Channel</.button>
</:actions>
</.simple_form>

12
lib/pinchflat_web/controllers/media_sources/channel_html/edit.html.heex Normal file
View file

@ -0,0 +1,12 @@
<.header>
Edit Channel <%= @channel.id %>
<:subtitle>Use this form to manage channel records in your database.</:subtitle>
</.header>
<.channel_form
changeset={@changeset}
media_profiles={@media_profiles}
action={~p"/media_sources/channels/#{@channel}"}
/>
<.back navigate={~p"/media_sources/channels"}>Back to channels</.back>

28
lib/pinchflat_web/controllers/media_sources/channel_html/index.html.heex Normal file
View file

@ -0,0 +1,28 @@
<.header>
Listing Channels
<:actions>
<.link href={~p"/media_sources/channels/new"}>
<.button>New Channel</.button>
</.link>
</:actions>
</.header>
<.table id="channels" rows={@channels} row_click={&JS.navigate(~p"/media_sources/channels/#{&1}")}>
<:col :let={channel} label="Name"><%= channel.name %></:col>
<:col :let={channel} label="Channel"><%= channel.channel_id %></:col>
<:action :let={channel}>
<div class="sr-only">
<.link navigate={~p"/media_sources/channels/#{channel}"}>Show</.link>
</div>
<.link navigate={~p"/media_sources/channels/#{channel}/edit"}>Edit</.link>
</:action>
<:action :let={channel}>
<.link
href={~p"/media_sources/channels/#{channel}"}
method="delete"
data-confirm="Are you sure?"
>
Delete
</.link>
</:action>
</.table>

12
lib/pinchflat_web/controllers/media_sources/channel_html/new.html.heex Normal file
View file

@ -0,0 +1,12 @@
<.header>
New Channel
<:subtitle>Use this form to manage channel records in your database.</:subtitle>
</.header>
<.channel_form
changeset={@changeset}
media_profiles={@media_profiles}
action={~p"/media_sources/channels"}
/>
<.back navigate={~p"/media_sources/channels"}>Back to channels</.back>

17
lib/pinchflat_web/controllers/media_sources/channel_html/show.html.heex Normal file
View file

@ -0,0 +1,17 @@
<.header>
Channel <%= @channel.id %>
<:subtitle>This is a channel record from your database.</:subtitle>
<:actions>
<.link href={~p"/media_sources/channels/#{@channel}/edit"}>
<.button>Edit channel</.button>
</.link>
</:actions>
</.header>
<.list>
<:item title="Channel Name"><%= @channel.name %></:item>
<:item title="Channel ID"><%= @channel.channel_id %></:item>
<:item title="Original URL"><%= @channel.original_url %></:item>
</.list>
<.back navigate={~p"/media_sources/channels"}>Back to channels</.back>

6
lib/pinchflat_web/router.ex
View file

@ -18,6 +18,12 @@ defmodule PinchflatWeb.Router do
pipe_through :browser
get "/", PageController, :home
resources "/media_profiles", MediaProfiles.MediaProfileController
scope "/media_sources", MediaSources do
resources "/channels", ChannelController
end
end
# Other scopes may use custom stacks.

5
mix.exs
View file

@ -51,8 +51,11 @@ defmodule Pinchflat.MixProject do
{:jason, "~> 1.2"},
{:dns_cluster, "~> 0.1.1"},
{:plug_cowboy, "~> 2.5"},
{:oban, "~> 2.16"},
{:nimble_parsec, "~> 1.4"},
{:mox, "~> 1.0", only: :test},
{:credo, "~> 1.7", only: [:dev, :test], runtime: false}
{:credo, "~> 1.7", only: [:dev, :test], runtime: false},
{:faker, "~> 0.17", only: :test}
]
end

3
mix.lock
View file

@ -12,6 +12,7 @@
"ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"},
"esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"},
"expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"},
"faker": {:hex, :faker, "0.17.0", "671019d0652f63aefd8723b72167ecdb284baf7d47ad3a82a15e9b8a6df5d1fa", [:mix], [], "hexpm", "a7d4ad84a93fd25c5f5303510753789fc2433ff241bf3b4144d3f6f291658a6a"},
"file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
"finch": {:hex, :finch, "0.17.0", "17d06e1d44d891d20dbd437335eebe844e2426a0cd7e3a3e220b461127c73f70", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8d014a661bb6a437263d4b5abf0bcbd3cf0deb26b1e8596f2a271d22e48934c7"},
"floki": {:hex, :floki, "0.35.2", "87f8c75ed8654b9635b311774308b2760b47e9a579dabf2e4d5f1e1d42c39e0b", [:mix], [], "hexpm", "6b05289a8e9eac475f644f09c2e4ba7e19201fd002b89c28c1293e7bd16773d9"},
@ -22,7 +23,9 @@
"mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"},
"mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
"nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
"nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
"nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
"oban": {:hex, :oban, "2.17.3", "ddfd5710aadcd550d2e174c8d73ce5f1865601418cf54a91775f20443fb832b7", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "452eada8bfe0d0fefd0740ab5fa8cf3ef6c375df0b4a3c3805d179022a04738a"},
"phoenix": {:hex, :phoenix, "1.7.10", "02189140a61b2ce85bb633a9b6fd02dff705a5f1596869547aeb2b2b95edd729", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "cf784932e010fd736d656d7fead6a584a4498efefe5b8227e9f383bf15bb79d0"},
"phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"},
"phoenix_html": {:hex, :phoenix_html, "3.3.3", "380b8fb45912b5638d2f1d925a3771b4516b9a78587249cabe394e0a5d579dc9", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "923ebe6fec6e2e3b3e569dfbdc6560de932cd54b000ada0208b5f45024bdd76c"},

2
package.json
View file

@ -1,5 +1,5 @@
{
"description": "Prettier is used for linting of all files so this package has to live in the root of the project. Use the other package.json files for dependencies.",
"description": "Prettier is used for linting of all files so this package has to live in the root of the project. Use the other package.json files for dependencies. Also, look into making this global or something to remove the need for this file.",
"devDependencies": {
"prettier": "3.2.4"
}

14
priv/repo/migrations/20240122030944_create_media_profiles.exs Normal file
View file

@ -0,0 +1,14 @@
defmodule Pinchflat.Repo.Migrations.CreateMediaProfiles do
use Ecto.Migration
def change do
create table(:media_profiles) do
add :name, :string, null: false
add :output_path_template, :string, null: false
timestamps(type: :utc_datetime)
end
create unique_index(:media_profiles, [:name])
end
end

17
priv/repo/migrations/20240123174417_create_channels.exs Normal file
View file

@ -0,0 +1,17 @@
defmodule Pinchflat.Repo.Migrations.CreateChannels do
use Ecto.Migration
def change do
create table(:channels) do
add :name, :string, null: false
add :channel_id, :string, null: false
add :original_url, :string, null: false
add :media_profile_id, references(:media_profiles, on_delete: :restrict), null: false
timestamps(type: :utc_datetime)
end
create index(:channels, [:media_profile_id])
create unique_index(:channels, [:channel_id, :media_profile_id])
end
end

17
priv/repo/migrations/20240125025325_create_media_items.exs Normal file
View file

@ -0,0 +1,17 @@
defmodule Pinchflat.Repo.Migrations.CreateMediaItems do
use Ecto.Migration
def change do
create table(:media_items) do
add :media_id, :string, null: false
add :title, :string
add :video_filepath, :string
add :channel_id, references(:channels, on_delete: :restrict), null: false
timestamps(type: :utc_datetime)
end
create index(:media_items, [:channel_id])
create unique_index(:media_items, [:media_id, :channel_id])
end
end

13
priv/repo/migrations/20240125043813_add_oban_jobs_table.exs Normal file
View file

@ -0,0 +1,13 @@
defmodule Pinchflat.Repo.Migrations.AddObanJobsTable do
use Ecto.Migration
def up do
Oban.Migration.up(version: 11)
end
# We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if
# necessary, regardless of which version we've migrated `up` to.
def down do
Oban.Migration.down(version: 1)
end
end

9
priv/repo/migrations/20240125165519_add_index_frequency_to_channels.exs Normal file
View file

@ -0,0 +1,9 @@
defmodule Pinchflat.Repo.Migrations.AddIndexFrequencyToChannels do
use Ecto.Migration
def change do
alter table(:channels) do
add :index_frequency_minutes, :integer, default: 60 * 24, null: false
end
end
end

16
priv/repo/migrations/20240125212753_create_tasks.exs Normal file
View file

@ -0,0 +1,16 @@
defmodule Pinchflat.Repo.Migrations.CreateTasks do
use Ecto.Migration
def change do
create table(:tasks) do
add :job_id, references(:oban_jobs, on_delete: :delete_all), null: false
# `restrict` because we need to be sure to delete pending tasks when a channel is deleted
add :channel_id, references(:channels, on_delete: :restrict), null: true
timestamps(type: :utc_datetime)
end
create index(:tasks, [:job_id])
create index(:tasks, [:channel_id])
end
end

15
priv/repo/migrations/20240129015810_create_media_metadata.exs Normal file
View file

@ -0,0 +1,15 @@
defmodule Pinchflat.Repo.Migrations.CreateMediaMetadata do
use Ecto.Migration
def change do
create table(:media_metadata) do
add :client_response, :jsonb, null: false
add :media_item_id, references(:media_items, on_delete: :delete_all), null: false
timestamps(type: :utc_datetime)
end
create unique_index(:media_metadata, [:media_item_id])
create index(:media_metadata, [:client_response], using: :gin)
end
end

12
priv/repo/migrations/20240130161316_add_media_item_to_tasks.exs Normal file
View file

@ -0,0 +1,12 @@
defmodule Pinchflat.Repo.Migrations.AddMediaItemToTasks do
use Ecto.Migration
def change do
alter table(:tasks) do
# `restrict` because we need to be sure to delete pending tasks when a channel is deleted
add :media_item_id, references(:media_items, on_delete: :restrict), null: true
end
create index(:tasks, [:media_item_id])
end
end

44
test/pinchflat/downloader_backends/yt_dlp/video_collection_test.exs
View file

@ -1,44 +0,0 @@
defmodule Pinchflat.DownloaderBackends.YtDlp.VideoCollectionTest do
use ExUnit.Case, async: true
import Mox
alias Pinchflat.DownloaderBackends.YtDlp.VideoCollection, as: VideoCollection
@channel_url "https://www.youtube.com/@TheUselessTrials"
setup :verify_on_exit!
describe "get_video_ids/2" do
test "returns a list of video ids with no blank elements" do
expect(CommandRunnerMock, :run, fn _url, _opts -> {:ok, "id1\nid2\n\nid3\n"} end)
assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url)
end
test "it passes the expected default args" do
expect(CommandRunnerMock, :run, fn _url, opts ->
assert opts == [:simulate, :skip_download, :get_id]
{:ok, ""}
end)
assert {:ok, _} = VideoCollection.get_video_ids(@channel_url)
end
test "it passes the expected custom args" do
expect(CommandRunnerMock, :run, fn _url, opts ->
assert opts == [:custom_arg, :simulate, :skip_download, :get_id]
{:ok, ""}
end)
assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg])
end
test "returns the error straight through when the command fails" do
expect(CommandRunnerMock, :run, fn _url, _opts -> {:error, "Big issue", 1} end)
assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url)
end
end
end

45
test/pinchflat/media_client/backends/yt_dlp/channel_test.exs Normal file
View file

@ -0,0 +1,45 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.ChannelTest do
use ExUnit.Case, async: true
import Mox
alias Pinchflat.MediaClient.ChannelDetails
alias Pinchflat.MediaClient.Backends.YtDlp.Channel
@channel_url "https://www.youtube.com/c/TheUselessTrials"
setup :verify_on_exit!
describe "get_channel_details/1" do
test "it returns a %ChannelDetails{} with data on success" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, "{\"channel\": \"TheUselessTrials\", \"channel_id\": \"UCQH2\"}"}
end)
assert {:ok, res} = Channel.get_channel_details(@channel_url)
assert %ChannelDetails{id: "UCQH2", name: "TheUselessTrials"} = res
end
test "it passes the expected args to the backend runner" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [playlist_end: 1]
assert ot == "%(.{channel,channel_id})j"
{:ok, "{}"}
end)
assert {:ok, _} = Channel.get_channel_details(@channel_url)
end
test "it returns an error if the runner returns an error" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
assert {:error, "Big issue", 1} = Channel.get_channel_details(@channel_url)
end
test "it returns an error if the output is not JSON" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "Not JSON"} end)
assert {:error, %Jason.DecodeError{}} = Channel.get_channel_details(@channel_url)
end
end
end

31
test/pinchflat/downloader_backends/yt_dlp/command_runner_test.exs → test/pinchflat/media_client/backends/yt_dlp/command_runner_test.exs
View file

@ -1,10 +1,10 @@
defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunnerTest do
defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunnerTest do
use ExUnit.Case, async: true
alias Pinchflat.DownloaderBackends.YtDlp.CommandRunner, as: Runner
alias Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, as: Runner
@original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
@video_url "https://www.youtube.com/watch?v=9bZkp7q19f0"
@video_url "https://www.youtube.com/watch?v=-LHXuyzpex0"
setup do
on_exit(&reset_executable/0)
@ -12,50 +12,49 @@ defmodule Pinchflat.DownloaderBackends.YtDlp.CommandRunnerTest do
describe "run/2" do
test "it returns the output and status when the command succeeds" do
assert {:ok, _output} = Runner.run(@video_url, [])
assert {:ok, _output} = Runner.run(@video_url, [], "")
end
test "it converts symbol k-v arg keys to kebab case" do
assert {:ok, output} = Runner.run(@video_url, buffer_size: 1024)
assert {:ok, output} = Runner.run(@video_url, [buffer_size: 1024], "")
assert String.contains?(output, "--buffer-size 1024")
end
test "it keeps string k-v arg keys untouched" do
assert {:ok, output} = Runner.run(@video_url, [{"--under_score", 1024}])
assert {:ok, output} = Runner.run(@video_url, [{"--under_score", 1024}], "")
assert String.contains?(output, "--under_score 1024")
end
test "it converts symbol arg keys to kebab case" do
assert {:ok, output} = Runner.run(@video_url, [:ignore_errors])
assert {:ok, output} = Runner.run(@video_url, [:ignore_errors], "")
assert String.contains?(output, "--ignore-errors")
end
test "it keeps string arg keys untouched" do
assert {:ok, output} = Runner.run(@video_url, ["-v"])
assert {:ok, output} = Runner.run(@video_url, ["-v"], "")
assert String.contains?(output, "-v")
refute String.contains?(output, "--v")
end
test "it places arg keys (flags) at the beginning of the command" do
assert {:ok, output} =
Runner.run(@video_url, [{"--under_score", 1024}, :ignore_errors])
test "it includes the video url as the first argument" do
assert {:ok, output} = Runner.run(@video_url, [:ignore_errors], "")
assert String.contains?(output, "--ignore-errors --under_score 1024")
assert String.contains?(output, "#{@video_url} --ignore-errors")
end
test "it includes the video url as the last argument" do
assert {:ok, output} = Runner.run(@video_url, [:ignore_errors])
test "it automatically includes the --print-to-file flag" do
assert {:ok, output} = Runner.run(@video_url, [], "%(id)s")
assert String.contains?(output, "--ignore-errors #{@video_url}\n")
assert String.contains?(output, "--print-to-file %(id)s /tmp/")
end
test "it returns the output and status when the command fails" do
wrap_executable("/bin/false", fn ->
assert {:error, "", 1} = Runner.run(@video_url, [])
assert {:error, "", 1} = Runner.run(@video_url, [], "")
end)
end
end

45
test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs Normal file
View file

@ -0,0 +1,45 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
use ExUnit.Case, async: true
alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: Parser
setup do
json_filepath =
Path.join([
File.cwd!(),
"test",
"support",
"files",
"media_metadata.json"
])
{:ok, file_body} = File.read(json_filepath)
{:ok, parsed_json} = Phoenix.json_library().decode(file_body)
{:ok,
%{
metadata: parsed_json
}}
end
describe "parse_for_media_item/1" do
test "it extracts the video filepath", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert String.contains?(result.video_filepath, "bwRHIkYqYJo")
assert String.ends_with?(result.video_filepath, ".mkv")
end
test "it extracts the title", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert result.title == "Trying to Wheelie Without the Rear Brake"
end
test "it returns the metadata as a map", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert result.metadata.client_response == metadata
end
end
end

25
test/pinchflat/media_client/backends/yt_dlp/output_path_builder_test.exs Normal file
View file

@ -0,0 +1,25 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.OutputPathBuilderTest do
use ExUnit.Case, async: true
alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
describe "build/1" do
test "it expands 'standard' curly brace variables in the template" do
assert {:ok, res} = OutputPathBuilder.build("/videos/{{ title }}.{{ ext }}")
assert res == "/videos/%(title)S.%(ext)S"
end
test "it expands 'custom' curly brace variables in the template" do
assert {:ok, res} = OutputPathBuilder.build("/videos/{{ upload_year }}.{{ ext }}")
assert res == "/videos/%(upload_date>%Y)S.%(ext)S"
end
test "it leaves yt-dlp variables alone" do
assert {:ok, res} = OutputPathBuilder.build("/videos/%(title)s.%(ext)s")
assert res == "/videos/%(title)s.%(ext)s"
end
end
end

49
test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs Normal file
View file

@ -0,0 +1,49 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
use ExUnit.Case, async: true
import Mox
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection
@channel_url "https://www.youtube.com/@TheUselessTrials"
defmodule VideoCollectionUser do
use VideoCollection
end
setup :verify_on_exit!
describe "get_video_ids/2" do
test "returns a list of video ids with no blank elements" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "id1\nid2\n\nid3\n"} end)
assert {:ok, ["id1", "id2", "id3"]} = VideoCollectionUser.get_video_ids(@channel_url)
end
test "it passes the expected default args" do
expect(YtDlpRunnerMock, :run, fn _url, opts, ot ->
assert opts == [:simulate, :skip_download]
assert ot == "%(id)s"
{:ok, ""}
end)
assert {:ok, _} = VideoCollectionUser.get_video_ids(@channel_url)
end
test "it passes the expected custom args" do
expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
assert opts == [:custom_arg, :simulate, :skip_download]
{:ok, ""}
end)
assert {:ok, _} = VideoCollectionUser.get_video_ids(@channel_url, [:custom_arg])
end
test "returns the error straight through when the command fails" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
assert {:error, "Big issue", 1} = VideoCollectionUser.get_video_ids(@channel_url)
end
end
end

56
test/pinchflat/media_client/backends/yt_dlp/video_test.exs Normal file
View file

@ -0,0 +1,56 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoTest do
use Pinchflat.DataCase
import Mox
alias Pinchflat.MediaClient.Backends.YtDlp.Video
@video_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
setup :verify_on_exit!
# expect(YtDlpRunnerMock, :run, fn _url, [_, _, json_output_path | _] ->
# copy_metadata(json_output_path)
# {:ok, ""}
# end)
describe "download/2" do
test "it calls the backend runner with the expected arguments" do
expect(YtDlpRunnerMock, :run, fn @video_url, opts, ot ->
assert [:no_simulate] = opts
assert "after_move:%()j" = ot
{:ok, render_metadata(:media_metadata)}
end)
assert {:ok, _} = Video.download(@video_url)
end
test "it passes along additional options" do
expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
assert [:no_simulate, :custom_arg] = opts
{:ok, "{}"}
end)
assert {:ok, _} = Video.download(@video_url, [:custom_arg])
end
test "it parses and returns the generated file as JSON" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
assert {:ok, %{"title" => "Trying to Wheelie Without the Rear Brake"}} =
Video.download(@video_url)
end
test "it returns errors" do
expect(YtDlpRunnerMock, :run, fn _url, _opt, _ot ->
{:error, "something"}
end)
assert {:error, "something"} = Video.download(@video_url)
end
end
end

60
test/pinchflat/media_client/channel_details_test.exs Normal file
View file

@ -0,0 +1,60 @@
defmodule Pinchflat.MediaClient.ChannelDetailsTest do
use ExUnit.Case, async: true
import Mox
alias Pinchflat.MediaClient.ChannelDetails
@channel_url "https://www.youtube.com/c/TheUselessTrials"
setup :verify_on_exit!
describe "new/2" do
test "it returns a struct with the given values" do
assert %ChannelDetails{id: "UCQH2", name: "TheUselessTrials"} =
ChannelDetails.new("UCQH2", "TheUselessTrials")
end
end
describe "get_channel_details/2" do
test "it passes the expected arguments to the backend" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [playlist_end: 1]
assert ot == "%(.{channel,channel_id})j"
{:ok, "{\"channel\": \"TheUselessTrials\", \"channel_id\": \"UCQH2\"}"}
end)
assert {:ok, _} = ChannelDetails.get_channel_details(@channel_url)
end
test "it returns a struct composed of the returned data" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, "{\"channel\": \"TheUselessTrials\", \"channel_id\": \"UCQH2\"}"}
end)
assert {:ok, res} = ChannelDetails.get_channel_details(@channel_url)
assert %ChannelDetails{id: "UCQH2", name: "TheUselessTrials"} = res
end
end
describe "get_video_ids/2" do
test "it passes the expected arguments to the backend" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [:simulate, :skip_download]
assert ot == "%(id)s"
{:ok, ""}
end)
assert {:ok, _} = ChannelDetails.get_video_ids(@channel_url)
end
test "it returns a list of strings" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, "video1\nvideo2\nvideo3"}
end)
assert {:ok, ["video1", "video2", "video3"]} = ChannelDetails.get_video_ids(@channel_url)
end
end
end

61
test/pinchflat/media_client/video_downloader_test.exs Normal file
View file

@ -0,0 +1,61 @@
defmodule Pinchflat.MediaClient.VideoDownloaderTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
alias Pinchflat.MediaClient.VideoDownloader
setup :verify_on_exit!
setup do
media_item =
Repo.preload(
media_item_fixture(%{title: nil, video_filepath: nil}),
[:metadata, channel: :media_profile]
)
{:ok, %{media_item: media_item}}
end
describe "download_for_media_item/3" do
test "it calls the backend runner", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, ot ->
assert ot == "after_move:%()j"
{:ok, render_metadata(:media_metadata)}
end)
assert {:ok, _} = VideoDownloader.download_for_media_item(media_item)
end
test "it writes attributes to the media item", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
assert %{video_filepath: nil, title: nil} = media_item
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert updated_media_item.video_filepath
assert updated_media_item.title
end
test "it saves the metadata to the database", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
assert is_nil(media_item.metadata)
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert updated_media_item.metadata
assert is_map(updated_media_item.metadata.client_response)
end
test "errors are passed through", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:error, :some_error}
end)
assert {:error, :some_error} = VideoDownloader.download_for_media_item(media_item)
end
end
end

313
test/pinchflat/media_source_test.exs Normal file
View file

@ -0,0 +1,313 @@
defmodule Pinchflat.MediaSourceTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.TasksFixtures
import Pinchflat.ProfilesFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.MediaSource
alias Pinchflat.Media.MediaItem
alias Pinchflat.MediaSource.Channel
alias Pinchflat.Workers.MediaIndexingWorker
@invalid_channel_attrs %{name: nil, channel_id: nil}
setup :verify_on_exit!
describe "list_channels/0" do
test "it returns all channels" do
channel = channel_fixture()
assert MediaSource.list_channels() == [channel]
end
end
describe "get_channel!/1" do
test "it returns the channel with given id" do
channel = channel_fixture()
assert MediaSource.get_channel!(channel.id) == channel
end
end
describe "create_channel/1" do
test "creates a channel and adds name + ID from runner response" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
valid_attrs = %{
media_profile_id: media_profile_fixture().id,
original_url: "https://www.youtube.com/channel/abc123"
}
assert {:ok, %Channel{} = channel} = MediaSource.create_channel(valid_attrs)
assert channel.name == "some name"
assert String.starts_with?(channel.channel_id, "some_channel_id_")
end
test "creation with invalid data returns error changeset" do
assert {:error, %Ecto.Changeset{}} = MediaSource.create_channel(@invalid_channel_attrs)
end
test "creation enforces uniqueness of channel_id scoped to the media_profile" do
expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
{:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_channel_id_12345678"
})}
end)
valid_once_attrs = %{
media_profile_id: media_profile_fixture().id,
original_url: "https://www.youtube.com/channel/abc123"
}
assert {:ok, %Channel{}} = MediaSource.create_channel(valid_once_attrs)
assert {:error, %Ecto.Changeset{}} = MediaSource.create_channel(valid_once_attrs)
end
test "creation lets you duplicate channel_ids as long as the media profile is different" do
expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
{:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_channel_id_12345678"
})}
end)
valid_attrs = %{
name: "some name",
original_url: "https://www.youtube.com/channel/abc123"
}
channel_1_attrs = Map.merge(valid_attrs, %{media_profile_id: media_profile_fixture().id})
channel_2_attrs = Map.merge(valid_attrs, %{media_profile_id: media_profile_fixture().id})
assert {:ok, %Channel{}} = MediaSource.create_channel(channel_1_attrs)
assert {:ok, %Channel{}} = MediaSource.create_channel(channel_2_attrs)
end
test "creation will schedule the indexing task" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
valid_attrs = %{
media_profile_id: media_profile_fixture().id,
original_url: "https://www.youtube.com/channel/abc123"
}
assert {:ok, %Channel{} = channel} = MediaSource.create_channel(valid_attrs)
assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => channel.id})
end
end
describe "index_media_items/1" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2\nvideo3"} end)
{:ok, [channel: channel_fixture()]}
end
test "it creates a media_item record for each media ID returned", %{channel: channel} do
assert media_items = MediaSource.index_media_items(channel)
assert Enum.count(media_items) == 3
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
end
test "it attaches all media_items to the given channel", %{channel: channel} do
channel_id = channel.id
assert media_items = MediaSource.index_media_items(channel)
assert Enum.count(media_items) == 3
assert Enum.all?(media_items, fn %MediaItem{channel_id: ^channel_id} -> true end)
end
test "it won't duplicate media_items based on media_id and channel", %{channel: channel} do
_first_run = MediaSource.index_media_items(channel)
_duplicate_run = MediaSource.index_media_items(channel)
media_items = Repo.preload(channel, :media_items).media_items
assert Enum.count(media_items) == 3
end
test "it can duplicate media_ids for different channels", %{channel: channel} do
other_channel = channel_fixture()
media_items = MediaSource.index_media_items(channel)
media_items_other_channel = MediaSource.index_media_items(other_channel)
assert Enum.count(media_items) == 3
assert Enum.count(media_items_other_channel) == 3
assert Enum.map(media_items, & &1.media_id) ==
Enum.map(media_items_other_channel, & &1.media_id)
end
test "it returns a list of media_items or changesets", %{channel: channel} do
first_run = MediaSource.index_media_items(channel)
duplicate_run = MediaSource.index_media_items(channel)
assert Enum.all?(first_run, fn %MediaItem{} -> true end)
assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
end
end
describe "update_channel/2" do
test "updates with valid data updates the channel" do
channel = channel_fixture()
update_attrs = %{name: "some updated name"}
assert {:ok, %Channel{} = channel} = MediaSource.update_channel(channel, update_attrs)
assert channel.name == "some updated name"
end
test "updating the original_url will re-fetch the channel details" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
channel = channel_fixture()
update_attrs = %{original_url: "https://www.youtube.com/channel/abc123"}
assert {:ok, %Channel{} = channel} = MediaSource.update_channel(channel, update_attrs)
assert channel.name == "some name"
assert String.starts_with?(channel.channel_id, "some_channel_id_")
end
test "not updating the original_url will not re-fetch the channel details" do
expect(YtDlpRunnerMock, :run, 0, &runner_function_mock/3)
channel = channel_fixture()
update_attrs = %{name: "some updated name"}
assert {:ok, %Channel{}} = MediaSource.update_channel(channel, update_attrs)
end
test "updating the index frequency will re-schedule the indexing task" do
channel = channel_fixture()
update_attrs = %{index_frequency_minutes: 123}
assert {:ok, %Channel{} = channel} = MediaSource.update_channel(channel, update_attrs)
assert channel.index_frequency_minutes == 123
assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => channel.id})
end
test "not updating the index frequency will not re-schedule the indexing task" do
channel = channel_fixture()
update_attrs = %{name: "some updated name"}
assert {:ok, %Channel{}} = MediaSource.update_channel(channel, update_attrs)
refute_enqueued(worker: MediaIndexingWorker, args: %{"id" => channel.id})
end
test "updates with invalid data returns error changeset" do
channel = channel_fixture()
assert {:error, %Ecto.Changeset{}} =
MediaSource.update_channel(channel, @invalid_channel_attrs)
assert channel == MediaSource.get_channel!(channel.id)
end
end
describe "delete_channel/1" do
test "it deletes the channel" do
channel = channel_fixture()
assert {:ok, %Channel{}} = MediaSource.delete_channel(channel)
assert_raise Ecto.NoResultsError, fn -> MediaSource.get_channel!(channel.id) end
end
test "it returns a channel changeset" do
channel = channel_fixture()
assert %Ecto.Changeset{} = MediaSource.change_channel(channel)
end
test "deletion also deletes all associated tasks" do
channel = channel_fixture()
task = task_fixture(channel_id: channel.id)
assert {:ok, %Channel{}} = MediaSource.delete_channel(channel)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
describe "change_channel/2" do
test "it returns a changeset" do
channel = channel_fixture()
assert %Ecto.Changeset{} = MediaSource.change_channel(channel)
end
end
describe "change_channel_from_url/2" do
test "it returns a changeset" do
stub(YtDlpRunnerMock, :run, &runner_function_mock/3)
channel = channel_fixture()
assert %Ecto.Changeset{} = MediaSource.change_channel_from_url(channel, %{})
end
test "it does not fetch channel details if the original_url isn't in the changeset" do
expect(YtDlpRunnerMock, :run, 0, &runner_function_mock/3)
changeset = MediaSource.change_channel_from_url(%Channel{}, %{name: "some updated name"})
assert %Ecto.Changeset{} = changeset
end
test "it fetches channel details if the original_url is in the changeset" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
changeset =
MediaSource.change_channel_from_url(%Channel{}, %{
original_url: "https://www.youtube.com/channel/abc123"
})
assert %Ecto.Changeset{} = changeset
end
test "it adds channel details to the changeset, keeping the orignal details" do
expect(YtDlpRunnerMock, :run, &runner_function_mock/3)
media_profile = media_profile_fixture()
media_profile_id = media_profile.id
changeset =
MediaSource.change_channel_from_url(%Channel{}, %{
original_url: "https://www.youtube.com/channel/abc123",
media_profile_id: media_profile.id
})
assert %Ecto.Changeset{} = changeset
assert String.starts_with?(changeset.changes.channel_id, "some_channel_id_")
assert %{
name: "some name",
media_profile_id: ^media_profile_id,
original_url: "https://www.youtube.com/channel/abc123"
} = changeset.changes
end
test "it adds an error to the changeset if the runner fails" do
expect(YtDlpRunnerMock, :run, 1, fn _url, _opts, _ot ->
{:error, "some error", 1}
end)
changeset =
MediaSource.change_channel_from_url(%Channel{}, %{
original_url: "https://www.youtube.com/channel/abc123"
})
assert %Ecto.Changeset{} = changeset
assert errors_on(changeset).original_url == ["could not fetch channel details from URL"]
end
end
defp runner_function_mock(_url, _opts, _ot) do
{
:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_channel_id_#{:rand.uniform(1_000_000)}"
})
}
end
end

126
test/pinchflat/media_test.exs Normal file
View file

@ -0,0 +1,126 @@
defmodule Pinchflat.MediaTest do
use Pinchflat.DataCase
import Pinchflat.TasksFixtures
import Pinchflat.MediaFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.Media
alias Pinchflat.Media.MediaItem
@invalid_attrs %{title: nil, media_id: nil, video_filepath: nil}
describe "schema" do
test "media_metadata is deleted when media_item is deleted" do
media_item = media_item_fixture(%{metadata: %{client_response: %{foo: "bar"}}})
metadata = media_item.metadata
assert {:ok, %MediaItem{}} = Media.delete_media_item(media_item)
assert_raise Ecto.NoResultsError, fn ->
Repo.reload!(metadata)
end
end
end
describe "list_media_items/0" do
test "it returns all media_items" do
media_item = media_item_fixture()
assert Media.list_media_items() == [media_item]
end
end
describe "list_pending_media_items_for/1" do
test "it returns pending media_items for a given channel" do
channel = channel_fixture()
media_item = media_item_fixture(%{channel_id: channel.id, video_filepath: nil})
assert Media.list_pending_media_items_for(channel) == [media_item]
end
test "it does not return media_items with video_filepath" do
channel = channel_fixture()
_media_item =
media_item_fixture(%{
channel_id: channel.id,
video_filepath: "/video/#{Faker.File.file_name(:video)}"
})
assert Media.list_pending_media_items_for(channel) == []
end
end
describe "get_media_item!/1" do
test "it returns the media_item with given id" do
media_item = media_item_fixture()
assert Media.get_media_item!(media_item.id) == media_item
end
end
describe "create_media_item/1" do
test "creating with valid data creates a media_item" do
valid_attrs = %{
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
video_filepath: "/video/#{Faker.File.file_name(:video)}",
channel_id: channel_fixture().id
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
assert media_item.title == valid_attrs.title
assert media_item.media_id == valid_attrs.media_id
assert media_item.video_filepath == valid_attrs.video_filepath
end
test "creating with invalid data returns error changeset" do
assert {:error, %Ecto.Changeset{}} = Media.create_media_item(@invalid_attrs)
end
end
describe "update_media_item/2" do
test "updating with valid data updates the media_item" do
media_item = media_item_fixture()
update_attrs = %{
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
video_filepath: "/video/#{Faker.File.file_name(:video)}",
channel_id: channel_fixture().id
}
assert {:ok, %MediaItem{} = media_item} = Media.update_media_item(media_item, update_attrs)
assert media_item.title == update_attrs.title
assert media_item.media_id == update_attrs.media_id
assert media_item.video_filepath == update_attrs.video_filepath
end
test "updating with invalid data returns error changeset" do
media_item = media_item_fixture()
assert {:error, %Ecto.Changeset{}} = Media.update_media_item(media_item, @invalid_attrs)
assert media_item == Media.get_media_item!(media_item.id)
end
end
describe "delete_media_item/1" do
test "deletion deletes the media_item" do
media_item = media_item_fixture()
assert {:ok, %MediaItem{}} = Media.delete_media_item(media_item)
assert_raise Ecto.NoResultsError, fn -> Media.get_media_item!(media_item.id) end
end
test "it also deletes attached tasks" do
media_item = media_item_fixture()
task = task_fixture(%{media_item_id: media_item.id})
assert {:ok, %MediaItem{}} = Media.delete_media_item(media_item)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
describe "change_media_item/1" do
test "change_media_item/1 returns a media_item changeset" do
media_item = media_item_fixture()
assert %Ecto.Changeset{} = Media.change_media_item(media_item)
end
end
end

18
test/pinchflat/profiles/options/yt_dlp/option_builder_test.exs Normal file
View file

@ -0,0 +1,18 @@
defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilderTest do
use ExUnit.Case, async: true
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder
@media_profile %MediaProfile{
output_path_template: "{{ title }}.%(ext)s"
}
describe "build/1" do
test "it generates an expanded output path based on the given template" do
assert {:ok, res} = OptionBuilder.build(@media_profile)
assert {:output, "/tmp/videos/%(title)S.%(ext)s"} in res
end
end
end

78
test/pinchflat/profiles_test.exs Normal file
View file

@ -0,0 +1,78 @@
defmodule Pinchflat.ProfilesTest do
use Pinchflat.DataCase
alias Pinchflat.Profiles
alias Pinchflat.Profiles.MediaProfile
import Pinchflat.ProfilesFixtures
@invalid_attrs %{name: nil, output_path_template: nil}
describe "list_media_profiles/0" do
test "it returns all media_profiles" do
media_profile = media_profile_fixture()
assert Profiles.list_media_profiles() == [media_profile]
end
end
describe "get_media_profile!/1" do
test "it returns the media_profile with given id" do
media_profile = media_profile_fixture()
assert Profiles.get_media_profile!(media_profile.id) == media_profile
end
end
describe "create_media_profile/1" do
test "creation with valid data creates a media_profile" do
valid_attrs = %{name: "some name", output_path_template: "some output_path_template"}
assert {:ok, %MediaProfile{} = media_profile} = Profiles.create_media_profile(valid_attrs)
assert media_profile.name == "some name"
assert media_profile.output_path_template == "some output_path_template"
end
test "creation with invalid data returns error changeset" do
assert {:error, %Ecto.Changeset{}} = Profiles.create_media_profile(@invalid_attrs)
end
end
describe "update_media_profile/2" do
test "updating with valid data updates the media_profile" do
media_profile = media_profile_fixture()
update_attrs = %{
name: "some updated name",
output_path_template: "some updated output_path_template"
}
assert {:ok, %MediaProfile{} = media_profile} =
Profiles.update_media_profile(media_profile, update_attrs)
assert media_profile.name == "some updated name"
assert media_profile.output_path_template == "some updated output_path_template"
end
test "updating with invalid data returns error changeset" do
media_profile = media_profile_fixture()
assert {:error, %Ecto.Changeset{}} =
Profiles.update_media_profile(media_profile, @invalid_attrs)
assert media_profile == Profiles.get_media_profile!(media_profile.id)
end
end
describe "delete_media_profile/1" do
test "deletion deletes the media_profile" do
media_profile = media_profile_fixture()
assert {:ok, %MediaProfile{}} = Profiles.delete_media_profile(media_profile)
assert_raise Ecto.NoResultsError, fn -> Profiles.get_media_profile!(media_profile.id) end
end
end
describe "change_media_profile/1" do
test "it returns a media_profile changeset" do
media_profile = media_profile_fixture()
assert %Ecto.Changeset{} = Profiles.change_media_profile(media_profile)
end
end
end

40
test/pinchflat/rendered_string/parser_test.exs Normal file
View file

@ -0,0 +1,40 @@
defmodule Pinchflat.RenderedString.ParserTest do
use ExUnit.Case, async: true
alias Pinchflat.RenderedString.Parser
describe "parse/2" do
test "it returns the rendered string when the string is valid" do
assert {:ok, "bar"} = Parser.parse("{{ foo }}", %{"foo" => "bar"})
end
test "it works with filepath-like strings" do
assert {:ok, "bar/baz"} =
Parser.parse("{{ foo }}/{{ bar }}", %{"foo" => "bar", "bar" => "baz"})
end
test "it works when mixing text and variables" do
assert {:ok, "bar text baz"} =
Parser.parse("{{ foo }} text {{ bar }}", %{"foo" => "bar", "bar" => "baz"})
end
test "it removes the placeholder but doesn't blow up when the variable isn't provided" do
assert {:ok, ""} = Parser.parse("{{ foo }}", %{})
end
test "it accepts any number of spaces between open and closing tags" do
assert {:ok, "bar"} = Parser.parse("{{foo}}", %{"foo" => "bar"})
assert {:ok, "bar"} = Parser.parse("{{ foo}}", %{"foo" => "bar"})
assert {:ok, "bar"} = Parser.parse("{{foo }}", %{"foo" => "bar"})
assert {:ok, "bar"} = Parser.parse("{{ foo }}", %{"foo" => "bar"})
end
test "it doesn't interpret single braces as variables" do
assert {:ok, "{foo}"} = Parser.parse("{foo}", %{})
end
test "it returns an error when the string is invalid" do
assert {:error, "expected end of string"} = Parser.parse("{{ 1-1 }", %{})
end
end
end

26
test/pinchflat/repo_test.exs Normal file
View file

@ -0,0 +1,26 @@
defmodule Pinchflat.RepoTest do
use Pinchflat.DataCase
alias Pinchflat.JobFixtures.TestJobWorker
describe "insert_unique_job/1" do
test "returns {:ok, job} if there is no conflict" do
job = TestJobWorker.new(%{})
assert {:ok, %Oban.Job{}} = Pinchflat.Repo.insert_unique_job(job)
end
test "returns {:duplicate, original_job} if there is a conflict" do
job = TestJobWorker.new(%{foo: "bar"}, unique: [period: :infinity])
{:ok, saved_job_1} = Pinchflat.Repo.insert_unique_job(job)
assert {:duplicate, saved_job_2} = Pinchflat.Repo.insert_unique_job(job)
assert saved_job_1.id == saved_job_2.id
end
test "returns the error if there is an error" do
assert {:error, _} = Pinchflat.Repo.insert_unique_job(%Ecto.Changeset{})
end
end
end

45
test/pinchflat/tasks/channel_tasks_test.exs Normal file
View file

@ -0,0 +1,45 @@
defmodule Pinchflat.Tasks.ChannelTasksTest do
use Pinchflat.DataCase
import Pinchflat.TasksFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.Tasks.Task
alias Pinchflat.Tasks.ChannelTasks
alias Pinchflat.Workers.MediaIndexingWorker
describe "kickoff_indexing_task/1" do
test "it does not schedule a job if the interval is <= 0" do
channel = channel_fixture(index_frequency_minutes: -1)
assert {:ok, :should_not_index} = ChannelTasks.kickoff_indexing_task(channel)
refute_enqueued(worker: MediaIndexingWorker, args: %{"id" => channel.id})
end
test "it schedules a job if the interval is > 0" do
channel = channel_fixture(index_frequency_minutes: 1)
assert {:ok, _} = ChannelTasks.kickoff_indexing_task(channel)
assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => channel.id})
end
test "it creates and attaches a task if the interval is > 0" do
channel = channel_fixture(index_frequency_minutes: 1)
assert {:ok, %Task{} = task} = ChannelTasks.kickoff_indexing_task(channel)
assert task.channel_id == channel.id
end
test "it deletes any pending tasks for the channel" do
channel = channel_fixture()
task = task_fixture(channel_id: channel.id)
assert {:ok, _} = ChannelTasks.kickoff_indexing_task(channel)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
end

211
test/pinchflat/tasks_test.exs Normal file
View file

@ -0,0 +1,211 @@
defmodule Pinchflat.TasksTest do
use Pinchflat.DataCase
import Pinchflat.JobFixtures
import Pinchflat.TasksFixtures
import Pinchflat.MediaFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.Tasks
alias Pinchflat.Tasks.Task
alias Pinchflat.JobFixtures.TestJobWorker
@invalid_attrs %{job_id: nil}
describe "schema" do
test "it deletes a task when the job gets deleted" do
task = Repo.preload(task_fixture(), [:job])
{:ok, _} = Repo.delete(task.job)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
test "it does not delete the other record when a job gets deleted" do
task = Repo.preload(task_fixture(), [:channel, :job])
{:ok, _} = Repo.delete(task.job)
assert Repo.reload!(task.channel)
end
end
describe "list_tasks/0" do
test "it returns all tasks" do
task = task_fixture()
assert Tasks.list_tasks() == [task]
end
end
describe "list_tasks_for/3" do
test "it lets you specify which record type/ID to join on" do
task = task_fixture()
assert Tasks.list_tasks_for(:channel_id, task.channel_id) == [task]
end
test "it lets you specify which job states to include" do
task = task_fixture()
assert Tasks.list_tasks_for(:channel_id, task.channel_id, [:available]) == [task]
assert Tasks.list_tasks_for(:channel_id, task.channel_id, [:cancelled]) == []
end
end
describe "list_pending_tasks_for/2" do
test "it lists pending tasks" do
task = task_fixture()
assert Tasks.list_pending_tasks_for(:channel_id, task.channel_id) == [task]
end
test "it does not list non-pending tasks" do
task = Repo.preload(task_fixture(), :job)
:ok = Oban.cancel_job(task.job)
assert Tasks.list_pending_tasks_for(:channel_id, task.channel_id) == []
end
end
describe "get_task!/1" do
test "it returns the task with given id" do
task = task_fixture()
assert Tasks.get_task!(task.id) == task
end
end
describe "create_task/1" do
test "creation with valid data creates a task" do
valid_attrs = %{job_id: job_fixture().id}
assert {:ok, %Task{} = _task} = Tasks.create_task(valid_attrs)
end
test "creation with invalid data returns error changeset" do
assert {:error, %Ecto.Changeset{}} = Tasks.create_task(@invalid_attrs)
end
test "accepts a job and channel" do
job = job_fixture()
channel = channel_fixture()
assert {:ok, %Task{} = task} = Tasks.create_task(job, channel)
assert task.job_id == job.id
assert task.channel_id == channel.id
end
test "accepts a job and media item" do
job = job_fixture()
media_item = media_item_fixture()
assert {:ok, %Task{} = task} = Tasks.create_task(job, media_item)
assert task.job_id == job.id
assert task.media_item_id == media_item.id
end
end
describe "create_job_with_task/2" do
test "it enqueues the given job" do
media_item = media_item_fixture()
refute_enqueued(worker: TestJobWorker)
assert {:ok, %Task{}} = Tasks.create_job_with_task(TestJobWorker.new(%{}), media_item)
assert_enqueued(worker: TestJobWorker)
end
test "it creates a task record if successful" do
channel = channel_fixture()
assert {:ok, %Task{} = task} = Tasks.create_job_with_task(TestJobWorker.new(%{}), channel)
assert task.channel_id == channel.id
end
test "it returns an error if the job already exists" do
channel = channel_fixture()
job = TestJobWorker.new(%{foo: "bar"}, unique: [period: :infinity])
assert {:ok, %Task{}} = Tasks.create_job_with_task(job, channel)
assert {:error, :duplicate_job} = Tasks.create_job_with_task(job, channel)
end
test "it returns an error if the job fails to enqueue" do
channel = channel_fixture()
assert {:error, %Ecto.Changeset{}} = Tasks.create_job_with_task(%Ecto.Changeset{}, channel)
end
end
describe "delete_task/1" do
test "deletion deletes the task" do
task = task_fixture()
assert {:ok, %Task{}} = Tasks.delete_task(task)
assert_raise Ecto.NoResultsError, fn -> Tasks.get_task!(task.id) end
end
test "deletion also cancels the attached job" do
task = Repo.preload(task_fixture(), :job)
assert {:ok, %Task{}} = Tasks.delete_task(task)
job = Repo.reload!(task.job)
assert job.state == "cancelled"
end
end
describe "delete_tasks_for/1" do
test "it deletes tasks attached to a channel" do
channel = channel_fixture()
task = task_fixture(channel_id: channel.id)
assert :ok = Tasks.delete_tasks_for(channel)
assert_raise Ecto.NoResultsError, fn -> Tasks.get_task!(task.id) end
end
test "it deletes the tasks attached to a media_item" do
media_item = media_item_fixture()
task = task_fixture(media_item_id: media_item.id)
assert :ok = Tasks.delete_tasks_for(media_item)
assert_raise Ecto.NoResultsError, fn -> Tasks.get_task!(task.id) end
end
end
describe "delete_pending_tasks_for/1" do
test "it deletes pending tasks attached to a channel" do
channel = channel_fixture()
task = task_fixture(channel_id: channel.id)
assert :ok = Tasks.delete_pending_tasks_for(channel)
assert_raise Ecto.NoResultsError, fn -> Tasks.get_task!(task.id) end
end
test "it does not delete non-pending tasks" do
channel = channel_fixture()
task = Repo.preload(task_fixture(channel_id: channel.id), :job)
:ok = Oban.cancel_job(task.job)
assert :ok = Tasks.delete_pending_tasks_for(channel)
assert Tasks.get_task!(task.id)
end
test "it works on media_items" do
media_item = media_item_fixture()
pending_task = task_fixture(media_item_id: media_item.id)
cancelled_task = Repo.preload(task_fixture(media_item_id: media_item.id), :job)
:ok = Oban.cancel_job(cancelled_task.job)
assert :ok = Tasks.delete_pending_tasks_for(media_item)
assert Tasks.get_task!(cancelled_task.id)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(pending_task) end
end
end
describe "change_task/1" do
test "it returns a task changeset" do
task = task_fixture()
assert %Ecto.Changeset{} = Tasks.change_task(task)
end
end
end

17
test/pinchflat/utils/string_utils_test.exs
View file

@ -1,7 +1,7 @@
defmodule Pinchflat.Utils.StringUtilsTest do
use ExUnit.Case, async: true
alias Pinchflat.Utils.StringUtils, as: StringUtils
alias Pinchflat.Utils.StringUtils
describe "to_kebab_case/1" do
test "converts a space-delimited string to kebab-case" do
@ -12,4 +12,19 @@ defmodule Pinchflat.Utils.StringUtilsTest do
assert StringUtils.to_kebab_case("hello_world") == "hello-world"
end
end
describe "random_string/1" do
test "generates a random string" do
assert is_binary(StringUtils.random_string())
assert StringUtils.random_string() != StringUtils.random_string()
end
test "has a defined default length" do
assert String.length(StringUtils.random_string()) == 32
end
test "can generate a string of a given length" do
assert String.length(StringUtils.random_string(64)) == 64
end
end
end

110
test/pinchflat/workers/media_indexing_worker_test.exs Normal file
View file

@ -0,0 +1,110 @@
defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.Tasks
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.VideoDownloadWorker
setup :verify_on_exit!
describe "perform/1" do
test "it does not do any indexing if the channel shouldn't be indexed" do
expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot -> {:ok, ""} end)
channel = channel_fixture(index_frequency_minutes: -1)
perform_job(MediaIndexingWorker, %{id: channel.id})
end
test "it does not reschedule if the channel shouldn't be indexed" do
expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot -> {:ok, ""} end)
channel = channel_fixture(index_frequency_minutes: -1)
perform_job(MediaIndexingWorker, %{id: channel.id})
refute_enqueued(worker: MediaIndexingWorker, args: %{"id" => channel.id})
end
test "it indexes the channel if it should be indexed" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, ""} end)
channel = channel_fixture(index_frequency_minutes: 10)
perform_job(MediaIndexingWorker, %{id: channel.id})
end
test "it kicks off a download job for each pending media item" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
channel = channel_fixture(index_frequency_minutes: 10)
perform_job(MediaIndexingWorker, %{id: channel.id})
assert [_] = all_enqueued(worker: VideoDownloadWorker)
end
test "it starts a job for any pending media item even if it's from another run" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
channel = channel_fixture(index_frequency_minutes: 10)
media_item_fixture(%{channel_id: channel.id, video_filepath: nil})
perform_job(MediaIndexingWorker, %{id: channel.id})
assert [_, _] = all_enqueued(worker: VideoDownloadWorker)
end
test "it does not kick off a job for media items that could not be saved" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo1"} end)
channel = channel_fixture(index_frequency_minutes: 10)
perform_job(MediaIndexingWorker, %{id: channel.id})
# Only one job should be enqueued, since the second video is a duplicate
assert [_] = all_enqueued(worker: VideoDownloadWorker)
end
test "it reschedules the job based on the index frequency" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, ""} end)
channel = channel_fixture(index_frequency_minutes: 10)
perform_job(MediaIndexingWorker, %{id: channel.id})
assert_enqueued(
worker: MediaIndexingWorker,
args: %{"id" => channel.id},
scheduled_at: now_plus(channel.index_frequency_minutes, :minutes)
)
end
test "it creates a task for the rescheduled job" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, ""} end)
channel = channel_fixture(index_frequency_minutes: 10)
task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
perform_job(MediaIndexingWorker, %{id: channel.id})
end)
end
test "it creates the basic media_item records" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2"} end)
channel = channel_fixture(index_frequency_minutes: 10)
media_item_fetcher = fn ->
channel
|> Repo.preload(:media_items)
|> Map.get(:media_items)
|> Enum.map(fn media_item -> media_item.media_id end)
end
assert_changed([from: [], to: ["video1", "video2"]], media_item_fetcher, fn ->
perform_job(MediaIndexingWorker, %{id: channel.id})
end)
end
end
end

59
test/pinchflat/workers/video_download_worker_test.exs Normal file
View file

@ -0,0 +1,59 @@
defmodule Pinchflat.Workers.VideoDownloadWorkerTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
alias Pinchflat.Workers.VideoDownloadWorker
setup :verify_on_exit!
setup do
media_item =
Repo.preload(
media_item_fixture(%{video_filepath: nil}),
[:metadata, channel: :media_profile]
)
{:ok, %{media_item: media_item}}
end
describe "perform/1" do
test "it saves attributes to the media_item", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
assert media_item.video_filepath == nil
perform_job(VideoDownloadWorker, %{id: media_item.id})
assert Repo.reload(media_item).video_filepath != nil
end
test "it saves the metadata to the media_item", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, render_metadata(:media_metadata)}
end)
assert media_item.metadata == nil
perform_job(VideoDownloadWorker, %{id: media_item.id})
assert Repo.reload(media_item).metadata != nil
end
test "it won't double-schedule downloading jobs", %{media_item: media_item} do
Oban.insert(VideoDownloadWorker.new(%{id: media_item.id}))
Oban.insert(VideoDownloadWorker.new(%{id: media_item.id}))
assert [_] = all_enqueued(worker: VideoDownloadWorker)
end
test "it sets the job to retryable if the download fails", %{media_item: media_item} do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "error"} end)
Oban.Testing.with_testing_mode(:inline, fn ->
{:ok, job} = Oban.insert(VideoDownloadWorker.new(%{id: media_item.id}))
assert job.state == "retryable"
end)
end
end
end

119
test/pinchflat_web/controllers/channel_controller_test.exs Normal file
View file

@ -0,0 +1,119 @@
defmodule PinchflatWeb.ChannelControllerTest do
use PinchflatWeb.ConnCase
import Mox
import Pinchflat.ProfilesFixtures
import Pinchflat.MediaSourceFixtures
setup do
media_profile = media_profile_fixture()
{
:ok,
%{
create_attrs: %{
media_profile_id: media_profile.id,
original_url: "https://www.youtube.com/channel/abc123"
},
update_attrs: %{
original_url: "https://www.youtube.com/channel/321xyz"
},
invalid_attrs: %{original_url: nil, media_profile_id: nil}
}
}
end
setup :verify_on_exit!
describe "index" do
test "lists all channels", %{conn: conn} do
conn = get(conn, ~p"/media_sources/channels")
assert html_response(conn, 200) =~ "Listing Channels"
end
end
describe "new channel" do
test "renders form", %{conn: conn} do
conn = get(conn, ~p"/media_sources/channels/new")
assert html_response(conn, 200) =~ "New Channel"
end
end
describe "create channel" do
test "redirects to show when data is valid", %{conn: conn, create_attrs: create_attrs} do
expect(YtDlpRunnerMock, :run, 1, &runner_function_mock/3)
conn = post(conn, ~p"/media_sources/channels", channel: create_attrs)
assert %{id: id} = redirected_params(conn)
assert redirected_to(conn) == ~p"/media_sources/channels/#{id}"
conn = get(conn, ~p"/media_sources/channels/#{id}")
assert html_response(conn, 200) =~ "Channel #{id}"
end
test "renders errors when data is invalid", %{conn: conn, invalid_attrs: invalid_attrs} do
conn = post(conn, ~p"/media_sources/channels", channel: invalid_attrs)
assert html_response(conn, 200) =~ "New Channel"
end
end
describe "edit channel" do
setup [:create_channel]
test "renders form for editing chosen channel", %{conn: conn, channel: channel} do
conn = get(conn, ~p"/media_sources/channels/#{channel}/edit")
assert html_response(conn, 200) =~ "Edit Channel"
end
end
describe "update channel" do
setup [:create_channel]
test "redirects when data is valid", %{conn: conn, channel: channel, update_attrs: update_attrs} do
expect(YtDlpRunnerMock, :run, 1, &runner_function_mock/3)
conn = put(conn, ~p"/media_sources/channels/#{channel}", channel: update_attrs)
assert redirected_to(conn) == ~p"/media_sources/channels/#{channel}"
conn = get(conn, ~p"/media_sources/channels/#{channel}")
assert html_response(conn, 200) =~ "https://www.youtube.com/channel/321xyz"
end
test "renders errors when data is invalid", %{
conn: conn,
channel: channel,
invalid_attrs: invalid_attrs
} do
conn = put(conn, ~p"/media_sources/channels/#{channel}", channel: invalid_attrs)
assert html_response(conn, 200) =~ "Edit Channel"
end
end
describe "delete channel" do
setup [:create_channel]
test "deletes chosen channel", %{conn: conn, channel: channel} do
conn = delete(conn, ~p"/media_sources/channels/#{channel}")
assert redirected_to(conn) == ~p"/media_sources/channels"
assert_error_sent 404, fn ->
get(conn, ~p"/media_sources/channels/#{channel}")
end
end
end
defp create_channel(_) do
channel = channel_fixture()
%{channel: channel}
end
defp runner_function_mock(_url, _opts, _ot) do
{
:ok,
Phoenix.json_library().encode!(%{
channel: "some name",
channel_id: "some_channel_id_#{:rand.uniform(1_000_000)}"
})
}
end
end

90
test/pinchflat_web/controllers/media_profile_controller_test.exs Normal file
View file

@ -0,0 +1,90 @@
defmodule PinchflatWeb.MediaProfileControllerTest do
use PinchflatWeb.ConnCase
import Pinchflat.ProfilesFixtures
@create_attrs %{name: "some name", output_path_template: "some output_path_template"}
@update_attrs %{
name: "some updated name",
output_path_template: "some updated output_path_template"
}
@invalid_attrs %{name: nil, output_path_template: nil}
describe "index" do
test "lists all media_profiles", %{conn: conn} do
conn = get(conn, ~p"/media_profiles")
assert html_response(conn, 200) =~ "Listing Media profiles"
end
end
describe "new media_profile" do
test "renders form", %{conn: conn} do
conn = get(conn, ~p"/media_profiles/new")
assert html_response(conn, 200) =~ "New Media profile"
end
end
describe "create media_profile" do
test "redirects to show when data is valid", %{conn: conn} do
conn = post(conn, ~p"/media_profiles", media_profile: @create_attrs)
assert %{id: id} = redirected_params(conn)
assert redirected_to(conn) == ~p"/media_profiles/#{id}"
conn = get(conn, ~p"/media_profiles/#{id}")
assert html_response(conn, 200) =~ "Media profile #{id}"
end
test "renders errors when data is invalid", %{conn: conn} do
conn = post(conn, ~p"/media_profiles", media_profile: @invalid_attrs)
assert html_response(conn, 200) =~ "New Media profile"
end
end
describe "edit media_profile" do
setup [:create_media_profile]
test "renders form for editing chosen media_profile", %{
conn: conn,
media_profile: media_profile
} do
conn = get(conn, ~p"/media_profiles/#{media_profile}/edit")
assert html_response(conn, 200) =~ "Edit Media profile"
end
end
describe "update media_profile" do
setup [:create_media_profile]
test "redirects when data is valid", %{conn: conn, media_profile: media_profile} do
conn = put(conn, ~p"/media_profiles/#{media_profile}", media_profile: @update_attrs)
assert redirected_to(conn) == ~p"/media_profiles/#{media_profile}"
conn = get(conn, ~p"/media_profiles/#{media_profile}")
assert html_response(conn, 200) =~ "some updated name"
end
test "renders errors when data is invalid", %{conn: conn, media_profile: media_profile} do
conn = put(conn, ~p"/media_profiles/#{media_profile}", media_profile: @invalid_attrs)
assert html_response(conn, 200) =~ "Edit Media profile"
end
end
describe "delete media_profile" do
setup [:create_media_profile]
test "deletes chosen media_profile", %{conn: conn, media_profile: media_profile} do
conn = delete(conn, ~p"/media_profiles/#{media_profile}")
assert redirected_to(conn) == ~p"/media_profiles"
assert_error_sent 404, fn ->
get(conn, ~p"/media_profiles/#{media_profile}")
end
end
end
defp create_media_profile(_) do
media_profile = media_profile_fixture()
%{media_profile: media_profile}
end
end

1
test/support/conn_case.ex
View file

@ -28,6 +28,7 @@ defmodule PinchflatWeb.ConnCase do
import Plug.Conn
import Phoenix.ConnTest
import PinchflatWeb.ConnCase
import Pinchflat.TestingHelperMethods
end
end

3
test/support/data_case.ex
View file

@ -20,10 +20,13 @@ defmodule Pinchflat.DataCase do
quote do
alias Pinchflat.Repo
use Oban.Testing, repo: Repo
import Ecto
import Ecto.Changeset
import Ecto.Query
import Pinchflat.DataCase
import Pinchflat.TestingHelperMethods
end
end

5772
test/support/files/media_metadata.json Normal file
View file

File diff suppressed because one or more lines are too long

19
test/support/fixtures/job_fixtures.ex Normal file
View file

@ -0,0 +1,19 @@
defmodule Pinchflat.JobFixtures do
@moduledoc false
defmodule TestJobWorker do
@moduledoc false
use Oban.Worker, queue: :default
@impl Oban.Worker
def perform(%Oban.Job{}) do
:ok
end
end
def job_fixture() do
{:ok, job} = Oban.insert(TestJobWorker.new(%{}))
job
end
end

45
test/support/fixtures/media_fixtures.ex Normal file
View file

@ -0,0 +1,45 @@
defmodule Pinchflat.MediaFixtures do
@moduledoc """
This module defines test helpers for creating
entities via the `Pinchflat.Media` context.
"""
alias Pinchflat.MediaSourceFixtures
@doc """
Generate a media_item.
"""
def media_item_fixture(attrs \\ %{}) do
{:ok, media_item} =
attrs
|> Enum.into(%{
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
video_filepath: "/video/#{Faker.File.file_name(:video)}",
channel_id: MediaSourceFixtures.channel_fixture().id
})
|> Pinchflat.Media.create_media_item()
media_item
end
@doc """
Generate a media_item with metadata.
"""
def media_item_with_metadata(attrs \\ %{}) do
json_filepath =
Path.join([
Path.dirname(__ENV__.file),
"support",
"fixtures",
"files",
"media_metadata.json"
])
{:ok, file_body} = File.read(json_filepath)
{:ok, parsed_json} = Phoenix.json_library().decode(file_body)
merged_attrs = Map.merge(attrs, %{metadata: %{client_respinse: parsed_json}})
media_item_fixture(merged_attrs)
end
end

30
test/support/fixtures/media_source_fixtures.ex Normal file
View file

@ -0,0 +1,30 @@
defmodule Pinchflat.MediaSourceFixtures do
@moduledoc """
This module defines test helpers for creating
entities via the `Pinchflat.MediaSource` context.
"""
alias Pinchflat.Repo
alias Pinchflat.ProfilesFixtures
alias Pinchflat.MediaSource.Channel
@doc """
Generate a channel.
"""
def channel_fixture(attrs \\ %{}) do
{:ok, channel} =
%Channel{}
|> Channel.changeset(
Enum.into(attrs, %{
name: "Channel ##{:rand.uniform(1_000_000)}",
channel_id: Base.encode16(:crypto.hash(:md5, "#{:rand.uniform(1_000_000)}")),
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
media_profile_id: ProfilesFixtures.media_profile_fixture().id,
index_frequency_minutes: 60
})
)
|> Repo.insert()
channel
end
end

21
test/support/fixtures/profiles_fixtures.ex Normal file
View file

@ -0,0 +1,21 @@
defmodule Pinchflat.ProfilesFixtures do
@moduledoc """
This module defines test helpers for creating
entities via the `Pinchflat.Profiles` context.
"""
@doc """
Generate a media_profile.
"""
def media_profile_fixture(attrs \\ %{}) do
{:ok, media_profile} =
attrs
|> Enum.into(%{
name: "Media Profile ##{:rand.uniform(1_000_000)}",
output_path_template: "{{title}}.{{ext}}"
})
|> Pinchflat.Profiles.create_media_profile()
media_profile
end
end

24
test/support/fixtures/tasks_fixtures.ex Normal file
View file

@ -0,0 +1,24 @@
defmodule Pinchflat.TasksFixtures do
@moduledoc """
This module defines test helpers for creating
entities via the `Pinchflat.Tasks` context.
"""
alias Pinchflat.JobFixtures
alias Pinchflat.MediaSourceFixtures
@doc """
Generate a task.
"""
def task_fixture(attrs \\ %{}) do
{:ok, task} =
attrs
|> Enum.into(%{
channel_id: MediaSourceFixtures.channel_fixture().id,
job_id: JobFixtures.job_fixture().id
})
|> Pinchflat.Tasks.create_task()
task
end
end

19
test/support/scripts/yt-dlp-mocks/repeater.sh
View file

@ -1,7 +1,16 @@
#!/bin/bash
if [[ "$@" == *"--dump-json"* ]]; then
echo '{ "args": "'$@'"}'
else
echo $@
fi
# Args come in the format of "<unknown number of args> --print-to-file <output template> <file location> <unknown number of args>".
# I need to extract <file location> and write all args to it.
# Extract the file location (in an unknown position BUT it's 2 args after --print-to-file).
for ((i = 1; i <= $#; i++)); do
if [ "${!i}" == "--print-to-file" ]; then
# Extract the file location.
file_location="${@:i+2:1}"
break
fi
done
# Write all args to the file
echo "$@" >"$file_location"

43
test/support/testing_helper_methods.ex Normal file
View file

@ -0,0 +1,43 @@
defmodule Pinchflat.TestingHelperMethods do
@moduledoc false
use ExUnit.CaseTemplate
def now do
DateTime.utc_now()
end
def now_plus(offset, unit) when unit in [:minute, :minutes] do
DateTime.add(now(), offset, :minute)
end
def assert_changed(checker_fun, action_fn) do
before_res = checker_fun.()
action_fn.()
after_res = checker_fun.()
assert before_res != after_res
end
def assert_changed([from: from, to: to], checker_fun, action_fn) do
before_res = checker_fun.()
action_fn.()
after_res = checker_fun.()
assert before_res == from
assert after_res == to
end
def render_metadata(metadata_name) do
json_filepath =
Path.join([
File.cwd!(),
"test",
"support",
"files",
"#{metadata_name}.json"
])
File.read!(json_filepath)
end
end

5
test/test_helper.exs
View file

@ -1,5 +1,6 @@
Mox.defmock(CommandRunnerMock, for: Pinchflat.DownloaderBackends.BackendCommandRunner)
Application.put_env(:pinchflat, :yt_dlp_runner, CommandRunnerMock)
Mox.defmock(YtDlpRunnerMock, for: Pinchflat.MediaClient.Backends.BackendCommandRunner)
Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
ExUnit.start()
Ecto.Adapters.SQL.Sandbox.mode(Pinchflat.Repo, :manual)
Faker.start()