danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor modules into contexts (#78)

Browse source 
* [WIP] break out a few contexts, start refactoring fast index modules

* [WIP] more contexts, this time around slow indexing and downloads

* [WIP] got all tests passing

* [WIP] Added moduledocs

* Built a genserver to rename old jobs on boot

* Added a module naming check; moved things around

* Fixed specs
This commit is contained in:
Kieran 2024-03-12 17:54:55 -07:00 committed by GitHub
parent 7fa5c4da63
commit 33baa99aae
66 changed files with 1092 additions and 568 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.credo.exs
View file

@ -157,7 +157,16 @@
{Credo.Check.Warning.UnusedRegexOperation, []},
{Credo.Check.Warning.UnusedStringOperation, []},
{Credo.Check.Warning.UnusedTupleOperation, []},
{Credo.Check.Warning.WrongTestFileExtension, []}
{Credo.Check.Warning.WrongTestFileExtension, []},
#
## Naming Checks
#
{CredoNaming.Check.Consistency.ModuleFilename,
[
priority: :normal,
excluded_paths: [~r/test\/support/, ~r/priv/, ~r/lib\/pinchflat_web/, ~r/test\/pinchflat_web/]
]}
],
disabled: [
#

13
.iex.exs
View file

@ -4,8 +4,7 @@ alias Pinchflat.Repo
alias Pinchflat.Tasks.Task
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaItem
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Media.MediaMetadata
alias Pinchflat.Metadata.MediaMetadata
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Tasks
@ -14,14 +13,14 @@ alias Pinchflat.Profiles
alias Pinchflat.Sources
alias Pinchflat.Settings
alias Pinchflat.MediaClient.MediaDownloader
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
alias Pinchflat.YtDlp.Backend.MediaCollection, as: YtDlpCollection
alias Pinchflat.Downloading.MediaDownloader
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
alias Pinchflat.YtDlp.MediaCollection, as: YtDlpCollection
alias Pinchflat.Api.YoutubeRss
alias Pinchflat.FastIndexing.YoutubeRss
alias Pinchflat.Metadata.MetadataFileHelpers
alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
alias Pinchflat.SlowIndexing.FileFollowerServer
defmodule IexHelpers do
def playlist_url do

2
config/config.exs
View file

@ -12,7 +12,7 @@ config :pinchflat,
generators: [timestamp_type: :utc_datetime],
# Specifying backend data here makes mocking and local testing SUPER easy
yt_dlp_executable: System.find_executable("yt-dlp"),
yt_dlp_runner: Pinchflat.YtDlp.Backend.CommandRunner,
yt_dlp_runner: Pinchflat.YtDlp.CommandRunner,
media_directory: "/downloads",
# The user may or may not store metadata for their needs, but the app will always store its copy
metadata_directory: "/config/metadata",

401
dump_all.sql Normal file
View file

File diff suppressed because one or more lines are too long

3
lib/pinchflat/application.ex
View file

@ -11,8 +11,9 @@ defmodule Pinchflat.Application do
PinchflatWeb.Telemetry,
Pinchflat.Repo,
# Must be before startup tasks
Pinchflat.Boot.PreJobStartupTasks,
{Oban, Application.fetch_env!(:pinchflat, Oban)},
Pinchflat.StartupTasks,
Pinchflat.Boot.PostJobStartupTasks,
{DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
{Phoenix.PubSub, name: Pinchflat.PubSub},
# Start the Finch HTTP client for sending emails

4
lib/pinchflat/workers/data_backfill_worker.ex → lib/pinchflat/boot/data_backfill_worker.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.DataBackfillWorker do
defmodule Pinchflat.Boot.DataBackfillWorker do
@moduledoc false
use Oban.Worker,
@ -28,7 +28,7 @@ defmodule Pinchflat.Workers.DataBackfillWorker do
"""
def cancel_pending_backfill_jobs do
Oban.Job
|> where(worker: "Pinchflat.Workers.DataBackfillWorker")
|> where(worker: "Pinchflat.Boot.DataBackfillWorker")
|> Oban.cancel_all_jobs()
end

14
lib/pinchflat/startup_tasks.ex → lib/pinchflat/boot/post_job_startup_tasks.ex
View file

@ -1,6 +1,7 @@
defmodule Pinchflat.StartupTasks do
defmodule Pinchflat.Boot.PostJobStartupTasks do
@moduledoc """
This module is responsible for running startup tasks on app boot.
This module is responsible for running startup tasks on app boot
AFTER the job runner has initiallized.
It's a GenServer because that plays REALLY nicely with the existing
Phoenix supervision tree.
@ -11,8 +12,7 @@ defmodule Pinchflat.StartupTasks do
import Ecto.Query, warn: false
alias Pinchflat.Repo
alias Pinchflat.Settings
alias Pinchflat.Workers.DataBackfillWorker
alias Pinchflat.Boot.DataBackfillWorker
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, %{}, opts)
@ -29,17 +29,11 @@ defmodule Pinchflat.StartupTasks do
"""
@impl true
def init(state) do
apply_default_settings()
enqueue_backfill_worker()
{:ok, state}
end
defp apply_default_settings do
Settings.fetch!(:onboarding, true)
Settings.fetch!(:pro_enabled, false)
end
defp enqueue_backfill_worker do
DataBackfillWorker.cancel_pending_backfill_jobs()

73
lib/pinchflat/boot/pre_job_startup_tasks.ex Normal file
View file

@ -0,0 +1,73 @@
defmodule Pinchflat.Boot.PreJobStartupTasks do
@moduledoc """
This module is responsible for running startup tasks on app boot
BEFORE the job runner has initiallized.
It's a GenServer because that plays REALLY nicely with the existing
Phoenix supervision tree.
"""
# restart: :temporary means that this process will never be restarted (ie: will run once and then die)
use GenServer, restart: :temporary
import Ecto.Query, warn: false
require Logger
alias Pinchflat.Repo
alias Pinchflat.Settings
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, %{}, opts)
end
@doc """
Runs application startup tasks.
Any code defined here will run every time the application starts. You must
make sure that the code is idempotent and safe to run multiple times.
This is a good place to set up default settings, create initial records, stuff like that.
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
"""
@impl true
def init(state) do
apply_default_settings()
rename_old_job_workers()
{:ok, state}
end
defp apply_default_settings do
Settings.fetch!(:onboarding, true)
Settings.fetch!(:pro_enabled, false)
end
# As part of a large refactor, I ended up moving a bunch of workers around. This
# is a problem because the workers are stored in the database and the runner
# will try to run the OLD jobs. This is also why these tasks run before the job
# runner starts up.
#
# Can be removed after a few months (created: 2024-03-12)
defp rename_old_job_workers do
# [ [old_name, new_name], ...]
rename_map = [
["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
["Pinchflat.Workers.FilesystemDataWorker", "Pinchflat.Filesystem.FilesystemDataWorker"],
["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]
]
jobs_renamed =
Enum.reduce(rename_map, 0, fn [old_name, new_name], acc ->
{count, _} =
Oban.Job
|> where(worker: ^old_name)
|> Repo.update_all(set: [worker: new_name])
acc + count
end)
Logger.info("Renamed #{jobs_renamed} old job workers")
end
end

4
lib/pinchflat/yt_dlp/download_option_builder.ex → lib/pinchflat/downloading/download_option_builder.ex
View file

@ -1,10 +1,10 @@
defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
defmodule Pinchflat.Downloading.DownloadOptionBuilder do
@moduledoc """
Builds the options for yt-dlp to download media based on the given media profile.
"""
alias Pinchflat.Media.MediaItem
alias Pinchflat.Profiles.OutputPathBuilder
alias Pinchflat.Downloading.OutputPathBuilder
@doc """
Builds the options for yt-dlp to download media based on the given media's profile.

50
lib/pinchflat/downloading/downloading_helpers.ex Normal file
View file

@ -0,0 +1,50 @@
defmodule Pinchflat.Downloading.DownloadingHelpers do
@moduledoc """
Methods for helping download media
Many of these methods are made to be kickoff or be consumed by workers.
"""
require Logger
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.Downloading.MediaDownloadWorker
@doc """
Starts tasks for downloading media for any of a sources _pending_ media items.
Jobs are not enqueued if the source is set to not download media. This will return :ok.
NOTE: this starts a download for each media item that is pending,
not just the ones that were indexed in this job run. This should ensure
that any stragglers are caught if, for some reason, they weren't enqueued
or somehow got de-queued.
Returns :ok
"""
def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end)
end
def enqueue_pending_download_tasks(%Source{download_media: false}) do
:ok
end
@doc """
Deletes ALL pending tasks for a source's media items.
Returns :ok
"""
def dequeue_pending_download_tasks(%Source{} = source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(&Tasks.delete_pending_tasks_for/1)
end
end

6
lib/pinchflat/workers/media_download_worker.ex → lib/pinchflat/downloading/media_download_worker.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.MediaDownloadWorker do
defmodule Pinchflat.Downloading.MediaDownloadWorker do
@moduledoc false
use Oban.Worker,
@ -9,8 +9,8 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.MediaClient.MediaDownloader
alias Pinchflat.Workers.FilesystemDataWorker
alias Pinchflat.Downloading.MediaDownloader
alias Pinchflat.Filesystem.FilesystemDataWorker
@impl Oban.Worker
@doc """

6
lib/pinchflat/yt_dlp/media_downloader.ex → lib/pinchflat/downloading/media_downloader.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.MediaClient.MediaDownloader do
defmodule Pinchflat.Downloading.MediaDownloader do
@moduledoc """
This is the integration layer for actually downloading media.
It takes into account the media profile's settings in order
@ -9,8 +9,8 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
alias Pinchflat.Media
alias Pinchflat.Media.MediaItem
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
alias Pinchflat.Downloading.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers

4
lib/pinchflat/rendered_string/base.ex → lib/pinchflat/downloading/output_path/base.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.RenderedString.Base do
defmodule Pinchflat.Downloading.OutputPath.Base do
@moduledoc """
A base module for parsing rendered strings, designed as a macro to be used
in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
@ -6,7 +6,7 @@ defmodule Pinchflat.RenderedString.Base do
NOTE: if the needs here get any more complicated, look into using a Liquid
template parser. No need to reinvent the wheel any more than I already have.
NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
"""
defmacro __using__(_opts) do

4
lib/pinchflat/rendered_string/parser.ex → lib/pinchflat/downloading/output_path/parser.ex
View file

@ -1,11 +1,11 @@
defmodule Pinchflat.RenderedString.Parser do
defmodule Pinchflat.Downloading.OutputPath.Parser do
@moduledoc """
Parses liquid-ish-style strings into a rendered string
Used for turning filepath templates into real filepaths
"""
use Pinchflat.RenderedString.Base
use Pinchflat.Downloading.OutputPath.Base
@doc """
Parses a string into a rendered string, using the provided variables. Optionally

4
lib/pinchflat/profiles/output_path_builder.ex → lib/pinchflat/downloading/output_path_builder.ex
View file

@ -1,9 +1,9 @@
defmodule Pinchflat.Profiles.OutputPathBuilder do
defmodule Pinchflat.Downloading.OutputPathBuilder do
@moduledoc """
Builds yt-dlp-friendly output paths for downloaded media
"""
alias Pinchflat.RenderedString.Parser, as: TemplateParser
alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
@doc """
Builds the actual final filepath from a given template. Optionally, you can pass in

94
lib/pinchflat/fast_indexing/fast_indexing_helpers.ex Normal file
View file

@ -0,0 +1,94 @@
defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
@moduledoc """
Methods for performing fast indexing tasks and managing the fast indexing process.
Many of these methods are made to be kickoff or be consumed by workers.
"""
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.FastIndexing.YoutubeRss
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
Starts tasks for running a fast indexing task for a source's media
regardless of the source's fast_index state. It's assumed the
caller will check for fast_index.
This is used for running fast index tasks on update. On creation, the
fast index is enqueued after the slow index is complete.
Returns {:ok, %Task{}}.
"""
def kickoff_fast_indexing_task(%Source{} = source) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
%{id: source.id}
# Schedule this one immediately, but future ones will be on an interval
|> FastIndexingWorker.new()
|> Tasks.create_job_with_task(source)
end
@doc """
Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
for any new media items. See comments in `MediaIndexingWorker` for more info on the
order of operations and how this fits into the indexing process.
Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
`kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
function starts individual indexing tasks for each new media item. I think it does
make sense grammatically, but I could see how that's confusing.
Returns :ok
"""
def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
{:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
Enum.each(new_media_ids, fn media_id ->
url = "https://www.youtube.com/watch?v=#{media_id}"
%{id: source.id, media_url: url}
|> MediaIndexingWorker.new()
|> Tasks.create_job_with_task(source)
end)
end
@doc """
Indexes a single media item for a source and enqueues a download job if the
media should be downloaded. This method creates the media item record so it's
the one-stop-shop for adding a media item (and possibly downloading it) just
by a URL and source.
Returns {:ok, media_item} | {:error, any()}
"""
def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
maybe_media_item = create_media_item_from_url(source, url)
case maybe_media_item do
{:ok, media_item} ->
if source.download_media && Media.pending_download?(media_item) do
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end
{:ok, media_item}
err ->
err
end
end
defp create_media_item_from_url(source, url) do
{:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
Media.create_media_item_from_backend_attrs(source, media_attrs)
end
end

6
lib/pinchflat/workers/fast_indexing_worker.ex → lib/pinchflat/fast_indexing/fast_indexing_worker.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.FastIndexingWorker do
defmodule Pinchflat.FastIndexing.FastIndexingWorker do
@moduledoc false
use Oban.Worker,
@ -10,7 +10,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
alias Pinchflat.Tasks
alias Pinchflat.Sources
alias Pinchflat.Sources.Source
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.FastIndexing.FastIndexingHelpers
@impl Oban.Worker
@doc """
@ -24,7 +24,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
source = Sources.get_source!(source_id)
if source.fast_index do
SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
reschedule_indexing(source)
else

8
lib/pinchflat/workers/media_indexing_worker.ex → lib/pinchflat/fast_indexing/media_indexing_worker.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.MediaIndexingWorker do
defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
@moduledoc false
use Oban.Worker,
@ -9,7 +9,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
require Logger
alias Pinchflat.Sources
alias Pinchflat.Tasks.MediaItemTasks
alias Pinchflat.FastIndexing.FastIndexingHelpers
@impl Oban.Worker
@doc """
@ -23,7 +23,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
and the media matches the profile's format preferences)
Order of operations:
1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
in its own worker) periodically checks the YouTube RSS feed for new media
2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
item
@ -40,7 +40,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
source = Sources.get_source!(source_id)
case MediaItemTasks.index_and_enqueue_download_for_media_item(source, media_url) do
case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
{:ok, media_item} ->
Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")

2
lib/pinchflat/api/youtube_rss.ex → lib/pinchflat/fast_indexing/youtube_rss.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Api.YoutubeRss do
defmodule Pinchflat.FastIndexing.YoutubeRss do
@moduledoc """
Methods for interacting with YouTube RSS feeds
"""

6
lib/pinchflat/workers/filesystem_data_worker.ex → lib/pinchflat/filesystem/filesystem_data_worker.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.FilesystemDataWorker do
defmodule Pinchflat.Filesystem.FilesystemDataWorker do
@moduledoc false
use Oban.Worker,
@ -7,7 +7,7 @@ defmodule Pinchflat.Workers.FilesystemDataWorker do
max_attempts: 1
alias Pinchflat.Media
alias Pinchflat.Tasks.MediaItemTasks
alias Pinchflat.Filesystem.FilesystemHelpers
@impl Oban.Worker
@doc """
@ -18,7 +18,7 @@ defmodule Pinchflat.Workers.FilesystemDataWorker do
def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
media_item = Media.get_media_item!(media_item_id)
MediaItemTasks.compute_and_save_media_filesize(media_item)
FilesystemHelpers.compute_and_save_media_filesize(media_item)
# Don't retry on failure - if it didn't work immediately there's no
# reason to believe it will work later.

19
lib/pinchflat/utils/filesystem_utils.ex → lib/pinchflat/filesystem/filesystem_helpers.ex
View file

@ -1,8 +1,8 @@
defmodule Pinchflat.Utils.FilesystemUtils do
defmodule Pinchflat.Filesystem.FilesystemHelpers do
@moduledoc """
Utility methods for working with the filesystem
"""
alias Pinchflat.Media
alias Pinchflat.Utils.StringUtils
@doc """
@ -20,4 +20,19 @@ defmodule Pinchflat.Utils.FilesystemUtils do
filepath
end
@doc """
Fetches the file size of a media item and saves it to the database.
Returns {:ok, media_item} | {:error, any()}
"""
def compute_and_save_media_filesize(media_item) do
case File.stat(media_item.media_filepath) do
{:ok, %{size: size}} ->
Media.update_media_item(media_item, %{media_size_bytes: size})
err ->
err
end
end
end

2
lib/pinchflat/media.ex → lib/pinchflat/media/media.ex
View file

@ -9,7 +9,7 @@ defmodule Pinchflat.Media do
alias Pinchflat.Tasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaMetadata
alias Pinchflat.Metadata.MediaMetadata
@doc """
Returns the list of media_items.

6
lib/pinchflat/media/media_item.ex
View file

@ -8,8 +8,8 @@ defmodule Pinchflat.Media.MediaItem do
alias Pinchflat.Tasks.Task
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaMetadata
alias Pinchflat.Media.MediaItemSearchIndex
alias Pinchflat.Metadata.MediaMetadata
alias Pinchflat.Media.MediaItemsSearchIndex
@allowed_fields [
# these fields are captured on indexing (and again on download)
@ -64,7 +64,7 @@ defmodule Pinchflat.Media.MediaItem do
belongs_to :source, Source
has_one :metadata, MediaMetadata, on_replace: :update
has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
has_one :media_items_search_index, MediaItemsSearchIndex, foreign_key: :id
has_many :tasks, Task

2
lib/pinchflat/media/media_items_search_index.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Media.MediaItemSearchIndex do
defmodule Pinchflat.Media.MediaItemsSearchIndex do
@moduledoc """
The MediaItem fts5 search index. Not made to be directly interacted with,
but I figured it'd be better to have it in-app so it's not a mystery.

2
lib/pinchflat/media/media_metadata.ex → lib/pinchflat/metadata/media_metadata.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Media.MediaMetadata do
defmodule Pinchflat.Metadata.MediaMetadata do
@moduledoc """
The MediaMetadata schema.

0
lib/pinchflat/profiles.ex → lib/pinchflat/profiles/profiles.ex
View file

0
lib/pinchflat/settings.ex → lib/pinchflat/settings/settings.ex
View file

2
lib/pinchflat/utils/filesystem_utils/file_follower_server.ex → lib/pinchflat/slow_indexing/file_follower_server.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
defmodule Pinchflat.SlowIndexing.FileFollowerServer do
@moduledoc """
A GenServer that watches a file for new lines and processes them as they come in.
This is useful for tailing log files and other similar tasks. If there's no activity

14
lib/pinchflat/workers/media_collection_indexing_worker.ex → lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
@moduledoc false
use Oban.Worker,
@ -10,8 +10,8 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
alias Pinchflat.Tasks
alias Pinchflat.Sources
alias Pinchflat.Sources.Source
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
@impl Oban.Worker
@doc """
@ -40,7 +40,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
by the `download_media` field on the source as well as the profile's
shorts/livestream behaviour. At this step we also attach a file reader
to the `yt-dlp` output file so we can create media items as they come in
for a little speedup (see SourceTasks comments for more)
for a little speedup (see {Fast,Slow}IndexingHelpers comments for more)
4. If this job is meant to reschedule (ie: has an index frequency > 0),
it reschedules itself. If not, it runs once and does not reschedule
5. If the source uses fast indexing, that job is kicked off as well. It
@ -56,8 +56,6 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
NOTE: Since indexing can take a LONG time, I should check what happens if an
application restart occurs while a job is running. Will the job be lost?
IDEA: Should I use paging and do indexing in chunks? Is that even faster?
Returns :ok | {:ok, %Task{}}
"""
def perform(%Oban.Job{args: %{"id" => source_id}}) do
@ -66,14 +64,14 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
case {source.index_frequency_minutes, source.last_indexed_at} do
{index_freq, _} when index_freq > 0 ->
# If the indexing is on a schedule simply run indexing and reschedule
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
maybe_enqueue_fast_indexing_task(source)
reschedule_indexing(source)
{_, nil} ->
# If the source has never been indexed, index it once
# even if it's not meant to reschedule
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
:ok
_ ->

123
lib/pinchflat/tasks/source_tasks.ex → lib/pinchflat/slow_indexing/slow_indexing_helpers.ex
View file

@ -1,9 +1,8 @@
defmodule Pinchflat.Tasks.SourceTasks do
defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
@moduledoc """
Contains methods used by OR used to create/manage tasks for sources.
Methods for performing slow indexing tasks and managing the indexing process.
Tasks/workers are meant to be thin wrappers so most of the actual work they
do is also defined here. Essentially, a one-stop-shop for source-related tasks/workers.
Many of these methods are made to be kickoff or be consumed by workers.
"""
require Logger
@ -12,16 +11,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
alias Pinchflat.Tasks
alias Pinchflat.Sources
alias Pinchflat.Sources.Source
alias Pinchflat.Api.YoutubeRss
alias Pinchflat.Media.MediaItem
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.YtDlp.Backend.MediaCollection
alias Pinchflat.Workers.MediaCollectionIndexingWorker
alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
alias Pinchflat.YtDlp.MediaCollection
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.SlowIndexing.FileFollowerServer
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
Starts tasks for indexing a source's media regardless of the source's indexing
@ -40,46 +37,6 @@ defmodule Pinchflat.Tasks.SourceTasks do
|> Tasks.create_job_with_task(source)
end
@doc """
Starts tasks for running a fast indexing task for a source's media
regardless of the source's fast_index state. It's assumed the
caller will check for fast_index.
This is used for running fast index tasks on update. On creation, the
fast index is enqueued after the slow index is complete.
Returns {:ok, %Task{}}.
"""
def kickoff_fast_indexing_task(%Source{} = source) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
%{id: source.id}
# Schedule this one immediately, but future ones will be on an interval
|> FastIndexingWorker.new()
|> Tasks.create_job_with_task(source)
end
@doc """
Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
for any new media items. See comments in `MediaIndexingWorker` for more info on the
order of operations and how this fits into the indexing process.
Returns :ok
"""
def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
{:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
Enum.each(new_media_ids, fn media_id ->
url = "https://www.youtube.com/watch?v=#{media_id}"
%{id: source.id, media_url: url}
|> MediaIndexingWorker.new()
|> Tasks.create_job_with_task(source)
end)
end
@doc """
Given a media source, creates (indexes) the media by creating media_items for each
media ID in the source. Afterward, kicks off a download task for each pending media
@ -105,50 +62,21 @@ defmodule Pinchflat.Tasks.SourceTasks do
# See the method definition below for more info on how file watchers work
# (important reading if you're not familiar with it)
{:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
result =
Enum.map(media_attributes, fn media_attrs ->
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
{:ok, media_item} -> media_item
{:error, changeset} -> changeset
end
end)
Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
enqueue_pending_media_tasks(source)
DownloadingHelpers.enqueue_pending_download_tasks(source)
result
end
@doc """
Starts tasks for downloading media for any of a sources _pending_ media items.
Jobs are not enqueued if the source is set to not download media. This will return :ok.
NOTE: this starts a download for each media item that is pending,
not just the ones that were indexed in this job run. This should ensure
that any stragglers are caught if, for some reason, they weren't enqueued
or somehow got de-queued.
Returns :ok
"""
def enqueue_pending_media_tasks(%Source{download_media: true} = source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end)
end
def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
:ok
end
@doc """
Deletes ALL pending tasks for a source's media items.
Returns :ok
"""
def dequeue_pending_media_tasks(%Source{} = source) do
source
|> Media.list_pending_media_items_for()
|> Enum.each(&Tasks.delete_pending_tasks_for/1)
end
# The file follower is a GenServer that watches a file for new lines and
# processes them. This works well, but we have to be resilliant to partially-written
# lines (ie: you should gracefully fail if you can't parse a line).
@ -191,10 +119,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
end
defp create_media_item_and_enqueue_download(source, media_attrs) do
maybe_media_item = create_media_item_from_attributes(source, media_attrs)
case maybe_media_item do
%MediaItem{} = media_item ->
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
{:ok, %MediaItem{} = media_item} ->
if source.download_media && Media.pending_download?(media_item) do
Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
@ -203,15 +129,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
|> Tasks.create_job_with_task(media_item)
end
changeset ->
{:error, changeset} ->
changeset
end
end
defp create_media_item_from_attributes(source, media_attrs) do
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
{:ok, media_item} -> media_item
{:error, changeset} -> changeset
end
end
end

18
lib/pinchflat/sources.ex → lib/pinchflat/sources/sources.ex
View file

@ -9,9 +9,11 @@ defmodule Pinchflat.Sources do
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.YtDlp.Backend.MediaCollection
alias Pinchflat.YtDlp.MediaCollection
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.FastIndexing.FastIndexingHelpers
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
@doc """
Returns the list of sources. Returns [%Source{}, ...]
@ -67,7 +69,7 @@ defmodule Pinchflat.Sources do
media if the indexing frequency has been changed.
Existing indexing tasks will be cancelled if the indexing frequency has been
changed (logic in `SourceTasks.kickoff_indexing_task`)
changed (logic in `SlowIndexingHelpers.kickoff_indexing_task`)
Runs an initial `change_source` check to ensure most of the source is valid
before making an expensive API call. Runs it through `Repo.update` even
@ -205,10 +207,10 @@ defmodule Pinchflat.Sources do
defp maybe_handle_media_tasks(changeset, source) do
case {changeset.data, changeset.changes} do
{%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
SourceTasks.enqueue_pending_media_tasks(source)
DownloadingHelpers.enqueue_pending_download_tasks(source)
{%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
SourceTasks.dequeue_pending_media_tasks(source)
DownloadingHelpers.dequeue_pending_download_tasks(source)
_ ->
:ok
@ -221,7 +223,7 @@ defmodule Pinchflat.Sources do
case changeset.data do
# If the changeset is new (not persisted), attempt indexing no matter what
%{__meta__: %{state: :built}} ->
SourceTasks.kickoff_indexing_task(source)
SlowIndexingHelpers.kickoff_indexing_task(source)
# If the record has been persisted, only run indexing if the
# indexing frequency has been changed and is now greater than 0
@ -236,7 +238,7 @@ defmodule Pinchflat.Sources do
defp maybe_update_slow_indexing_task(changeset, source) do
case changeset.changes do
%{index_frequency_minutes: mins} when mins > 0 ->
SourceTasks.kickoff_indexing_task(source)
SlowIndexingHelpers.kickoff_indexing_task(source)
%{index_frequency_minutes: _} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
@ -251,7 +253,7 @@ defmodule Pinchflat.Sources do
defp maybe_update_fast_indexing_task(changeset, source) do
case changeset.changes do
%{fast_index: true} ->
SourceTasks.kickoff_fast_indexing_task(source)
FastIndexingHelpers.kickoff_fast_indexing_task(source)
%{fast_index: false} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

61
lib/pinchflat/tasks/media_item_tasks.ex
View file

@ -1,61 +0,0 @@
defmodule Pinchflat.Tasks.MediaItemTasks do
@moduledoc """
Contains methods used by OR used to create/manage tasks for media items.
Tasks/workers are meant to be thin wrappers so most of the actual work they
do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
"""
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
@doc """
Fetches the file size of a media item and saves it to the database.
Returns {:ok, media_item} | {:error, any()}
"""
def compute_and_save_media_filesize(media_item) do
case File.stat(media_item.media_filepath) do
{:ok, %{size: size}} ->
Media.update_media_item(media_item, %{media_size_bytes: size})
err ->
err
end
end
@doc """
Indexes a single media item for a source and enqueues a download job if the
media should be downloaded. This method creates the media item record so it's
the one-stop-shop for adding a media item (and possibly downloading it) just
by a URL and source.
Returns {:ok, media_item} | {:error, any()}
"""
def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
maybe_media_item = create_media_item_from_url(source, url)
case maybe_media_item do
{:ok, media_item} ->
if source.download_media && Media.pending_download?(media_item) do
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end
{:ok, media_item}
err ->
err
end
end
defp create_media_item_from_url(source, url) do
{:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
Media.create_media_item_from_backend_attrs(source, media_attrs)
end
end

0
lib/pinchflat/tasks.ex → lib/pinchflat/tasks/tasks.ex
View file

2
lib/pinchflat/yt_dlp/backend/backend_command_runner.ex → lib/pinchflat/yt_dlp/backend_command_runner.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
defmodule Pinchflat.YtDlp.BackendCommandRunner do
@moduledoc """
A behaviour for running CLI commands against a downloader backend (yt-dlp).

6
lib/pinchflat/yt_dlp/backend/command_runner.ex → lib/pinchflat/yt_dlp/command_runner.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.YtDlp.Backend.CommandRunner do
defmodule Pinchflat.YtDlp.CommandRunner do
@moduledoc """
Runs yt-dlp commands using the `System.cmd/3` function
"""
@ -6,8 +6,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
require Logger
alias Pinchflat.Utils.StringUtils
alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
alias Pinchflat.YtDlp.Backend.BackendCommandRunner
alias Pinchflat.Filesystem.FilesystemHelpers, as: FSUtils
alias Pinchflat.YtDlp.BackendCommandRunner
@behaviour BackendCommandRunner

2
lib/pinchflat/yt_dlp/backend/media.ex → lib/pinchflat/yt_dlp/media.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.YtDlp.Backend.Media do
defmodule Pinchflat.YtDlp.Media do
@moduledoc """
Contains utilities for working with singular pieces of media
"""

8
lib/pinchflat/yt_dlp/backend/media_collection.ex → lib/pinchflat/yt_dlp/media_collection.ex
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.YtDlp.Backend.MediaCollection do
defmodule Pinchflat.YtDlp.MediaCollection do
@moduledoc """
Contains utilities for working with collections of
media (aka: a source [ie: channels, playlists]).
@ -7,8 +7,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
require Logger
alias Pinchflat.Utils.FunctionUtils
alias Pinchflat.Utils.FilesystemUtils
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
alias Pinchflat.Filesystem.FilesystemHelpers
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
Returns a list of maps representing the media in the collection.
@ -24,7 +24,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
command_opts = [:simulate, :skip_download]
output_template = YtDlpMedia.indexing_output_template()
output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
if file_listener_handler do

1
mix.exs
View file

@ -59,6 +59,7 @@ defmodule Pinchflat.MixProject do
{:nimble_parsec, "~> 1.4"},
{:mox, "~> 1.0", only: :test},
{:credo, "~> 1.7", only: [:dev, :test], runtime: false},
{:credo_naming, "~> 2.1", only: [:dev, :test], runtime: false},
{:ex_check, "~> 0.14.0", only: [:dev, :test], runtime: false},
{:faker, "~> 0.17", only: :test},
{:sobelow, "~> 0.13", only: [:dev, :test], runtime: false}

1
mix.lock
View file

@ -6,6 +6,7 @@
"cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
"cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
"credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"},
"credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
"db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
"decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
"dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"},

6
test/pinchflat/workers/data_backfill_worker_test.exs → test/pinchflat/boot/data_backfill_worker_test.exs
View file

@ -1,10 +1,10 @@
defmodule Pinchflat.Workers.DataBackfillWorkerTest do
defmodule Pinchflat.Boot.DataBackfillWorkerTest do
use Pinchflat.DataCase
import Pinchflat.MediaFixtures
alias Pinchflat.Workers.DataBackfillWorker
alias Pinchflat.Workers.FilesystemDataWorker
alias Pinchflat.Boot.DataBackfillWorker
alias Pinchflat.Filesystem.FilesystemDataWorker
describe "cancel_pending_backfill_jobs/0" do
test "cancels all pending backfill jobs" do

25
test/pinchflat/boot/pre_job_startup_tasks_test.exs Normal file
View file

@ -0,0 +1,25 @@
defmodule Pinchflat.Boot.PreJobStartupTasksTest do
use Pinchflat.DataCase
alias Pinchflat.Settings
alias Pinchflat.Settings.Setting
alias Pinchflat.Boot.PreJobStartupTasks
describe "apply_default_settings" do
setup do
Repo.delete_all(Setting)
:ok
end
test "sets default settings" do
assert_raise Ecto.NoResultsError, fn -> Settings.get!(:onboarding) end
assert_raise Ecto.NoResultsError, fn -> Settings.get!(:pro_enabled) end
PreJobStartupTasks.start_link()
assert Settings.get!(:onboarding)
refute Settings.get!(:pro_enabled)
end
end
end

4
test/pinchflat/yt_dlp/download_option_builder_test.exs → test/pinchflat/downloading/download_option_builder_test.exs
View file

@ -1,11 +1,11 @@
defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
use Pinchflat.DataCase
import Pinchflat.MediaFixtures
import Pinchflat.ProfilesFixtures
import Pinchflat.SourcesFixtures
alias Pinchflat.Profiles
alias Pinchflat.YtDlp.DownloadOptionBuilder
alias Pinchflat.Downloading.DownloadOptionBuilder
setup do
media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})

75
test/pinchflat/downloading/downloading_helpers_test.exs Normal file
View file

@ -0,0 +1,75 @@
defmodule Pinchflat.Downloading.DownloadingHelpersTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
alias Pinchflat.Tasks
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.Downloading.MediaDownloadWorker
setup :verify_on_exit!
describe "enqueue_pending_download_tasks/1" do
test "it enqueues a job for each pending media item" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
test "it does not enqueue a job for media items with a filepath" do
source = source_fixture()
_media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
refute_enqueued(worker: MediaDownloadWorker)
end
test "it attaches a task to each enqueued job" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
test "it does not create a job if the source is set to not download" do
source = source_fixture(download_media: false)
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
refute_enqueued(worker: MediaDownloadWorker)
end
test "it does not attach tasks if the source is set to not download" do
source = source_fixture(download_media: false)
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
end
describe "dequeue_pending_download_tasks/1" do
test "it deletes all pending tasks for a source's media items" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
DownloadingHelpers.enqueue_pending_download_tasks(source)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
assert :ok = DownloadingHelpers.dequeue_pending_download_tasks(source)
refute_enqueued(worker: MediaDownloadWorker)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
end
end

6
test/pinchflat/workers/media_download_worker_test.exs → test/pinchflat/downloading/media_download_worker_test.exs
View file

@ -1,12 +1,12 @@
defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
alias Pinchflat.Sources
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.Workers.FilesystemDataWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.Filesystem.FilesystemDataWorker
setup :verify_on_exit!

4
test/pinchflat/yt_dlp/media_downloader_test.exs → test/pinchflat/downloading/media_downloader_test.exs
View file

@ -1,9 +1,9 @@
defmodule Pinchflat.MediaClient.MediaDownloaderTest do
defmodule Pinchflat.Downloading.MediaDownloaderTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
alias Pinchflat.MediaClient.MediaDownloader
alias Pinchflat.Downloading.MediaDownloader
setup :verify_on_exit!

4
test/pinchflat/rendered_string/parser_test.exs → test/pinchflat/downloading/output_path/parser_test.exs
View file

@ -1,7 +1,7 @@
defmodule Pinchflat.RenderedString.ParserTest do
defmodule Pinchflat.Downloading.OutputPath.ParserTest do
use ExUnit.Case, async: true
alias Pinchflat.RenderedString.Parser
alias Pinchflat.Downloading.OutputPath.Parser
describe "parse/3" do
test "it returns the rendered string when the string is valid" do

4
test/profiles/output_path_builder_test.exs → test/pinchflat/downloading/output_path_builder_test.exs
View file

@ -1,7 +1,7 @@
defmodule Pinchflat.Profiles.OutputPathBuilderTest do
defmodule Pinchflat.Downloading.OutputPathBuilderTest do
use Pinchflat.DataCase
alias Pinchflat.Profiles.OutputPathBuilder
alias Pinchflat.Downloading.OutputPathBuilder
describe "build/2" do
test "it expands 'standard' curly brace variables in the template" do

142
test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs Normal file
View file

@ -0,0 +1,142 @@
defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.TasksFixtures
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
import Pinchflat.ProfilesFixtures
alias Pinchflat.Tasks
alias Pinchflat.Tasks.Task
alias Pinchflat.Media.MediaItem
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingHelpers
alias Pinchflat.FastIndexing.FastIndexingWorker
setup :verify_on_exit!
@media_url "https://www.youtube.com/watch?v=test_1"
describe "kickoff_fast_indexing_task/1" do
test "it schedules a job" do
source = source_fixture()
assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
end
test "it creates and attaches a task" do
source = source_fixture()
assert {:ok, %Task{} = task} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
assert task.source_id == source.id
end
test "it deletes any fast indexing tasks for the source" do
source = source_fixture()
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
setup do
{:ok, [source: source_fixture()]}
end
test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
assert [worker] = all_enqueued(worker: MediaIndexingWorker)
assert worker.args["id"] == source.id
assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
end
test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
media_item_fixture(source_id: source.id, media_id: "test_1")
assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaIndexingWorker)
end
end
describe "index_and_enqueue_download_for_media_item/2" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, media_attributes_return_fixture()}
end)
{:ok, [source: source_fixture()]}
end
test "creates a new media item based on the URL", %{source: source} do
assert Repo.aggregate(MediaItem, :count) == 0
assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert Repo.aggregate(MediaItem, :count) == 1
end
test "won't duplicate media_items based on media_id and source", %{source: source} do
assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert Repo.aggregate(MediaItem, :count) == 1
assert mi_1.id == mi_2.id
end
test "enqueues a download job", %{source: source} do
assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
test "creates a download task record", %{source: source} do
assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
end
test "does not enqueue a download job if the source does not allow it" do
source = source_fixture(%{download_media: false})
assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
refute_enqueued(worker: MediaDownloadWorker)
end
test "does not enqueue a download job if the media item does not match the format rules" do
profile = media_profile_fixture(%{shorts_behaviour: :exclude})
source = source_fixture(%{media_profile_id: profile.id})
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
output =
Phoenix.json_library().encode!(%{
id: "video2",
title: "Video 2",
webpage_url: "https://example.com/shorts/video2",
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67,
upload_date: "20210101"
})
{:ok, output}
end)
assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
refute_enqueued(worker: MediaDownloadWorker)
end
end
end

4
test/pinchflat/workers/fast_indexing_worker_test.exs → test/pinchflat/fast_indexing/fast_indexing_worker_test.exs
View file

@ -1,11 +1,11 @@
defmodule Pinchflat.Workers.FastIndexingWorkerTest do
defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.SourcesFixtures
alias Pinchflat.Sources.Source
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingWorker
setup :verify_on_exit!

6
test/pinchflat/workers/media_indexing_worker_test.exs → test/pinchflat/fast_indexing/media_indexing_worker_test.exs
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
use Pinchflat.DataCase
import Mox
@ -6,8 +6,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
import Pinchflat.SourcesFixtures
alias Pinchflat.Media.MediaItem
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
@media_url "https://www.youtube.com/watch?v=1234567890"

4
test/pinchflat/api/youtube_rss_test.exs → test/pinchflat/fast_indexing/youtube_rss_test.exs
View file

@ -1,9 +1,9 @@
defmodule Pinchflat.Api.YoutubeRssTest do
defmodule Pinchflat.FastIndexing.YoutubeRssTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.SourcesFixtures
alias Pinchflat.Api.YoutubeRss
alias Pinchflat.FastIndexing.YoutubeRss
setup :verify_on_exit!

4
test/pinchflat/workers/filesystem_data_worker_test.exs → test/pinchflat/filesystem/filesystem_data_worker_test.exs
View file

@ -1,9 +1,9 @@
defmodule Pinchflat.Workers.FilesystemDataWorkerTest do
defmodule Pinchflat.Filesystem.FilesystemDataWorkerTest do
use Pinchflat.DataCase
import Pinchflat.MediaFixtures
alias Pinchflat.Workers.FilesystemDataWorker
alias Pinchflat.Filesystem.FilesystemDataWorker
describe "perform/1" do
test "Computes and stores the media file size" do

36
test/pinchflat/filesystem/filesystem_helpers_test.exs Normal file
View file

@ -0,0 +1,36 @@
defmodule Pinchflat.Filesystem.FilesystemHelpersTest do
use Pinchflat.DataCase
import Pinchflat.MediaFixtures
alias Pinchflat.Filesystem.FilesystemHelpers
describe "generate_metadata_tmpfile/1" do
test "creates a tmpfile and returns its path" do
res = FilesystemHelpers.generate_metadata_tmpfile(:json)
assert String.ends_with?(res, ".json")
assert File.exists?(res)
File.rm!(res)
end
end
describe "compute_and_save_media_filesize/1" do
test "updates the media item with the file size" do
media_item = media_item_with_attachments()
refute media_item.media_size_bytes
assert {:ok, media_item} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
assert Repo.reload!(media_item).media_size_bytes
end
test "returns the error if operation fails" do
media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
assert {:error, _} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
end
end
end

2
test/pinchflat/media_test.exs
View file

@ -11,7 +11,7 @@ defmodule Pinchflat.MediaTest do
alias Pinchflat.Media.MediaItem
alias Pinchflat.Metadata.MetadataFileHelpers
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
setup :verify_on_exit!

2
test/pinchflat/metadata/metadata_parser_test.exs
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
defmodule Pinchflat.Metadata.MetadataParserTest do
use Pinchflat.DataCase
alias Pinchflat.Metadata.MetadataParser, as: Parser

8
test/pinchflat/utils/filesystem_utils/file_follower_server_test.exs → test/pinchflat/slow_indexing/file_follower_server_test.exs
View file

@ -1,12 +1,12 @@
defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServerTest do
defmodule Pinchflat.SlowIndexing.FileFollowerServerTest do
use ExUnit.Case, async: true
alias alias Pinchflat.Utils.FilesystemUtils
alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
alias alias Pinchflat.Filesystem.FilesystemHelpers
alias Pinchflat.SlowIndexing.FileFollowerServer
setup do
{:ok, pid} = FileFollowerServer.start_link()
tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
tmpfile = FilesystemHelpers.generate_metadata_tmpfile(:txt)
{:ok, %{pid: pid, tmpfile: tmpfile}}
end

8
test/pinchflat/workers/media_collection_indexing_worker_test.exs → test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
use Pinchflat.DataCase
import Mox
@ -8,9 +8,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.Workers.MediaCollectionIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
setup :verify_on_exit!

169
test/pinchflat/tasks/source_tasks_test.exs → test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs
View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Tasks.SourceTasksTest do
defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
use Pinchflat.DataCase
import Mox
@ -9,12 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
alias Pinchflat.Tasks
alias Pinchflat.Tasks.Task
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.MediaCollectionIndexingWorker
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
setup :verify_on_exit!
@ -22,7 +22,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
test "it schedules a job" do
source = source_fixture(index_frequency_minutes: 1)
assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
end
@ -30,7 +30,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
test "it creates and attaches a task" do
source = source_fixture(index_frequency_minutes: 1)
assert {:ok, %Task{} = task} = SourceTasks.kickoff_indexing_task(source)
assert {:ok, %Task{} = task} = SlowIndexingHelpers.kickoff_indexing_task(source)
assert task.source_id == source.id
end
@ -40,7 +40,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
{:ok, job} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
@ -50,7 +50,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
{:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
@ -60,65 +60,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
describe "kickoff_fast_indexing_task/1" do
test "it schedules a job" do
source = source_fixture()
assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
end
test "it creates and attaches a task" do
source = source_fixture()
assert {:ok, %Task{} = task} = SourceTasks.kickoff_fast_indexing_task(source)
assert task.source_id == source.id
end
test "it deletes any fast indexing tasks for the source" do
source = source_fixture()
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
task = task_fixture(source_id: source.id, job_id: job.id)
assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
end
end
describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
setup do
{:ok, [source: source_fixture()]}
end
test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
assert [worker] = all_enqueued(worker: MediaIndexingWorker)
assert worker.args["id"] == source.id
assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
end
test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
media_item_fixture(source_id: source.id, media_id: "test_1")
assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
refute_enqueued(worker: MediaIndexingWorker)
end
end
describe "index_and_enqueue_download_for_media_items/1" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
@ -129,7 +76,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
end
test "it creates a media_item record for each media ID returned", %{source: source} do
assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert Enum.count(media_items) == 3
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
@ -141,15 +88,15 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
test "it attaches all media_items to the given source", %{source: source} do
source_id = source.id
assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert Enum.count(media_items) == 3
assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
end
test "it won't duplicate media_items based on media_id and source", %{source: source} do
_first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
_duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
_first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
_duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
media_items = Repo.preload(source, :media_items).media_items
assert Enum.count(media_items) == 3
@ -158,8 +105,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
test "it can duplicate media_ids for different sources", %{source: source} do
other_source = source_fixture()
media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
media_items_other_source = SourceTasks.index_and_enqueue_download_for_media_items(other_source)
media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
media_items_other_source = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(other_source)
assert Enum.count(media_items) == 3
assert Enum.count(media_items_other_source) == 3
@ -169,8 +116,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
end
test "it returns a list of media_items", %{source: source} do
first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
first_ids = Enum.map(first_run, & &1.id)
duplicate_ids = Enum.map(duplicate_run, & &1.id)
@ -181,7 +128,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
test "it updates the source's last_indexed_at field", %{source: source} do
assert source.last_indexed_at == nil
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
source = Repo.reload!(source)
assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 2
@ -191,7 +138,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
@ -200,7 +147,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
source = source_fixture(download_media: false)
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
@ -226,7 +173,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
end)
assert Repo.aggregate(MediaItem, :count, :id) == 0
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert Repo.aggregate(MediaItem, :count, :id) == 3
end
@ -245,7 +192,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
end)
refute_enqueued(worker: MediaDownloadWorker)
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert_enqueued(worker: MediaDownloadWorker)
end
@ -264,7 +211,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
{:ok, ""}
end)
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
refute_enqueued(worker: MediaDownloadWorker)
end
@ -297,7 +244,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
{:ok, ""}
end)
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
refute_enqueued(worker: MediaDownloadWorker)
end
@ -315,71 +262,9 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
{:ok, source_attributes_return_fixture()}
end)
SourceTasks.index_and_enqueue_download_for_media_items(source)
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
assert Repo.aggregate(MediaItem, :count, :id) == 3
assert [_, _, _] = all_enqueued(worker: MediaDownloadWorker)
end
end
describe "enqueue_pending_media_tasks/1" do
test "it enqueues a job for each pending media item" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
test "it does not enqueue a job for media items with a filepath" do
source = source_fixture()
_media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
refute_enqueued(worker: MediaDownloadWorker)
end
test "it attaches a task to each enqueued job" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
test "it does not create a job if the source is set to not download" do
source = source_fixture(download_media: false)
assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
refute_enqueued(worker: MediaDownloadWorker)
end
test "it does not attach tasks if the source is set to not download" do
source = source_fixture(download_media: false)
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
end
describe "dequeue_pending_media_tasks/1" do
test "it deletes all pending tasks for a source's media items" do
source = source_fixture()
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
SourceTasks.enqueue_pending_media_tasks(source)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
assert :ok = SourceTasks.dequeue_pending_media_tasks(source)
refute_enqueued(worker: MediaDownloadWorker)
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
end
end
end

12
test/pinchflat/sources_test.exs
View file

@ -7,12 +7,12 @@ defmodule Pinchflat.SourcesTest do
import Pinchflat.SourcesFixtures
alias Pinchflat.Sources
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Sources.Source
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.MediaCollectionIndexingWorker
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.MediaDownloadWorker
alias Pinchflat.FastIndexing.MediaIndexingWorker
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
@invalid_source_attrs %{name: nil, collection_id: nil}
@ -331,7 +331,7 @@ defmodule Pinchflat.SourcesTest do
source = source_fixture(download_media: true)
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
update_attrs = %{download_media: false}
SourceTasks.enqueue_pending_media_tasks(source)
DownloadingHelpers.enqueue_pending_download_tasks(source)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)

16
test/pinchflat/startup_tasks_test.exs
View file

@ -1,16 +0,0 @@
defmodule Pinchflat.StartupTasksTest do
use Pinchflat.DataCase
alias Pinchflat.Settings
# Since this runs on app boot (even in the test env),
# any actions in the `init/1` function will already have
# run. So we can only test the side effects of those actions,
# rather than the actions themselves.
describe "apply_default_settings" do
test "sets default settings" do
assert Settings.get!(:onboarding) == true
end
end
end

103
test/pinchflat/tasks/media_items_tasks_test.exs
View file

@ -1,103 +0,0 @@
defmodule Pinchflat.Tasks.MediaItemTasksTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
import Pinchflat.ProfilesFixtures
alias Pinchflat.Tasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.Tasks.MediaItemTasks
alias Pinchflat.Workers.MediaDownloadWorker
setup :verify_on_exit!
@media_url "https://www.youtube.com/watch?v=1234"
describe "compute_and_save_media_filesize/1" do
test "updates the media item with the file size" do
media_item = media_item_with_attachments()
refute media_item.media_size_bytes
assert {:ok, media_item} = MediaItemTasks.compute_and_save_media_filesize(media_item)
assert Repo.reload!(media_item).media_size_bytes
end
test "returns the error if operation fails" do
media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
assert {:error, _} = MediaItemTasks.compute_and_save_media_filesize(media_item)
end
end
describe "index_and_enqueue_download_for_media_item/2" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, media_attributes_return_fixture()}
end)
{:ok, [source: source_fixture()]}
end
test "creates a new media item based on the URL", %{source: source} do
assert Repo.aggregate(MediaItem, :count) == 0
assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert Repo.aggregate(MediaItem, :count) == 1
end
test "won't duplicate media_items based on media_id and source", %{source: source} do
assert {:ok, mi_1} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert {:ok, mi_2} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert Repo.aggregate(MediaItem, :count) == 1
assert mi_1.id == mi_2.id
end
test "enqueues a download job", %{source: source} do
assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
end
test "creates a download task record", %{source: source} do
assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
end
test "does not enqueue a download job if the source does not allow it" do
source = source_fixture(%{download_media: false})
assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
refute_enqueued(worker: MediaDownloadWorker)
end
test "does not enqueue a download job if the media item does not match the format rules" do
profile = media_profile_fixture(%{shorts_behaviour: :exclude})
source = source_fixture(%{media_profile_id: profile.id})
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
output =
Phoenix.json_library().encode!(%{
id: "video2",
title: "Video 2",
webpage_url: "https://example.com/shorts/video2",
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67,
upload_date: "20210101"
})
{:ok, output}
end)
assert {:ok, _media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
refute_enqueued(worker: MediaDownloadWorker)
end
end
end

16
test/pinchflat/utils/filesystem_utils_test.exs
View file

@ -1,16 +0,0 @@
defmodule Pinchflat.Utils.FilesystemUtilsTest do
use ExUnit.Case, async: true
alias Pinchflat.Utils.FilesystemUtils
describe "generate_metadata_tmpfile/1" do
test "creates a tmpfile and returns its path" do
res = FilesystemUtils.generate_metadata_tmpfile(:json)
assert String.ends_with?(res, ".json")
assert File.exists?(res)
File.rm!(res)
end
end
end

4
test/pinchflat/yt_dlp/backend/command_runner_test.exs → test/pinchflat/yt_dlp/command_runner_test.exs
View file

@ -1,7 +1,7 @@
defmodule Pinchflat.YtDlp.Backend.CommandRunnerTest do
defmodule Pinchflat.YtDlp.CommandRunnerTest do
use ExUnit.Case, async: true
alias Pinchflat.YtDlp.Backend.CommandRunner, as: Runner
alias Pinchflat.YtDlp.CommandRunner, as: Runner
@original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
@media_url "https://www.youtube.com/watch?v=-LHXuyzpex0"

6
test/pinchflat/yt_dlp/backend/media_collection_test.exs → test/pinchflat/yt_dlp/media_collection_test.exs
View file

@ -1,10 +1,10 @@
defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
defmodule Pinchflat.YtDlp.MediaCollectionTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.SourcesFixtures
alias Pinchflat.YtDlp.Backend.Media
alias Pinchflat.YtDlp.Backend.MediaCollection
alias Pinchflat.YtDlp.Media
alias Pinchflat.YtDlp.MediaCollection
@channel_url "https://www.youtube.com/c/TheUselessTrials"

4
test/pinchflat/yt_dlp/backend/media_test.exs → test/pinchflat/yt_dlp/media_test.exs
View file

@ -1,9 +1,9 @@
defmodule Pinchflat.YtDlp.Backend.MediaTest do
defmodule Pinchflat.YtDlp.MediaTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.MediaFixtures
alias Pinchflat.YtDlp.Backend.Media
alias Pinchflat.YtDlp.Media
@media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"

2
test/test_helper.exs
View file

@ -1,4 +1,4 @@
Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.Backend.BackendCommandRunner)
Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.BackendCommandRunner)
Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour)