Redo indexing mechanism (#16)

* Bumped up the line length because I fear no man

* Refactored indexing

Previously, indexing worked by collecting the video IDs of only videos
that matched indexing criteria. This new model instead stores ALL videos
for a given source, but will only _download_ videos that meet that criteria.
This lets us backfill without indexing, makes it easier to add in other
backends, lets us download one-off videos for a source that don't quite
meet criteria, you name it.

* Updated media finders to respect format filters; Added credo file

.credo.exs

@@ -0,0 +1,215 @@
+# This file contains the configuration for Credo and you are probably reading
+# this after creating it with `mix credo.gen.config`.
+#
+# If you find anything wrong or unclear in this file, please report an
+# issue on GitHub: https://github.com/rrrene/credo/issues
+#
+%{
+  #
+  # You can have as many configs as you like in the `configs:` field.
+  configs: [
+    %{
+      #
+      # Run any config using `mix credo -C <name>`. If no config name is given
+      # "default" is used.
+      #
+      name: "default",
+      #
+      # These are the files included in the analysis:
+      files: %{
+        #
+        # You can give explicit globs or simply directories.
+        # In the latter case `**/*.{ex,exs}` will be used.
+        #
+        included: [
+          "lib/",
+          "src/",
+          "test/",
+          "web/",
+          "apps/*/lib/",
+          "apps/*/src/",
+          "apps/*/test/",
+          "apps/*/web/"
+        ],
+        excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"]
+      },
+      #
+      # Load and configure plugins here:
+      #
+      plugins: [],
+      #
+      # If you create your own checks, you must specify the source files for
+      # them here, so they can be loaded by Credo before running the analysis.
+      #
+      requires: [],
+      #
+      # If you want to enforce a style guide and need a more traditional linting
+      # experience, you can change `strict` to `true` below:
+      #
+      strict: false,
+      #
+      # To modify the timeout for parsing files, change this value:
+      #
+      parse_timeout: 5000,
+      #
+      # If you want to use uncolored output by default, you can change `color`
+      # to `false` below:
+      #
+      color: true,
+      #
+      # You can customize the parameters of any check by adding a second element
+      # to the tuple.
+      #
+      # To disable a check put `false` as second element:
+      #
+      #     {Credo.Check.Design.DuplicatedCode, false}
+      #
+      checks: %{
+        enabled: [
+          #
+          ## Consistency Checks
+          #
+          {Credo.Check.Consistency.ExceptionNames, []},
+          {Credo.Check.Consistency.LineEndings, []},
+          {Credo.Check.Consistency.ParameterPatternMatching, []},
+          {Credo.Check.Consistency.SpaceAroundOperators, []},
+          {Credo.Check.Consistency.SpaceInParentheses, []},
+          {Credo.Check.Consistency.TabsOrSpaces, []},
+
+          #
+          ## Design Checks
+          #
+          # You can customize the priority of any check
+          # Priority values are: `low, normal, high, higher`
+          #
+          {Credo.Check.Design.AliasUsage, [priority: :low, if_nested_deeper_than: 2, if_called_more_often_than: 0]},
+          {Credo.Check.Design.TagFIXME, []},
+          # You can also customize the exit_status of each check.
+          # If you don't want TODO comments to cause `mix credo` to fail, just
+          # set this value to 0 (zero).
+          #
+          {Credo.Check.Design.TagTODO, [exit_status: 2]},
+
+          #
+          ## Readability Checks
+          #
+          {Credo.Check.Readability.AliasOrder, []},
+          {Credo.Check.Readability.FunctionNames, []},
+          {Credo.Check.Readability.LargeNumbers, []},
+          {Credo.Check.Readability.MaxLineLength, [priority: :low, max_length: 120]},
+          {Credo.Check.Readability.ModuleAttributeNames, []},
+          {Credo.Check.Readability.ModuleDoc, []},
+          {Credo.Check.Readability.ModuleNames, []},
+          {Credo.Check.Readability.ParenthesesInCondition, []},
+          {Credo.Check.Readability.ParenthesesOnZeroArityDefs, []},
+          {Credo.Check.Readability.PipeIntoAnonymousFunctions, []},
+          {Credo.Check.Readability.PredicateFunctionNames, []},
+          {Credo.Check.Readability.PreferImplicitTry, []},
+          {Credo.Check.Readability.RedundantBlankLines, []},
+          {Credo.Check.Readability.Semicolons, []},
+          {Credo.Check.Readability.SpaceAfterCommas, []},
+          {Credo.Check.Readability.StringSigils, []},
+          {Credo.Check.Readability.TrailingBlankLine, []},
+          {Credo.Check.Readability.TrailingWhiteSpace, []},
+          {Credo.Check.Readability.UnnecessaryAliasExpansion, []},
+          {Credo.Check.Readability.VariableNames, []},
+          {Credo.Check.Readability.WithSingleClause, []},
+
+          #
+          ## Refactoring Opportunities
+          #
+          {Credo.Check.Refactor.Apply, []},
+          {Credo.Check.Refactor.CondStatements, []},
+          {Credo.Check.Refactor.FilterCount, []},
+          {Credo.Check.Refactor.FilterFilter, []},
+          {Credo.Check.Refactor.FunctionArity, []},
+          {Credo.Check.Refactor.LongQuoteBlocks, []},
+          {Credo.Check.Refactor.MapJoin, []},
+          {Credo.Check.Refactor.MatchInCondition, []},
+          {Credo.Check.Refactor.NegatedConditionsInUnless, []},
+          {Credo.Check.Refactor.NegatedConditionsWithElse, []},
+          {Credo.Check.Refactor.Nesting, []},
+          {Credo.Check.Refactor.RedundantWithClauseResult, []},
+          {Credo.Check.Refactor.RejectReject, []},
+          {Credo.Check.Refactor.UnlessWithElse, []},
+          {Credo.Check.Refactor.WithClauses, []},
+
+          #
+          ## Warnings
+          #
+          {Credo.Check.Warning.ApplicationConfigInModuleAttribute, []},
+          {Credo.Check.Warning.BoolOperationOnSameValues, []},
+          {Credo.Check.Warning.Dbg, []},
+          {Credo.Check.Warning.ExpensiveEmptyEnumCheck, []},
+          {Credo.Check.Warning.IExPry, []},
+          {Credo.Check.Warning.IoInspect, []},
+          {Credo.Check.Warning.MissedMetadataKeyInLoggerConfig, []},
+          {Credo.Check.Warning.OperationOnSameValues, []},
+          {Credo.Check.Warning.OperationWithConstantResult, []},
+          {Credo.Check.Warning.RaiseInsideRescue, []},
+          {Credo.Check.Warning.SpecWithStruct, []},
+          {Credo.Check.Warning.UnsafeExec, []},
+          {Credo.Check.Warning.UnusedEnumOperation, []},
+          {Credo.Check.Warning.UnusedFileOperation, []},
+          {Credo.Check.Warning.UnusedKeywordOperation, []},
+          {Credo.Check.Warning.UnusedListOperation, []},
+          {Credo.Check.Warning.UnusedPathOperation, []},
+          {Credo.Check.Warning.UnusedRegexOperation, []},
+          {Credo.Check.Warning.UnusedStringOperation, []},
+          {Credo.Check.Warning.UnusedTupleOperation, []},
+          {Credo.Check.Warning.WrongTestFileExtension, []}
+        ],
+        disabled: [
+          #
+          # Checks scheduled for next check update (opt-in for now, just replace `false` with `[]`)
+
+          #
+          # Controversial and experimental checks (opt-in, just move the check to `:enabled`
+          #   and be sure to use `mix credo --strict` to see low priority checks)
+          #
+          {Credo.Check.Refactor.CyclomaticComplexity, []},
+          {Credo.Check.Consistency.MultiAliasImportRequireUse, []},
+          {Credo.Check.Consistency.UnusedVariableNames, []},
+          {Credo.Check.Design.DuplicatedCode, []},
+          {Credo.Check.Design.SkipTestWithoutComment, []},
+          {Credo.Check.Readability.AliasAs, []},
+          {Credo.Check.Readability.BlockPipe, []},
+          {Credo.Check.Readability.ImplTrue, []},
+          {Credo.Check.Readability.MultiAlias, []},
+          {Credo.Check.Readability.NestedFunctionCalls, []},
+          {Credo.Check.Readability.OneArityFunctionInPipe, []},
+          {Credo.Check.Readability.OnePipePerLine, []},
+          {Credo.Check.Readability.SeparateAliasRequire, []},
+          {Credo.Check.Readability.SingleFunctionToBlockPipe, []},
+          {Credo.Check.Readability.SinglePipe, []},
+          {Credo.Check.Readability.Specs, []},
+          {Credo.Check.Readability.StrictModuleLayout, []},
+          {Credo.Check.Readability.WithCustomTaggedTuple, []},
+          {Credo.Check.Refactor.ABCSize, []},
+          {Credo.Check.Refactor.AppendSingleItem, []},
+          {Credo.Check.Refactor.DoubleBooleanNegation, []},
+          {Credo.Check.Refactor.FilterReject, []},
+          {Credo.Check.Refactor.IoPuts, []},
+          {Credo.Check.Refactor.MapMap, []},
+          {Credo.Check.Refactor.ModuleDependencies, []},
+          {Credo.Check.Refactor.NegatedIsNil, []},
+          {Credo.Check.Refactor.PassAsyncInTestCases, []},
+          {Credo.Check.Refactor.PipeChainStart, []},
+          {Credo.Check.Refactor.RejectFilter, []},
+          {Credo.Check.Refactor.VariableRebinding, []},
+          {Credo.Check.Warning.LazyLogging, []},
+          {Credo.Check.Warning.LeakyEnvironment, []},
+          {Credo.Check.Warning.MapGetUnsafePass, []},
+          {Credo.Check.Warning.MixEnv, []},
+          {Credo.Check.Warning.UnsafeToAtom, []}
+
+          # {Credo.Check.Refactor.MapInto, []},
+
+          #
+          # Custom checks can be created using `mix credo.gen.check`.
+          #
+        ]
+      }
+    }
+  ]
+}

.formatter.exs

@@ -3,5 +3,5 @@
   subdirectories: ["priv/*/migrations"],
   plugins: [Phoenix.LiveView.HTMLFormatter],
   inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
-  line_length: 100
+  line_length: 120
 ]

.iex.exs

@@ -44,7 +44,7 @@ defmodule IexHelpers do
         :channel -> channel_url()
       end
 
-    SourceDetails.get_video_ids(source)
+    SourceDetails.get_media_attributes(source)
   end
 end
 

lib/pinchflat/media.ex

@@ -19,15 +19,31 @@ defmodule Pinchflat.Media do
 
   @doc """
   Returns a list of pending media_items for a given source, where
-  pending means the `media_filepath` is `nil`.
+  pending means the `media_filepath` is `nil` AND the media_item
+  matches the format selection rules of the parent media_profile.
+
+  See `build_format_clauses` but tl;dr is it _may_ filter based
+  on shorts or livestreams depending on the media_profile settings.
 
   Returns [%MediaItem{}, ...].
   """
   def list_pending_media_items_for(%Source{} = source) do
-    from(
+    media_profile = Repo.preload(source, :media_profile).media_profile
-      m in MediaItem,
+
-      where: m.source_id == ^source.id and is_nil(m.media_filepath)
+    MediaItem
-    )
+    |> where([mi], mi.source_id == ^source.id and is_nil(mi.media_filepath))
+    |> where(^build_format_clauses(media_profile))
+    |> Repo.all()
+  end
+
+  @doc """
+  Returns a list of downloaded media_items for a given source.
+
+  Returns [%MediaItem{}, ...].
+  """
+  def list_downloaded_media_items_for(%Source{} = source) do
+    MediaItem
+    |> where([mi], mi.source_id == ^source.id and not is_nil(mi.media_filepath))
     |> Repo.all()
   end
 
@@ -72,4 +88,39 @@ defmodule Pinchflat.Media do
   def change_media_item(%MediaItem{} = media_item, attrs \\ %{}) do
     MediaItem.changeset(media_item, attrs)
   end
+
+  defp build_format_clauses(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, dynamic(true), fn attr, dynamic ->
+      case {attr, media_profile} do
+        {{:shorts_behaviour, :only}, %{livestream_behaviour: :only}} ->
+          dynamic([mi], ^dynamic and (mi.livestream == true or fragment("? ILIKE ?", mi.original_url, "%/shorts/%")))
+
+        # Technically redundant, but makes the other clauses easier to parse
+        # (redundant because this condition is the same as the condition above, just flipped)
+        {{:livestream_behaviour, :only}, %{shorts_behaviour: :only}} ->
+          dynamic
+
+        {{:shorts_behaviour, :only}, _} ->
+          # return records with /shorts/ in the original_url
+          dynamic([mi], ^dynamic and fragment("? ILIKE ?", mi.original_url, "%/shorts/%"))
+
+        {{:livestream_behaviour, :only}, _} ->
+          # return records with livestream: true
+          dynamic([mi], ^dynamic and mi.livestream == true)
+
+        {{:shorts_behaviour, :exclude}, %{livestream_behaviour: lb}} when lb != :only ->
+          # return records without /shorts/ in the original_url
+          dynamic([mi], ^dynamic and fragment("? NOT ILIKE ?", mi.original_url, "%/shorts/%"))
+
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: sb}} when sb != :only ->
+          # return records with livestream: false
+          dynamic([mi], ^dynamic and mi.livestream == false)
+
+        _ ->
+          dynamic
+      end
+    end)
+  end
 end

lib/pinchflat/media/media_item.ex

@@ -13,17 +13,21 @@ defmodule Pinchflat.Media.MediaItem do
   @allowed_fields ~w(
     title
     media_id
+    original_url
+    livestream
     media_filepath
     source_id
     subtitle_filepaths
     thumbnail_filepath
     metadata_filepath
   )a
-  @required_fields ~w(media_id source_id)a
+  @required_fields ~w(title original_url livestream media_id source_id)a
 
   schema "media_items" do
     field :title, :string
     field :media_id, :string
+    field :original_url, :string
+    field :livestream, :boolean, default: false
     field :media_filepath, :string
     field :thumbnail_filepath, :string
     field :metadata_filepath, :string

lib/pinchflat/media_client/backends/yt_dlp/video_collection.ex

@@ -4,18 +4,26 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
   videos (aka: a source [ie: channels, playlists]).
   """
 
-  @doc """
+  alias Pinchflat.Utils.FunctionUtils
-  Returns a list of strings representing the video ids in the collection.
 
-  Returns {:ok, [binary()]} | {:error, any, ...}.
+  @doc """
+  Returns a list of maps representing the videos in the collection.
+
+  Returns {:ok, [map()]} | {:error, any, ...}.
   """
-  def get_video_ids(url, command_opts \\ []) do
+  def get_media_attributes(url, command_opts \\ []) do
     runner = Application.get_env(:pinchflat, :yt_dlp_runner)
     opts = command_opts ++ [:simulate, :skip_download]
 
-    case runner.run(url, opts, "%(id)s") do
+    case runner.run(url, opts, "%(.{id,title,was_live,original_url})j") do
-      {:ok, output} -> {:ok, String.split(output, "\n", trim: true)}
+      {:ok, output} ->
-      res -> res
+        output
+        |> String.split("\n", trim: true)
+        |> Enum.map(&Phoenix.json_library().decode!/1)
+        |> FunctionUtils.wrap_ok()
+
+      res ->
+        res
     end
   end
 

lib/pinchflat/media_client/source_details.ex

@@ -6,11 +6,8 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   it open-ish for future expansion (just in case).
   """
 
-  alias Pinchflat.Repo
   alias Pinchflat.MediaSource.Source
-
   alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
-  alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder, as: YtDlpIndexOptionBuilder
 
   @doc """
   Gets a source's ID and name from its URL using the given backend.
@@ -22,24 +19,19 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   end
 
   @doc """
-  Returns a list of video IDs for the given source URL OR source record using the given backend.
+  Returns a list of basic video data mapsfor the given source URL OR
+  source record using the given backend.
 
-  If passing a source record, the call to the backend may have custom options applied based on
+  Returns {:ok, [map()]} | {:error, any, ...}.
-  the `option_builder`.
-
-  Returns {:ok, list(binary())} | {:error, any, ...}.
   """
-  def get_video_ids(sourceable, backend \\ :yt_dlp)
+  def get_media_attributes(sourceable, backend \\ :yt_dlp)
 
-  def get_video_ids(%Source{} = source, backend) do
+  def get_media_attributes(%Source{} = source, backend) do
-    media_profile = Repo.preload(source, :media_profile).media_profile
+    source_module(backend).get_media_attributes(source.collection_id)
-    {:ok, options} = option_builder(backend).build(media_profile)
-
-    source_module(backend).get_video_ids(source.collection_id, options)
   end
 
-  def get_video_ids(source_url, backend) when is_binary(source_url) do
+  def get_media_attributes(source_url, backend) when is_binary(source_url) do
-    source_module(backend).get_video_ids(source_url)
+    source_module(backend).get_media_attributes(source_url)
   end
 
   defp source_module(backend) do
@@ -47,10 +39,4 @@ defmodule Pinchflat.MediaClient.SourceDetails do
       :yt_dlp -> YtDlpSource
     end
   end
-
-  defp option_builder(backend) do
-    case backend do
-      :yt_dlp -> YtDlpIndexOptionBuilder
-    end
-  end
 end

lib/pinchflat/media_source.ex

@@ -7,7 +7,6 @@ defmodule Pinchflat.MediaSource do
   alias Pinchflat.Repo
 
   alias Pinchflat.Tasks
-  alias Pinchflat.Media
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.MediaClient.SourceDetails
@@ -39,26 +38,6 @@ defmodule Pinchflat.MediaSource do
     |> commit_and_start_indexing()
   end
 
-  @doc """
-  Given a media source, creates (indexes) the media by creating media_items for each
-  media ID in the source.
-
-  Returns [%MediaItem{}, ...] | [%Ecto.Changeset{}, ...]
-  """
-  def index_media_items(%Source{} = source) do
-    {:ok, media_ids} = SourceDetails.get_video_ids(source.original_url)
-
-    media_ids
-    |> Enum.map(fn media_id ->
-      attrs = %{source_id: source.id, media_id: media_id}
-
-      case Media.create_media_item(attrs) do
-        {:ok, media_item} -> media_item
-        {:error, changeset} -> changeset
-      end
-    end)
-  end
-
   @doc """
   Updates a source. May attempt to pull additional source details from the
   original_url (if changed). May attempt to start indexing the source's
@@ -101,6 +80,9 @@ defmodule Pinchflat.MediaSource do
   This means that it'll go for it even if a changeset is otherwise invalid. This
   is pretty easy to change, but for MVP I'm not concerned.
 
+  NOTE: When operating in the ideal path, this effectively adds an API call
+  to the source creation/update process. Should be used only when needed.
+
   IDEA: Maybe I could discern `collection_type` based on the original URL?
   It also seems like it's a channel when the returned yt-dlp channel_id is the
   same as the playlist_id - maybe could use that?

lib/pinchflat/profiles/media_profile.ex

@@ -45,7 +45,9 @@ defmodule Pinchflat.Profiles.MediaProfile do
     # through the entire collection to determine if a video is a short or
     # a livestream.
     # NOTE: these can BOTH be set to :only which will download shorts and
-    # livestreams _only_ and ignore regular videos.
+    # livestreams _only_ and ignore regular videos. The redundant case
+    # is when one is set to :only and the other is set to :exclude.
+    # See `build_format_clauses` in the Media context for more.
     field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
     field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
 

lib/pinchflat/profiles/options/yt_dlp/index_option_builder.ex

@@ -1,52 +0,0 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
-  @moduledoc """
-  Builds the options for yt-dlp to index a media source based on the given media profile.
-  """
-
-  alias Pinchflat.Profiles.MediaProfile
-
-  @doc """
-  Builds the options for yt-dlp to index a media source based on the given media profile.
-  """
-  def build(%MediaProfile{} = media_profile) do
-    built_options = release_type_options(media_profile)
-
-    {:ok, built_options}
-  end
-
-  defp release_type_options(media_profile) do
-    mapped_struct = Map.from_struct(media_profile)
-
-    # Appending multiple match filters treats them as an OR condition,
-    # so we have to be careful around combining `only` and `exclude` options.
-    # eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
-    # which will return all shorts AND normal videos.
-    Enum.reduce(mapped_struct, [], fn attr, acc ->
-      case {attr, media_profile} do
-        {{:shorts_behaviour, :only}, _} ->
-          acc ++ [match_filter: "original_url*=/shorts/"]
-
-        {{:livestream_behaviour, :only}, _} ->
-          acc ++ [match_filter: "was_live"]
-
-        # Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
-        # other type is set to `only`. There is also special behaviour if they're both excludes,
-        # hence why these check against `:include` alone.
-        {{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
-          acc ++ [match_filter: "original_url!*=/shorts/"]
-
-        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
-          acc ++ [match_filter: "!was_live"]
-
-        # Again, since it's an OR, there's a special syntax if they're both excluded
-        # to make it an AND. Note that I'm not checking for the other permutation of
-        # both excluding since this MUST get hit so adding the other version would double up.
-        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
-          acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
-
-        _ ->
-          acc
-      end
-    end)
-  end
-end

lib/pinchflat/tasks/source_tasks.ex

@@ -6,6 +6,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.MediaSource.Source
+  alias Pinchflat.MediaClient.SourceDetails
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.Workers.VideoDownloadWorker
 
@@ -33,6 +34,32 @@ defmodule Pinchflat.Tasks.SourceTasks do
     end
   end
 
+  @doc """
+  Given a media source, creates (indexes) the media by creating media_items for each
+  media ID in the source.
+
+  Returns [%MediaItem{}, ...] | [%Ecto.Changeset{}, ...]
+  """
+  def index_media_items(%Source{} = source) do
+    {:ok, media_attributes} = SourceDetails.get_media_attributes(source.original_url)
+
+    media_attributes
+    |> Enum.map(fn media_attrs ->
+      attrs = %{
+        source_id: source.id,
+        title: media_attrs["title"],
+        media_id: media_attrs["id"],
+        original_url: media_attrs["original_url"],
+        livestream: media_attrs["was_live"]
+      }
+
+      case Media.create_media_item(attrs) do
+        {:ok, media_item} -> media_item
+        {:error, changeset} -> changeset
+      end
+    end)
+  end
+
   @doc """
   Starts tasks for downloading videos for any of a sources _pending_ media items.
   Jobs are not enqueued if the source is set to not download media. This will return :ok.

lib/pinchflat/utils/function_utils.ex

@@ -0,0 +1,15 @@
+defmodule Pinchflat.Utils.FunctionUtils do
+  @moduledoc """
+  Utility functions for working with functions
+  """
+
+  @doc """
+  Wraps the provided term in an :ok tuple. Useful for fulfilling a contract, but
+  other usage should be assessed to see if it's the right fit.
+
+  Returns {:ok, term}
+  """
+  def wrap_ok(value) do
+    {:ok, value}
+  end
+end

lib/pinchflat/workers/media_indexing_worker.ex

@@ -47,7 +47,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   end
 
   defp index_media_and_reschedule(source) do
-    MediaSource.index_media_items(source)
+    SourceTasks.index_media_items(source)
     SourceTasks.enqueue_pending_media_downloads(source)
 
     source

lib/pinchflat_web/components/core_components.ex

@@ -168,8 +168,7 @@ defmodule PinchflatWeb.CoreComponents do
         phx-connected={hide("#server-error")}
         hidden
       >
-        Hang in there while we get back on track
+        Hang in there while we get back on track <.icon name="hero-arrow-path" class="ml-1 h-3 w-3 animate-spin" />
-        <.icon name="hero-arrow-path" class="ml-1 h-3 w-3 animate-spin" />
       </.flash>
     </div>
     """
@@ -245,8 +244,7 @@ defmodule PinchflatWeb.CoreComponents do
     values: ~w(checkbox color date datetime-local email file hidden month number password
                toggle range radio search select tel text textarea time url week)
 
-  attr :field, Phoenix.HTML.FormField,
+  attr :field, Phoenix.HTML.FormField, doc: "a form field struct retrieved from the form, for example: @form[:email]"
-    doc: "a form field struct retrieved from the form, for example: @form[:email]"
 
   attr :errors, :list, default: []
   attr :checked, :boolean, doc: "the checked flag for checkbox inputs"
@@ -254,8 +252,7 @@ defmodule PinchflatWeb.CoreComponents do
   attr :options, :list, doc: "the options to pass to Phoenix.HTML.Form.options_for_select/2"
   attr :multiple, :boolean, default: false, doc: "the multiple flag for select inputs"
 
-  attr :rest, :global,
+  attr :rest, :global, include: ~w(accept autocomplete capture cols disabled form list max maxlength min minlength
-    include: ~w(accept autocomplete capture cols disabled form list max maxlength min minlength
                 multiple pattern placeholder readonly required rows size step)
 
   slot :inner_block
@@ -279,15 +276,7 @@ defmodule PinchflatWeb.CoreComponents do
     <div phx-feedback-for={@name}>
       <label class="flex items-center gap-4 text-sm leading-6">
         <input type="hidden" name={@name} value="false" />
-        <input
+        <input type="checkbox" id={@id} name={@name} value="true" checked={@checked} class="rounded focus:ring-0" {@rest} />
-          type="checkbox"
-          id={@id}
-          name={@name}
-          value="true"
-          checked={@checked}
-          class="rounded focus:ring-0"
-          {@rest}
-        />
         <%= @label %>
       </label>
       <.help :if={@help}><%= @help %></.help>
@@ -318,8 +307,7 @@ defmodule PinchflatWeb.CoreComponents do
           {@rest}
         />
         <div class="inline-block cursor-pointer" @click="enabled = !enabled">
-          <div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black">
+          <div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
-          </div>
           <div
             x-bind:class="enabled && '!right-1 !translate-x-full'"
             class="absolute left-1 top-1 flex h-6 w-6 items-center justify-center rounded-full bg-white transition"
@@ -529,10 +517,7 @@ defmodule PinchflatWeb.CoreComponents do
           <td :if={@action != []} class="relative w-14 p-0">
             <div class="relative whitespace-nowrap py-4 text-right text-sm font-medium">
               <span class="absolute -inset-y-px -right-4 left-0 group-hover:bg-zinc-50 sm:rounded-r-xl" />
-              <span
+              <span :for={action <- @action} class="relative ml-4 font-semibold leading-6 text-zinc-900 hover:text-zinc-700">
-                :for={action <- @action}
-                class="relative ml-4 font-semibold leading-6 text-zinc-900 hover:text-zinc-700"
-              >
                 <%= render_slot(action, @row_item.(row)) %>
               </span>
             </div>
@@ -607,10 +592,7 @@ defmodule PinchflatWeb.CoreComponents do
   def back(assigns) do
     ~H"""
     <div class="mt-16">
-      <.link
+      <.link navigate={@navigate} class="text-sm font-semibold leading-6 text-zinc-900 hover:text-zinc-700">
-        navigate={@navigate}
-        class="text-sm font-semibold leading-6 text-zinc-900 hover:text-zinc-700"
-      >
         <.icon name="hero-arrow-left-solid" class="h-3 w-3" />
         <%= render_slot(@inner_block) %>
       </.link>
@@ -651,8 +633,7 @@ defmodule PinchflatWeb.CoreComponents do
     JS.show(js,
       to: selector,
       transition:
-        {"transition-all transform ease-out duration-300",
+        {"transition-all transform ease-out duration-300", "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95",
-         "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95",
          "opacity-100 translate-y-0 sm:scale-100"}
     )
   end

lib/pinchflat_web/components/layouts/partials/sidebar.html.heex

@@ -17,11 +17,7 @@
         <ul class="mb-6 flex flex-col gap-1.5">
           <.sidebar_item icon="hero-tv" text="Sources" navigate={~p"/sources"} />
 
-          <.sidebar_item
+          <.sidebar_item icon="hero-adjustments-vertical" text="Media Profiles" navigate={~p"/media_profiles"} />
-            icon="hero-adjustments-vertical"
-            text="Media Profiles"
-            navigate={~p"/media_profiles"}
-          />
         </ul>
       </div>
     </nav>

lib/pinchflat_web/controllers/media_sources/source_html/edit.html.heex

@@ -8,11 +8,7 @@
 <div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">
   <div class="max-w-full overflow-x-auto">
     <div class="flex flex-col gap-10">
-      <.source_form
+      <.source_form changeset={@changeset} media_profiles={@media_profiles} action={~p"/sources/#{@source}"} />
-        changeset={@changeset}
-        media_profiles={@media_profiles}
-        action={~p"/sources/#{@source}"}
-      />
     </div>
   </div>
 </div>

lib/pinchflat_web/controllers/media_sources/source_html/index.html.heex

@@ -29,16 +29,10 @@
           </.link>
         </:col>
         <:col :let={source} label="" class="flex place-content-evenly">
-          <.link
+          <.link navigate={~p"/sources/#{source.id}"} class="hover:text-secondary duration-200 ease-in-out mx-0.5">
-            navigate={~p"/sources/#{source.id}"}
-            class="hover:text-secondary duration-200 ease-in-out mx-0.5"
-          >
             <.icon name="hero-eye" />
           </.link>
-          <.link
+          <.link navigate={~p"/sources/#{source.id}/edit"} class="hover:text-secondary duration-200 ease-in-out mx-0.5">
-            navigate={~p"/sources/#{source.id}/edit"}
-            class="hover:text-secondary duration-200 ease-in-out mx-0.5"
-          >
             <.icon name="hero-pencil-square" />
           </.link>
         </:col>

lib/pinchflat_web/controllers/media_sources/source_html/source_form.html.heex

@@ -3,12 +3,7 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
-  <.input
+  <.input field={f[:original_url]} type="text" label="Source URL" help="URL of a channel or playlist (required)" />
-    field={f[:original_url]}
-    type="text"
-    label="Source URL"
-    help="URL of a channel or playlist (required)"
-  />
 
   <.input field={f[:friendly_name]} type="text" label="Friendly Name" />
 
@@ -19,12 +14,7 @@
     label="Media Profile"
   />
 
-  <.input
+  <.input field={f[:collection_type]} options={friendly_collection_types()} type="select" label="Source Type" />
-    field={f[:collection_type]}
-    options={friendly_collection_types()}
-    type="select"
-    label="Source Type"
-  />
 
   <.input
     field={f[:index_frequency_minutes]}

lib/pinchflat_web/controllers/page_html/home.html.heex

@@ -136,11 +136,7 @@
               href="https://twitter.com/elixirphoenix"
               class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
             >
-              <svg
+              <svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
-                viewBox="0 0 16 16"
-                aria-hidden="true"
-                class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
-              >
                 <path d="M5.403 14c5.283 0 8.172-4.617 8.172-8.62 0-.131 0-.262-.008-.391A6.033 6.033 0 0 0 15 3.419a5.503 5.503 0 0 1-1.65.477 3.018 3.018 0 0 0 1.263-1.676 5.579 5.579 0 0 1-1.824.736 2.832 2.832 0 0 0-1.63-.916 2.746 2.746 0 0 0-1.821.319A2.973 2.973 0 0 0 8.076 3.78a3.185 3.185 0 0 0-.182 1.938 7.826 7.826 0 0 1-3.279-.918 8.253 8.253 0 0 1-2.64-2.247 3.176 3.176 0 0 0-.315 2.208 3.037 3.037 0 0 0 1.203 1.836A2.739 2.739 0 0 1 1.56 6.22v.038c0 .7.23 1.377.65 1.919.42.54 1.004.912 1.654 1.05-.423.122-.866.14-1.297.052.184.602.541 1.129 1.022 1.506a2.78 2.78 0 0 0 1.662.598 5.656 5.656 0 0 1-2.007 1.074A5.475 5.475 0 0 1 1 12.64a7.827 7.827 0 0 0 4.403 1.358" />
               </svg>
               Follow on Twitter
@@ -151,11 +147,7 @@
               href="https://elixirforum.com"
               class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
             >
-              <svg
+              <svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
-                viewBox="0 0 16 16"
-                aria-hidden="true"
-                class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
-              >
                 <path d="M8 13.833c3.866 0 7-2.873 7-6.416C15 3.873 11.866 1 8 1S1 3.873 1 7.417c0 1.081.292 2.1.808 2.995.606 1.05.806 2.399.086 3.375l-.208.283c-.285.386-.01.905.465.85.852-.098 2.048-.318 3.137-.81a3.717 3.717 0 0 1 1.91-.318c.263.027.53.041.802.041Z" />
               </svg>
               Discuss on the Elixir Forum
@@ -166,11 +158,7 @@
               href="https://web.libera.chat/#elixir"
               class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
             >
-              <svg
+              <svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
-                viewBox="0 0 16 16"
-                aria-hidden="true"
-                class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
-              >
                 <path
                   fill-rule="evenodd"
                   clip-rule="evenodd"
@@ -190,11 +178,7 @@
               href="https://discord.gg/elixir"
               class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
             >
-              <svg
+              <svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
-                viewBox="0 0 16 16"
-                aria-hidden="true"
-                class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
-              >
                 <path d="M13.545 2.995c-1.02-.46-2.114-.8-3.257-.994a.05.05 0 0 0-.052.024c-.141.246-.297.567-.406.82a12.377 12.377 0 0 0-3.658 0 8.238 8.238 0 0 0-.412-.82.052.052 0 0 0-.052-.024 13.315 13.315 0 0 0-3.257.994.046.046 0 0 0-.021.018C.356 6.063-.213 9.036.066 11.973c.001.015.01.029.02.038a13.353 13.353 0 0 0 3.996 1.987.052.052 0 0 0 .056-.018c.308-.414.582-.85.818-1.309a.05.05 0 0 0-.028-.069 8.808 8.808 0 0 1-1.248-.585.05.05 0 0 1-.005-.084c.084-.062.168-.126.248-.191a.05.05 0 0 1 .051-.007c2.619 1.176 5.454 1.176 8.041 0a.05.05 0 0 1 .053.006c.08.065.164.13.248.192a.05.05 0 0 1-.004.084c-.399.23-.813.423-1.249.585a.05.05 0 0 0-.027.07c.24.457.514.893.817 1.307a.051.051 0 0 0 .056.019 13.31 13.31 0 0 0 4.001-1.987.05.05 0 0 0 .021-.037c.334-3.396-.559-6.345-2.365-8.96a.04.04 0 0 0-.021-.02Zm-8.198 7.19c-.789 0-1.438-.712-1.438-1.587 0-.874.637-1.586 1.438-1.586.807 0 1.45.718 1.438 1.586 0 .875-.637 1.587-1.438 1.587Zm5.316 0c-.788 0-1.438-.712-1.438-1.587 0-.874.637-1.586 1.438-1.586.807 0 1.45.718 1.438 1.586 0 .875-.63 1.587-1.438 1.587Z" />
               </svg>
               Join our Discord server
@@ -205,11 +189,7 @@
               href="https://fly.io/docs/elixir/getting-started/"
               class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
             >
-              <svg
+              <svg viewBox="0 0 20 20" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
-                viewBox="0 0 20 20"
-                aria-hidden="true"
-                class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
-              >
                 <path d="M1 12.5A4.5 4.5 0 005.5 17H15a4 4 0 001.866-7.539 3.504 3.504 0 00-4.504-4.272A4.5 4.5 0 004.06 8.235 4.502 4.502 0 001 12.5z" />
               </svg>
               Deploy your application

priv/repo/migrations/20240208194729_add_indexing_attributes_to_media_items.exs

@@ -0,0 +1,10 @@
+defmodule Pinchflat.Repo.Migrations.AddIndexingAttributesToMediaItems do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :livestream, :boolean, default: false, null: false
+      add :original_url, :string, null: false
+    end
+  end
+end

test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs

@@ -1,6 +1,7 @@
 defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
   use ExUnit.Case, async: true
   import Mox
+  import Pinchflat.MediaSourceFixtures
 
   alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection
 
@@ -8,22 +9,23 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
 
   setup :verify_on_exit!
 
-  describe "get_video_ids/2" do
+  describe "get_media_attributes/2" do
-    test "returns a list of video ids with no blank elements" do
+    test "returns a list of video attributes with no blank elements" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "id1\nid2\n\nid3\n"} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture() <> "\n\n"} end)
 
-      assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url)
+      assert {:ok, [%{"id" => "video1"}, %{"id" => "video2"}, %{"id" => "video3"}]} =
+               VideoCollection.get_media_attributes(@channel_url)
     end
 
     test "it passes the expected default args" do
       expect(YtDlpRunnerMock, :run, fn _url, opts, ot ->
         assert opts == [:simulate, :skip_download]
-        assert ot == "%(id)s"
+        assert ot == "%(.{id,title,was_live,original_url})j"
 
         {:ok, ""}
       end)
 
-      assert {:ok, _} = VideoCollection.get_video_ids(@channel_url)
+      assert {:ok, _} = VideoCollection.get_media_attributes(@channel_url)
     end
 
     test "it passes the expected custom args" do
@@ -33,13 +35,13 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
         {:ok, ""}
       end)
 
-      assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg])
+      assert {:ok, _} = VideoCollection.get_media_attributes(@channel_url, [:custom_arg])
     end
 
     test "returns the error straight through when the command fails" do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
 
-      assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url)
+      assert {:error, "Big issue", 1} = VideoCollection.get_media_attributes(@channel_url)
     end
   end
 

test/pinchflat/media_client/source_details_test.exs

@@ -22,7 +22,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
       assert {:ok, _} = SourceDetails.get_source_details(@channel_url)
     end
 
-    test "it returns a struct composed of the returned data" do
+    test "it returns a map composed of the returned data" do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
         Phoenix.json_library().encode(%{
           channel: "TheUselessTrials",
@@ -43,42 +43,42 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
     end
   end
 
-  describe "get_video_ids/2 when passed a string" do
+  describe "get_media_attributes/2 when passed a string" do
     test "it passes the expected arguments to the backend" do
       expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
         assert opts == [:simulate, :skip_download]
-        assert ot == "%(id)s"
+        assert ot == "%(.{id,title,was_live,original_url})j"
 
         {:ok, ""}
       end)
 
-      assert {:ok, _} = SourceDetails.get_video_ids(@channel_url)
+      assert {:ok, _} = SourceDetails.get_media_attributes(@channel_url)
     end
 
-    test "it returns a list of strings" do
+    test "it returns a list of maps" do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, "video1\nvideo2\nvideo3"}
+        {:ok, source_attributes_return_fixture()}
       end)
 
-      assert {:ok, ["video1", "video2", "video3"]} = SourceDetails.get_video_ids(@channel_url)
+      assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes(@channel_url)
     end
   end
 
-  describe "get_video_ids/2 when passed a Source record" do
+  describe "get_media_attributes/2 when passed a Source record" do
     test "it calls the backend with the source's collection ID" do
       source = source_fixture()
 
       expect(YtDlpRunnerMock, :run, fn url, _opts, _ot ->
         assert source.collection_id == url
-        {:ok, "video1\nvideo2\nvideo3"}
+        {:ok, source_attributes_return_fixture()}
       end)
 
-      assert {:ok, _} = SourceDetails.get_video_ids(source)
+      assert {:ok, _} = SourceDetails.get_media_attributes(source)
     end
 
     test "it builds options based on the source's media profile" do
       expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
-        assert opts == [{:match_filter, "!was_live"}, :simulate, :skip_download]
+        assert opts == [:simulate, :skip_download]
         {:ok, ""}
       end)
 
@@ -89,7 +89,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
         )
 
       source = source_fixture(media_profile_id: media_profile.id)
-      assert {:ok, _} = SourceDetails.get_video_ids(source)
+      assert {:ok, _} = SourceDetails.get_media_attributes(source)
     end
   end
 end

test/pinchflat/media_client/video_downloader_test.exs

@@ -10,7 +10,7 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
   setup do
     media_item =
       Repo.preload(
-        media_item_fixture(%{title: nil, media_filepath: nil}),
+        media_item_fixture(%{title: "Something", media_filepath: nil}),
         [:metadata, source: :media_profile]
       )
 
@@ -58,7 +58,6 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
     end
 
     test "it extracts the title", %{media_item: media_item} do
-      assert media_item.title == nil
       assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
       assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake"
     end

test/pinchflat/media_source_test.exs

@@ -6,7 +6,6 @@ defmodule Pinchflat.MediaSourceTest do
   import Pinchflat.MediaSourceFixtures
 
   alias Pinchflat.MediaSource
-  alias Pinchflat.Media.MediaItem
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.Workers.MediaIndexingWorker
 
@@ -117,59 +116,6 @@ defmodule Pinchflat.MediaSourceTest do
     end
   end
 
-  describe "index_media_items/1" do
-    setup do
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2\nvideo3"} end)
-
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "it creates a media_item record for each media ID returned", %{source: source} do
-      assert media_items = MediaSource.index_media_items(source)
-
-      assert Enum.count(media_items) == 3
-      assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
-      assert Enum.all?(media_items, fn %MediaItem{} -> true end)
-    end
-
-    test "it attaches all media_items to the given source", %{source: source} do
-      source_id = source.id
-      assert media_items = MediaSource.index_media_items(source)
-
-      assert Enum.count(media_items) == 3
-      assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
-    end
-
-    test "it won't duplicate media_items based on media_id and source", %{source: source} do
-      _first_run = MediaSource.index_media_items(source)
-      _duplicate_run = MediaSource.index_media_items(source)
-
-      media_items = Repo.preload(source, :media_items).media_items
-      assert Enum.count(media_items) == 3
-    end
-
-    test "it can duplicate media_ids for different sources", %{source: source} do
-      other_source = source_fixture()
-
-      media_items = MediaSource.index_media_items(source)
-      media_items_other_source = MediaSource.index_media_items(other_source)
-
-      assert Enum.count(media_items) == 3
-      assert Enum.count(media_items_other_source) == 3
-
-      assert Enum.map(media_items, & &1.media_id) ==
-               Enum.map(media_items_other_source, & &1.media_id)
-    end
-
-    test "it returns a list of media_items or changesets", %{source: source} do
-      first_run = MediaSource.index_media_items(source)
-      duplicate_run = MediaSource.index_media_items(source)
-
-      assert Enum.all?(first_run, fn %MediaItem{} -> true end)
-      assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
-    end
-  end
-
   describe "update_source/2" do
     test "updates with valid data updates the source" do
       source = source_fixture()

test/pinchflat/media_test.exs

@@ -3,6 +3,7 @@ defmodule Pinchflat.MediaTest do
 
   import Pinchflat.TasksFixtures
   import Pinchflat.MediaFixtures
+  import Pinchflat.ProfilesFixtures
   import Pinchflat.MediaSourceFixtures
 
   alias Pinchflat.Media
@@ -30,7 +31,7 @@ defmodule Pinchflat.MediaTest do
   end
 
   describe "list_pending_media_items_for/1" do
-    test "it returns pending media_items for a given source" do
+    test "it returns pending without a filepath for a given source" do
       source = source_fixture()
       media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
 
@@ -50,6 +51,138 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "list_pending_media_items_for/1 when testing shorts" do
+    test "returns shorts and normal media when shorts_behaviour is :include" do
+      source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :include}).id})
+      normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [normal, short]
+    end
+
+    test "returns only shorts when shorts_behaviour is :only" do
+      source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :only}).id})
+      _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [short]
+    end
+
+    test "returns only normal media when shorts_behaviour is :exclude" do
+      source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :exclude}).id})
+      normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      _short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [normal]
+    end
+  end
+
+  describe "list_pending_media_items_for/1 when testing livestreams" do
+    test "returns livestreams and normal media when livestream_behaviour is :include" do
+      source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :include}).id})
+      normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+
+      assert Media.list_pending_media_items_for(source) == [normal, livestream]
+    end
+
+    test "returns only livestreams when livestream_behaviour is :only" do
+      source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :only}).id})
+      _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+
+      assert Media.list_pending_media_items_for(source) == [livestream]
+    end
+
+    test "returns only normal media when livestream_behaviour is :exclude" do
+      source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :exclude}).id})
+      normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      _livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+
+      assert Media.list_pending_media_items_for(source) == [normal]
+    end
+  end
+
+  describe "list_pending_media_items_for/1 when testing all format options" do
+    test "returns livestreams, shorts, and normal media when behaviour is :include" do
+      source =
+        source_fixture(%{
+          media_profile_id:
+            media_profile_fixture(%{
+              shorts_behaviour: :include,
+              livestream_behaviour: :include
+            }).id
+        })
+
+      normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [normal, livestream, short]
+    end
+
+    test "returns only livestreams and shorts when behaviour is :only" do
+      source =
+        source_fixture(%{
+          media_profile_id:
+            media_profile_fixture(%{
+              shorts_behaviour: :only,
+              livestream_behaviour: :only
+            }).id
+        })
+
+      _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [livestream, short]
+    end
+
+    test "returns only normal media when behaviour is :exclude" do
+      source =
+        source_fixture(%{
+          media_profile_id:
+            media_profile_fixture(%{
+              shorts_behaviour: :exclude,
+              livestream_behaviour: :exclude
+            }).id
+        })
+
+      normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      _livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+      _short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [normal]
+    end
+
+    test ":only and :exclude return the expected results" do
+      source =
+        source_fixture(%{
+          media_profile_id:
+            media_profile_fixture(%{
+              shorts_behaviour: :only,
+              livestream_behaviour: :exclude
+            }).id
+        })
+
+      _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      _livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+
+      assert Media.list_pending_media_items_for(source) == [short]
+    end
+  end
+
+  describe "list_downloaded_media_items_for/1" do
+    test "returns only media items with a media_filepath" do
+      source = source_fixture()
+      _media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      media_item = media_item_fixture(%{source_id: source.id, media_filepath: "/video/#{Faker.File.file_name(:video)}"})
+
+      assert Media.list_downloaded_media_items_for(source) == [media_item]
+    end
+  end
+
   describe "get_media_item!/1" do
     test "it returns the media_item with given id" do
       media_item = media_item_fixture()
@@ -63,7 +196,8 @@ defmodule Pinchflat.MediaTest do
         media_id: Faker.String.base64(12),
         title: Faker.Commerce.product_name(),
         media_filepath: "/video/#{Faker.File.file_name(:video)}",
-        source_id: source_fixture().id
+        source_id: source_fixture().id,
+        original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}"
       }
 
       assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)

test/pinchflat/profiles/options/yt_dlp/index_option_builder_test.exs

@@ -1,104 +0,0 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilderTest do
-  use ExUnit.Case, async: true
-
-  alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder
-
-  @media_profile %MediaProfile{
-    output_path_template: "{{ title }}.%(ext)s",
-    shorts_behaviour: :include,
-    livestream_behaviour: :include
-  }
-
-  describe "build/1 when testing release type options" do
-    test "adds correct filter when shorts_behaviour is :only" do
-      media_profile = %MediaProfile{@media_profile | shorts_behaviour: :only}
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "original_url*=/shorts/"} in res
-      refute {:match_filter, "original_url!*=/shorts/"} in res
-      refute {:match_filter, "!was_live"} in res
-      refute {:match_filter, "was_live"} in res
-    end
-
-    test "adds correct filter when livestream_behaviour is :only" do
-      media_profile = %MediaProfile{@media_profile | livestream_behaviour: :only}
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "was_live"} in res
-      refute {:match_filter, "!was_live"} in res
-      refute {:match_filter, "!original_url*=/shorts/"} in res
-      refute {:match_filter, "original_url*=/shorts/"} in res
-    end
-
-    test "adds correct filter when both livestreams and shorts are :only" do
-      media_profile = %MediaProfile{
-        @media_profile
-        | shorts_behaviour: :only,
-          livestream_behaviour: :only
-      }
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "original_url*=/shorts/"} in res
-      assert {:match_filter, "was_live"} in res
-      refute {:match_filter, "original_url!*=/shorts/"} in res
-      refute {:match_filter, "!was_live"} in res
-    end
-
-    test "adds correct filter when shorts_behaviour is :exclude" do
-      media_profile = %MediaProfile{@media_profile | shorts_behaviour: :exclude}
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "original_url!*=/shorts/"} in res
-      refute {:match_filter, "original_url*=/shorts/"} in res
-      refute {:match_filter, "was_live"} in res
-      refute {:match_filter, "!was_live"} in res
-    end
-
-    test "adds correct filter when livestream_behaviour is :exclude" do
-      media_profile = %MediaProfile{@media_profile | livestream_behaviour: :exclude}
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "!was_live"} in res
-      refute {:match_filter, "was_live"} in res
-      refute {:match_filter, "original_url!*=/shorts/"} in res
-      refute {:match_filter, "original_url*=/shorts/"} in res
-    end
-
-    test "adds correct filter when shorts and livestreams are both exclude" do
-      media_profile = %MediaProfile{
-        @media_profile
-        | shorts_behaviour: :exclude,
-          livestream_behaviour: :exclude
-      }
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "!was_live & original_url!*=/shorts/"} in res
-      refute {:match_filter, "original_url!*=/shorts/"} in res
-      refute {:match_filter, "!was_live"} in res
-      refute {:match_filter, "original_url*=/shorts/"} in res
-      refute {:match_filter, "was_live"} in res
-    end
-
-    test "does not add exclusion filter if one is excluded and the other is only" do
-      media_profile = %MediaProfile{
-        @media_profile
-        | shorts_behaviour: :exclude,
-          livestream_behaviour: :only
-      }
-
-      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
-
-      assert {:match_filter, "was_live"} in res
-      refute {:match_filter, "original_url!*=/shorts/"} in res
-      refute {:match_filter, "original_url*=/shorts/"} in res
-      refute {:match_filter, "!was_live"} in res
-    end
-  end
-end

test/pinchflat/tasks/source_tasks_test.exs

@@ -1,6 +1,7 @@
 defmodule Pinchflat.Tasks.SourceTasksTest do
   use Pinchflat.DataCase
 
+  import Mox
   import Pinchflat.TasksFixtures
   import Pinchflat.MediaFixtures
   import Pinchflat.MediaSourceFixtures
@@ -8,9 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
   alias Pinchflat.Tasks
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.Workers.VideoDownloadWorker
 
+  setup :verify_on_exit!
+
   describe "kickoff_indexing_task/1" do
     test "it does not schedule a job if the interval is <= 0" do
       source = source_fixture(index_frequency_minutes: -1)
@@ -46,6 +50,61 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
   end
 
+  describe "index_media_items/1" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
+
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "it creates a media_item record for each media ID returned", %{source: source} do
+      assert media_items = SourceTasks.index_media_items(source)
+
+      assert Enum.count(media_items) == 3
+      assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
+      assert ["Video 1", "Video 2", "Video 3"] == Enum.map(media_items, & &1.title)
+      assert Enum.all?(media_items, fn mi -> mi.original_url end)
+      assert Enum.all?(media_items, fn %MediaItem{} -> true end)
+    end
+
+    test "it attaches all media_items to the given source", %{source: source} do
+      source_id = source.id
+      assert media_items = SourceTasks.index_media_items(source)
+
+      assert Enum.count(media_items) == 3
+      assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
+    end
+
+    test "it won't duplicate media_items based on media_id and source", %{source: source} do
+      _first_run = SourceTasks.index_media_items(source)
+      _duplicate_run = SourceTasks.index_media_items(source)
+
+      media_items = Repo.preload(source, :media_items).media_items
+      assert Enum.count(media_items) == 3
+    end
+
+    test "it can duplicate media_ids for different sources", %{source: source} do
+      other_source = source_fixture()
+
+      media_items = SourceTasks.index_media_items(source)
+      media_items_other_source = SourceTasks.index_media_items(other_source)
+
+      assert Enum.count(media_items) == 3
+      assert Enum.count(media_items_other_source) == 3
+
+      assert Enum.map(media_items, & &1.media_id) ==
+               Enum.map(media_items_other_source, & &1.media_id)
+    end
+
+    test "it returns a list of media_items or changesets", %{source: source} do
+      first_run = SourceTasks.index_media_items(source)
+      duplicate_run = SourceTasks.index_media_items(source)
+
+      assert Enum.all?(first_run, fn %MediaItem{} -> true end)
+      assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
+    end
+  end
+
   describe "enqueue_pending_media_downloads/1" do
     test "it enqueues a job for each pending media item" do
       source = source_fixture()

test/pinchflat/utils/function_utils_test.exs

@@ -0,0 +1,11 @@
+defmodule Pinchflat.Utils.FunctionUtilsTest do
+  use ExUnit.Case, async: true
+
+  alias Pinchflat.Utils.FunctionUtils
+
+  describe "wrap_ok/1" do
+    test "wraps the provided term in an :ok tuple" do
+      assert FunctionUtils.wrap_ok("hello") == {:ok, "hello"}
+    end
+  end
+end

test/pinchflat/workers/media_indexing_worker_test.exs

@@ -38,32 +38,33 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
     end
 
     test "it kicks off a download job for each pending media item" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
 
       source = source_fixture(index_frequency_minutes: 10)
       perform_job(MediaIndexingWorker, %{id: source.id})
 
-      assert [_] = all_enqueued(worker: VideoDownloadWorker)
+      assert length(all_enqueued(worker: VideoDownloadWorker)) == 3
     end
 
     test "it starts a job for any pending media item even if it's from another run" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
 
       source = source_fixture(index_frequency_minutes: 10)
       media_item_fixture(%{source_id: source.id, media_filepath: nil})
       perform_job(MediaIndexingWorker, %{id: source.id})
 
-      assert [_, _] = all_enqueued(worker: VideoDownloadWorker)
+      assert length(all_enqueued(worker: VideoDownloadWorker)) == 4
     end
 
     test "it does not kick off a job for media items that could not be saved" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo1"} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
 
       source = source_fixture(index_frequency_minutes: 10)
+      media_item_fixture(%{source_id: source.id, media_filepath: nil, media_id: "video1"})
       perform_job(MediaIndexingWorker, %{id: source.id})
 
-      # Only one job should be enqueued, since the second video is a duplicate
+      # Only 3 jobs should be enqueued, since the first video is a duplicate
-      assert [_] = all_enqueued(worker: VideoDownloadWorker)
+      assert length(all_enqueued(worker: VideoDownloadWorker))
     end
 
     test "it reschedules the job based on the index frequency" do
@@ -91,7 +92,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
     end
 
     test "it creates the basic media_item records" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2"} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
 
       source = source_fixture(index_frequency_minutes: 10)
 
@@ -102,7 +103,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
         |> Enum.map(fn media_item -> media_item.media_id end)
       end
 
-      assert_changed([from: [], to: ["video1", "video2"]], media_item_fetcher, fn ->
+      assert_changed([from: [], to: ["video1", "video2", "video3"]], media_item_fetcher, fn ->
         perform_job(MediaIndexingWorker, %{id: source.id})
       end)
     end

test/support/fixtures/media_fixtures.ex

@@ -15,6 +15,8 @@ defmodule Pinchflat.MediaFixtures do
       |> Enum.into(%{
         media_id: Faker.String.base64(12),
         title: Faker.Commerce.product_name(),
+        original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
+        livestream: false,
         media_filepath: "/video/#{Faker.File.file_name(:video)}",
         source_id: MediaSourceFixtures.source_fixture().id
       })

test/support/fixtures/media_source_fixtures.ex

@@ -29,4 +29,30 @@ defmodule Pinchflat.MediaSourceFixtures do
 
     source
   end
+
+  def source_attributes_return_fixture do
+    source_attributes = [
+      %{
+        id: "video1",
+        title: "Video 1",
+        original_url: "https://example.com/video1",
+        was_live: false
+      },
+      %{
+        id: "video2",
+        title: "Video 2",
+        original_url: "https://example.com/video2",
+        was_live: true
+      },
+      %{
+        id: "video3",
+        title: "Video 3",
+        original_url: "https://example.com/video3",
+        was_live: false
+      }
+    ]
+
+    source_attributes
+    |> Enum.map_join("\n", &Phoenix.json_library().encode!(&1))
+  end
 end