[Enhancement] Added search to source forms (#259)

* Changed sqlite FTS to use a trigram tokenizer

* Added search UI to source tables

* Fix bug with special chars in search form

* improved main search results form

* Improved centering for media table header elements

lib/pinchflat/media/media_query.ex

@@ -125,6 +125,18 @@ defmodule Pinchflat.Media.MediaQuery do
     )
   end
 
+  def matches_search_term(nil), do: dynamic([mi], true)
+
+  def matches_search_term(term) do
+    escaped_term = clean_search_term(term)
+
+    # Matching on `term` instead of `escaped_term` because the latter can mangle empty strings
+    case String.trim(term) do
+      "" -> dynamic([mi], true)
+      _ -> dynamic([mi], fragment("media_items_search_index MATCH ?", ^escaped_term))
+    end
+  end
+
   def require_assoc(query, identifier) do
     if has_named_binding?(query, identifier) do
       query
@@ -133,6 +145,10 @@ defmodule Pinchflat.Media.MediaQuery do
     end
   end
 
+  defp do_require_assoc(query, :media_items_search_index) do
+    from(mi in query, join: s in assoc(mi, :media_items_search_index), as: :media_items_search_index)
+  end
+
   defp do_require_assoc(query, :source) do
     from(mi in query, join: s in assoc(mi, :source), as: :source)
   end
@@ -148,9 +164,11 @@ defmodule Pinchflat.Media.MediaQuery do
   def matching_search_term(query, nil), do: query
 
   def matching_search_term(query, term) do
+    escaped_term = clean_search_term(term)
+
     from(mi in query,
       join: mi_search_index in assoc(mi, :media_items_search_index),
-      where: fragment("media_items_search_index MATCH ?", ^term),
+      where: fragment("media_items_search_index MATCH ?", ^escaped_term),
       select_merge: %{
         matching_search_term:
           fragment("""
@@ -162,4 +180,26 @@ defmodule Pinchflat.Media.MediaQuery do
       order_by: [desc: fragment("rank")]
     )
   end
+
+  # SQLite's FTS5 is very picky about what it will accept as a search term.
+  # To that end, we need to clean up the search term before passing it to the
+  # MATCH clause.
+  # This method:
+  #   - Trims leading and trailing whitespace
+  #   - Collapses multiple spaces into a single space
+  #   - Removes quote characters
+  #   - Wraps any word in quotes (must happen after the double quote replacement)
+  #
+  # This allows for works with apostrophes and quotes to be searched for correctly
+  defp clean_search_term(nil), do: ""
+  defp clean_search_term(""), do: ""
+
+  defp clean_search_term(term) do
+    term
+    |> String.trim()
+    |> String.replace(~r/\s+/, " ")
+    |> String.split(~r/\s+/)
+    |> Enum.map(fn str -> String.replace(str, ~s("), "") end)
+    |> Enum.map_join(" ", fn str -> ~s("#{str}") end)
+  end
 end

lib/pinchflat_web/components/layouts/partials/header.html.heex

@@ -22,7 +22,7 @@
             type="text"
             name="q"
             value={@params["q"]}
-            placeholder="Type to search..."
+            placeholder="Search all media..."
             class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none"
           />
         </form>

lib/pinchflat_web/controllers/searches/search_html/show.html.heex

@@ -10,19 +10,13 @@
       <%= if match?([_|_], @search_results) do %>
         <.table rows={@search_results} table_class="text-black dark:text-white">
           <:col :let={result} label="Title">
-            <%= result.title %>
+            <.subtle_link href={~p"/sources/#{result.source_id}/media/#{result.id}"}>
+              <%= StringUtils.truncate(result.title, 35) %>
+            </.subtle_link>
           </:col>
           <:col :let={result} label="Excerpt">
             <.highlight_search_terms text={result.matching_search_term} />
           </:col>
-          <:col :let={result} label="" class="flex place-content-evenly">
-            <.link
-              href={~p"/sources/#{result.source_id}/media/#{result.id}"}
-              class="hover:text-secondary duration-200 ease-in-out mx-0.5"
-            >
-              <.icon name="hero-eye" />
-            </.link>
-          </:col>
         </.table>
       <% else %>
         <p class="font-bold text-lg text-center text-black dark:text-white">No results found</p>

lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex

@@ -8,7 +8,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
 
   @limit 10
 
-  def render(%{records: []} = assigns) do
+  def render(%{total_record_count: 0} = assigns) do
     ~H"""
     <div class="mb-4 flex items-center">
       <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" />
@@ -20,10 +20,29 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   def render(assigns) do
     ~H"""
     <div>
-      <span class="mb-4 flex items-center">
-        <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" tooltip="Refresh" />
-        <span class="ml-2">Showing <%= length(@records) %> of <%= @total_record_count %></span>
-      </span>
+      <header class="flex justify-between items-center mb-4">
+        <span class="flex items-center">
+          <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" tooltip="Refresh" />
+          <span class="ml-2">Showing <%= length(@records) %> of <%= @filtered_record_count %></span>
+        </span>
+        <div class="bg-meta-4 rounded-md">
+          <div class="relative">
+            <span class="absolute left-2 top-1/2 -translate-y-1/2 flex">
+              <.icon name="hero-magnifying-glass" />
+            </span>
+            <form phx-change="search_term" phx-submit="search_term">
+              <input
+                type="text"
+                name="q"
+                value={@search_term}
+                placeholder="Search in table..."
+                class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none"
+                phx-debounce="200"
+              />
+            </form>
+          </div>
+        </div>
+      </header>
       <.table rows={@records} table_class="text-white">
         <:col :let={media_item} label="Title">
           <.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
@@ -51,7 +70,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
     media_state = session["media_state"]
     source = Sources.get_source!(session["source_id"])
     base_query = generate_base_query(source, media_state)
-    pagination_attrs = fetch_pagination_attributes(base_query, page)
+    pagination_attrs = fetch_pagination_attributes(base_query, page, nil)
 
     new_assigns =
       Map.merge(
@@ -69,7 +88,14 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   def handle_event("page_change", %{"direction" => direction}, %{assigns: assigns} = socket) do
     direction = if direction == "inc", do: 1, else: -1
     new_page = assigns.page + direction
-    new_assigns = fetch_pagination_attributes(assigns.base_query, new_page)
+    new_assigns = fetch_pagination_attributes(assigns.base_query, new_page, assigns.search_term)
+
+    {:noreply, assign(socket, new_assigns)}
+  end
+
+  def handle_event("search_term", params, socket) do
+    search_term = Map.get(params, "q", nil)
+    new_assigns = fetch_pagination_attributes(socket.assigns.base_query, 1, search_term)
 
     {:noreply, assign(socket, new_assigns)}
   end
@@ -83,18 +109,28 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   end
 
   def handle_info(%{topic: "media_table", event: "reload"}, %{assigns: assigns} = socket) do
-    new_assigns = fetch_pagination_attributes(assigns.base_query, assigns.page)
+    new_assigns = fetch_pagination_attributes(assigns.base_query, assigns.page, assigns.search_term)
 
     {:noreply, assign(socket, new_assigns)}
   end
 
-  defp fetch_pagination_attributes(base_query, page) do
-    total_record_count = Repo.aggregate(base_query, :count, :id)
-    total_pages = max(ceil(total_record_count / @limit), 1)
-    page = NumberUtils.clamp(page, 1, total_pages)
-    records = fetch_records(base_query, page)
+  defp fetch_pagination_attributes(base_query, page, search_term) do
+    filtered_base_query = filter_base_query(base_query, search_term)
 
-    %{page: page, total_pages: total_pages, records: records, total_record_count: total_record_count}
+    total_record_count = Repo.aggregate(base_query, :count, :id)
+    filtered_record_count = Repo.aggregate(filtered_base_query, :count, :id)
+    total_pages = max(ceil(filtered_record_count / @limit), 1)
+    page = NumberUtils.clamp(page, 1, total_pages)
+    records = fetch_records(filtered_base_query, page)
+
+    %{
+      page: page,
+      total_pages: total_pages,
+      records: records,
+      search_term: search_term,
+      total_record_count: total_record_count,
+      filtered_record_count: filtered_record_count
+    }
   end
 
   defp fetch_records(base_query, page) do
@@ -109,25 +145,33 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   defp generate_base_query(source, "pending") do
     MediaQuery.new()
     |> MediaQuery.require_assoc(:media_profile)
+    |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending()))
-    |> order_by(desc: :id)
+    |> order_by(desc: fragment("rank"), desc: :id)
   end
 
   defp generate_base_query(source, "downloaded") do
     MediaQuery.new()
+    |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
-    |> order_by(desc: :id)
+    |> order_by(desc: fragment("rank"), desc: :id)
   end
 
   defp generate_base_query(source, "other") do
     MediaQuery.new()
     |> MediaQuery.require_assoc(:media_profile)
+    |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(
       ^dynamic(
         ^MediaQuery.for_source(source) and
           (not (^MediaQuery.downloaded()) and not (^MediaQuery.pending()))
       )
     )
-    |> order_by(desc: :id)
+    |> order_by(desc: fragment("rank"), desc: :id)
+  end
+
+  defp filter_base_query(base_query, search_term) do
+    base_query
+    |> where(^MediaQuery.matches_search_term(search_term))
   end
 end

priv/repo/migrations/20240523183003_change_media_items_search_index_tokenizer.exs

@@ -0,0 +1,60 @@
+defmodule Pinchflat.Repo.Migrations.ChangeMediaItemsSearchIndexTokenizer do
+  use Ecto.Migration
+
+  def up do
+    # These all need to run as part of separate `execute` blocks. Do NOT ask me why.
+    execute "DROP TRIGGER IF EXISTS media_items_search_index_insert;"
+    execute "DROP TRIGGER IF EXISTS media_items_search_index_update;"
+    execute "DROP TRIGGER IF EXISTS media_items_search_index_delete;"
+    execute "DROP TABLE IF EXISTS media_items_search_index;"
+
+    execute """
+      CREATE VIRTUAL TABLE media_items_search_index USING fts5(
+        title,
+        description,
+        tokenize=trigram
+      );
+    """
+
+    execute """
+      CREATE TRIGGER media_items_search_index_insert AFTER INSERT ON media_items BEGIN
+        INSERT INTO media_items_search_index(
+          rowid,
+          title,
+          description
+        )
+        VALUES(
+          new.id,
+          new.title,
+          new.description
+        );
+      END;
+    """
+
+    execute """
+      CREATE TRIGGER media_items_search_index_update AFTER UPDATE ON media_items BEGIN
+        UPDATE media_items_search_index SET
+          title = new.title,
+          description = new.description
+        WHERE
+          rowid = old.id;
+      END;
+    """
+
+    execute """
+      CREATE TRIGGER media_items_search_index_delete AFTER DELETE ON media_items BEGIN
+        DELETE FROM media_items_search_index WHERE rowid = old.id;
+      END;
+    """
+
+    # Fully re-index the media_items table
+    execute """
+      INSERT INTO media_items_search_index(rowid, title, description)
+      SELECT id, title, description FROM media_items;
+    """
+  end
+
+  def down do
+    execute "DROP TABLE media_items_search_index;"
+  end
+end

test/pinchflat/media_test.exs

@@ -727,6 +727,17 @@ defmodule Pinchflat.MediaTest do
     test "returns an empty list when the search term is nil" do
       assert [] = Media.search(nil)
     end
+
+    test "doesn't blow up if there's an apostrophe or quotes in the search term" do
+      assert [] = Media.search("don't expl'ode")
+      assert [] = Media.search(~s(dont expl"o"de))
+      assert [] = Media.search(~s(dont explo"de))
+    end
+
+    test "doesn't blow up if there is a trailing operand" do
+      assert [] = Media.search("foo OR")
+      assert [] = Media.search("foo AND")
+    end
   end
 
   describe "get_media_item!/1" do