diff --git a/lib/pinchflat/media/media_query.ex b/lib/pinchflat/media/media_query.ex index c3a1a65..2eeb5f9 100644 --- a/lib/pinchflat/media/media_query.ex +++ b/lib/pinchflat/media/media_query.ex @@ -125,6 +125,18 @@ defmodule Pinchflat.Media.MediaQuery do ) end + def matches_search_term(nil), do: dynamic([mi], true) + + def matches_search_term(term) do + escaped_term = clean_search_term(term) + + # Matching on `term` instead of `escaped_term` because the latter can mangle empty strings + case String.trim(term) do + "" -> dynamic([mi], true) + _ -> dynamic([mi], fragment("media_items_search_index MATCH ?", ^escaped_term)) + end + end + def require_assoc(query, identifier) do if has_named_binding?(query, identifier) do query @@ -133,6 +145,10 @@ defmodule Pinchflat.Media.MediaQuery do end end + defp do_require_assoc(query, :media_items_search_index) do + from(mi in query, join: s in assoc(mi, :media_items_search_index), as: :media_items_search_index) + end + defp do_require_assoc(query, :source) do from(mi in query, join: s in assoc(mi, :source), as: :source) end @@ -148,9 +164,11 @@ defmodule Pinchflat.Media.MediaQuery do def matching_search_term(query, nil), do: query def matching_search_term(query, term) do + escaped_term = clean_search_term(term) + from(mi in query, join: mi_search_index in assoc(mi, :media_items_search_index), - where: fragment("media_items_search_index MATCH ?", ^term), + where: fragment("media_items_search_index MATCH ?", ^escaped_term), select_merge: %{ matching_search_term: fragment(""" @@ -162,4 +180,26 @@ defmodule Pinchflat.Media.MediaQuery do order_by: [desc: fragment("rank")] ) end + + # SQLite's FTS5 is very picky about what it will accept as a search term. + # To that end, we need to clean up the search term before passing it to the + # MATCH clause. + # This method: + # - Trims leading and trailing whitespace + # - Collapses multiple spaces into a single space + # - Removes quote characters + # - Wraps any word in quotes (must happen after the double quote replacement) + # + # This allows for works with apostrophes and quotes to be searched for correctly + defp clean_search_term(nil), do: "" + defp clean_search_term(""), do: "" + + defp clean_search_term(term) do + term + |> String.trim() + |> String.replace(~r/\s+/, " ") + |> String.split(~r/\s+/) + |> Enum.map(fn str -> String.replace(str, ~s("), "") end) + |> Enum.map_join(" ", fn str -> ~s("#{str}") end) + end end diff --git a/lib/pinchflat_web/components/layouts/partials/header.html.heex b/lib/pinchflat_web/components/layouts/partials/header.html.heex index 1c03f08..ebbfd8d 100644 --- a/lib/pinchflat_web/components/layouts/partials/header.html.heex +++ b/lib/pinchflat_web/components/layouts/partials/header.html.heex @@ -22,7 +22,7 @@ type="text" name="q" value={@params["q"]} - placeholder="Type to search..." + placeholder="Search all media..." class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none" /> diff --git a/lib/pinchflat_web/controllers/searches/search_html/show.html.heex b/lib/pinchflat_web/controllers/searches/search_html/show.html.heex index 22ea5dd..47a09fb 100644 --- a/lib/pinchflat_web/controllers/searches/search_html/show.html.heex +++ b/lib/pinchflat_web/controllers/searches/search_html/show.html.heex @@ -10,19 +10,13 @@ <%= if match?([_|_], @search_results) do %> <.table rows={@search_results} table_class="text-black dark:text-white"> <:col :let={result} label="Title"> - <%= result.title %> + <.subtle_link href={~p"/sources/#{result.source_id}/media/#{result.id}"}> + <%= StringUtils.truncate(result.title, 35) %> + <:col :let={result} label="Excerpt"> <.highlight_search_terms text={result.matching_search_term} /> - <:col :let={result} label="" class="flex place-content-evenly"> - <.link - href={~p"/sources/#{result.source_id}/media/#{result.id}"} - class="hover:text-secondary duration-200 ease-in-out mx-0.5" - > - <.icon name="hero-eye" /> - - <% else %>

No results found

diff --git a/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex b/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex index 48e766c..9912137 100644 --- a/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex +++ b/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex @@ -8,7 +8,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do @limit 10 - def render(%{records: []} = assigns) do + def render(%{total_record_count: 0} = assigns) do ~H"""
<.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" /> @@ -20,10 +20,29 @@ defmodule Pinchflat.Sources.MediaItemTableLive do def render(assigns) do ~H"""
- - <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" tooltip="Refresh" /> - Showing <%= length(@records) %> of <%= @total_record_count %> - +
+ + <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" tooltip="Refresh" /> + Showing <%= length(@records) %> of <%= @filtered_record_count %> + +
+
+ + <.icon name="hero-magnifying-glass" /> + +
+ +
+
+
+
<.table rows={@records} table_class="text-white"> <:col :let={media_item} label="Title"> <.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}> @@ -51,7 +70,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do media_state = session["media_state"] source = Sources.get_source!(session["source_id"]) base_query = generate_base_query(source, media_state) - pagination_attrs = fetch_pagination_attributes(base_query, page) + pagination_attrs = fetch_pagination_attributes(base_query, page, nil) new_assigns = Map.merge( @@ -69,7 +88,14 @@ defmodule Pinchflat.Sources.MediaItemTableLive do def handle_event("page_change", %{"direction" => direction}, %{assigns: assigns} = socket) do direction = if direction == "inc", do: 1, else: -1 new_page = assigns.page + direction - new_assigns = fetch_pagination_attributes(assigns.base_query, new_page) + new_assigns = fetch_pagination_attributes(assigns.base_query, new_page, assigns.search_term) + + {:noreply, assign(socket, new_assigns)} + end + + def handle_event("search_term", params, socket) do + search_term = Map.get(params, "q", nil) + new_assigns = fetch_pagination_attributes(socket.assigns.base_query, 1, search_term) {:noreply, assign(socket, new_assigns)} end @@ -83,18 +109,28 @@ defmodule Pinchflat.Sources.MediaItemTableLive do end def handle_info(%{topic: "media_table", event: "reload"}, %{assigns: assigns} = socket) do - new_assigns = fetch_pagination_attributes(assigns.base_query, assigns.page) + new_assigns = fetch_pagination_attributes(assigns.base_query, assigns.page, assigns.search_term) {:noreply, assign(socket, new_assigns)} end - defp fetch_pagination_attributes(base_query, page) do - total_record_count = Repo.aggregate(base_query, :count, :id) - total_pages = max(ceil(total_record_count / @limit), 1) - page = NumberUtils.clamp(page, 1, total_pages) - records = fetch_records(base_query, page) + defp fetch_pagination_attributes(base_query, page, search_term) do + filtered_base_query = filter_base_query(base_query, search_term) - %{page: page, total_pages: total_pages, records: records, total_record_count: total_record_count} + total_record_count = Repo.aggregate(base_query, :count, :id) + filtered_record_count = Repo.aggregate(filtered_base_query, :count, :id) + total_pages = max(ceil(filtered_record_count / @limit), 1) + page = NumberUtils.clamp(page, 1, total_pages) + records = fetch_records(filtered_base_query, page) + + %{ + page: page, + total_pages: total_pages, + records: records, + search_term: search_term, + total_record_count: total_record_count, + filtered_record_count: filtered_record_count + } end defp fetch_records(base_query, page) do @@ -109,25 +145,33 @@ defmodule Pinchflat.Sources.MediaItemTableLive do defp generate_base_query(source, "pending") do MediaQuery.new() |> MediaQuery.require_assoc(:media_profile) + |> MediaQuery.require_assoc(:media_items_search_index) |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending())) - |> order_by(desc: :id) + |> order_by(desc: fragment("rank"), desc: :id) end defp generate_base_query(source, "downloaded") do MediaQuery.new() + |> MediaQuery.require_assoc(:media_items_search_index) |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded())) - |> order_by(desc: :id) + |> order_by(desc: fragment("rank"), desc: :id) end defp generate_base_query(source, "other") do MediaQuery.new() |> MediaQuery.require_assoc(:media_profile) + |> MediaQuery.require_assoc(:media_items_search_index) |> where( ^dynamic( ^MediaQuery.for_source(source) and (not (^MediaQuery.downloaded()) and not (^MediaQuery.pending())) ) ) - |> order_by(desc: :id) + |> order_by(desc: fragment("rank"), desc: :id) + end + + defp filter_base_query(base_query, search_term) do + base_query + |> where(^MediaQuery.matches_search_term(search_term)) end end diff --git a/priv/repo/erd.png b/priv/repo/erd.png index 4e1e6e6..8ae455e 100644 Binary files a/priv/repo/erd.png and b/priv/repo/erd.png differ diff --git a/priv/repo/migrations/20240523183003_change_media_items_search_index_tokenizer.exs b/priv/repo/migrations/20240523183003_change_media_items_search_index_tokenizer.exs new file mode 100644 index 0000000..b254a5d --- /dev/null +++ b/priv/repo/migrations/20240523183003_change_media_items_search_index_tokenizer.exs @@ -0,0 +1,60 @@ +defmodule Pinchflat.Repo.Migrations.ChangeMediaItemsSearchIndexTokenizer do + use Ecto.Migration + + def up do + # These all need to run as part of separate `execute` blocks. Do NOT ask me why. + execute "DROP TRIGGER IF EXISTS media_items_search_index_insert;" + execute "DROP TRIGGER IF EXISTS media_items_search_index_update;" + execute "DROP TRIGGER IF EXISTS media_items_search_index_delete;" + execute "DROP TABLE IF EXISTS media_items_search_index;" + + execute """ + CREATE VIRTUAL TABLE media_items_search_index USING fts5( + title, + description, + tokenize=trigram + ); + """ + + execute """ + CREATE TRIGGER media_items_search_index_insert AFTER INSERT ON media_items BEGIN + INSERT INTO media_items_search_index( + rowid, + title, + description + ) + VALUES( + new.id, + new.title, + new.description + ); + END; + """ + + execute """ + CREATE TRIGGER media_items_search_index_update AFTER UPDATE ON media_items BEGIN + UPDATE media_items_search_index SET + title = new.title, + description = new.description + WHERE + rowid = old.id; + END; + """ + + execute """ + CREATE TRIGGER media_items_search_index_delete AFTER DELETE ON media_items BEGIN + DELETE FROM media_items_search_index WHERE rowid = old.id; + END; + """ + + # Fully re-index the media_items table + execute """ + INSERT INTO media_items_search_index(rowid, title, description) + SELECT id, title, description FROM media_items; + """ + end + + def down do + execute "DROP TABLE media_items_search_index;" + end +end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index f98afb1..515b7f9 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -727,6 +727,17 @@ defmodule Pinchflat.MediaTest do test "returns an empty list when the search term is nil" do assert [] = Media.search(nil) end + + test "doesn't blow up if there's an apostrophe or quotes in the search term" do + assert [] = Media.search("don't expl'ode") + assert [] = Media.search(~s(dont expl"o"de)) + assert [] = Media.search(~s(dont explo"de)) + end + + test "doesn't blow up if there is a trailing operand" do + assert [] = Media.search("foo OR") + assert [] = Media.search("foo AND") + end end describe "get_media_item!/1" do