danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Search v1 (#19)

Browse source 
* Adds description to media items; hooks it up to indexing/media downloading

* Added a search method using postgres fulltext search

* Hooked up search functionality to the search form

* Added persistence to the search form when on search page
This commit is contained in:
Kieran 2024-02-13 17:46:14 -08:00 committed by GitHub
parent 850cf2db73
commit 58771ee37b
25 changed files with 261 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
lib/pinchflat/media.ex
View file

@ -47,6 +47,40 @@ defmodule Pinchflat.Media do
|> Repo.all()
end
@doc """
Returns a list of media_items that match the search term. Adds a `matching_search_term`
virtual field to the result set.
Returns [%MediaItem{}, ...].
"""
def search(search_term, opts \\ []) do
limit = Keyword.get(opts, :limit, 50)
from(mi in MediaItem,
where: fragment("searchable @@ websearch_to_tsquery(?)", ^search_term),
select_merge: %{
matching_search_term:
fragment(
"""
ts_headline(
'english',
CONCAT(title, ' ', description),
websearch_to_tsquery(?),
'StartSel=[PF_HIGHLIGHT],StopSel=[/PF_HIGHLIGHT]'
)
""",
^search_term
)
},
order_by: {
:desc,
fragment("ts_rank_cd(searchable, websearch_to_tsquery(?), 0)", ^search_term)
},
limit: ^limit
)
|> Repo.all()
end
@doc """
Gets a single media_item.

4
lib/pinchflat/media/media_item.ex
View file

@ -13,6 +13,7 @@ defmodule Pinchflat.Media.MediaItem do
@allowed_fields ~w(
title
media_id
description
original_url
livestream
media_downloaded_at
@ -27,6 +28,7 @@ defmodule Pinchflat.Media.MediaItem do
schema "media_items" do
field :title, :string
field :media_id, :string
field :description, :string
field :original_url, :string
field :livestream, :boolean, default: false
field :media_downloaded_at, :utc_datetime
@ -39,6 +41,8 @@ defmodule Pinchflat.Media.MediaItem do
# Will very likely revisit because I can't leave well-enough alone.
field :subtitle_filepaths, {:array, {:array, :string}}, default: []
field :matching_search_term, :string, virtual: true
belongs_to :source, Source
has_one :metadata, MediaMetadata, on_replace: :update

1
lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex
View file

@ -32,6 +32,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
defp parse_media_metadata(metadata) do
%{
title: metadata["title"],
description: metadata["description"],
media_filepath: metadata["filepath"]
}
end

3
lib/pinchflat/media_client/backends/yt_dlp/video_collection.ex
View file

@ -14,8 +14,9 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
def get_media_attributes(url, command_opts \\ []) do
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
opts = command_opts ++ [:simulate, :skip_download]
output_template = "%(.{id,title,was_live,original_url,description})j"
case runner.run(url, opts, "%(.{id,title,was_live,original_url})j") do
case runner.run(url, opts, output_template) do
{:ok, output} ->
output
|> String.split("\n", trim: true)

3
lib/pinchflat/tasks/source_tasks.ex
View file

@ -52,7 +52,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
title: media_attrs["title"],
media_id: media_attrs["id"],
original_url: media_attrs["original_url"],
livestream: media_attrs["was_live"]
livestream: media_attrs["was_live"],
description: media_attrs["description"]
}
case Media.create_media_item(attrs) do

17
lib/pinchflat/utils/string_utils.ex
View file

@ -24,4 +24,21 @@ defmodule Pinchflat.Utils.StringUtils do
|> Base.encode16(case: :lower)
|> String.slice(0..(length - 1))
end
@doc """
Truncates a string to the given length and adds `...` if the string is longer than the given length.
Will break on a word boundary. Nothing happens if the string is shorter than the given length.
Returns binary()
"""
def truncate(string, length) do
if String.length(string) > length do
string
|> String.slice(0..(length - 1))
|> String.replace(~r/\s+\S*$/, "")
|> Kernel.<>("...")
else
string
end
end
end

2
lib/pinchflat_web.ex
View file

@ -89,6 +89,8 @@ defmodule PinchflatWeb do
import PinchflatWeb.CustomComponents.TableComponents
import PinchflatWeb.CustomComponents.ButtonComponents
alias Pinchflat.Utils.StringUtils
# Shortcut for generating JS commands
alias Phoenix.LiveView.JS

2
lib/pinchflat_web/components/layouts/app.html.heex
View file

@ -2,7 +2,7 @@
<.sidebar />
<div class="relative flex flex-1 flex-col overflow-y-auto overflow-x-hidden">
<.header />
<.header params={@conn.params} />
<main>
<div class="mx-auto max-w-screen-2xl p-4 md:p-6 2xl:p-10">
<.flash_group flash={@flash} />

22
lib/pinchflat_web/components/layouts/partials/header.html.heex
View file

@ -1,5 +1,5 @@
<header class="sticky top-0 z-999 flex w-full bg-white drop-shadow-1 dark:bg-boxdark dark:drop-shadow-none">
<div class="flex flex-grow items-center justify-end px-4 py-4 shadow-2 md:px-6 2xl:px-11">
<div class="flex flex-grow items-center justify-between lg:justify-end px-4 py-4 shadow-2 md:px-6 2xl:px-11">
<div class="flex items-center gap-2 sm:gap-4 lg:hidden">
<button
class="z-99999 block rounded-sm border border-stroke bg-white p-1.5 shadow-sm dark:border-strokedark dark:bg-boxdark lg:hidden"
@ -7,22 +7,24 @@
>
<.icon name="hero-bars-3" />
</button>
<a class="block flex-shrink-0 lg:hidden" href="#">
<a class="hidden sm:block flex-shrink-0 lg:hidden" href="#">
<h2 class="text-title-md2 font-bold text-white">Pinchflat</h2>
</a>
</div>
<div class="hidden sm:block bg-meta-4 rounded-md">
<%!-- Aspirational (for now) --%>
<div class="bg-meta-4 rounded-md">
<div class="relative">
<span class="absolute left-2 top-1/2 -translate-y-1/2 flex">
<.icon name="hero-magnifying-glass" />
</span>
<input
type="text"
placeholder="Type to search..."
class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none xl:w-125"
/>
<form action={~p"/search"}>
<input
type="text"
name="q"
value={@params["q"]}
placeholder="Type to search..."
class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none xl:w-125"
/>
</form>
</div>
</div>
</div>

10
lib/pinchflat_web/controllers/media_sources/source_html/show.html.heex
View file

@ -16,7 +16,7 @@
</.link>
</nav>
</div>
<div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">
<div class="rounded-sm border border-stroke bg-white px-5 py-5 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5">
<div class="max-w-full overflow-x-auto">
<div class="flex flex-col gap-10 dark:text-white">
<h3 class="mt-14 font-bold text-xl">Relationships</h3>
@ -35,10 +35,10 @@
<.list_items_from_map map={Map.from_struct(@source)} />
<h3 class="font-bold text-xl">Downloaded Media</h3>
<%= if length(@downloaded_media) > 0 do %>
<%= if match?([_|_], @downloaded_media) do %>
<.table rows={@downloaded_media} table_class="text-black dark:text-white">
<:col :let={media_item} label="Title">
<%= String.slice(media_item.title, 0..50) %>...
<%= StringUtils.truncate(media_item.title, 50) %>
</:col>
<:col :let={media_item} label="" class="flex place-content-evenly">
<.link
@ -54,10 +54,10 @@
<% end %>
<h3 class="font-bold text-xl">Pending Media</h3>
<%= if length(@pending_media) > 0 do %>
<%= if match?([_|_], @pending_media) do %>
<.table rows={@pending_media} table_class="text-black dark:text-white">
<:col :let={media_item} label="Title">
<%= String.slice(media_item.title, 0..50) %>...
<%= StringUtils.truncate(media_item.title, 50) %>
</:col>
<:col :let={media_item} label="" class="flex place-content-evenly">
<.link

12
lib/pinchflat_web/controllers/searches/search_controller.ex Normal file
View file

@ -0,0 +1,12 @@
defmodule PinchflatWeb.Searches.SearchController do
use PinchflatWeb, :controller
alias Pinchflat.Media
def show(conn, params) do
search_term = Map.get(params, "q", "")
search_results = Media.search(search_term)
render(conn, :show, search_term: search_term, search_results: search_results)
end
end

25
lib/pinchflat_web/controllers/searches/search_html.ex Normal file
View file

@ -0,0 +1,25 @@
defmodule PinchflatWeb.Searches.SearchHTML do
use PinchflatWeb, :html
embed_templates "search_html/*"
@doc """
Highlight search terms in a string of text based on `[PF_HIGHLIGHT]` and `[/PF_HIGHLIGHT]` tags
"""
attr :text, :string, required: true
def highlight_search_terms(assigns) do
split_string = String.split(assigns.text, ~r{\[PF_HIGHLIGHT\]|\[/PF_HIGHLIGHT\]}, include_captures: true)
assigns = assign(assigns, split_string: split_string)
~H"""
<%= for fragment <- @split_string do %>
<%= render_fragment(fragment) %>
<% end %>
"""
end
defp render_fragment("[PF_HIGHLIGHT]"), do: raw(~s(<span class="font-bold italic">))
defp render_fragment("[/PF_HIGHLIGHT]"), do: raw("</span>")
defp render_fragment(text), do: text
end

29
lib/pinchflat_web/controllers/searches/search_html/show.html.heex Normal file
View file

@ -0,0 +1,29 @@
<div class="mb-6 flex gap-3 flex-row items-center justify-between">
<h2 class="text-title-md2 font-bold text-black dark:text-white">
<span class="hidden sm:inline">Search </span>Results for "<%= StringUtils.truncate(@search_term, 50) %>"
</h2>
</div>
<div class="rounded-sm border border-stroke bg-white px-5 py-5 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5">
<div class="max-w-full overflow-x-auto">
<div class="flex flex-col gap-10 dark:text-white">
<%= if match?([_|_], @search_results) do %>
<.table rows={@search_results} table_class="text-black dark:text-white">
<:col :let={result} label="Title">
<%= StringUtils.truncate(result.title, 40) %>
</:col>
<:col :let={result} label="Excerpt">
<.highlight_search_terms text={result.matching_search_term} />
</:col>
<:col :let={result} label="" class="flex place-content-evenly">
<.link navigate={~p"/media/#{result.id}"} class="hover:text-secondary duration-200 ease-in-out mx-0.5">
<.icon name="hero-eye" />
</.link>
</:col>
</.table>
<% else %>
<p class="font-bold text-lg text-center text-black dark:text-white">No results found</p>
<% end %>
</div>
</div>
</div>

1
lib/pinchflat_web/router.ex
View file

@ -21,6 +21,7 @@ defmodule PinchflatWeb.Router do
resources "/media_profiles", MediaProfiles.MediaProfileController
resources "/media", Media.MediaItemController, only: [:show]
resources "/search", Searches.SearchController, only: [:show], singleton: true
resources "/sources", MediaSources.SourceController do
resources "/media", Media.MediaItemController, only: [:show]

9
priv/repo/migrations/20240210045410_add_description_to_media_items.exs Normal file
View file

@ -0,0 +1,9 @@
defmodule Pinchflat.Repo.Migrations.AddDescriptionToMediaItems do
use Ecto.Migration
def change do
alter table(:media_items) do
add :description, :text
end
end
end

28
priv/repo/migrations/20240210051038_add_search_field_to_media_items.exs Normal file
View file

@ -0,0 +1,28 @@
defmodule Pinchflat.Repo.Migrations.AddSearchFieldToMediaItems do
use Ecto.Migration
def up do
execute """
ALTER TABLE media_items
ADD COLUMN searchable tsvector
GENERATED ALWAYS AS (
setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
setweight(to_tsvector('english', coalesce(description, '')), 'B')
) STORED;
"""
execute """
CREATE INDEX media_items_searchable_idx ON media_items USING gin(searchable);
"""
end
def down do
execute """
DROP INDEX media_items_searchable_idx;
"""
alter table(:media_items) do
remove :searchable
end
end
end

6
test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs
View file

@ -36,6 +36,12 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
assert result.title == "Trying to Wheelie Without the Rear Brake"
end
test "it extracts the description", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)
assert is_binary(result.description)
end
test "it returns the metadata as a map", %{metadata: metadata} do
result = Parser.parse_for_media_item(metadata)

2
test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs
View file

@ -20,7 +20,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
test "it passes the expected default args" do
expect(YtDlpRunnerMock, :run, fn _url, opts, ot ->
assert opts == [:simulate, :skip_download]
assert ot == "%(.{id,title,was_live,original_url})j"
assert ot == "%(.{id,title,was_live,original_url,description})j"
{:ok, ""}
end)

2
test/pinchflat/media_client/source_details_test.exs
View file

@ -47,7 +47,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
test "it passes the expected arguments to the backend" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [:simulate, :skip_download]
assert ot == "%(.{id,title,was_live,original_url})j"
assert ot == "%(.{id,title,was_live,original_url,description})j"
{:ok, ""}
end)

7
test/pinchflat/media_client/video_downloader_test.exs
View file

@ -60,7 +60,7 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
test "it sets the media_downloaded_at", %{media_item: media_item} do
assert media_item.media_downloaded_at == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert DateTime.diff(DateTime.utc_now(), updated_media_item.media_downloaded_at) < 1
assert DateTime.diff(DateTime.utc_now(), updated_media_item.media_downloaded_at) < 2
end
test "it extracts the title", %{media_item: media_item} do
@ -68,6 +68,11 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake"
end
test "it extracts the description", %{media_item: media_item} do
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert is_binary(updated_media_item.description)
end
test "it extracts the media_filepath", %{media_item: media_item} do
assert media_item.media_filepath == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)

31
test/pinchflat/media_test.exs
View file

@ -183,6 +183,37 @@ defmodule Pinchflat.MediaTest do
end
end
describe "search/1" do
setup do
media_item =
media_item_fixture(%{
title: "The quick brown fox",
description: "jumps over the lazy dog"
})
{:ok, %{media_item_id: media_item.id}}
end
test "searches based on title", %{media_item_id: media_item_id} do
assert [%{id: ^media_item_id}] = Media.search("quick")
end
test "searches based on description", %{media_item_id: media_item_id} do
assert [%{id: ^media_item_id}] = Media.search("lazy")
end
test "adds a matching_search_term attribute with the relevant text" do
assert [res] = Media.search("quick")
assert String.contains?(res.matching_search_term, "The [PF_HIGHLIGHT]quick[/PF_HIGHLIGHT] brown fox")
end
test "optionally lets you specify a limit" do
media_item_fixture(%{title: "The small gray dog"})
assert [_] = Media.search("dog", limit: 1)
end
end
describe "get_media_item!/1" do
test "it returns the media_item with given id" do
media_item = media_item_fixture()

3
test/pinchflat/tasks/source_tasks_test.exs
View file

@ -63,6 +63,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
assert Enum.count(media_items) == 3
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
assert ["Video 1", "Video 2", "Video 3"] == Enum.map(media_items, & &1.title)
assert ["desc1", "desc2", "desc3"] == Enum.map(media_items, & &1.description)
assert Enum.all?(media_items, fn mi -> mi.original_url end)
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
end
@ -110,7 +111,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
SourceTasks.index_media_items(source)
source = Repo.reload!(source)
assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 1
assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 2
end
end

14
test/pinchflat/utils/string_utils_test.exs
View file

@ -27,4 +27,18 @@ defmodule Pinchflat.Utils.StringUtilsTest do
assert String.length(StringUtils.random_string(64)) == 64
end
end
describe "truncate/2" do
test "truncates a string to the given length and adds ..." do
assert StringUtils.truncate("hello world", 5) == "hello..."
end
test "breaks on a word boundary" do
assert StringUtils.truncate("hello world", 7) == "hello..."
end
test "does not truncate a string shorter than the given length" do
assert StringUtils.truncate("hello", 10) == "hello"
end
end
end

10
test/pinchflat_web/controllers/search_controller_test.exs Normal file
View file

@ -0,0 +1,10 @@
defmodule PinchflatWeb.SearchControllerTest do
use PinchflatWeb.ConnCase
describe "show search" do
test "renders the page", %{conn: conn} do
conn = get(conn, ~p"/search")
assert html_response(conn, 200) =~ "Results"
end
end
end

9
test/support/fixtures/media_source_fixtures.ex
View file

@ -36,19 +36,22 @@ defmodule Pinchflat.MediaSourceFixtures do
id: "video1",
title: "Video 1",
original_url: "https://example.com/video1",
was_live: false
was_live: false,
description: "desc1"
},
%{
id: "video2",
title: "Video 2",
original_url: "https://example.com/video2",
was_live: true
was_live: true,
description: "desc2"
},
%{
id: "video3",
title: "Video 3",
original_url: "https://example.com/video3",
was_live: false
was_live: false,
description: "desc3"
}
]