diff --git a/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex b/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex index 1407e8c..a0a1a3a 100644 --- a/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex +++ b/lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex @@ -117,14 +117,40 @@ defmodule Pinchflat.Sources.MediaItemTableLive do {:noreply, assign(socket, new_assigns)} end + defp fetch_pagination_attributes(base_query, page, ""), do: fetch_pagination_attributes(base_query, page, nil) + + defp fetch_pagination_attributes(base_query, page, nil) do + total_record_count = Repo.aggregate(base_query, :count, :id) + total_pages = max(ceil(total_record_count / @limit), 1) + page = NumberUtils.clamp(page, 1, total_pages) + + records = + fetch_records(base_query, page) + |> order_by(desc: :uploaded_at) + |> Repo.all() + + %{ + page: page, + total_pages: total_pages, + records: records, + search_term: nil, + total_record_count: total_record_count, + filtered_record_count: total_record_count + } + end + defp fetch_pagination_attributes(base_query, page, search_term) do - filtered_base_query = filter_base_query(base_query, search_term) + filtered_base_query = filtered_base_query(base_query, search_term) total_record_count = Repo.aggregate(base_query, :count, :id) filtered_record_count = Repo.aggregate(filtered_base_query, :count, :id) total_pages = max(ceil(filtered_record_count / @limit), 1) page = NumberUtils.clamp(page, 1, total_pages) - records = fetch_records(filtered_base_query, page) + + records = + fetch_records(filtered_base_query, page) + |> order_by(desc: fragment("rank"), desc: :uploaded_at) + |> Repo.all() %{ page: page, @@ -142,39 +168,41 @@ defmodule Pinchflat.Sources.MediaItemTableLive do base_query |> limit(^@limit) |> offset(^offset) - |> Repo.all() end defp generate_base_query(source, "pending") do MediaQuery.new() + |> select(^select_fields()) |> MediaQuery.require_assoc(:media_profile) - |> MediaQuery.require_assoc(:media_items_search_index) |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending())) - |> order_by(desc: fragment("rank"), desc: :uploaded_at) end defp generate_base_query(source, "downloaded") do MediaQuery.new() - |> MediaQuery.require_assoc(:media_items_search_index) + |> select(^select_fields()) |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded())) - |> order_by(desc: fragment("rank"), desc: :uploaded_at) end defp generate_base_query(source, "other") do MediaQuery.new() + |> select(^select_fields()) |> MediaQuery.require_assoc(:media_profile) - |> MediaQuery.require_assoc(:media_items_search_index) |> where( ^dynamic( ^MediaQuery.for_source(source) and (not (^MediaQuery.downloaded()) and not (^MediaQuery.pending())) ) ) - |> order_by(desc: fragment("rank"), desc: :uploaded_at) end - defp filter_base_query(base_query, search_term) do + defp filtered_base_query(base_query, search_term) do base_query + |> MediaQuery.require_assoc(:media_items_search_index) |> where(^MediaQuery.matches_search_term(search_term)) end + + # Selecting only what we need GREATLY speeds up queries on large tables + defp select_fields do + [:id, :title, :uploaded_at, :prevent_download] + end end diff --git a/priv/repo/erd.png b/priv/repo/erd.png index 5f66b70..30d3e8a 100644 Binary files a/priv/repo/erd.png and b/priv/repo/erd.png differ diff --git a/priv/repo/migrations/20240529000015_modify_upload_date_index.exs b/priv/repo/migrations/20240529000015_modify_upload_date_index.exs new file mode 100644 index 0000000..a9be455 --- /dev/null +++ b/priv/repo/migrations/20240529000015_modify_upload_date_index.exs @@ -0,0 +1,8 @@ +defmodule Pinchflat.Repo.Migrations.ModifyUploadDateIndex do + use Ecto.Migration + + def change do + drop index("media_items", [:upload_date]) + create index("media_items", [:uploaded_at]) + end +end diff --git a/priv/repo/migrations/20240529153838_add_indexes_for_testing.exs b/priv/repo/migrations/20240529153838_add_indexes_for_testing.exs new file mode 100644 index 0000000..e96d20a --- /dev/null +++ b/priv/repo/migrations/20240529153838_add_indexes_for_testing.exs @@ -0,0 +1,19 @@ +defmodule Pinchflat.Repo.Migrations.AddIndexesForTesting do + use Ecto.Migration + + def change do + create index( + "media_items", + [ + :source_id, + :media_filepath, + :uploaded_at, + :prevent_download, + :livestream, + :short_form_content, + :title + ], + name: "media_items_pending_and_downloaded_index" + ) + end +end