From d26c33129a4661be7c8a78b234bad069c6bcd1f1 Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Sat, 18 May 2024 12:02:02 -0700 Subject: [PATCH] [WIP] Got query kinda working, now to refactor other queries --- lib/pinchflat/media/media.ex | 5 +- lib/pinchflat/media/media_query.ex | 71 +++++++++++++++++++ .../controllers/sources/source_controller.ex | 35 ++++++--- .../sources/source_html/index.html.heex | 8 +-- mix.exs | 1 + mix.lock | 2 +- test/pinchflat/media_test.exs | 2 + 7 files changed, 108 insertions(+), 16 deletions(-) diff --git a/lib/pinchflat/media/media.ex b/lib/pinchflat/media/media.ex index c5e9a38..f27ff7a 100644 --- a/lib/pinchflat/media/media.ex +++ b/lib/pinchflat/media/media.ex @@ -73,8 +73,9 @@ defmodule Pinchflat.Media do """ def list_pending_media_items_for(%Source{} = source) do MediaQuery.new() - |> MediaQuery.for_source(source) - |> MediaQuery.where_pending_download() + |> join(:inner, [m], s in assoc(m, :source)) + |> join(:inner, [m, s], mp in assoc(s, :media_profile)) + |> where([m, s, mp], ^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending?())) |> Repo.all() end diff --git a/lib/pinchflat/media/media_query.ex b/lib/pinchflat/media/media_query.ex index 0ae4777..00b1119 100644 --- a/lib/pinchflat/media/media_query.ex +++ b/lib/pinchflat/media/media_query.ex @@ -38,6 +38,77 @@ defmodule Pinchflat.Media.MediaQuery do MediaItem end + # Queries below this line are dynamic query methods (which I want to move to) + + def for_source(source_id) when is_integer(source_id) do + dynamic([mi], mi.source_id == ^source_id) + end + + def for_source(source) do + dynamic([mi], mi.source_id == ^source.id) + end + + def downloaded? do + dynamic([mi], not is_nil(mi.media_downloaded_at)) + end + + def download_not_prevented do + dynamic([mi], mi.prevent_download == false) + end + + def no_media_filepath do + dynamic([mi], is_nil(mi.media_filepath)) + end + + def upload_date_after_source_cutoff do + dynamic([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date) + end + + def format_matching_profile_preference do + dynamic( + [mi, source, media_profile], + fragment(""" + CASE + WHEN shorts_behaviour = 'only' AND livestream_behaviour = 'only' THEN + livestream = true OR short_form_content = true + WHEN shorts_behaviour = 'only' THEN + short_form_content = true + WHEN livestream_behaviour = 'only' THEN + livestream = true + WHEN shorts_behaviour = 'exclude' AND livestream_behaviour = 'exclude' THEN + short_form_content = false AND livestream = false + WHEN shorts_behaviour = 'exclude' THEN + short_form_content = false + WHEN livestream_behaviour = 'exclude' THEN + livestream = false + ELSE + true + END + """) + ) + end + + def matches_source_title_regex do + dynamic( + [mi, source], + is_nil(source.title_filter_regex) or fragment("regexp_like(?, ?)", mi.title, source.title_filter_regex) + ) + end + + # TODO: figure out how to do something like `require_assoc` here + def pending? do + dynamic( + [mi], + ^download_not_prevented() and + ^no_media_filepath() and + ^upload_date_after_source_cutoff() and + ^format_matching_profile_preference() and + ^matches_source_title_regex() + ) + end + + # Queries below this line are the "legacy" query methods (which I want to move from) + def for_source(query, source_id) when is_integer(source_id) do where(query, [mi], mi.source_id == ^source_id) end diff --git a/lib/pinchflat_web/controllers/sources/source_controller.ex b/lib/pinchflat_web/controllers/sources/source_controller.ex index 3b9c5b2..97da1f2 100644 --- a/lib/pinchflat_web/controllers/sources/source_controller.ex +++ b/lib/pinchflat_web/controllers/sources/source_controller.ex @@ -1,25 +1,44 @@ defmodule PinchflatWeb.Sources.SourceController do use PinchflatWeb, :controller - - import Ecto.Query, warn: false + use Pinchflat.Media.MediaQuery alias Pinchflat.Repo alias Pinchflat.Tasks alias Pinchflat.Sources alias Pinchflat.Sources.Source + alias Pinchflat.Media.MediaItem alias Pinchflat.Profiles.MediaProfile alias Pinchflat.Downloading.DownloadingHelpers alias Pinchflat.SlowIndexing.SlowIndexingHelpers alias Pinchflat.Metadata.SourceMetadataStorageWorker def index(conn, _params) do - sources = - Source - |> order_by(asc: :custom_name) - |> Repo.all() - |> Repo.preload(:media_profile) + source_query = + from s in Source, + as: :source, + inner_join: mp in assoc(s, :media_profile), + preload: [media_profile: mp], + order_by: [asc: s.custom_name], + select: map(s, ^Source.__schema__(:fields)), + select_merge: %{ + downloaded_count: + subquery( + from m in MediaItem, + where: m.source_id == parent_as(:source).id, + where: ^MediaQuery.downloaded?(), + select: count(m.id) + ), + pending_count: + subquery( + from m in MediaItem, + join: s in assoc(m, :source), + where: m.source_id == parent_as(:source).id, + where: ^MediaQuery.pending?(), + select: count(m.id) + ) + } - render(conn, :index, sources: sources) + render(conn, :index, sources: Repo.all(source_query)) end def new(conn, _params) do diff --git a/lib/pinchflat_web/controllers/sources/source_html/index.html.heex b/lib/pinchflat_web/controllers/sources/source_html/index.html.heex index c912cfb..675601c 100644 --- a/lib/pinchflat_web/controllers/sources/source_html/index.html.heex +++ b/lib/pinchflat_web/controllers/sources/source_html/index.html.heex @@ -15,13 +15,12 @@ <.table rows={@sources} table_class="text-black dark:text-white"> <:col :let={source} label="Name"> <.subtle_link href={~p"/sources/#{source.id}"}> - <%= source.custom_name || source.collection_name %> + <%= StringUtils.truncate(source.custom_name || source.collection_name, 35) %> <:col :let={source} label="Type"><%= source.collection_type %> - <:col :let={source} label="Should Download?"> - <.icon name={if source.download_media, do: "hero-check", else: "hero-x-mark"} /> - + <:col :let={source} label="Downloaded"><%= source.downloaded_count %> + <:col :let={source} label="Pending"><%= source.pending_count %> <:col :let={source} label="Retention"> <%= if source.retention_period_days && source.retention_period_days > 0 do %> <%= source.retention_period_days %> day(s) @@ -35,7 +34,6 @@ <:col :let={source} label="" class="flex place-content-evenly"> - <.icon_link href={~p"/sources/#{source.id}"} icon="hero-eye" class="mx-1" /> <.icon_link href={~p"/sources/#{source.id}/edit"} icon="hero-pencil-square" class="mx-1" /> diff --git a/mix.exs b/mix.exs index a1b6028..e74e6b1 100644 --- a/mix.exs +++ b/mix.exs @@ -48,6 +48,7 @@ defmodule Pinchflat.MixProject do [ {:phoenix, "~> 1.7.10"}, {:phoenix_ecto, "~> 4.4"}, + {:ecto, "~> 3.11.2"}, {:ecto_sql, "~> 3.10"}, {:ecto_sqlite3, ">= 0.0.0"}, {:ecto_sqlite3_extras, "~> 1.2.0"}, diff --git a/mix.lock b/mix.lock index 7fe77ff..e986d1b 100644 --- a/mix.lock +++ b/mix.lock @@ -12,7 +12,7 @@ "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"}, - "ecto": {:hex, :ecto, "3.11.1", "4b4972b717e7ca83d30121b12998f5fcdc62ba0ed4f20fd390f16f3270d85c3e", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebd3d3772cd0dfcd8d772659e41ed527c28b2a8bde4b00fe03e0463da0f1983b"}, + "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"}, "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.15.1", "40f2fbd9e246455f8c42e7e0a77009ef806caa1b3ce6f717b2a0a80e8432fcfd", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.11", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.19", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "28b16e177123c688948357176662bf9ff9084daddf950ef5b6baf3ee93707064"}, "ecto_sqlite3_extras": {:hex, :ecto_sqlite3_extras, "1.2.2", "36e60b561a11441d15f26c791817999269fb578b985162207ebb08b04ca71e40", [:mix], [{:exqlite, ">= 0.13.2", [hex: :exqlite, repo: "hexpm", optional: false]}, {:table_rex, "~> 4.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "2b66ba7246bb4f7e39e2578acd4a0e4e4be54f60ff52d450a01be95eeb78ff1e"}, diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index 412b9e3..f98afb1 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -405,7 +405,9 @@ defmodule Pinchflat.MediaTest do describe "list_pending_media_items_for/1" do test "it returns pending without a filepath for a given source" do source = source_fixture() + other_source = source_fixture() media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil}) + _other_media_item = media_item_fixture(%{source_id: other_source.id, media_filepath: nil}) assert Media.list_pending_media_items_for(source) == [media_item] end