From c78444294a234e3486f7a4cbe4ac0a90680059d5 Mon Sep 17 00:00:00 2001
From: Kieran Eglin <kieran.eglin@gmail.com>
Date: Wed, 14 Aug 2024 12:28:57 -0700
Subject: [PATCH] Updated certain Oban queues to have a configurable limit

---
 .iex.exs           |  8 ++++++++
 config/config.exs  | 10 ----------
 config/runtime.exs | 14 ++++++++++++++
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/.iex.exs b/.iex.exs
index ebe1a24..8533f83 100644
--- a/.iex.exs
+++ b/.iex.exs
@@ -23,3 +23,11 @@ alias Pinchflat.Metadata.MetadataFileHelpers
 alias Pinchflat.SlowIndexing.FileFollowerServer
 
 Pinchflat.Release.check_file_permissions()
+
+defmodule IexHelpers do
+  def restart do
+    :init.restart()
+  end
+end
+
+import IexHelpers
diff --git a/config/config.exs b/config/config.exs
index aab4d10..d0b5a58 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -57,16 +57,6 @@ config :pinchflat, Oban,
        {"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
        {"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
      ]}
-  ],
-  # TODO: consider making this an env var or something?
-  queues: [
-    default: 10,
-    fast_indexing: 6,
-    media_indexing: 2,
-    media_collection_indexing: 2,
-    media_fetching: 2,
-    local_data: 8,
-    remote_metadata: 4
   ]
 
 # Configures the mailer
diff --git a/config/runtime.exs b/config/runtime.exs
index 06aa731..1a3c0de 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -40,6 +40,20 @@ config :pinchflat, Pinchflat.Repo,
     Path.join([:code.priv_dir(:pinchflat), "repo", "extensions", "sqlean-linux-#{system_arch}", "sqlean"])
   ]
 
+# Some users may want to increase the number of workers that use yt-dlp to improve speeds
+# Others may want to decrease the number of these workers to lessen the chance of an IP ban
+{yt_dlp_worker_count, _} = Integer.parse(System.get_env("YT_DLP_WORKER_CONCURRENCY", "2"))
+
+config :pinchflat, Oban,
+  queues: [
+    default: 10,
+    fast_indexing: 6,
+    media_collection_indexing: yt_dlp_worker_count,
+    media_fetching: yt_dlp_worker_count,
+    remote_metadata: yt_dlp_worker_count,
+    local_data: 8
+  ]
+
 if config_env() == :prod do
   config_path = "/config"
   db_path = System.get_env("DATABASE_PATH", Path.join([config_path, "db", "pinchflat.db"]))