From 74e6d2cf95cc381f9b600f0aeea9ef70f2caf720 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:08:51 +0000 Subject: [PATCH] don't add torrents straight to debrid provider, add to queue first, dequeue in `TorrentRunner` --- client/src/app/models/torrent.model.ts | 1 + client/src/app/torrent-status.pipe.ts | 2 + server/RdtClient.Data/Data/ITorrentData.cs | 3 +- server/RdtClient.Data/Data/TorrentData.cs | 22 +++- server/RdtClient.Data/Enums/TorrentStatus.cs | 2 + .../Models/Internal/DbSettings.cs | 4 + .../BackgroundServices/WatchFolderChecker.cs | 4 +- .../RdtClient.Service/Services/QBittorrent.cs | 4 +- .../Services/TorrentRunner.cs | 23 ++++ server/RdtClient.Service/Services/Torrents.cs | 123 +++++++++++------- .../Controllers/TorrentsController.cs | 4 +- 11 files changed, 139 insertions(+), 53 deletions(-) diff --git a/client/src/app/models/torrent.model.ts b/client/src/app/models/torrent.model.ts index 36abb26..0397dbb 100644 --- a/client/src/app/models/torrent.model.ts +++ b/client/src/app/models/torrent.model.ts @@ -62,6 +62,7 @@ export class TorrentFileAvailability { } export enum RealDebridStatus { + NotYetAdded = -1, Processing = 0, WaitingForFileSelection = 1, Downloading = 2, diff --git a/client/src/app/torrent-status.pipe.ts b/client/src/app/torrent-status.pipe.ts index 0eb2ed9..3e46a3c 100644 --- a/client/src/app/torrent-status.pipe.ts +++ b/client/src/app/torrent-status.pipe.ts @@ -87,6 +87,8 @@ export class TorrentStatusPipe implements PipeTransform { } switch (torrent.rdStatus) { + case RealDebridStatus.NotYetAdded: + return "Not Yet Added to Provider" case RealDebridStatus.Downloading: if (torrent.rdSeeders < 1) { return `Torrent stalled`; diff --git a/server/RdtClient.Data/Data/ITorrentData.cs b/server/RdtClient.Data/Data/ITorrentData.cs index 4bbaa24..9c286e3 100644 --- a/server/RdtClient.Data/Data/ITorrentData.cs +++ b/server/RdtClient.Data/Data/ITorrentData.cs @@ -9,7 +9,7 @@ public interface ITorrentData Task GetById(Guid torrentId); Task GetByHash(String hash); - Task Add(String rdId, + Task Add(String? rdId, String hash, String? fileOrMagnetContents, Boolean isFile, @@ -17,6 +17,7 @@ public interface ITorrentData Torrent torrent); Task UpdateRdData(Torrent torrent); + Task UpdateRdId(Torrent torrent, String rdId); Task Update(Torrent torrent); Task UpdateCategory(Guid torrentId, String? category); Task UpdateComplete(Guid torrentId, String? error, DateTimeOffset? datetime, Boolean retry); diff --git a/server/RdtClient.Data/Data/TorrentData.cs b/server/RdtClient.Data/Data/TorrentData.cs index 4578348..5b74f63 100644 --- a/server/RdtClient.Data/Data/TorrentData.cs +++ b/server/RdtClient.Data/Data/TorrentData.cs @@ -71,7 +71,7 @@ public class TorrentData(DataContext dataContext) : ITorrentData return dbTorrent; } - public async Task Add(String rdId, + public async Task Add(String? rdId, String hash, String? fileOrMagnetContents, Boolean isFile, @@ -99,7 +99,9 @@ public class TorrentData(DataContext dataContext) : ITorrentData TorrentRetryAttempts = torrent.TorrentRetryAttempts, DownloadRetryAttempts = torrent.DownloadRetryAttempts, DeleteOnError = torrent.DeleteOnError, - Lifetime = torrent.Lifetime + Lifetime = torrent.Lifetime, + RdStatus = torrent.RdStatus, + RdName = torrent.RdName }; await dataContext.Torrents.AddAsync(newTorrent); @@ -138,6 +140,22 @@ public class TorrentData(DataContext dataContext) : ITorrentData await VoidCache(); } + public async Task UpdateRdId(Torrent torrent, String rdId) + { + var dbTorrent = await dataContext.Torrents.FirstOrDefaultAsync(m => m.TorrentId == torrent.TorrentId); + + if (dbTorrent == null) + { + return; + } + + dbTorrent.RdId = rdId; + + await dataContext.SaveChangesAsync(); + + await VoidCache(); + } + public async Task Update(Torrent torrent) { var dbTorrent = await dataContext.Torrents.FirstOrDefaultAsync(m => m.TorrentId == torrent.TorrentId); diff --git a/server/RdtClient.Data/Enums/TorrentStatus.cs b/server/RdtClient.Data/Enums/TorrentStatus.cs index d4de133..16b556e 100644 --- a/server/RdtClient.Data/Enums/TorrentStatus.cs +++ b/server/RdtClient.Data/Enums/TorrentStatus.cs @@ -2,6 +2,8 @@ public enum TorrentStatus { + NotYetAdded = -1, + Processing = 0, WaitingForFileSelection = 1, Downloading = 2, diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index e90e258..5cf177d 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -193,6 +193,10 @@ or [Description("The interval to check the torrents info on the providers API. Minumum is 3 seconds. When there are no active downloads this limit is increased * 3.")] public Int32 CheckInterval { get; set; } = 10; + [DisplayName("Max parallel downloads")] + [Description("Limits the number of torrents that will be sent for downloading on the debrid provider at the same time. If set to 0, all downloads will be sent immediately without queuing.")] + public Int32 MaxParallelDownloads { get; set; } = 0; + [DisplayName("Auto Import Defaults")] public DbSettingsDefaultsWithCategory Default { get; set; } = new(); } diff --git a/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs b/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs index 417dee5..4c1f243 100644 --- a/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs +++ b/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs @@ -99,12 +99,12 @@ public class WatchFolderChecker(ILogger logger, IServiceProv if (fileInfo.Extension == ".torrent") { var torrentFileContents = await File.ReadAllBytesAsync(torrentFile, stoppingToken); - await torrentService.UploadFile(torrentFileContents, torrent); + await torrentService.AddFileToDebridQueue(torrentFileContents, torrent); } else if (fileInfo.Extension == ".magnet") { var magnetLink = await File.ReadAllTextAsync(torrentFile, stoppingToken); - await torrentService.UploadMagnet(magnetLink, torrent); + await torrentService.AddMagnetToDebridQueue(magnetLink, torrent); } if (!Directory.Exists(processedStorePath)) diff --git a/server/RdtClient.Service/Services/QBittorrent.cs b/server/RdtClient.Service/Services/QBittorrent.cs index 3c12685..081762e 100644 --- a/server/RdtClient.Service/Services/QBittorrent.cs +++ b/server/RdtClient.Service/Services/QBittorrent.cs @@ -474,7 +474,7 @@ public class QBittorrent(ILogger logger, Settings settings, Authent Priority = priority ?? (Settings.Get.Integrations.Default.Priority > 0 ? Settings.Get.Integrations.Default.Priority : null) }; - await torrents.UploadMagnet(magnetLink, torrent); + await torrents.AddMagnetToDebridQueue(magnetLink, torrent); } public async Task TorrentsAddFile(Byte[] fileBytes, String? category, Int32? priority) @@ -498,7 +498,7 @@ public class QBittorrent(ILogger logger, Settings settings, Authent Priority = priority ?? (Settings.Get.Integrations.Default.Priority > 0 ? Settings.Get.Integrations.Default.Priority : null) }; - await torrents.UploadFile(fileBytes, torrent); + await torrents.AddFileToDebridQueue(fileBytes, torrent); } public async Task TorrentsSetCategory(String hash, String? category) diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index a7e7cb6..a518620 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -314,6 +314,29 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow await torrents.UpdateComplete(torrent.TorrentId, $"Torrent lifetime of {torrent.Lifetime} minutes reached", DateTimeOffset.UtcNow, false); } + // Process torrents in DebridQueue + var torrentsToAddToProvider = allTorrents.Where(m => m.RdId == null && m.RdAdded == null && m.FileOrMagnet != null && m.RdStatus == TorrentStatus.NotYetAdded) + .ToList(); + + if (torrentsToAddToProvider.Count != 0) + { + var downloadingTorrentsCount = allTorrents.Count(m => m.RdStatus is not (TorrentStatus.NotYetAdded or TorrentStatus.Finished or TorrentStatus.Error)); + + var maxParallelDownloads = Settings.Get.Provider.MaxParallelDownloads; + + logger.LogDebug("Currently downloading {downloadingTorrentCount}/{maxParallelDownloads} torrents, {queuedCount} queued.", + downloadingTorrentsCount, + maxParallelDownloads, + torrentsToAddToProvider.Count); + + var dequeueCount = maxParallelDownloads == 0 ? torrentsToAddToProvider.Count : maxParallelDownloads - downloadingTorrentsCount; + + foreach (var torrent in torrentsToAddToProvider.Take(dequeueCount)) + { + await torrents.DequeueFromDebridQueue(torrent); + } + } + allTorrents = await torrents.Get(); allTorrents = allTorrents.Where(m => m.Completed == null).ToList(); diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index a88087a..505e1af 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -107,7 +107,7 @@ public class Torrents( await torrentData.UpdateCategory(torrent.TorrentId, category); } - public async Task UploadMagnet(String magnetLink, Torrent torrent) + public async Task AddMagnetToDebridQueue(String magnetLink, Torrent torrent) { MagnetLink magnet; @@ -121,42 +121,21 @@ public class Torrents( throw new($"{ex.Message}, trying to parse {magnetLink}"); } - var id = await TorrentClient.AddMagnet(magnetLink); + torrent.RdStatus = TorrentStatus.NotYetAdded; + torrent.RdName = magnet.Name; var hash = magnet.InfoHashes.V1OrV2.ToHex(); - var newTorrent = await Add(id, hash, magnetLink, false, torrent); + var newTorrent = await AddQueued(hash, magnetLink, false, torrent); - Log($"Adding {hash} magnet link {magnetLink}", newTorrent); + Log($"Adding {hash} (magnet link) to queue", newTorrent); - if (!String.IsNullOrWhiteSpace(Settings.Get.General.CopyAddedTorrents)) - { - try - { - if (!Directory.Exists(Settings.Get.General.CopyAddedTorrents)) - { - Directory.CreateDirectory(Settings.Get.General.CopyAddedTorrents); - } - - var copyFileName = Path.Combine(Settings.Get.General.CopyAddedTorrents, $"{FileHelper.RemoveInvalidFileNameChars(magnet.Name!)}.magnet"); - - if (File.Exists(copyFileName)) - { - File.Delete(copyFileName); - } - - await File.WriteAllTextAsync(copyFileName, magnetLink); - } - catch (Exception ex) - { - logger.LogError(ex, $"Unable to create torrent blackhole directory: {Settings.Get.General.CopyAddedTorrents}: {ex.Message}"); - } - } + await CopyAddedTorrent(magnet.Name!, magnetLink); return newTorrent; } - public async Task UploadFile(Byte[] bytes, Torrent torrent) + public async Task AddFileToDebridQueue(Byte[] bytes, Torrent torrent) { MonoTorrent.Torrent monoTorrent; @@ -172,14 +151,22 @@ public class Torrents( throw new($"{ex.Message}, trying to parse {fileAsBase64}"); } - var id = await TorrentClient.AddFile(bytes); + torrent.RdStatus = TorrentStatus.NotYetAdded; + torrent.RdName = monoTorrent.Name; var hash = monoTorrent.InfoHashes.V1OrV2.ToHex(); - var newTorrent = await Add(id, hash, fileAsBase64, true, torrent); + var newTorrent = await AddQueued(hash, fileAsBase64, true, torrent); - Log($"Adding {hash} torrent file", newTorrent); + Log($"Adding {hash} (torrent file) to queue", newTorrent); + await CopyAddedTorrent(monoTorrent.Name, bytes); + + return newTorrent; + } + + private async Task CopyAddedTorrent(String torrentName, Object fileOrMagnet) + { if (!String.IsNullOrWhiteSpace(Settings.Get.General.CopyAddedTorrents)) { try @@ -189,22 +176,73 @@ public class Torrents( Directory.CreateDirectory(Settings.Get.General.CopyAddedTorrents); } - var copyFileName = Path.Combine(Settings.Get.General.CopyAddedTorrents, $"{FileHelper.RemoveInvalidFileNameChars(monoTorrent.Name)}.torrent"); + var copyFileName = Path.Combine(Settings.Get.General.CopyAddedTorrents, FileHelper.RemoveInvalidFileNameChars(torrentName)); + switch (fileOrMagnet) + { + case String: + copyFileName = $"{copyFileName}.magnet"; + break; + case Byte[]: + copyFileName = $"{copyFileName}.torrent"; + break; + default: + throw new ArgumentException("Unexpected type for fileOrMagnet"); + } + if (File.Exists(copyFileName)) { File.Delete(copyFileName); } - await File.WriteAllBytesAsync(copyFileName, bytes); + switch (fileOrMagnet) + { + case String magnetLink: + await File.WriteAllTextAsync(copyFileName, magnetLink); + break; + case Byte[] torrentFile: + await File.WriteAllBytesAsync(copyFileName, torrentFile); + break; + } } catch (Exception ex) { logger.LogError(ex, $"Unable to create torrent blackhole directory: {Settings.Get.General.CopyAddedTorrents}: {ex.Message}"); } } + } - return newTorrent; + /// + /// Adds torrent in database to debrid provider and updates database accordingly. + /// + /// The torrent from the database to upload to the debrid provider + /// Updated torrent + /// When RdId is not null or FileOrMagnet is null. + public async Task DequeueFromDebridQueue(Torrent torrent) + { + if (torrent.RdId != null) + { + throw new("Torrent already added to debrid provider, cannot dequeue"); + } + + if (torrent.FileOrMagnet == null) + { + throw new("Torrent has no torrent file or magnet link"); + } + + logger.LogDebug("Adding {hash} to debrid provider {torrentInfo}", torrent.Hash, torrent.ToLog()); + + var id = (torrent.IsFile) switch + { + true => await TorrentClient.AddFile(Convert.FromBase64String(torrent.FileOrMagnet)), + false => await TorrentClient.AddMagnet(torrent.FileOrMagnet) + }; + + await torrentData.UpdateRdId(torrent, id); + + await UpdateTorrentClientData(torrent); + + return torrent; } public async Task> GetAvailableFiles(String hash) @@ -538,11 +576,11 @@ public class Torrents( { var bytes = Convert.FromBase64String(torrent.FileOrMagnet); - newTorrent = await UploadFile(bytes, torrent); + newTorrent = await AddFileToDebridQueue(bytes, torrent); } else { - newTorrent = await UploadMagnet(torrent.FileOrMagnet, torrent); + newTorrent = await AddMagnetToDebridQueue(torrent.FileOrMagnet, torrent); } await torrentData.UpdateRetry(newTorrent.TorrentId, null, retryCount); @@ -670,11 +708,10 @@ public class Torrents( return settingDownloadPath; } - private async Task Add(String rdTorrentId, - String infoHash, - String fileOrMagnetContents, - Boolean isFile, - Torrent torrent) + private async Task AddQueued(String infoHash, + String fileOrMagnetContents, + Boolean isFile, + Torrent torrent) { await RealDebridUpdateLock.WaitAsync(); @@ -687,15 +724,13 @@ public class Torrents( return existingTorrent; } - var newTorrent = await torrentData.Add(rdTorrentId, + var newTorrent = await torrentData.Add(null, infoHash, fileOrMagnetContents, isFile, torrent.DownloadClient, torrent); - await UpdateTorrentClientData(newTorrent); - return newTorrent; } finally diff --git a/server/RdtClient.Web/Controllers/TorrentsController.cs b/server/RdtClient.Web/Controllers/TorrentsController.cs index 69536b5..f713985 100644 --- a/server/RdtClient.Web/Controllers/TorrentsController.cs +++ b/server/RdtClient.Web/Controllers/TorrentsController.cs @@ -84,7 +84,7 @@ public class TorrentsController(ILogger logger, Torrents tor var bytes = memoryStream.ToArray(); - await torrents.UploadFile(bytes, formData.Torrent); + await torrents.AddFileToDebridQueue(bytes, formData.Torrent); return Ok(); } @@ -110,7 +110,7 @@ public class TorrentsController(ILogger logger, Torrents tor logger.LogDebug($"Add magnet"); - await torrents.UploadMagnet(request.MagnetLink, request.Torrent); + await torrents.AddMagnetToDebridQueue(request.MagnetLink, request.Torrent); return Ok(); }