From 337b70ba01bbfe31680a608621d3b442faa3f90f Mon Sep 17 00:00:00 2001 From: Bryan Reagan Date: Fri, 15 Oct 2021 10:11:50 -0500 Subject: [PATCH 01/31] Rootless Podman and CIFS Connections First, thanks for the awesome utility! Some feedback on getting the client running on Arch using rootless Podman. Works OOB with normal paths on the host and following the tips from LinuxServer.io, but some funny errors working with paths available through a CIFS mount. Still having some bugs I'm working out, but thought to get a PR going if you thought these notes would be helpful to others. These issues seem like an edge case; let me know if you want issues created for these. ## Issues - [x] Access Denied Errors in Web GUI Tests ([dotnet runtime issue link](https://github.com/dotnet/runtime/issues/42790)) - [ ] Directory Does Not Exist To Extract To --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 59b00ab..70c23af 100644 --- a/README.md +++ b/README.md @@ -162,3 +162,14 @@ Notice: the progress and ETA reported in Sonarr's Activity tab will not be accur 1. To stop: `docker stop rdtclient` 1. To remove: `docker rm rdtclient` 1. Or use `docker-build.bat` + +## Misc Install Notes + +### Rootless Podman, Linux Host, and CIFS Connections + +RDT Client read and write permission tests fail if the CIFS connection is not setup properly, despite permissions working inspection. In the Web GUI, it will report access denied, and in the log file you will see exceptions like this ([dotnet issue](https://github.com/dotnet/runtime/issues/42790)): +``` +System.IO.IOException: Permission denied +``` +The **nobrl** has to be specified in your CIFS connection - [man page](https://linux.die.net/man/8/mount.cifs). +Example: ```Options=_netdev,credentials=/etc/samba/credentials/600file,rw,uid=SUBUID,gid=SBUGID,nobrl,file_mode=0770,dir_mode=0770,noperm``` From f21fd6d8f3a3a67d9b9f47f63a39c6288b61ac9f Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Mon, 17 Feb 2025 22:38:10 +0000 Subject: [PATCH 02/31] Add `dotnet test` GitHub Action --- .github/workflows/dotnet-test.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/dotnet-test.yml diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml new file mode 100644 index 0000000..5d104d3 --- /dev/null +++ b/.github/workflows/dotnet-test.yml @@ -0,0 +1,24 @@ +name: dotnet test + +on: + push: + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 9.0.x + - name: Restore dependencies + run: dotnet restore server + - name: Build + run: dotnet build --no-restore server + - name: Test + run: dotnet test --no-build --verbosity normal server From e77623627006b39ed8c4eb178379173e632d332b Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:12:16 +0000 Subject: [PATCH 03/31] Use richer `GetDownloadInfos` instead of `GetDownloadLinks` *Most* debrid providers know the filename as soon as they know the restricted links. By allowing them to tell the program this at an earlier stage, we don't need the `GetFileName` call for most debrid providers --- server/RdtClient.Data/Data/DownloadData.cs | 6 ++++-- server/RdtClient.Data/Models/Data/Download.cs | 18 +++++++++++++++++- server/RdtClient.Service/Services/Downloads.cs | 5 +++-- .../RdtClient.Service/Services/IDownloads.cs | 2 +- .../Services/TorrentClients/ITorrentClient.cs | 10 ++++++++-- .../Services/TorrentRunner.cs | 7 +++++-- server/RdtClient.Service/Services/Torrents.cs | 16 ++++++++-------- 7 files changed, 46 insertions(+), 18 deletions(-) diff --git a/server/RdtClient.Data/Data/DownloadData.cs b/server/RdtClient.Data/Data/DownloadData.cs index 9d141f9..3b9e303 100644 --- a/server/RdtClient.Data/Data/DownloadData.cs +++ b/server/RdtClient.Data/Data/DownloadData.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using RdtClient.Data.Models.Data; using Download = RdtClient.Data.Models.Data.Download; namespace RdtClient.Data.Data; @@ -29,13 +30,14 @@ public class DownloadData(DataContext dataContext) .FirstOrDefaultAsync(m => m.TorrentId == torrentId && m.Path == path); } - public async Task Add(Guid torrentId, String path) + public async Task Add(Guid torrentId, DownloadInfo downloadInfo) { var download = new Download { DownloadId = Guid.NewGuid(), TorrentId = torrentId, - Path = path, + FileName = downloadInfo.FileName, + Path = downloadInfo.RestrictedLink, Added = DateTimeOffset.UtcNow, DownloadQueued = DateTimeOffset.UtcNow, RetryCount = 0 diff --git a/server/RdtClient.Data/Models/Data/Download.cs b/server/RdtClient.Data/Models/Data/Download.cs index d078356..6720415 100644 --- a/server/RdtClient.Data/Models/Data/Download.cs +++ b/server/RdtClient.Data/Models/Data/Download.cs @@ -26,7 +26,7 @@ public class Download public DateTimeOffset? Completed { get; set; } public Int32 RetryCount { get; set; } - + public String? Error { get; set; } public String? RemoteId { get; set; } @@ -41,4 +41,20 @@ public class Download [NotMapped] public Int64 Speed { get; set; } +} + +/// +/// Used to create s +/// +public class DownloadInfo +{ + /// + /// The name of the file. Should not include directory. + /// If the filename is not known, set tn null and `GetFileName` will be called with the unrestricted link. + /// + public required String? FileName; + /// + /// The restricted link to download this download. If the debrid serice in question does not have restricted links, use either a fake or the unrestricted link + /// + public required String RestrictedLink; } \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Downloads.cs b/server/RdtClient.Service/Services/Downloads.cs index 925c8e0..76677c9 100644 --- a/server/RdtClient.Service/Services/Downloads.cs +++ b/server/RdtClient.Service/Services/Downloads.cs @@ -1,4 +1,5 @@ using RdtClient.Data.Data; +using RdtClient.Data.Models.Data; using Download = RdtClient.Data.Models.Data.Download; namespace RdtClient.Service.Services; @@ -20,9 +21,9 @@ public class Downloads(DownloadData downloadData) : IDownloads return await downloadData.Get(torrentId, path); } - public async Task Add(Guid torrentId, String path) + public async Task Add(Guid torrentId, DownloadInfo downloadInfo) { - return await downloadData.Add(torrentId, path); + return await downloadData.Add(torrentId, downloadInfo); } public async Task UpdateUnrestrictedLink(Guid downloadId, String unrestrictedLink) diff --git a/server/RdtClient.Service/Services/IDownloads.cs b/server/RdtClient.Service/Services/IDownloads.cs index d0550b9..b79f59a 100644 --- a/server/RdtClient.Service/Services/IDownloads.cs +++ b/server/RdtClient.Service/Services/IDownloads.cs @@ -7,7 +7,7 @@ public interface IDownloads Task> GetForTorrent(Guid torrentId); Task GetById(Guid downloadId); Task Get(Guid torrentId, String path); - Task Add(Guid torrentId, String path); + Task Add(Guid torrentId, DownloadInfo downloadInfo); Task UpdateUnrestrictedLink(Guid downloadId, String unrestrictedLink); Task UpdateFileName(Guid downloadId, String fileName); Task UpdateDownloadStarted(Guid downloadId, DateTimeOffset? dateTime); diff --git a/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs index 38121ae..e0ca2c3 100644 --- a/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs @@ -14,6 +14,12 @@ public interface ITorrentClient Task Delete(String torrentId); Task Unrestrict(String link); Task UpdateData(Torrent torrent, TorrentClientTorrent? torrentClientTorrent); - Task?> GetDownloadLinks(Torrent torrent); - Task GetFileName(String downloadUrl); + Task?> GetDownloadInfos(Torrent torrent); + /// + /// To be called only when . is not set by + /// + /// + /// The download to get the filename of + /// The filename of the download + Task GetFileName(Download download); } \ No newline at end of file diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index 6151100..102b9b6 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -363,8 +363,11 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow var downloadLink = await torrents.UnrestrictLink(download.DownloadId); download.Link = downloadLink; - var fileName = await torrents.RetrieveFileName(download.DownloadId); - download.FileName = fileName; + if (download.FileName == null) + { + var fileName = await torrents.RetrieveFileName(download.DownloadId); + download.FileName = fileName; + } } } catch (Exception ex) diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index a88087a..9fe1cb6 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -235,14 +235,14 @@ public class Torrents( return; } - var downloadLinks = await TorrentClient.GetDownloadLinks(torrent); + var downloadInfos = await TorrentClient.GetDownloadInfos(torrent); - if (downloadLinks == null) + if (downloadInfos == null) { return; } - if (downloadLinks.Count == 0) + if (downloadInfos.Count == 0) { logger.LogInformation("All files excluded by filters (IncludeRegex: {includeRegex}, ExcludeRegex: {excludeRegex}, DownloadMinSize: {downloadMinSize} {torrentInfo}", torrent.IncludeRegex, @@ -256,14 +256,14 @@ public class Torrents( return; } - foreach (var downloadLink in downloadLinks) + foreach (var downloadInfo in downloadInfos) { // Make sure downloads don't get added multiple times - var downloadExists = await downloads.Get(torrent.TorrentId, downloadLink); + var downloadExists = await downloads.Get(torrent.TorrentId, downloadInfo.RestrictedLink); - if (downloadExists == null && !String.IsNullOrWhiteSpace(downloadLink)) + if (downloadExists == null && !String.IsNullOrWhiteSpace(downloadInfo.RestrictedLink)) { - await downloads.Add(torrent.TorrentId, downloadLink); + await downloads.Add(torrent.TorrentId, downloadInfo); } } } @@ -396,7 +396,7 @@ public class Torrents( Log($"Retrieving filename for", download, download.Torrent); - var fileName = await TorrentClient.GetFileName(download.Link!); + var fileName = await TorrentClient.GetFileName(download!); await downloads.UpdateFileName(downloadId, fileName); From 88af15b3bcda58fc82fad6f2db03f1723046bb08 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:13:24 +0000 Subject: [PATCH 04/31] [AD] `GetDownloadLinks` -> `GetDownloadInfos` Also makes the tests work wiith this new method --- .../AllDebridTorrentClientTest.cs | 18 ++++--- .../TorrentClients/AllDebridTorrentClient.cs | 48 +++++++------------ 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/server/RdtClient.Service.Test/Services/TorrentClients/AllDebridTorrentClientTest.cs b/server/RdtClient.Service.Test/Services/TorrentClients/AllDebridTorrentClientTest.cs index 29d740f..1ec1f88 100644 --- a/server/RdtClient.Service.Test/Services/TorrentClients/AllDebridTorrentClientTest.cs +++ b/server/RdtClient.Service.Test/Services/TorrentClients/AllDebridTorrentClientTest.cs @@ -405,7 +405,7 @@ public class AllDebridTorrentClientTest } [Fact] - public async Task GetDownloadLinks_WhenTorrentRdIdNull_ReturnsNull() + public async Task GetDownloadInfos_WhenTorrentRdIdNull_ReturnsNull() { // Arrange var mocks = new Mocks(); @@ -418,7 +418,7 @@ public class AllDebridTorrentClientTest var allDebridTorrentClient = new AllDebridTorrentClient(mocks.LoggerMock.Object, mocks.AllDebridClientFactoryMock.Object, mocks.FileFilterMock.Object); // Act - var result = await allDebridTorrentClient.GetDownloadLinks(torrent); + var result = await allDebridTorrentClient.GetDownloadInfos(torrent); // Assert Assert.Null(result); @@ -426,7 +426,7 @@ public class AllDebridTorrentClientTest } [Fact] - public async Task GetDownloadLinks_UsesFileFilter() + public async Task GetDownloadInfos_UsesFileFilter() { // Arrange var mocks = new Mocks(); @@ -467,16 +467,16 @@ public class AllDebridTorrentClientTest var allDebridTorrentClient = new AllDebridTorrentClient(mocks.LoggerMock.Object, mocks.AllDebridClientFactoryMock.Object, mocks.FileFilterMock.Object); // Act - var result = await allDebridTorrentClient.GetDownloadLinks(torrent); + var result = await allDebridTorrentClient.GetDownloadInfos(torrent); // Assert Assert.NotNull(result); Assert.Single(result); - Assert.Contains("https://fake.url/file1.txt", result); + Assert.Single(result.Where(info => info.RestrictedLink == "https://fake.url/file1.txt")); } [Fact] - public async Task GetDownloadLinks_WhenAllFilesExcluded_ReturnsAllFiles() + public async Task GetDownloadInfos_WhenAllFilesExcluded_ReturnsEmptyList() { // Arrange var mocks = new Mocks(); @@ -502,19 +502,17 @@ public class AllDebridTorrentClientTest } ]; - var expectedLinksSet = new HashSet(files.Select(n => n.DownloadLink)!); mocks.AllDebridClientMock.Setup(c => c.Magnet.FilesAsync(Int64.Parse(torrent.RdId), It.IsAny())).ReturnsAsync(files); mocks.FileFilterMock.Setup(f => f.IsDownloadable(torrent, It.IsAny(), It.IsAny())).Returns(false); var allDebridTorrentClient = new AllDebridTorrentClient(mocks.LoggerMock.Object, mocks.AllDebridClientFactoryMock.Object, mocks.FileFilterMock.Object); // Act - var result = await allDebridTorrentClient.GetDownloadLinks(torrent); + var result = await allDebridTorrentClient.GetDownloadInfos(torrent); // Assert Assert.NotNull(result); - var linksSet = new HashSet(result); - Assert.Equal(expectedLinksSet, linksSet); + Assert.Empty(result); mocks.FileFilterMock.Verify(f => f.IsDownloadable(torrent, "file-1.txt", 100)); mocks.FileFilterMock.Verify(f => f.IsDownloadable(torrent, "file-2.txt", 100)); diff --git a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs index b1415cc..da4998b 100644 --- a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs @@ -1,10 +1,10 @@ -using AllDebridNET; +using System.Diagnostics; +using AllDebridNET; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using RdtClient.Data.Enums; using RdtClient.Data.Models.TorrentClient; using RdtClient.Service.Helpers; -using System.Web; using RdtClient.Data.Models.Data; using File = AllDebridNET.File; using Torrent = RdtClient.Data.Models.Data.Torrent; @@ -245,48 +245,32 @@ public class AllDebridTorrentClient(ILogger logger, IAll return torrent; } - public async Task?> GetDownloadLinks(Torrent torrent) + public async Task?> GetDownloadInfos(Torrent torrent) { if (torrent.RdId == null) { return null; } - + + Log($"Getting download links", torrent); + var allFiles = await allDebridNetClientFactory.GetClient().Magnet.FilesAsync(Int64.Parse(torrent.RdId)); var files = GetFiles(allFiles); - - files = files.Where(f => fileFilter.IsDownloadable(torrent, f.Path, f.Bytes)).ToList(); - - Log($"Getting download links", torrent); - - if (files.Count == 0) + + return files.Where(f => fileFilter.IsDownloadable(torrent, f.Path, f.Bytes) && f.DownloadLink != null).Select(f => new DownloadInfo { - Log($"Filtered all files out! Downloading ALL files instead!", torrent); - - files = GetFiles(allFiles); - } - - Log($"Selecting links:"); - - foreach (var file in files) - { - Log($"{file.Path} ({file.Bytes}b) {file.DownloadLink}"); - } - - return files.Where(m => m.DownloadLink != null).Select(m => m.DownloadLink!.ToString()).ToList(); + FileName = Path.GetFileName(f.Path), + RestrictedLink = f.DownloadLink! + }).ToList(); } - public Task GetFileName(String downloadUrl) + public Task GetFileName(Download download) { - if (String.IsNullOrWhiteSpace(downloadUrl)) - { - return Task.FromResult(""); - } - - var uri = new Uri(downloadUrl); - - return Task.FromResult(HttpUtility.UrlDecode(uri.Segments.Last())); + // FileName is set in GetDownlaadInfos + Debug.Assert(download.FileName != null); + + return Task.FromResult(download.FileName); } private async Task GetInfo(String torrentId) From 2b2c5462f105d1b5458f56b6cf2703851068dcb5 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:13:44 +0000 Subject: [PATCH 05/31] [DL] `GetDownloadLinks` -> `GetDownloadInfos` --- .../TorrentClients/DebridLinkTorrentClient.cs | 44 ++++++++----------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs index a4c4026..3e81729 100644 --- a/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs @@ -1,12 +1,13 @@ -using Microsoft.Extensions.Logging; +using System.Diagnostics; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using DebridLinkFrNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.TorrentClient; using RdtClient.Service.Helpers; -using DebridLinkFrNET.Models; -using System.Web; +using RdtClient.Data.Models.Data; using Download = RdtClient.Data.Models.Data.Download; +using Torrent = DebridLinkFrNET.Models.Torrent; namespace RdtClient.Service.Services.TorrentClients; @@ -234,7 +235,7 @@ public class DebridLinkClient(ILogger logger, IHttpClientFacto return torrent; } - public async Task?> GetDownloadLinks(Data.Models.Data.Torrent torrent) + public async Task?> GetDownloadInfos(Data.Models.Data.Torrent torrent) { if (torrent.RdId == null) { @@ -248,19 +249,14 @@ public class DebridLinkClient(ILogger logger, IHttpClientFacto return null; } - var downloadLinks = rdTorrent.Files? - .Where(m => fileFilter.IsDownloadable(torrent, m.Path, m.Bytes) && m.DownloadLink != null) - .Select(m => m.DownloadLink!) - .ToList() ?? []; - - Log($"Found {downloadLinks.Count} links", torrent); - - foreach (var link in downloadLinks) - { - Log($"{link}", torrent); - } - - return downloadLinks; + return rdTorrent.Files? + .Where(m => fileFilter.IsDownloadable(torrent, m.Path, m.Bytes) && m.DownloadLink != null) + .Select(m => new DownloadInfo + { + RestrictedLink = m.DownloadLink!, + FileName = Path.GetFileName(m.Path) + }) + .ToList(); } private async Task GetInfo(String torrentId) @@ -280,16 +276,12 @@ public class DebridLinkClient(ILogger logger, IHttpClientFacto logger.LogDebug(message); } - public Task GetFileName(String downloadUrl) + public Task GetFileName(Download download) { - if (String.IsNullOrWhiteSpace(downloadUrl)) - { - return Task.FromResult(""); - } - - var uri = new Uri(downloadUrl); - - return Task.FromResult(HttpUtility.UrlDecode(uri.Segments.Last())); + // FileName is set in GetDownlaadInfos + Debug.Assert(download.FileName != null); + + return Task.FromResult(download.FileName); } public static String? GetSymlinkPath(Data.Models.Data.Torrent torrent, Download download) From 7615883fdb87a202b614e7f4df4e9401d631b598 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:14:04 +0000 Subject: [PATCH 06/31] [PM] `GetDownloadLinks` -> `GetDownloadInfos` --- .../TorrentClients/PremiumizeTorrentClient.cs | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs index 5682002..8c9d62b 100644 --- a/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs @@ -1,10 +1,11 @@ -using Microsoft.Extensions.Logging; +using System.Diagnostics; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using PremiumizeNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.TorrentClient; using RdtClient.Service.Helpers; -using System.Web; +using RdtClient.Data.Models.Data; using Torrent = RdtClient.Data.Models.Data.Torrent; namespace RdtClient.Service.Services.TorrentClients; @@ -197,7 +198,7 @@ public class PremiumizeTorrentClient(ILogger logger, IH return torrent; } - public async Task?> GetDownloadLinks(Torrent torrent) + public async Task?> GetDownloadInfos(Torrent torrent) { if (torrent.RdId == null) { @@ -212,7 +213,7 @@ public class PremiumizeTorrentClient(ILogger logger, IH Log($"Found transfer {transfer.Name} ({transfer.Id})", torrent); - var downloadLinks = await GetAllDownloadLinks(torrent, transfer.FolderId); + var downloadInfos = await GetAllDownloadInfos(torrent, transfer.FolderId); if (!String.IsNullOrWhiteSpace(transfer.FileId)) { @@ -225,34 +226,31 @@ public class PremiumizeTorrentClient(ILogger logger, IH Log($"File {file.Name} ({file.Id}) does not contain a link", torrent); } - downloadLinks.Add(file.Link); + downloadInfos.Add(new () {RestrictedLink = file.Link, FileName = file.Name }); } - if (downloadLinks.Count == 0) + if (downloadInfos.Count == 0) { Log($"No download links found for transfer {transfer.Name} ({transfer.Id})", torrent); return null; } - foreach (var link in downloadLinks) + foreach (var info in downloadInfos) { - Log($"Found {link}", torrent); + Log($"Found {info.RestrictedLink}", torrent); } - return downloadLinks; + return downloadInfos; } - public Task GetFileName(String downloadUrl) + /// + public Task GetFileName(Download download) { - if (String.IsNullOrWhiteSpace(downloadUrl)) - { - return Task.FromResult(""); - } - - var uri = new Uri(downloadUrl); - - return Task.FromResult(HttpUtility.UrlDecode(uri.Segments.Last())); + // FileName is set in GetDownlaadInfos + Debug.Assert(download.FileName != null); + + return Task.FromResult(download.FileName); } private async Task GetInfo(String id) @@ -263,13 +261,12 @@ public class PremiumizeTorrentClient(ILogger logger, IH return Map(result); } - private async Task> GetAllDownloadLinks(Torrent torrent, String folderId) + private async Task> GetAllDownloadInfos(Torrent torrent, String folderId) { - var downloadLinks = new List(); if (String.IsNullOrWhiteSpace(folderId)) { - return downloadLinks; + return []; } var folder = await GetClient().Folder.ListAsync(folderId); @@ -277,11 +274,13 @@ public class PremiumizeTorrentClient(ILogger logger, IH if (folder.Content == null) { Log($"Found no items in folder {folder.Name} ({folderId})", torrent); - return downloadLinks; + return []; } Log($"Found {folder.Content.Count} items in folder {folder.Name} ({folderId})", torrent); + var downloadInfos = new List(); + foreach (var item in folder.Content) { if (item.Type == "file") @@ -300,7 +299,7 @@ public class PremiumizeTorrentClient(ILogger logger, IH Log($"Found item {item.Name} in folder {folder.Name} ({folderId})", torrent); - downloadLinks.Add(item.Link); + downloadInfos.Add(new () { RestrictedLink = item.Link, FileName = item.Name}); } else if (item.Type == "folder") { @@ -311,8 +310,8 @@ public class PremiumizeTorrentClient(ILogger logger, IH } Log($"Found subfolder {item.Name} in {folder.Name} ({folderId}), searching subfolder for files", torrent); - var subDownloadLinks = await GetAllDownloadLinks(torrent, item.Id); - downloadLinks.AddRange(subDownloadLinks); + var subDownloadLinks = await GetAllDownloadInfos(torrent, item.Id); + downloadInfos.AddRange(subDownloadLinks); } else { @@ -320,7 +319,7 @@ public class PremiumizeTorrentClient(ILogger logger, IH } } - return downloadLinks; + return downloadInfos; } private void Log(String message, Torrent? torrent = null) From 0cc15d26c387ea1ddb951ef49ac438e13053d252 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:14:09 +0000 Subject: [PATCH 07/31] [RD] `GetDownloadLinks` -> `GetDownloadInfos` --- .../TorrentClients/RealDebridTorrentClient.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs index 5baa7e3..d1a43d0 100644 --- a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs @@ -3,8 +3,11 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using RDNET; using RdtClient.Data.Enums; +using RdtClient.Data.Models.Data; using RdtClient.Data.Models.TorrentClient; using RdtClient.Service.Helpers; +using Download = RdtClient.Data.Models.Data.Download; +using Torrent = RDNET.Torrent; namespace RdtClient.Service.Services.TorrentClients; @@ -280,7 +283,7 @@ public class RealDebridTorrentClient(ILogger logger, IH return torrent; } - public async Task?> GetDownloadLinks(Data.Models.Data.Torrent torrent) + public async Task?> GetDownloadInfos(Data.Models.Data.Torrent torrent) { if (torrent.RdId == null) { @@ -294,7 +297,14 @@ public class RealDebridTorrentClient(ILogger logger, IH return null; } - var downloadLinks = rdTorrent.Links.Where(m => !String.IsNullOrWhiteSpace(m)).ToList(); + var downloadLinks = rdTorrent.Links + .Where(m => !String.IsNullOrWhiteSpace(m)) + .Select(l => new DownloadInfo + { + RestrictedLink = l, + FileName = null + }) + .ToList(); Log($"Found {downloadLinks.Count} links", torrent); @@ -341,14 +351,14 @@ public class RealDebridTorrentClient(ILogger logger, IH return null; } - public Task GetFileName(String downloadUrl) + public Task GetFileName(Download download) { - if (String.IsNullOrWhiteSpace(downloadUrl)) + if (String.IsNullOrWhiteSpace(download.Link)) { return Task.FromResult(""); } - var uri = new Uri(downloadUrl); + var uri = new Uri(download.Link); return Task.FromResult(HttpUtility.UrlDecode(uri.Segments.Last())); } From c50afe9e646a910fc20b95e0b0d938b6fb4415bd Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:14:15 +0000 Subject: [PATCH 08/31] [TB] `GetDownloadLinks` -> `GetDownloadInfos` --- .../TorrentClients/TorBoxTorrentClient.cs | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs index 556faa7..26d95e1 100644 --- a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs @@ -1,9 +1,9 @@ +using System.Diagnostics; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using TorBoxNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.TorrentClient; -using System.Web; using RdtClient.Data.Models.Data; using RdtClient.Service.Helpers; @@ -284,39 +284,21 @@ public class TorBoxTorrentClient(ILogger logger, IHttpClien return torrent; } - public async Task?> GetDownloadLinks(Torrent torrent) + public async Task?> GetDownloadInfos(Torrent torrent) { var torrentId = await GetClient().Torrents.GetHashInfoAsync(torrent.Hash, skipCache: true); return torrent.Files.Where(file => fileFilter.IsDownloadable(torrent, file.Path, file.Bytes)) - .Select(file => $"https://torbox.app/fakedl/{torrentId?.Id}/{file.Id}") + .Select(file => new DownloadInfo { RestrictedLink = $"https://torbox.app/fakedl/{torrentId?.Id}/{file.Id}", FileName = Path.GetFileName(file.Path)}) .ToList(); } - public async Task GetFileName(String downloadUrl) + public Task GetFileName(Download download) { - if (String.IsNullOrWhiteSpace(downloadUrl)) - { - return ""; - } - - var uri = new Uri(downloadUrl); - - using (HttpClient client = new()) - { - var request = new HttpRequestMessage(HttpMethod.Head, uri); - var response = await client.SendAsync(request); - if (response.Content.Headers.ContentDisposition != null) - { - var fileName = response.Content.Headers.ContentDisposition.FileName; - if (!String.IsNullOrWhiteSpace(fileName)) - { - return fileName.Trim('"'); - } - } - } - - return HttpUtility.UrlDecode(uri.Segments.Last()); + // FileName is set in GetDownlaadInfos + Debug.Assert(download.FileName != null); + + return Task.FromResult(download.FileName); } private DateTimeOffset? ChangeTimeZone(DateTimeOffset? dateTimeOffset) From d6efff2fb546b7ba3a1d356279aed575272278a0 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:32:47 +0000 Subject: [PATCH 09/31] add rest of xmldoc for DownloadInfos --- .../Services/TorrentClients/AllDebridTorrentClient.cs | 1 + .../Services/TorrentClients/DebridLinkTorrentClient.cs | 1 + .../Services/TorrentClients/RealDebridTorrentClient.cs | 1 + .../Services/TorrentClients/TorBoxTorrentClient.cs | 1 + server/RdtClient.Service/Services/Torrents.cs | 4 ++++ 5 files changed, 8 insertions(+) diff --git a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs index da4998b..818222f 100644 --- a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs @@ -265,6 +265,7 @@ public class AllDebridTorrentClient(ILogger logger, IAll }).ToList(); } + /// public Task GetFileName(Download download) { // FileName is set in GetDownlaadInfos diff --git a/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs index 3e81729..baebc84 100644 --- a/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs @@ -276,6 +276,7 @@ public class DebridLinkClient(ILogger logger, IHttpClientFacto logger.LogDebug(message); } + /// public Task GetFileName(Download download) { // FileName is set in GetDownlaadInfos diff --git a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs index d1a43d0..f78c533 100644 --- a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs @@ -351,6 +351,7 @@ public class RealDebridTorrentClient(ILogger logger, IH return null; } + /// public Task GetFileName(Download download) { if (String.IsNullOrWhiteSpace(download.Link)) diff --git a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs index 26d95e1..463fb28 100644 --- a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs @@ -293,6 +293,7 @@ public class TorBoxTorrentClient(ILogger logger, IHttpClien .ToList(); } + /// public Task GetFileName(Download download) { // FileName is set in GetDownlaadInfos diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 9fe1cb6..f374b11 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -390,6 +390,10 @@ public class Torrents( return unrestrictedLink; } + /// + /// To be called only when . is not set by + /// + /// public async Task RetrieveFileName(Guid downloadId) { var download = await downloads.GetById(downloadId) ?? throw new($"Download with ID {downloadId} not found"); From e6f7db07630b7e952bf35cbce7a6e3885435ebce Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Sun, 16 Mar 2025 15:49:46 +0000 Subject: [PATCH 10/31] fix merge --- .../Services/TorrentClients/TorBoxTorrentClient.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs index 22a6994..0dccb07 100644 --- a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs @@ -4,7 +4,6 @@ using Newtonsoft.Json; using TorBoxNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.TorrentClient; -using System.Web; using RdtClient.Data.Models.Data; using RdtClient.Service.Helpers; From ab5a013edb6abb79649ee3de30f1d7133f1ceab3 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:23:05 +0000 Subject: [PATCH 11/31] [TB] use `torrentClientTorrent` (if it's passed) instead of fetching in `UpdateData` Makes `ProviderUpdater` only cause 2 API requests each run rather than `O(n)` where `n` is the number of torrents --- .../Services/TorrentClients/TorBoxTorrentClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs index 5cfcd3f..95d8718 100644 --- a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs @@ -206,7 +206,7 @@ public class TorBoxTorrentClient(ILogger logger, IHttpClien return torrent; } - var rdTorrent = await GetInfo(torrent.Hash) ?? throw new($"Resource not found"); + var rdTorrent = torrentClientTorrent ?? await GetInfo(torrent.Hash) ?? throw new($"Resource not found"); if (!String.IsNullOrWhiteSpace(rdTorrent.Filename)) { From 74e6d2cf95cc381f9b600f0aeea9ef70f2caf720 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:08:51 +0000 Subject: [PATCH 12/31] don't add torrents straight to debrid provider, add to queue first, dequeue in `TorrentRunner` --- client/src/app/models/torrent.model.ts | 1 + client/src/app/torrent-status.pipe.ts | 2 + server/RdtClient.Data/Data/ITorrentData.cs | 3 +- server/RdtClient.Data/Data/TorrentData.cs | 22 +++- server/RdtClient.Data/Enums/TorrentStatus.cs | 2 + .../Models/Internal/DbSettings.cs | 4 + .../BackgroundServices/WatchFolderChecker.cs | 4 +- .../RdtClient.Service/Services/QBittorrent.cs | 4 +- .../Services/TorrentRunner.cs | 23 ++++ server/RdtClient.Service/Services/Torrents.cs | 123 +++++++++++------- .../Controllers/TorrentsController.cs | 4 +- 11 files changed, 139 insertions(+), 53 deletions(-) diff --git a/client/src/app/models/torrent.model.ts b/client/src/app/models/torrent.model.ts index 36abb26..0397dbb 100644 --- a/client/src/app/models/torrent.model.ts +++ b/client/src/app/models/torrent.model.ts @@ -62,6 +62,7 @@ export class TorrentFileAvailability { } export enum RealDebridStatus { + NotYetAdded = -1, Processing = 0, WaitingForFileSelection = 1, Downloading = 2, diff --git a/client/src/app/torrent-status.pipe.ts b/client/src/app/torrent-status.pipe.ts index 0eb2ed9..3e46a3c 100644 --- a/client/src/app/torrent-status.pipe.ts +++ b/client/src/app/torrent-status.pipe.ts @@ -87,6 +87,8 @@ export class TorrentStatusPipe implements PipeTransform { } switch (torrent.rdStatus) { + case RealDebridStatus.NotYetAdded: + return "Not Yet Added to Provider" case RealDebridStatus.Downloading: if (torrent.rdSeeders < 1) { return `Torrent stalled`; diff --git a/server/RdtClient.Data/Data/ITorrentData.cs b/server/RdtClient.Data/Data/ITorrentData.cs index 4bbaa24..9c286e3 100644 --- a/server/RdtClient.Data/Data/ITorrentData.cs +++ b/server/RdtClient.Data/Data/ITorrentData.cs @@ -9,7 +9,7 @@ public interface ITorrentData Task GetById(Guid torrentId); Task GetByHash(String hash); - Task Add(String rdId, + Task Add(String? rdId, String hash, String? fileOrMagnetContents, Boolean isFile, @@ -17,6 +17,7 @@ public interface ITorrentData Torrent torrent); Task UpdateRdData(Torrent torrent); + Task UpdateRdId(Torrent torrent, String rdId); Task Update(Torrent torrent); Task UpdateCategory(Guid torrentId, String? category); Task UpdateComplete(Guid torrentId, String? error, DateTimeOffset? datetime, Boolean retry); diff --git a/server/RdtClient.Data/Data/TorrentData.cs b/server/RdtClient.Data/Data/TorrentData.cs index 4578348..5b74f63 100644 --- a/server/RdtClient.Data/Data/TorrentData.cs +++ b/server/RdtClient.Data/Data/TorrentData.cs @@ -71,7 +71,7 @@ public class TorrentData(DataContext dataContext) : ITorrentData return dbTorrent; } - public async Task Add(String rdId, + public async Task Add(String? rdId, String hash, String? fileOrMagnetContents, Boolean isFile, @@ -99,7 +99,9 @@ public class TorrentData(DataContext dataContext) : ITorrentData TorrentRetryAttempts = torrent.TorrentRetryAttempts, DownloadRetryAttempts = torrent.DownloadRetryAttempts, DeleteOnError = torrent.DeleteOnError, - Lifetime = torrent.Lifetime + Lifetime = torrent.Lifetime, + RdStatus = torrent.RdStatus, + RdName = torrent.RdName }; await dataContext.Torrents.AddAsync(newTorrent); @@ -138,6 +140,22 @@ public class TorrentData(DataContext dataContext) : ITorrentData await VoidCache(); } + public async Task UpdateRdId(Torrent torrent, String rdId) + { + var dbTorrent = await dataContext.Torrents.FirstOrDefaultAsync(m => m.TorrentId == torrent.TorrentId); + + if (dbTorrent == null) + { + return; + } + + dbTorrent.RdId = rdId; + + await dataContext.SaveChangesAsync(); + + await VoidCache(); + } + public async Task Update(Torrent torrent) { var dbTorrent = await dataContext.Torrents.FirstOrDefaultAsync(m => m.TorrentId == torrent.TorrentId); diff --git a/server/RdtClient.Data/Enums/TorrentStatus.cs b/server/RdtClient.Data/Enums/TorrentStatus.cs index d4de133..16b556e 100644 --- a/server/RdtClient.Data/Enums/TorrentStatus.cs +++ b/server/RdtClient.Data/Enums/TorrentStatus.cs @@ -2,6 +2,8 @@ public enum TorrentStatus { + NotYetAdded = -1, + Processing = 0, WaitingForFileSelection = 1, Downloading = 2, diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index e90e258..5cf177d 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -193,6 +193,10 @@ or [Description("The interval to check the torrents info on the providers API. Minumum is 3 seconds. When there are no active downloads this limit is increased * 3.")] public Int32 CheckInterval { get; set; } = 10; + [DisplayName("Max parallel downloads")] + [Description("Limits the number of torrents that will be sent for downloading on the debrid provider at the same time. If set to 0, all downloads will be sent immediately without queuing.")] + public Int32 MaxParallelDownloads { get; set; } = 0; + [DisplayName("Auto Import Defaults")] public DbSettingsDefaultsWithCategory Default { get; set; } = new(); } diff --git a/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs b/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs index 417dee5..4c1f243 100644 --- a/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs +++ b/server/RdtClient.Service/BackgroundServices/WatchFolderChecker.cs @@ -99,12 +99,12 @@ public class WatchFolderChecker(ILogger logger, IServiceProv if (fileInfo.Extension == ".torrent") { var torrentFileContents = await File.ReadAllBytesAsync(torrentFile, stoppingToken); - await torrentService.UploadFile(torrentFileContents, torrent); + await torrentService.AddFileToDebridQueue(torrentFileContents, torrent); } else if (fileInfo.Extension == ".magnet") { var magnetLink = await File.ReadAllTextAsync(torrentFile, stoppingToken); - await torrentService.UploadMagnet(magnetLink, torrent); + await torrentService.AddMagnetToDebridQueue(magnetLink, torrent); } if (!Directory.Exists(processedStorePath)) diff --git a/server/RdtClient.Service/Services/QBittorrent.cs b/server/RdtClient.Service/Services/QBittorrent.cs index 3c12685..081762e 100644 --- a/server/RdtClient.Service/Services/QBittorrent.cs +++ b/server/RdtClient.Service/Services/QBittorrent.cs @@ -474,7 +474,7 @@ public class QBittorrent(ILogger logger, Settings settings, Authent Priority = priority ?? (Settings.Get.Integrations.Default.Priority > 0 ? Settings.Get.Integrations.Default.Priority : null) }; - await torrents.UploadMagnet(magnetLink, torrent); + await torrents.AddMagnetToDebridQueue(magnetLink, torrent); } public async Task TorrentsAddFile(Byte[] fileBytes, String? category, Int32? priority) @@ -498,7 +498,7 @@ public class QBittorrent(ILogger logger, Settings settings, Authent Priority = priority ?? (Settings.Get.Integrations.Default.Priority > 0 ? Settings.Get.Integrations.Default.Priority : null) }; - await torrents.UploadFile(fileBytes, torrent); + await torrents.AddFileToDebridQueue(fileBytes, torrent); } public async Task TorrentsSetCategory(String hash, String? category) diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index a7e7cb6..a518620 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -314,6 +314,29 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow await torrents.UpdateComplete(torrent.TorrentId, $"Torrent lifetime of {torrent.Lifetime} minutes reached", DateTimeOffset.UtcNow, false); } + // Process torrents in DebridQueue + var torrentsToAddToProvider = allTorrents.Where(m => m.RdId == null && m.RdAdded == null && m.FileOrMagnet != null && m.RdStatus == TorrentStatus.NotYetAdded) + .ToList(); + + if (torrentsToAddToProvider.Count != 0) + { + var downloadingTorrentsCount = allTorrents.Count(m => m.RdStatus is not (TorrentStatus.NotYetAdded or TorrentStatus.Finished or TorrentStatus.Error)); + + var maxParallelDownloads = Settings.Get.Provider.MaxParallelDownloads; + + logger.LogDebug("Currently downloading {downloadingTorrentCount}/{maxParallelDownloads} torrents, {queuedCount} queued.", + downloadingTorrentsCount, + maxParallelDownloads, + torrentsToAddToProvider.Count); + + var dequeueCount = maxParallelDownloads == 0 ? torrentsToAddToProvider.Count : maxParallelDownloads - downloadingTorrentsCount; + + foreach (var torrent in torrentsToAddToProvider.Take(dequeueCount)) + { + await torrents.DequeueFromDebridQueue(torrent); + } + } + allTorrents = await torrents.Get(); allTorrents = allTorrents.Where(m => m.Completed == null).ToList(); diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index a88087a..505e1af 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -107,7 +107,7 @@ public class Torrents( await torrentData.UpdateCategory(torrent.TorrentId, category); } - public async Task UploadMagnet(String magnetLink, Torrent torrent) + public async Task AddMagnetToDebridQueue(String magnetLink, Torrent torrent) { MagnetLink magnet; @@ -121,42 +121,21 @@ public class Torrents( throw new($"{ex.Message}, trying to parse {magnetLink}"); } - var id = await TorrentClient.AddMagnet(magnetLink); + torrent.RdStatus = TorrentStatus.NotYetAdded; + torrent.RdName = magnet.Name; var hash = magnet.InfoHashes.V1OrV2.ToHex(); - var newTorrent = await Add(id, hash, magnetLink, false, torrent); + var newTorrent = await AddQueued(hash, magnetLink, false, torrent); - Log($"Adding {hash} magnet link {magnetLink}", newTorrent); + Log($"Adding {hash} (magnet link) to queue", newTorrent); - if (!String.IsNullOrWhiteSpace(Settings.Get.General.CopyAddedTorrents)) - { - try - { - if (!Directory.Exists(Settings.Get.General.CopyAddedTorrents)) - { - Directory.CreateDirectory(Settings.Get.General.CopyAddedTorrents); - } - - var copyFileName = Path.Combine(Settings.Get.General.CopyAddedTorrents, $"{FileHelper.RemoveInvalidFileNameChars(magnet.Name!)}.magnet"); - - if (File.Exists(copyFileName)) - { - File.Delete(copyFileName); - } - - await File.WriteAllTextAsync(copyFileName, magnetLink); - } - catch (Exception ex) - { - logger.LogError(ex, $"Unable to create torrent blackhole directory: {Settings.Get.General.CopyAddedTorrents}: {ex.Message}"); - } - } + await CopyAddedTorrent(magnet.Name!, magnetLink); return newTorrent; } - public async Task UploadFile(Byte[] bytes, Torrent torrent) + public async Task AddFileToDebridQueue(Byte[] bytes, Torrent torrent) { MonoTorrent.Torrent monoTorrent; @@ -172,14 +151,22 @@ public class Torrents( throw new($"{ex.Message}, trying to parse {fileAsBase64}"); } - var id = await TorrentClient.AddFile(bytes); + torrent.RdStatus = TorrentStatus.NotYetAdded; + torrent.RdName = monoTorrent.Name; var hash = monoTorrent.InfoHashes.V1OrV2.ToHex(); - var newTorrent = await Add(id, hash, fileAsBase64, true, torrent); + var newTorrent = await AddQueued(hash, fileAsBase64, true, torrent); - Log($"Adding {hash} torrent file", newTorrent); + Log($"Adding {hash} (torrent file) to queue", newTorrent); + await CopyAddedTorrent(monoTorrent.Name, bytes); + + return newTorrent; + } + + private async Task CopyAddedTorrent(String torrentName, Object fileOrMagnet) + { if (!String.IsNullOrWhiteSpace(Settings.Get.General.CopyAddedTorrents)) { try @@ -189,22 +176,73 @@ public class Torrents( Directory.CreateDirectory(Settings.Get.General.CopyAddedTorrents); } - var copyFileName = Path.Combine(Settings.Get.General.CopyAddedTorrents, $"{FileHelper.RemoveInvalidFileNameChars(monoTorrent.Name)}.torrent"); + var copyFileName = Path.Combine(Settings.Get.General.CopyAddedTorrents, FileHelper.RemoveInvalidFileNameChars(torrentName)); + switch (fileOrMagnet) + { + case String: + copyFileName = $"{copyFileName}.magnet"; + break; + case Byte[]: + copyFileName = $"{copyFileName}.torrent"; + break; + default: + throw new ArgumentException("Unexpected type for fileOrMagnet"); + } + if (File.Exists(copyFileName)) { File.Delete(copyFileName); } - await File.WriteAllBytesAsync(copyFileName, bytes); + switch (fileOrMagnet) + { + case String magnetLink: + await File.WriteAllTextAsync(copyFileName, magnetLink); + break; + case Byte[] torrentFile: + await File.WriteAllBytesAsync(copyFileName, torrentFile); + break; + } } catch (Exception ex) { logger.LogError(ex, $"Unable to create torrent blackhole directory: {Settings.Get.General.CopyAddedTorrents}: {ex.Message}"); } } + } - return newTorrent; + /// + /// Adds torrent in database to debrid provider and updates database accordingly. + /// + /// The torrent from the database to upload to the debrid provider + /// Updated torrent + /// When RdId is not null or FileOrMagnet is null. + public async Task DequeueFromDebridQueue(Torrent torrent) + { + if (torrent.RdId != null) + { + throw new("Torrent already added to debrid provider, cannot dequeue"); + } + + if (torrent.FileOrMagnet == null) + { + throw new("Torrent has no torrent file or magnet link"); + } + + logger.LogDebug("Adding {hash} to debrid provider {torrentInfo}", torrent.Hash, torrent.ToLog()); + + var id = (torrent.IsFile) switch + { + true => await TorrentClient.AddFile(Convert.FromBase64String(torrent.FileOrMagnet)), + false => await TorrentClient.AddMagnet(torrent.FileOrMagnet) + }; + + await torrentData.UpdateRdId(torrent, id); + + await UpdateTorrentClientData(torrent); + + return torrent; } public async Task> GetAvailableFiles(String hash) @@ -538,11 +576,11 @@ public class Torrents( { var bytes = Convert.FromBase64String(torrent.FileOrMagnet); - newTorrent = await UploadFile(bytes, torrent); + newTorrent = await AddFileToDebridQueue(bytes, torrent); } else { - newTorrent = await UploadMagnet(torrent.FileOrMagnet, torrent); + newTorrent = await AddMagnetToDebridQueue(torrent.FileOrMagnet, torrent); } await torrentData.UpdateRetry(newTorrent.TorrentId, null, retryCount); @@ -670,11 +708,10 @@ public class Torrents( return settingDownloadPath; } - private async Task Add(String rdTorrentId, - String infoHash, - String fileOrMagnetContents, - Boolean isFile, - Torrent torrent) + private async Task AddQueued(String infoHash, + String fileOrMagnetContents, + Boolean isFile, + Torrent torrent) { await RealDebridUpdateLock.WaitAsync(); @@ -687,15 +724,13 @@ public class Torrents( return existingTorrent; } - var newTorrent = await torrentData.Add(rdTorrentId, + var newTorrent = await torrentData.Add(null, infoHash, fileOrMagnetContents, isFile, torrent.DownloadClient, torrent); - await UpdateTorrentClientData(newTorrent); - return newTorrent; } finally diff --git a/server/RdtClient.Web/Controllers/TorrentsController.cs b/server/RdtClient.Web/Controllers/TorrentsController.cs index 69536b5..f713985 100644 --- a/server/RdtClient.Web/Controllers/TorrentsController.cs +++ b/server/RdtClient.Web/Controllers/TorrentsController.cs @@ -84,7 +84,7 @@ public class TorrentsController(ILogger logger, Torrents tor var bytes = memoryStream.ToArray(); - await torrents.UploadFile(bytes, formData.Torrent); + await torrents.AddFileToDebridQueue(bytes, formData.Torrent); return Ok(); } @@ -110,7 +110,7 @@ public class TorrentsController(ILogger logger, Torrents tor logger.LogDebug($"Add magnet"); - await torrents.UploadMagnet(request.MagnetLink, request.Torrent); + await torrents.AddMagnetToDebridQueue(request.MagnetLink, request.Torrent); return Ok(); } From 7eb1a610b4676b4f54f68f23e20ec27f2c890ea6 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:35:51 +0000 Subject: [PATCH 13/31] use ternary not `(Boolean) switch` --- server/RdtClient.Service/Services/Torrents.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 505e1af..0c92c29 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -232,11 +232,9 @@ public class Torrents( logger.LogDebug("Adding {hash} to debrid provider {torrentInfo}", torrent.Hash, torrent.ToLog()); - var id = (torrent.IsFile) switch - { - true => await TorrentClient.AddFile(Convert.FromBase64String(torrent.FileOrMagnet)), - false => await TorrentClient.AddMagnet(torrent.FileOrMagnet) - }; + var id = torrent.IsFile + ? await TorrentClient.AddFile(Convert.FromBase64String(torrent.FileOrMagnet)) + : await TorrentClient.AddMagnet(torrent.FileOrMagnet); await torrentData.UpdateRdId(torrent, id); From 99ea3f008f289209097bab88d950392c555b8d85 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Sun, 23 Mar 2025 13:32:37 +0000 Subject: [PATCH 14/31] fix merge (again!) --- .../Services/TorrentClients/TorBoxTorrentClient.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs index 726e130..9f74b35 100644 --- a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs @@ -5,7 +5,6 @@ using Newtonsoft.Json; using TorBoxNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.TorrentClient; -using System.Web; using RdtClient.Data.Models.Data; using RdtClient.Service.Helpers; From 3780a954a225814a3a7a3220f6102d10047f928b Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Sun, 23 Mar 2025 14:08:27 +0000 Subject: [PATCH 15/31] add setting to change RD api hostname --- server/RdtClient.Data/Models/Internal/DbSettings.cs | 7 +++++++ .../Services/TorrentClients/RealDebridTorrentClient.cs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 5ad2a57..3295a1f 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -181,6 +181,13 @@ or https://debrid-link.com/webapp/apikey")] public String ApiKey { get; set; } = ""; + /// + /// API hostname to use for Real Debrid only + /// + [DisplayName("API Hostname (RD only)")] + [Description("Use this instead of the normal hostname for Real Debrid API requests. Only used by Real Debrid. Leave blank to use default.")] + public String? ApiHostname { get; set; } + [DisplayName("Automatically import and process torrents added to provider")] [Description("When selected, import downloads that are not added through RealDebridClient but have been directly added to your debrid provider.")] public Boolean AutoImport { get; set; } = false; diff --git a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs index efff0a4..2177df5 100644 --- a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs @@ -26,7 +26,7 @@ public class RealDebridTorrentClient(ILogger logger, IH var httpClient = httpClientFactory.CreateClient(DiConfig.RD_CLIENT); httpClient.Timeout = TimeSpan.FromSeconds(Settings.Get.Provider.Timeout); - var rdtNetClient = new RdNetClient(null, httpClient, 5); + var rdtNetClient = new RdNetClient(null, httpClient, 5, Settings.Get.Provider.ApiHostname); rdtNetClient.UseApiAuthentication(apiKey); // Get the server time to fix up the timezones on results From d267a3451a9db1706d8eee361424b6cf5720319f Mon Sep 17 00:00:00 2001 From: Morgan Courbet Date: Mon, 24 Mar 2025 13:26:38 +0100 Subject: [PATCH 16/31] docs: fix level for Synology Download Station --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 352cef1..7289574 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ Required configuration: Suggested configuration: - Automatic retry downloads > 3 -### Synology Download Station +#### Synology Download Station The Synology Download Station downloader uses an external Download Station server. You will need to set this up yourself. From aa3f24ff50e6d03f8587c97abbbf53766dfca3ce Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:13:05 +0000 Subject: [PATCH 17/31] Return number of selected files from `SelectFiles` so we can handle all files being excluded --- .../TorrentClients/AllDebridTorrentClient.cs | 5 +-- .../TorrentClients/DebridLinkTorrentClient.cs | 5 +-- .../Services/TorrentClients/ITorrentClient.cs | 10 +++++- .../TorrentClients/PremiumizeTorrentClient.cs | 7 ++-- .../TorrentClients/RealDebridTorrentClient.cs | 10 +++++- .../TorrentClients/TorBoxTorrentClient.cs | 5 +-- server/RdtClient.Service/Services/Torrents.cs | 32 +++++++++++++------ 7 files changed, 55 insertions(+), 19 deletions(-) diff --git a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs index 6fd5388..afc6ed6 100644 --- a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs @@ -155,9 +155,10 @@ public class AllDebridTorrentClient(ILogger logger, IAll return Task.FromResult>([]); } - public Task SelectFiles(Torrent torrent) + /// + public Task SelectFiles(Torrent torrent) { - return Task.CompletedTask; + return Task.FromResult(torrent.Files.Count); } public async Task Delete(String torrentId) diff --git a/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs index a4c4026..cf674d0 100644 --- a/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/DebridLinkTorrentClient.cs @@ -136,9 +136,10 @@ public class DebridLinkClient(ILogger logger, IHttpClientFacto return Task.FromResult>([]); } - public Task SelectFiles(Data.Models.Data.Torrent torrent) + /// + public Task SelectFiles(Data.Models.Data.Torrent torrent) { - return Task.CompletedTask; + return Task.FromResult(torrent.Files.Count); } public async Task Delete(String torrentId) diff --git a/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs index 38121ae..c08c774 100644 --- a/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs @@ -10,7 +10,15 @@ public interface ITorrentClient Task AddMagnet(String magnetLink); Task AddFile(Byte[] bytes); Task> GetAvailableFiles(String hash); - Task SelectFiles(Torrent torrent); + /// + /// Tell the debrid provider which files to download. + /// + /// + /// Not all providers support this feature. + /// + /// The torrent to select files for + /// Number of files selected + Task SelectFiles(Torrent torrent); Task Delete(String torrentId); Task Unrestrict(String link); Task UpdateData(Torrent torrent, TorrentClientTorrent? torrentClientTorrent); diff --git a/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs index 5682002..fd84b8b 100644 --- a/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/PremiumizeTorrentClient.cs @@ -121,9 +121,12 @@ public class PremiumizeTorrentClient(ILogger logger, IH return Task.FromResult>([]); } - public Task SelectFiles(Torrent torrent) + /// + public Task SelectFiles(Torrent torrent) { - return Task.CompletedTask; + // torrent.Files is not populated when this function is called + // by returning 1, we ensure no logic for "all files excluded" is followed + return Task.FromResult(1); } public async Task Delete(String id) diff --git a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs index efff0a4..ad343b3 100644 --- a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs @@ -142,7 +142,8 @@ public class RealDebridTorrentClient(ILogger logger, IH return Task.FromResult>([]); } - public async Task SelectFiles(Data.Models.Data.Torrent torrent) + /// + public async Task SelectFiles(Data.Models.Data.Torrent torrent) { List files; @@ -166,7 +167,14 @@ public class RealDebridTorrentClient(ILogger logger, IH var fileIds = files.Select(m => m.Id.ToString()).ToArray(); + if (fileIds.Length == 0) + { + return 0; + } + await GetClient().Torrents.SelectFilesAsync(torrent.RdId!, [.. fileIds]); + + return fileIds.Length; } public async Task Delete(String torrentId) diff --git a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs index 445f27e..d5968c2 100644 --- a/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs @@ -171,9 +171,10 @@ public class TorBoxTorrentClient(ILogger logger, IHttpClien return []; } - public Task SelectFiles(Torrent torrent) + /// + public Task SelectFiles(Torrent torrent) { - return Task.CompletedTask; + return Task.FromResult(torrent.Files.Count); } public async Task Delete(String torrentId) diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 9004cb3..34d80ab 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -223,7 +223,12 @@ public class Torrents( return; } - await TorrentClient.SelectFiles(torrent); + var selected = await TorrentClient.SelectFiles(torrent); + + if (selected == 0) + { + await MarkAllFilesExcluded(torrent); + } } public async Task CreateDownloads(Guid torrentId) @@ -244,14 +249,7 @@ public class Torrents( if (downloadLinks.Count == 0) { - logger.LogInformation("All files excluded by filters (IncludeRegex: {includeRegex}, ExcludeRegex: {excludeRegex}, DownloadMinSize: {downloadMinSize} {torrentInfo}", - torrent.IncludeRegex, - torrent.ExcludeRegex, - torrent.DownloadMinSize, - torrent.ToLog()); - - await torrentData.UpdateRetry(torrentId, null, torrent.TorrentRetryAttempts); - await torrentData.UpdateComplete(torrentId, "All files excluded", DateTimeOffset.Now, false); + await MarkAllFilesExcluded(torrent); return; } @@ -268,6 +266,22 @@ public class Torrents( } } + /// + /// Logs a message to the console, sets the error on the torrent and ensures it is not retried. + /// + /// The torrent to mark as "All files excluded" + private async Task MarkAllFilesExcluded(Torrent torrent) + { + logger.LogInformation("All files excluded by filters (IncludeRegex: {includeRegex}, ExcludeRegex: {excludeRegex}, DownloadMinSize: {downloadMinSize}) {torrentInfo}", + torrent.IncludeRegex, + torrent.ExcludeRegex, + torrent.DownloadMinSize, + torrent.ToLog()); + + await torrentData.UpdateRetry(torrent.TorrentId, null, torrent.TorrentRetryAttempts); + await torrentData.UpdateComplete(torrent.TorrentId, "All files excluded", DateTimeOffset.Now, false); + } + public async Task Delete(Guid torrentId, Boolean deleteData, Boolean deleteRdTorrent, Boolean deleteLocalFiles) { var torrent = await GetById(torrentId); From 6f67f386007c6b6aed37c1e06cf21c5db9490fa0 Mon Sep 17 00:00:00 2001 From: Eugene Kallivrousis Date: Tue, 8 Apr 2025 15:20:56 -0400 Subject: [PATCH 18/31] Adding a select all button for the delete torrent popup --- .../torrent-table/torrent-table.component.html | 15 +++++++++++---- .../app/torrent-table/torrent-table.component.ts | 12 ++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/client/src/app/torrent-table/torrent-table.component.html b/client/src/app/torrent-table/torrent-table.component.html index b728bdc..5524fce 100644 --- a/client/src/app/torrent-table/torrent-table.component.html +++ b/client/src/app/torrent-table/torrent-table.component.html @@ -106,20 +106,27 @@
-
diff --git a/client/src/app/torrent-table/torrent-table.component.ts b/client/src/app/torrent-table/torrent-table.component.ts index 3f17925..782b6c5 100644 --- a/client/src/app/torrent-table/torrent-table.component.ts +++ b/client/src/app/torrent-table/torrent-table.component.ts @@ -20,6 +20,7 @@ export class TorrentTableComponent implements OnInit { public isDeleteModalActive: boolean; public deleteError: string; public deleting: boolean; + public selectAll: boolean; public deleteData: boolean; public deleteRdTorrent: boolean; public deleteLocalFiles: boolean; @@ -247,4 +248,15 @@ export class TorrentTableComponent implements OnInit { }, }); } + // Called when "Select All" is toggled + toggleAllOptions() { + this.deleteData = this.selectAll; + this.deleteRdTorrent = this.selectAll; + this.deleteLocalFiles = this.selectAll; + } + + // Called when any individual checkbox is toggled + updateSelectAll() { + this.selectAll = this.deleteData && this.deleteRdTorrent && this.deleteLocalFiles; + } } From 953e208b18e72369c4ba77bfae9dfa89529ab39d Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Tue, 8 Apr 2025 20:49:06 +0100 Subject: [PATCH 19/31] run prettier --- client/src/app/torrent-table/torrent-table.component.html | 2 +- client/src/app/torrent-table/torrent-table.component.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/app/torrent-table/torrent-table.component.html b/client/src/app/torrent-table/torrent-table.component.html index 5524fce..fe243e1 100644 --- a/client/src/app/torrent-table/torrent-table.component.html +++ b/client/src/app/torrent-table/torrent-table.component.html @@ -106,7 +106,7 @@
-

-