From fe77391e7fafcdaba89979ae0a13e2e453a30128 Mon Sep 17 00:00:00 2001 From: Gaisberg Date: Wed, 27 Sep 2023 21:52:02 +0300 Subject: [PATCH] Symlink downloader performance fixes - Added concurrent downloading for symlink downloader - Use FinishAction configuration setting for QBittorrent delete endpoint - Add symlink downloader readme section --- README.md | 14 ++ server/RdtClient.Data/Data/DownloadData.cs | 26 ++- .../Services/DownloadClient.cs | 8 +- .../Services/Downloaders/SymlinkDownloader.cs | 86 +++----- .../RdtClient.Service/Services/Downloads.cs | 5 + .../RdtClient.Service/Services/QBittorrent.cs | 27 ++- .../Services/TorrentRunner.cs | 195 +++++++++++------- 7 files changed, 220 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index 47f1d02..4a075a4 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,20 @@ It has the following options: If Aria2c is selected, none of the above options for `Internal Downloader` are used, you have to configure Aria2c manually. +#### Symlink downloader + +Symlink downloader requires a rclone mount to be mounted into your filesystem. Be sure to keep the exact path to mounted files in other apps exactly +the same as used by rdt-client. Otherwise the symlinks wont resolve the file its trying to point to. + +If the mount path folder cant be found the client wont start downloading anything. + +Required configuration: +- Post Download Action = DO NOT SELECT REMOVE FROM PROVIDER +- Rclone mount path = /PATH_TO_YOUR_RCLONE_MOUNT/torrents/ + +Suggested configuration: +- Automatic retry downloads > 3 + ### Troubleshooting - If you forgot your logins simply delete the `rdtclient.db` and restart the service. diff --git a/server/RdtClient.Data/Data/DownloadData.cs b/server/RdtClient.Data/Data/DownloadData.cs index 27f80de..6c35041 100644 --- a/server/RdtClient.Data/Data/DownloadData.cs +++ b/server/RdtClient.Data/Data/DownloadData.cs @@ -210,17 +210,27 @@ public class DownloadData await TorrentData.VoidCache(); } - public async Task UpdateRemoteId(Guid downloadId, String remoteId) + public async Task UpdateRemoteId(Guid downloadId, string remoteId) { - var dbDownload = await _dataContext.Downloads - .FirstOrDefaultAsync(m => m.DownloadId == downloadId); - - if (dbDownload == null) + await UpdateRemoteIdRange(new Dictionary { - return; - } + { downloadId, remoteId } + }); - dbDownload.RemoteId = remoteId; + } + + public async Task UpdateRemoteIdRange(Dictionary remoteIdRange) + { + foreach (var entry in remoteIdRange) + { + var dbDownload = await _dataContext.Downloads.FirstOrDefaultAsync(m => m.DownloadId == entry.Key); + if (dbDownload == null) + { + continue; + } + + dbDownload.RemoteId = entry.Value; + } await _dataContext.SaveChangesAsync(); } diff --git a/server/RdtClient.Service/Services/DownloadClient.cs b/server/RdtClient.Service/Services/DownloadClient.cs index ddfd144..b4cddc5 100644 --- a/server/RdtClient.Service/Services/DownloadClient.cs +++ b/server/RdtClient.Service/Services/DownloadClient.cs @@ -23,11 +23,13 @@ public class DownloadClient public Int64 BytesTotal { get; private set; } public Int64 BytesDone { get; private set; } - public DownloadClient(Download download, Torrent torrent, String destinationPath) + public DownloadClient(Download download, Torrent torrent, string destinationPath) { _download = download; _torrent = torrent; _destinationPath = destinationPath; + + Type = Settings.Get.DownloadClient.Client; } public async Task Start() @@ -52,9 +54,7 @@ public class DownloadClient await FileHelper.Delete(filePath); - Type = Settings.Get.DownloadClient.Client; - - Downloader = Settings.Get.DownloadClient.Client switch + Downloader = Type switch { Data.Enums.DownloadClient.Internal => new InternalDownloader(_download.Link, filePath), Data.Enums.DownloadClient.Aria2c => new Aria2cDownloader(_download.RemoteId, _download.Link, filePath), diff --git a/server/RdtClient.Service/Services/Downloaders/SymlinkDownloader.cs b/server/RdtClient.Service/Services/Downloaders/SymlinkDownloader.cs index 5bf3830..909f8b3 100644 --- a/server/RdtClient.Service/Services/Downloaders/SymlinkDownloader.cs +++ b/server/RdtClient.Service/Services/Downloaders/SymlinkDownloader.cs @@ -8,8 +8,6 @@ public class SymlinkDownloader : IDownloader public event EventHandler? DownloadComplete; public event EventHandler? DownloadProgress; - private const Int32 RetryCount = 5; - private readonly String _filePath; private readonly String _uri; @@ -25,7 +23,7 @@ public class SymlinkDownloader : IDownloader _filePath = filePath; } - public async Task Download() + public Task Download() { _logger.Debug($"Starting download of {_uri}, writing to path: {_filePath}"); @@ -38,48 +36,34 @@ public class SymlinkDownloader : IDownloader Speed = 0 }); - var retryCount = 1; - while (retryCount < RetryCount) + _logger.Debug($"Searching {Settings.Get.DownloadClient.RcloneMountPath} for {fileName}"); + + // Recursively search for the fileName in the rclone mount location. + var foundFiles = Directory.GetFiles(Settings.Get.DownloadClient.RcloneMountPath, fileName, SearchOption.AllDirectories); + + if (foundFiles.Any()) { - _logger.Debug($"Searching {Settings.Get.DownloadClient.RcloneMountPath} for {fileName} ({retryCount}/{RetryCount}) "); - - // Recursively search for the fileName in the rclone mount location. - var foundFiles = Directory.GetFiles(Settings.Get.DownloadClient.RcloneMountPath, fileName, SearchOption.AllDirectories).ToList(); - - if (foundFiles.Any()) + if (foundFiles.Length > 1) { - if (foundFiles.Count > 1) - { - _logger.Warning($"Found {foundFiles.Count} files named {fileName}"); - } - - // Assume first matching filename is the one we want. - var actualFilePath = foundFiles.First(); - - var result = TryCreateSymbolicLink(actualFilePath, _filePath); - - if (result) - { - DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs()); - - return actualFilePath; - } + _logger.Warning($"Found {foundFiles.Length} files named {fileName}"); } - await Task.Delay(TimeSpan.FromSeconds(30), _cancellationToken.Token); + // Assume first matching filename is the one we want. + var actualFilePath = foundFiles.First(); - retryCount++; + var result = TryCreateSymbolicLink(actualFilePath, _filePath); + + if (result) + { + DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs()); + + return Task.FromResult(actualFilePath); + } } - _logger.Error($"File '{fileName}' not found after {RetryCount} attempts."); - - DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs - { - Error = $"File '{fileName}' not found after {RetryCount} attempts." - }); - - return null; + // Return null and try again next cycle. + return Task.FromResult(null); } public Task Cancel() @@ -101,37 +85,25 @@ public class SymlinkDownloader : IDownloader return Task.CompletedTask; } - private Boolean TryCreateSymbolicLink(String sourcePath, String symlinkPath) + private bool TryCreateSymbolicLink(string sourcePath, string symlinkPath) { try { - var process = new Process(); - process.StartInfo.FileName = "ln"; - process.StartInfo.Arguments = @$"-s ""{sourcePath}"" ""{symlinkPath}"""; - process.StartInfo.UseShellExecute = false; - process.StartInfo.RedirectStandardError = true; - - process.Start(); - - var errors = process.StandardError.ReadToEnd(); - - process.WaitForExit(); - - if (process.ExitCode == 0) + File.CreateSymbolicLink(symlinkPath, sourcePath); + if (File.Exists(symlinkPath)) // Double-check that the link was created { _logger.Information($"Created symbolic link from {sourcePath} to {symlinkPath}"); - return true; } - - _logger.Error($"Failed to create symbolic link: {process.ExitCode} - {errors}"); - - return false; + else + { + _logger.Error($"Failed to create symbolic link from {sourcePath} to {symlinkPath}"); + return false; + } } catch (Exception ex) { _logger.Error($"Error creating symbolic link from {sourcePath} to {symlinkPath}: {ex.Message}"); - return false; } } diff --git a/server/RdtClient.Service/Services/Downloads.cs b/server/RdtClient.Service/Services/Downloads.cs index 9acc847..efccc24 100644 --- a/server/RdtClient.Service/Services/Downloads.cs +++ b/server/RdtClient.Service/Services/Downloads.cs @@ -82,6 +82,11 @@ public class Downloads await _downloadData.UpdateRemoteId(downloadId, remoteId); } + public async Task UpdateRemoteIdRange(Dictionary updateDict) + { + await _downloadData.UpdateRemoteIdRange(updateDict); + } + public async Task DeleteForTorrent(Guid torrentId) { await _downloadData.DeleteForTorrent(torrentId); diff --git a/server/RdtClient.Service/Services/QBittorrent.cs b/server/RdtClient.Service/Services/QBittorrent.cs index 86a8e2b..08cd09f 100644 --- a/server/RdtClient.Service/Services/QBittorrent.cs +++ b/server/RdtClient.Service/Services/QBittorrent.cs @@ -421,7 +421,32 @@ public class QBittorrent return; } - await _torrents.Delete(torrent.TorrentId, true, true, deleteFiles); + switch (Settings.Get.Integrations.Default.FinishedAction) + { + case TorrentFinishedAction.RemoveAllTorrents: + _logger.LogDebug($"Removing torrents from Real-Debrid and Real-Debrid Client, no files", torrent); + await _torrents.Delete(torrent.TorrentId, true, true, false); + + break; + case TorrentFinishedAction.RemoveRealDebrid: + _logger.LogDebug($"Removing torrents from Real-Debrid, no files", torrent); + await _torrents.Delete(torrent.TorrentId, false, true, false); + + break; + case TorrentFinishedAction.RemoveClient: + _logger.LogDebug($"Removing torrents from client, no files", torrent); + await _torrents.Delete(torrent.TorrentId, true, false, false); + + break; + case TorrentFinishedAction.None: + _logger.LogDebug($"Not removing torrents or files", torrent); + + break; + default: + _logger.LogDebug($"Invalid torrent FinishedAction {torrent.FinishedAction}", torrent); + + break; + } } public async Task TorrentsAddMagnet(String magnetLink, String? category, Int32? priority) diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index ae4f2c4..05cf44c 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -22,6 +22,7 @@ public class TorrentRunner private readonly Downloads _downloads; private readonly RemoteService _remoteService; private readonly HttpClient _httpClient; + private readonly Dictionary _aggregatedDownloadResults; public TorrentRunner(ILogger logger, Torrents torrents, Downloads downloads, RemoteService remoteService) { @@ -29,6 +30,7 @@ public class TorrentRunner _torrents = torrents; _downloads = downloads; _remoteService = remoteService; + _aggregatedDownloadResults = new Dictionary(); _httpClient = new HttpClient { @@ -87,7 +89,18 @@ public class TorrentRunner Log($"No RealDebridApiKey set in settings"); return; } - + + if (Settings.Get.DownloadClient.Client == Data.Enums.DownloadClient.Symlink) + { + var rcloneMountPath = Settings.Get.DownloadClient.RcloneMountPath; + + if (!Directory.Exists(rcloneMountPath)) + { + Log($"Rclone mount path ({rcloneMountPath}) was not found!"); + return; + } + } + var settingDownloadLimit = Settings.Get.General.DownloadLimit; if (settingDownloadLimit < 1) { @@ -310,77 +323,15 @@ public class TorrentRunner .OrderBy(m => m.DownloadQueued) .ToList(); + _aggregatedDownloadResults.Clear(); foreach (var download in queuedDownloads) { - Log($"Processing to download", download, torrent); + await ProcessDownload(download, torrent, settingDownloadPath, settingDownloadLimit); + } + if (_aggregatedDownloadResults.Count > 0) + { + await _downloads.UpdateRemoteIdRange(_aggregatedDownloadResults); - if (ActiveDownloadClients.Count >= settingDownloadLimit) - { - Log($"Not starting download because there are already the max number of downloads active", download, torrent); - - continue; - } - - if (ActiveDownloadClients.ContainsKey(download.DownloadId)) - { - Log($"Not starting download because this download is already active", download, torrent); - - continue; - } - - try - { - Log($"Unrestricting links", download, torrent); - - var downloadLink = await _torrents.UnrestrictLink(download.DownloadId); - download.Link = downloadLink; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot unrestrict link: {ex.Message}", ex.Message); - - await _downloads.UpdateError(download.DownloadId, ex.Message); - await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); - download.Error = ex.Message; - download.Completed = DateTimeOffset.UtcNow; - - continue; - } - - Log($"Marking download as started", download, torrent); - - download.DownloadStarted = DateTime.UtcNow; - await _downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted); - - var downloadPath = settingDownloadPath; - - if (!String.IsNullOrWhiteSpace(torrent.Category)) - { - downloadPath = Path.Combine(downloadPath, torrent.Category); - } - - Log($"Setting download path to {downloadPath}", download, torrent); - - // Start the download process - var downloadClient = new DownloadClient(download, torrent, downloadPath); - - if (ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient)) - { - Log($"Starting download", download, torrent); - - var remoteId = await downloadClient.Start(); - - if (!String.IsNullOrWhiteSpace(remoteId) && download.RemoteId != remoteId) - { - Log($"Received ID {remoteId}", download, torrent); - - await _downloads.UpdateRemoteId(download.DownloadId, remoteId); - } - else - { - Log($"No ID received", download, torrent); - } - } } // Check if there are any unpacks that are queued and can be started. @@ -428,8 +379,18 @@ public class TorrentRunner continue; } - // Check if we have reached the download limit, if so queue the download, but don't start it. - if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit) + if (Settings.Get.DownloadClient.Client == Data.Enums.DownloadClient.Symlink) + { + Log("Lets not unzip with symlink downloader..."); + + await _downloads.UpdateError(download.DownloadId, "Will not unzip with SymlinkDownloader!"); + await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); + + continue; + } + + // Check if we have reached the download limit, if so queue the download, but don't start it. + if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit) { Log($"Not starting unpack because there are already the max number of unpacks active", download, torrent); @@ -628,4 +589,96 @@ public class TorrentRunner _logger.LogError(message); } + + private async Task ProcessDownload(Download download, Torrent torrent, string settingDownloadPath, int settingDownloadLimit) + { + if (ActiveDownloadClients.Count >= settingDownloadLimit) + { + Log($"Not starting download because there are already the max number of downloads active", download, torrent); + + return; + } + + if (ActiveDownloadClients.ContainsKey(download.DownloadId)) + { + Log($"Not starting download because this download is already active", download, torrent); + + return; + } + + try + { + if (download.Link == null) + { + Log($"Unrestricting links", download, torrent); + + var downloadLink = await _torrents.UnrestrictLink(download.DownloadId); + download.Link = downloadLink; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot unrestrict link: {ex.Message}", ex.Message); + + await _downloads.UpdateError(download.DownloadId, ex.Message); + await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); + download.Error = ex.Message; + download.Completed = DateTimeOffset.UtcNow; + + return; + } + + Log($"Marking download as started", download, torrent); + + download.DownloadStarted = DateTime.UtcNow; + await _downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted); + + var downloadPath = settingDownloadPath; + + if (!String.IsNullOrWhiteSpace(torrent.Category)) + { + downloadPath = Path.Combine(downloadPath, torrent.Category); + } + + Log($"Setting download path to {downloadPath}", download, torrent); + + var downloadClient = new DownloadClient(download, torrent, downloadPath); + + if (downloadClient.Type == Data.Enums.DownloadClient.Symlink) // Check if the type is "Symlink" + { + // If it's Symlink type, start the download concurrently + _ = Task.Run(async () => await StartDownload(download, torrent, downloadClient)); + } + else + { + await StartDownload(download, torrent, downloadClient); + } + } + + private async Task StartDownload(Download download, Torrent torrent, DownloadClient downloadClient) + { + if (ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient) || + (downloadClient.Type == Data.Enums.DownloadClient.Symlink && download.RetryCount > 0)) + { + Log($"Starting download", download, torrent); + + var remoteId = await downloadClient.Start(); + + if (!string.IsNullOrWhiteSpace(remoteId) && download.RemoteId != remoteId) + { + Log($"Received ID {remoteId}", download, torrent); + _aggregatedDownloadResults.Add(download.DownloadId, remoteId); + } + else + { + Log($"No ID received", download, torrent); + // Lets us redo the download next cycle + if (downloadClient.Type == Data.Enums.DownloadClient.Symlink) + { + Log($"Marking download as ended, so we can retry", download, torrent); + download.DownloadStarted = null; + } + } + } + } } \ No newline at end of file