danvics/rdt-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Symlink downloader performance fixes

Browse source 
- Added concurrent downloading for symlink downloader
- Use FinishAction configuration setting for QBittorrent delete endpoint
- Add symlink downloader readme section
This commit is contained in:
Gaisberg 2023-09-27 21:52:02 +03:00
parent 8d16f7fc57
commit fe77391e7f
7 changed files with 220 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
README.md
View file

@ -169,6 +169,20 @@ It has the following options:
If Aria2c is selected, none of the above options for `Internal Downloader` are used, you have to configure Aria2c manually.
#### Symlink downloader
Symlink downloader requires a rclone mount to be mounted into your filesystem. Be sure to keep the exact path to mounted files in other apps exactly
the same as used by rdt-client. Otherwise the symlinks wont resolve the file its trying to point to.
If the mount path folder cant be found the client wont start downloading anything.
Required configuration:
- Post Download Action = DO NOT SELECT REMOVE FROM PROVIDER
- Rclone mount path = /PATH_TO_YOUR_RCLONE_MOUNT/torrents/
Suggested configuration:
- Automatic retry downloads > 3
### Troubleshooting
- If you forgot your logins simply delete the `rdtclient.db` and restart the service.

26
server/RdtClient.Data/Data/DownloadData.cs
View file

@ -210,17 +210,27 @@ public class DownloadData
await TorrentData.VoidCache();
}
public async Task UpdateRemoteId(Guid downloadId, String remoteId)
public async Task UpdateRemoteId(Guid downloadId, string remoteId)
{
var dbDownload = await _dataContext.Downloads
.FirstOrDefaultAsync(m => m.DownloadId == downloadId);
if (dbDownload == null)
await UpdateRemoteIdRange(new Dictionary<Guid, string>
{
return;
}
{ downloadId, remoteId }
});
dbDownload.RemoteId = remoteId;
}
public async Task UpdateRemoteIdRange(Dictionary<Guid, string> remoteIdRange)
{
foreach (var entry in remoteIdRange)
{
var dbDownload = await _dataContext.Downloads.FirstOrDefaultAsync(m => m.DownloadId == entry.Key);
if (dbDownload == null)
{
continue;
}
dbDownload.RemoteId = entry.Value;
}
await _dataContext.SaveChangesAsync();
}

8
server/RdtClient.Service/Services/DownloadClient.cs
View file

@ -23,11 +23,13 @@ public class DownloadClient
public Int64 BytesTotal { get; private set; }
public Int64 BytesDone { get; private set; }
public DownloadClient(Download download, Torrent torrent, String destinationPath)
public DownloadClient(Download download, Torrent torrent, string destinationPath)
{
_download = download;
_torrent = torrent;
_destinationPath = destinationPath;
Type = Settings.Get.DownloadClient.Client;
}
public async Task<String?> Start()
@ -52,9 +54,7 @@ public class DownloadClient
await FileHelper.Delete(filePath);
Type = Settings.Get.DownloadClient.Client;
Downloader = Settings.Get.DownloadClient.Client switch
Downloader = Type switch
{
Data.Enums.DownloadClient.Internal => new InternalDownloader(_download.Link, filePath),
Data.Enums.DownloadClient.Aria2c => new Aria2cDownloader(_download.RemoteId, _download.Link, filePath),

86
server/RdtClient.Service/Services/Downloaders/SymlinkDownloader.cs
View file

@ -8,8 +8,6 @@ public class SymlinkDownloader : IDownloader
public event EventHandler<DownloadCompleteEventArgs>? DownloadComplete;
public event EventHandler<DownloadProgressEventArgs>? DownloadProgress;
private const Int32 RetryCount = 5;
private readonly String _filePath;
private readonly String _uri;
@ -25,7 +23,7 @@ public class SymlinkDownloader : IDownloader
_filePath = filePath;
}
public async Task<String?> Download()
public Task<string?> Download()
{
_logger.Debug($"Starting download of {_uri}, writing to path: {_filePath}");
@ -38,48 +36,34 @@ public class SymlinkDownloader : IDownloader
Speed = 0
});
var retryCount = 1;
while (retryCount < RetryCount)
_logger.Debug($"Searching {Settings.Get.DownloadClient.RcloneMountPath} for {fileName}");
// Recursively search for the fileName in the rclone mount location.
var foundFiles = Directory.GetFiles(Settings.Get.DownloadClient.RcloneMountPath, fileName, SearchOption.AllDirectories);
if (foundFiles.Any())
{
_logger.Debug($"Searching {Settings.Get.DownloadClient.RcloneMountPath} for {fileName} ({retryCount}/{RetryCount}) ");
// Recursively search for the fileName in the rclone mount location.
var foundFiles = Directory.GetFiles(Settings.Get.DownloadClient.RcloneMountPath, fileName, SearchOption.AllDirectories).ToList();
if (foundFiles.Any())
if (foundFiles.Length > 1)
{
if (foundFiles.Count > 1)
{
_logger.Warning($"Found {foundFiles.Count} files named {fileName}");
}
// Assume first matching filename is the one we want.
var actualFilePath = foundFiles.First();
var result = TryCreateSymbolicLink(actualFilePath, _filePath);
if (result)
{
DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs());
return actualFilePath;
}
_logger.Warning($"Found {foundFiles.Length} files named {fileName}");
}
await Task.Delay(TimeSpan.FromSeconds(30), _cancellationToken.Token);
// Assume first matching filename is the one we want.
var actualFilePath = foundFiles.First();
retryCount++;
var result = TryCreateSymbolicLink(actualFilePath, _filePath);
if (result)
{
DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs());
return Task.FromResult<string?>(actualFilePath);
}
}
_logger.Error($"File '{fileName}' not found after {RetryCount} attempts.");
DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs
{
Error = $"File '{fileName}' not found after {RetryCount} attempts."
});
return null;
// Return null and try again next cycle.
return Task.FromResult<string?>(null);
}
public Task Cancel()
@ -101,37 +85,25 @@ public class SymlinkDownloader : IDownloader
return Task.CompletedTask;
}
private Boolean TryCreateSymbolicLink(String sourcePath, String symlinkPath)
private bool TryCreateSymbolicLink(string sourcePath, string symlinkPath)
{
try
{
var process = new Process();
process.StartInfo.FileName = "ln";
process.StartInfo.Arguments = @$"-s ""{sourcePath}"" ""{symlinkPath}""";
process.StartInfo.UseShellExecute = false;
process.StartInfo.RedirectStandardError = true;
process.Start();
var errors = process.StandardError.ReadToEnd();
process.WaitForExit();
if (process.ExitCode == 0)
File.CreateSymbolicLink(symlinkPath, sourcePath);
if (File.Exists(symlinkPath)) // Double-check that the link was created
{
_logger.Information($"Created symbolic link from {sourcePath} to {symlinkPath}");
return true;
}
_logger.Error($"Failed to create symbolic link: {process.ExitCode} - {errors}");
return false;
else
{
_logger.Error($"Failed to create symbolic link from {sourcePath} to {symlinkPath}");
return false;
}
}
catch (Exception ex)
{
_logger.Error($"Error creating symbolic link from {sourcePath} to {symlinkPath}: {ex.Message}");
return false;
}
}

5
server/RdtClient.Service/Services/Downloads.cs
View file

@ -82,6 +82,11 @@ public class Downloads
await _downloadData.UpdateRemoteId(downloadId, remoteId);
}
public async Task UpdateRemoteIdRange(Dictionary<Guid, String> updateDict)
{
await _downloadData.UpdateRemoteIdRange(updateDict);
}
public async Task DeleteForTorrent(Guid torrentId)
{
await _downloadData.DeleteForTorrent(torrentId);

27
server/RdtClient.Service/Services/QBittorrent.cs
View file

@ -421,7 +421,32 @@ public class QBittorrent
return;
}
await _torrents.Delete(torrent.TorrentId, true, true, deleteFiles);
switch (Settings.Get.Integrations.Default.FinishedAction)
{
case TorrentFinishedAction.RemoveAllTorrents:
_logger.LogDebug($"Removing torrents from Real-Debrid and Real-Debrid Client, no files", torrent);
await _torrents.Delete(torrent.TorrentId, true, true, false);
break;
case TorrentFinishedAction.RemoveRealDebrid:
_logger.LogDebug($"Removing torrents from Real-Debrid, no files", torrent);
await _torrents.Delete(torrent.TorrentId, false, true, false);
break;
case TorrentFinishedAction.RemoveClient:
_logger.LogDebug($"Removing torrents from client, no files", torrent);
await _torrents.Delete(torrent.TorrentId, true, false, false);
break;
case TorrentFinishedAction.None:
_logger.LogDebug($"Not removing torrents or files", torrent);
break;
default:
_logger.LogDebug($"Invalid torrent FinishedAction {torrent.FinishedAction}", torrent);
break;
}
}
public async Task TorrentsAddMagnet(String magnetLink, String? category, Int32? priority)

195
server/RdtClient.Service/Services/TorrentRunner.cs
View file

@ -22,6 +22,7 @@ public class TorrentRunner
private readonly Downloads _downloads;
private readonly RemoteService _remoteService;
private readonly HttpClient _httpClient;
private readonly Dictionary<Guid, string> _aggregatedDownloadResults;
public TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Downloads downloads, RemoteService remoteService)
{
@ -29,6 +30,7 @@ public class TorrentRunner
_torrents = torrents;
_downloads = downloads;
_remoteService = remoteService;
_aggregatedDownloadResults = new Dictionary<Guid, string>();
_httpClient = new HttpClient
{
@ -87,7 +89,18 @@ public class TorrentRunner
Log($"No RealDebridApiKey set in settings");
return;
}
if (Settings.Get.DownloadClient.Client == Data.Enums.DownloadClient.Symlink)
{
var rcloneMountPath = Settings.Get.DownloadClient.RcloneMountPath;
if (!Directory.Exists(rcloneMountPath))
{
Log($"Rclone mount path ({rcloneMountPath}) was not found!");
return;
}
}
var settingDownloadLimit = Settings.Get.General.DownloadLimit;
if (settingDownloadLimit < 1)
{
@ -310,77 +323,15 @@ public class TorrentRunner
.OrderBy(m => m.DownloadQueued)
.ToList();
_aggregatedDownloadResults.Clear();
foreach (var download in queuedDownloads)
{
Log($"Processing to download", download, torrent);
await ProcessDownload(download, torrent, settingDownloadPath, settingDownloadLimit);
}
if (_aggregatedDownloadResults.Count > 0)
{
await _downloads.UpdateRemoteIdRange(_aggregatedDownloadResults);
if (ActiveDownloadClients.Count >= settingDownloadLimit)
{
Log($"Not starting download because there are already the max number of downloads active", download, torrent);
continue;
}
if (ActiveDownloadClients.ContainsKey(download.DownloadId))
{
Log($"Not starting download because this download is already active", download, torrent);
continue;
}
try
{
Log($"Unrestricting links", download, torrent);
var downloadLink = await _torrents.UnrestrictLink(download.DownloadId);
download.Link = downloadLink;
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot unrestrict link: {ex.Message}", ex.Message);
await _downloads.UpdateError(download.DownloadId, ex.Message);
await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow);
download.Error = ex.Message;
download.Completed = DateTimeOffset.UtcNow;
continue;
}
Log($"Marking download as started", download, torrent);
download.DownloadStarted = DateTime.UtcNow;
await _downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted);
var downloadPath = settingDownloadPath;
if (!String.IsNullOrWhiteSpace(torrent.Category))
{
downloadPath = Path.Combine(downloadPath, torrent.Category);
}
Log($"Setting download path to {downloadPath}", download, torrent);
// Start the download process
var downloadClient = new DownloadClient(download, torrent, downloadPath);
if (ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient))
{
Log($"Starting download", download, torrent);
var remoteId = await downloadClient.Start();
if (!String.IsNullOrWhiteSpace(remoteId) && download.RemoteId != remoteId)
{
Log($"Received ID {remoteId}", download, torrent);
await _downloads.UpdateRemoteId(download.DownloadId, remoteId);
}
else
{
Log($"No ID received", download, torrent);
}
}
}
// Check if there are any unpacks that are queued and can be started.
@ -428,8 +379,18 @@ public class TorrentRunner
continue;
}
// Check if we have reached the download limit, if so queue the download, but don't start it.
if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit)
if (Settings.Get.DownloadClient.Client == Data.Enums.DownloadClient.Symlink)
{
Log("Lets not unzip with symlink downloader...");
await _downloads.UpdateError(download.DownloadId, "Will not unzip with SymlinkDownloader!");
await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow);
continue;
}
// Check if we have reached the download limit, if so queue the download, but don't start it.
if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit)
{
Log($"Not starting unpack because there are already the max number of unpacks active", download, torrent);
@ -628,4 +589,96 @@ public class TorrentRunner
_logger.LogError(message);
}
private async Task ProcessDownload(Download download, Torrent torrent, string settingDownloadPath, int settingDownloadLimit)
{
if (ActiveDownloadClients.Count >= settingDownloadLimit)
{
Log($"Not starting download because there are already the max number of downloads active", download, torrent);
return;
}
if (ActiveDownloadClients.ContainsKey(download.DownloadId))
{
Log($"Not starting download because this download is already active", download, torrent);
return;
}
try
{
if (download.Link == null)
{
Log($"Unrestricting links", download, torrent);
var downloadLink = await _torrents.UnrestrictLink(download.DownloadId);
download.Link = downloadLink;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot unrestrict link: {ex.Message}", ex.Message);
await _downloads.UpdateError(download.DownloadId, ex.Message);
await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow);
download.Error = ex.Message;
download.Completed = DateTimeOffset.UtcNow;
return;
}
Log($"Marking download as started", download, torrent);
download.DownloadStarted = DateTime.UtcNow;
await _downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted);
var downloadPath = settingDownloadPath;
if (!String.IsNullOrWhiteSpace(torrent.Category))
{
downloadPath = Path.Combine(downloadPath, torrent.Category);
}
Log($"Setting download path to {downloadPath}", download, torrent);
var downloadClient = new DownloadClient(download, torrent, downloadPath);
if (downloadClient.Type == Data.Enums.DownloadClient.Symlink) // Check if the type is "Symlink"
{
// If it's Symlink type, start the download concurrently
_ = Task.Run(async () => await StartDownload(download, torrent, downloadClient));
}
else
{
await StartDownload(download, torrent, downloadClient);
}
}
private async Task StartDownload(Download download, Torrent torrent, DownloadClient downloadClient)
{
if (ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient) ||
(downloadClient.Type == Data.Enums.DownloadClient.Symlink && download.RetryCount > 0))
{
Log($"Starting download", download, torrent);
var remoteId = await downloadClient.Start();
if (!string.IsNullOrWhiteSpace(remoteId) && download.RemoteId != remoteId)
{
Log($"Received ID {remoteId}", download, torrent);
_aggregatedDownloadResults.Add(download.DownloadId, remoteId);
}
else
{
Log($"No ID received", download, torrent);
// Lets us redo the download next cycle
if (downloadClient.Type == Data.Enums.DownloadClient.Symlink)
{
Log($"Marking download as ended, so we can retry", download, torrent);
download.DownloadStarted = null;
}
}
}
}
}