danvics/rdt-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add fixes for the symlink downloader from https://github.com/Pukabyte/rdtclient .

Browse source
This commit is contained in:
Roger Far 2024-04-06 14:52:11 -06:00
parent 45c805eb00
commit e5dfb0e027
11 changed files with 218 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
server/RdtClient.Data/Data/DownloadData.cs
View file

@ -186,6 +186,24 @@ public class DownloadData(DataContext dataContext)
await TorrentData.VoidCache();
}
public async Task UpdateErrors(Dictionary<Guid, String> downloadIds)
{
foreach (var entry in downloadIds)
{
var dbDownload = await dataContext.Downloads
.FirstOrDefaultAsync(m => m.DownloadId == entry.Key);
if (dbDownload == null)
{
continue;
}
dbDownload.Error = entry.Value;
}
await dataContext.SaveChangesAsync();
}
public async Task UpdateRetryCount(Guid downloadId, Int32 retryCount)
{
var dbDownload = await dataContext.Downloads
@ -218,6 +236,22 @@ public class DownloadData(DataContext dataContext)
await dataContext.SaveChangesAsync();
}
public async Task UpdateRemoteIds(Dictionary<Guid, String> remoteIds)
{
foreach (var entry in remoteIds)
{
var dbDownload = await dataContext.Downloads.FirstOrDefaultAsync(m => m.DownloadId == entry.Key);
if (dbDownload == null)
{
continue;
}
dbDownload.RemoteId = entry.Value;
}
await dataContext.SaveChangesAsync();
}
public async Task DeleteForTorrent(Guid torrentId)
{
var downloads = await dataContext.Downloads
@ -234,12 +268,8 @@ public class DownloadData(DataContext dataContext)
public async Task Reset(Guid downloadId)
{
var dbDownload = await dataContext.Downloads
.FirstOrDefaultAsync(m => m.DownloadId == downloadId);
if (dbDownload == null)
{
throw new Exception($"Cannot find download with ID {downloadId}");
}
.FirstOrDefaultAsync(m => m.DownloadId == downloadId)
?? throw new Exception($"Cannot find download with ID {downloadId}");
dbDownload.RetryCount = 0;
dbDownload.Link = null;

4
server/RdtClient.Service/Middleware/RequestLoggingMiddleware.cs
View file

@ -24,11 +24,11 @@ public class RequestLoggingMiddleware(RequestDelegate next, ILoggerFactory logge
requestLog += $", QueryString: {context.Request.QueryString}";
}
if (context.Request.HasFormContentType && context.Request.Form.Any())
if (context.Request.HasFormContentType && context.Request.Form.Count == 0)
{
requestLog += $", Form: {String.Join(", ", context.Request.Form.Select(f => $"{f.Key}: {f.Value}"))}";
}
else if (context.Request.ContentType?.ToLower().Contains("application/json") == true)
else if (context.Request.ContentType?.Contains("application/json", StringComparison.CurrentCultureIgnoreCase) == true)
{
var body = await ReadRequestBodyAsync(context.Request);
requestLog += $", Body: {body}";

24
server/RdtClient.Service/Services/DownloadClient.cs
View file

@ -6,11 +6,6 @@ namespace RdtClient.Service.Services;
public class DownloadClient(Download download, Torrent torrent, String destinationPath)
{
private readonly String _destinationPath = destinationPath;
private readonly Download _download = download;
private readonly Torrent _torrent = torrent;
public IDownloader? Downloader;
public Data.Enums.DownloadClient Type { get; private set; }
@ -23,7 +18,7 @@ public class DownloadClient(Download download, Torrent torrent, String destinati
public Int64 BytesTotal { get; private set; }
public Int64 BytesDone { get; private set; }
public async Task<String?> Start()
public async Task<String> Start()
{
BytesDone = 0;
BytesTotal = 0;
@ -31,13 +26,13 @@ public class DownloadClient(Download download, Torrent torrent, String destinati
try
{
if (_download.Link == null)
if (download.Link == null)
{
throw new Exception($"Invalid download link");
}
var filePath = DownloadHelper.GetDownloadPath(_destinationPath, _torrent, _download);
var downloadPath = DownloadHelper.GetDownloadPath(_torrent, _download);
var filePath = DownloadHelper.GetDownloadPath(destinationPath, torrent, download);
var downloadPath = DownloadHelper.GetDownloadPath(torrent, download);
if (filePath == null || downloadPath == null)
{
@ -50,10 +45,10 @@ public class DownloadClient(Download download, Torrent torrent, String destinati
Downloader = Settings.Get.DownloadClient.Client switch
{
Data.Enums.DownloadClient.Internal => new InternalDownloader(_download.Link, filePath),
Data.Enums.DownloadClient.Bezzad => new BezzadDownloader(_download.Link, filePath),
Data.Enums.DownloadClient.Aria2c => new Aria2cDownloader(_download.RemoteId, _download.Link, filePath, downloadPath),
Data.Enums.DownloadClient.Symlink => new SymlinkDownloader(_download.Link, filePath),
Data.Enums.DownloadClient.Internal => new InternalDownloader(download.Link, filePath),
Data.Enums.DownloadClient.Bezzad => new BezzadDownloader(download.Link, filePath),
Data.Enums.DownloadClient.Aria2c => new Aria2cDownloader(download.RemoteId, download.Link, filePath, downloadPath),
Data.Enums.DownloadClient.Symlink => new SymlinkDownloader(download.Link, filePath),
_ => throw new Exception($"Unknown download client {Settings.Get.DownloadClient}")
};
@ -83,10 +78,9 @@ public class DownloadClient(Download download, Torrent torrent, String destinati
await Downloader.Cancel();
}
Error = $"An unexpected error occurred preparing download {_download.Link} for torrent {_torrent.RdName}: {ex.Message}";
Finished = true;
return null;
throw new($"An unexpected error occurred preparing download {download.Link} for torrent {torrent.RdName}: {ex.Message}");
}
}

2
server/RdtClient.Service/Services/Downloaders/Aria2cDownloader.cs
View file

@ -45,7 +45,7 @@ public class Aria2cDownloader : IDownloader
_aria2NetClient = new Aria2NetClient(Settings.Get.DownloadClient.Aria2cUrl, Settings.Get.DownloadClient.Aria2cSecret, httpClient, 10);
}
public async Task<String?> Download()
public async Task<String> Download()
{
var path = Path.GetDirectoryName(_remotePath) ?? throw new Exception($"Invalid file path {_filePath}");
var fileName = Path.GetFileName(_filePath);

4
server/RdtClient.Service/Services/Downloaders/BezzadDownloader.cs
View file

@ -97,7 +97,7 @@ public class BezzadDownloader : IDownloader
};
}
public Task<String?> Download()
public Task<String> Download()
{
_logger.Debug($"Starting download of {_uri}, writing to path: {_filePath}");
@ -108,7 +108,7 @@ public class BezzadDownloader : IDownloader
_ = Task.Run(StartTimer);
return Task.FromResult<String?>(Guid.NewGuid().ToString());
return Task.FromResult(Guid.NewGuid().ToString());
}
public Task Cancel()

2
server/RdtClient.Service/Services/Downloaders/IDownloader.cs
View file

@ -16,7 +16,7 @@ public interface IDownloader
{
event EventHandler<DownloadCompleteEventArgs>? DownloadComplete;
event EventHandler<DownloadProgressEventArgs>? DownloadProgress;
Task<String?> Download();
Task<String> Download();
Task Cancel();
Task Pause();
Task Resume();

2
server/RdtClient.Service/Services/Downloaders/InternalDownloader.cs
View file

@ -65,7 +65,7 @@ public class InternalDownloader : IDownloader
};
}
public async Task<String?> Download()
public async Task<String> Download()
{
_logger.Debug($"Starting download of {_uri}, writing to path: {_filePath}");

117
server/RdtClient.Service/Services/Downloaders/SymlinkDownloader.cs
View file

@ -2,76 +2,96 @@
namespace RdtClient.Service.Services.Downloaders;
public class SymlinkDownloader : IDownloader
public class SymlinkDownloader(String uri, String path) : IDownloader
{
public event EventHandler<DownloadCompleteEventArgs>? DownloadComplete;
public event EventHandler<DownloadProgressEventArgs>? DownloadProgress;
private readonly String _filePath;
private readonly String _uri;
private readonly CancellationTokenSource _cancellationToken = new();
private readonly ILogger _logger;
public SymlinkDownloader(String uri, String filePath)
private readonly ILogger _logger = Log.ForContext<SymlinkDownloader>();
public async Task<String> Download()
{
_logger = Log.ForContext<SymlinkDownloader>();
_logger.Debug($"Instantiated new Symlink Downloader for URI {uri} to filePath {filePath}");
_logger.Debug($"Starting symlink resolving of {uri}, writing to path: {path}");
_uri = uri;
_filePath = filePath;
}
var filePath = new DirectoryInfo(path);
public Task<String?> Download()
{
_logger.Debug($"Starting symlink resolving of {_uri}, writing to path: {_filePath}");
var fileName = filePath.Name;
var fileExtension = filePath.Extension;
var directoryName = Path.GetDirectoryName(filePath.FullName) ?? throw new($"Cannot get directory name for file {filePath.FullName}");
var fileDirectory = Path.GetFileName(directoryName) ?? throw new($"Cannot get directory name for file {directoryName}");
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName) ?? throw new($"Cannot get directory name for file {fileName}");
var fileDirectoryWithoutExtension = Path.GetFileNameWithoutExtension(fileDirectory) ?? throw new($"Cannot get directory name for file {fileDirectory}");
var fileName = Path.GetFileName(_filePath);
String[] folders =
[
fileNameWithoutExtension,
fileDirectoryWithoutExtension,
fileName,
fileDirectory
];
DownloadProgress?.Invoke(this, new DownloadProgressEventArgs
List<String> unWantedExtensions =
[
"zip",
"rar",
"tar"
];
if (unWantedExtensions.Any(m => fileExtension == m))
{
BytesDone = 0,
BytesTotal = 0,
Speed = 0
});
throw new($"Cant handle compressed files with symlink downloader");
}
_logger.Debug($"Searching {Settings.Get.DownloadClient.RcloneMountPath} for {fileName}");
DownloadProgress?.Invoke(this, new()
{
BytesDone = 0,
BytesTotal = 0,
Speed = 0
});
// Recursively search for the fileName in the rclone mount location.
var foundFiles = Directory.GetFiles(Settings.Get.DownloadClient.RcloneMountPath, fileName, SearchOption.AllDirectories).ToList();
FileInfo? file = null;
if (foundFiles.Count > 0)
var tries = 1;
while (file == null && tries <= 10)
{
if (foundFiles.Count > 1)
_logger.Debug($"Searching {Settings.Get.DownloadClient.RcloneMountPath} for {fileName} (attempt #{tries})...");
var dirInfo = new DirectoryInfo(Settings.Get.DownloadClient.RcloneMountPath);
file = dirInfo.EnumerateDirectories().FirstOrDefault(dir => folders.Contains(dir.Name))?.EnumerateFiles().FirstOrDefault(x => x.Name == fileName);
if (file == null)
{
_logger.Warning($"Found {foundFiles.Count} files named {fileName}");
}
await Task.Delay(1000 * tries);
// Assume first matching filename is the one we want.
var actualFilePath = foundFiles.First();
var result = TryCreateSymbolicLink(actualFilePath, _filePath);
if (result)
{
DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs());
_logger.Information($"File {fileName} found on {Settings.Get.DownloadClient.RcloneMountPath} at {actualFilePath}");
return Task.FromResult<String?>(actualFilePath);
tries++;
}
}
_logger.Information($"File {fileName} not found on {Settings.Get.DownloadClient.RcloneMountPath}!");
if (file == null)
{
throw new("Could not find file from rclone mount!");
}
_logger.Debug($"Found {file.FullName} after #{tries} attempts");
var result = TryCreateSymbolicLink(file.FullName, filePath.FullName);
if (!result)
{
throw new("Could not find file from rclone mount!");
}
DownloadComplete?.Invoke(this, new());
return file.FullName;
// Return null and try again next cycle.
return Task.FromResult<String?>(null);
}
public Task Cancel()
{
_logger.Debug($"Cancelling download {_uri}");
_cancellationToken.Cancel(false);
return Task.CompletedTask;
@ -91,22 +111,23 @@ public class SymlinkDownloader : IDownloader
{
try
{
_logger.Information($"Creating symbolic link from {sourcePath} to {symlinkPath}");
File.CreateSymbolicLink(symlinkPath, sourcePath);
if (File.Exists(symlinkPath))
if (File.Exists(symlinkPath)) // Double-check that the link was created
{
_logger.Information($"Created symbolic link from {sourcePath} to {symlinkPath}");
return true;
}
_logger.Error($"Failed to create symbolic link from {sourcePath} to {symlinkPath}");
return false;
}
catch (Exception ex)
{
_logger.Error($"Error creating symbolic link from {sourcePath} to {symlinkPath}: {ex.Message}");
return false;
}
}

10
server/RdtClient.Service/Services/Downloads.cs
View file

@ -65,6 +65,11 @@ public class Downloads(DownloadData downloadData)
await downloadData.UpdateError(downloadId, error);
}
public async Task UpdateErrors(Dictionary<Guid, String> downloadIds)
{
await downloadData.UpdateErrors(downloadIds);
}
public async Task UpdateRetryCount(Guid downloadId, Int32 retryCount)
{
await downloadData.UpdateRetryCount(downloadId, retryCount);
@ -75,6 +80,11 @@ public class Downloads(DownloadData downloadData)
await downloadData.UpdateRemoteId(downloadId, remoteId);
}
public async Task UpdateRemoteIds(Dictionary<Guid, String> downloadIds)
{
await downloadData.UpdateRemoteIds(downloadIds);
}
public async Task DeleteForTorrent(Guid torrentId)
{
await downloadData.DeleteForTorrent(torrentId);

13
server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs
View file

@ -76,7 +76,7 @@ public class RealDebridTorrentClient(ILogger<RealDebridTorrentClient> logger, IH
Progress = torrent.Progress,
Status = torrent.Status,
Added = ChangeTimeZone(torrent.Added)!.Value,
Files = (torrent.Files ?? new List<TorrentFile>()).Select(m => new TorrentClientFile
Files = (torrent.Files ?? []).Select(m => new TorrentClientFile
{
Path = m.Path,
Bytes = m.Bytes,
@ -173,7 +173,7 @@ public class RealDebridTorrentClient(ILogger<RealDebridTorrentClient> logger, IH
else if (torrent.DownloadAction == TorrentDownloadAction.DownloadAll)
{
Log("Selecting all files", torrent);
files = torrent.Files.ToList();
files = [.. torrent.Files];
}
else if (torrent.DownloadAction == TorrentDownloadAction.DownloadManual)
{
@ -258,7 +258,7 @@ public class RealDebridTorrentClient(ILogger<RealDebridTorrentClient> logger, IH
Log("", torrent);
await GetClient().Torrents.SelectFilesAsync(torrent.RdId!, fileIds.ToArray());
await GetClient().Torrents.SelectFilesAsync(torrent.RdId!, [.. fileIds]);
}
public async Task Delete(String torrentId)
@ -287,12 +287,7 @@ public class RealDebridTorrentClient(ILogger<RealDebridTorrentClient> logger, IH
return torrent;
}
var rdTorrent = await GetInfo(torrent.RdId);
if (rdTorrent == null)
{
throw new Exception($"Resource not found");
}
var rdTorrent = await GetInfo(torrent.RdId) ?? throw new Exception($"Resource not found");
if (!String.IsNullOrWhiteSpace(rdTorrent.Filename))
{

121
server/RdtClient.Service/Services/TorrentRunner.cs
View file

@ -17,6 +17,9 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
public static readonly ConcurrentDictionary<Guid, DownloadClient> ActiveDownloadClients = new();
public static readonly ConcurrentDictionary<Guid, UnpackClient> ActiveUnpackClients = new();
private readonly Dictionary<Guid, String> _aggregatedDownloadResults = [];
private readonly Dictionary<Guid, String> _aggregatedDownloadErrors = [];
private readonly HttpClient _httpClient = new()
{
Timeout = TimeSpan.FromSeconds(10)
@ -37,13 +40,13 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
}
// When starting up reset any pending downloads or unpackings so that they are restarted.
var torrents1 = await torrents.Get();
var allTorrents = await torrents.Get();
torrents1 = torrents1.Where(m => m.Completed == null).ToList();
allTorrents = allTorrents.Where(m => m.Completed == null).ToList();
Log($"Found {torrents1.Count} not completed torrents");
Log($"Found {allTorrents.Count} not completed torrents");
foreach (var torrent in torrents1)
foreach (var torrent in allTorrents)
{
foreach (var download in torrent.Downloads)
{
@ -93,6 +96,16 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
return;
}
if (Settings.Get.DownloadClient.Client == Data.Enums.DownloadClient.Symlink)
{
var rcloneMountPath = Settings.Get.DownloadClient.RcloneMountPath;
if (!Directory.Exists(rcloneMountPath))
{
throw new($"Rclone mount path ({rcloneMountPath}) was not found!");
}
}
var sw = new Stopwatch();
sw.Start();
@ -220,12 +233,12 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
}
}
var torrents1 = await torrents.Get();
var allTorrents = await torrents.Get();
// Check for deleted torrents that are stuck in the ActiveDownloads or ActiveUnpacks
foreach (var activeDownload in ActiveDownloadClients)
{
var download = torrents1.SelectMany(m => m.Downloads).FirstOrDefault(m => m.DownloadId == activeDownload.Key);
var download = allTorrents.SelectMany(m => m.Downloads).FirstOrDefault(m => m.DownloadId == activeDownload.Key);
if (download == null)
{
@ -237,7 +250,7 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
foreach (var activeUnpacks in ActiveUnpackClients)
{
var download = torrents1.SelectMany(m => m.Downloads).FirstOrDefault(m => m.DownloadId == activeUnpacks.Key);
var download = allTorrents.SelectMany(m => m.Downloads).FirstOrDefault(m => m.DownloadId == activeUnpacks.Key);
if (download == null)
{
@ -248,7 +261,7 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
}
// Process torrent retries
foreach (var torrent in torrents1.Where(m => m.Retry != null))
foreach (var torrent in allTorrents.Where(m => m.Retry != null))
{
try
{
@ -271,7 +284,7 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
}
// Process torrent errors
foreach (var torrent in torrents1.Where(m => m.Error != null && m.DeleteOnError > 0))
foreach (var torrent in allTorrents.Where(m => m.Error != null && m.DeleteOnError > 0))
{
if (torrent.Completed == null)
{
@ -289,7 +302,7 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
}
// Process torrent lifetime
foreach (var torrent in torrents1.Where(m => m.Downloads.Count == 0 && m.Completed == null && m.Lifetime > 0))
foreach (var torrent in allTorrents.Where(m => m.Downloads.Count == 0 && m.Completed == null && m.Lifetime > 0))
{
if (torrent.Added.AddMinutes(torrent.Lifetime) > DateTime.UtcNow)
{
@ -302,16 +315,16 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
await torrents.UpdateComplete(torrent.TorrentId, $"Torrent lifetime of {torrent.Lifetime} minutes reached", DateTimeOffset.UtcNow, false);
}
torrents1 = await torrents.Get();
allTorrents = await torrents.Get();
torrents1 = torrents1.Where(m => m.Completed == null).ToList();
allTorrents = allTorrents.Where(m => m.Completed == null).ToList();
if (torrents1.Count > 0)
if (allTorrents.Count > 0)
{
Log($"Processing {torrents1.Count} torrents");
Log($"Processing {allTorrents.Count} torrents");
}
foreach (var torrent in torrents1)
foreach (var torrent in allTorrents)
{
try
{
@ -321,6 +334,9 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
.OrderBy(m => m.DownloadQueued)
.ToList();
_aggregatedDownloadResults.Clear();
_aggregatedDownloadErrors.Clear();
Log($"Currently {queuedDownloads.Count} queued downloads and {ActiveDownloadClients.Count} total active downloads", torrent);
foreach (var download in queuedDownloads)
@ -343,10 +359,13 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
try
{
Log($"Unrestricting links", download, torrent);
if (download.Link == null)
{
Log($"Unrestricting links", download, torrent);
var downloadLink = await torrents.UnrestrictLink(download.DownloadId);
download.Link = downloadLink;
var downloadLink = await torrents.UnrestrictLink(download.DownloadId);
download.Link = downloadLink;
}
}
catch (Exception ex)
{
@ -360,6 +379,11 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
continue;
}
Log($"Marking download as started", download, torrent);
download.DownloadStarted = DateTime.UtcNow;
await downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted);
var downloadPath = settingDownloadPath;
if (!String.IsNullOrWhiteSpace(torrent.Category))
@ -372,29 +396,49 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
// Start the download process
var downloadClient = new DownloadClient(download, torrent, downloadPath);
Log($"Starting download", download, torrent);
var remoteId = await downloadClient.Start();
if (String.IsNullOrWhiteSpace(remoteId))
if (ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient))
{
Log($"No ID received", download, torrent);
continue;
Log($"Starting download", download, torrent);
try
{
var remoteId = await downloadClient.Start();
if (String.IsNullOrWhiteSpace(remoteId))
{
throw new($"No remote ID received from download client");
}
Log($"Received ID {remoteId}", download, torrent);
if (download.RemoteId != remoteId)
{
_aggregatedDownloadResults.Add(download.DownloadId, remoteId);
}
}
catch (Exception ex)
{
LogError($"Unable to start download: {ex.Message}", download, torrent);
_aggregatedDownloadErrors.Add(download.DownloadId, ex.Message);
}
Log($"Started download", download, torrent);
}
Log($"Received ID {remoteId}", download, torrent);
download.RemoteId = remoteId;
await downloads.UpdateRemoteId(download.DownloadId, remoteId);
Log($"Marking download as started", download, torrent);
download.DownloadStarted = DateTime.UtcNow;
await downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted);
ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient);
}
if (_aggregatedDownloadResults.Count > 0)
{
await downloads.UpdateRemoteIds(_aggregatedDownloadResults);
}
if (_aggregatedDownloadErrors.Count > 0)
{
await downloads.UpdateErrors(_aggregatedDownloadErrors);
}
_aggregatedDownloadResults.Clear();
_aggregatedDownloadErrors.Clear();
// Check if there are any unpacks that are queued and can be started.
var queuedUnpacks = torrent.Downloads
.Where(m => m.Completed == null && m.UnpackingQueued != null && m.UnpackingStarted == null && m.Error == null)
@ -425,7 +469,8 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
var extension = Path.GetExtension(fileName);
if (extension != ".rar" && extension != ".zip")
if ((extension != ".rar" && extension != ".zip") ||
torrent.DownloadClient == Data.Enums.DownloadClient.Symlink)
{
Log($"No need to unpack, setting it as unpacked", download, torrent);