From d16e7f0c884cf36c9ca32aa9016e4586cc1e0b99 Mon Sep 17 00:00:00 2001 From: Roger Far Date: Wed, 27 Oct 2021 14:55:46 -0600 Subject: [PATCH] Changed the Aria2 updates to be done in the main TorrentRunner loop instead of each downloader making their own connections. Added checks if Aria2 links actually got added. Improved debug messages and set Microsoft message to Warning. Abstracted the RealDebrid client into a ITorrentClient. Add retry mechanism for server errors from RealDebrid. --- CHANGELOG.md | 10 + client/src/app/navbar/navbar.component.html | 2 +- client/src/app/torrent-status.pipe.ts | 4 +- package.json | 2 +- server/RdtClient.Data/Data/DownloadData.cs | 35 + server/RdtClient.Data/RdtClient.Data.csproj | 10 +- server/RdtClient.Service/DiConfig.cs | 2 + .../RdtClient.Service/Helpers/FileHelper.cs | 45 ++ server/RdtClient.Service/Helpers/Logger.cs | 31 + .../TorrentClientAvailableFile.cs | 11 + .../TorrentClient/TorrentClientTorrent.cs | 33 + .../Models/TorrentClient/TorrentClientUser.cs | 10 + .../RdtClient.Service.csproj | 4 +- .../Services/DownloadClient.cs | 36 +- .../Services/Downloaders/Aria2cDownloader.cs | 148 ++-- .../Services/Downloaders/SimpleDownloader.cs | 4 +- .../RdtClient.Service/Services/Downloads.cs | 10 + .../RdtClient.Service/Services/QBittorrent.cs | 28 +- server/RdtClient.Service/Services/Settings.cs | 34 +- server/RdtClient.Service/Services/Startup.cs | 2 +- .../RdtClient.Service/Services/TaskRunner.cs | 6 - .../Services/TorrentClients/ITorrentClient.cs | 20 + .../TorrentClients/RealDebridTorrentClient.cs | 155 ++++ .../Services/TorrentRunner.cs | 700 +++++++++++------- server/RdtClient.Service/Services/Torrents.cs | 411 +++++----- .../Services/UnpackClient.cs | 15 +- .../Controllers/TorrentsController.cs | 2 +- server/RdtClient.Web/Program.cs | 8 +- server/RdtClient.Web/RdtClient.Web.csproj | 14 +- 29 files changed, 1132 insertions(+), 660 deletions(-) create mode 100644 server/RdtClient.Service/Helpers/FileHelper.cs create mode 100644 server/RdtClient.Service/Helpers/Logger.cs create mode 100644 server/RdtClient.Service/Models/TorrentClient/TorrentClientAvailableFile.cs create mode 100644 server/RdtClient.Service/Models/TorrentClient/TorrentClientTorrent.cs create mode 100644 server/RdtClient.Service/Models/TorrentClient/TorrentClientUser.cs create mode 100644 server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs create mode 100644 server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 60ed563..d228d37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.1] - 2021-10-24 +### Added +- Added automatic torrent retrying when RealDebrid reports an error on the torrent. + +### Changed +- Changed how Aria2 is polled, this will result in less RPC calls for a large amount of torrents. +- Add more and better logging for the torrent runner in Debug. +- Add Aria2 checks to see if links get added properly. +- Update Aria2.NET and RD.NET, added automatic retrying in case of server failures. + ## [1.9.0] - 2021-10-24 ### Added - Add priorty for torrents. You can set the priority when adding a new torrent. When added from Sonarr/Radarr it will assume no priority. diff --git a/client/src/app/navbar/navbar.component.html b/client/src/app/navbar/navbar.component.html index 4937688..983ccf6 100644 --- a/client/src/app/navbar/navbar.component.html +++ b/client/src/app/navbar/navbar.component.html @@ -55,7 +55,7 @@ Profile Logout
-
Version 1.9.0
+
Version 1.9.1
diff --git a/client/src/app/torrent-status.pipe.ts b/client/src/app/torrent-status.pipe.ts index 2b9b7d6..3d3cfa3 100644 --- a/client/src/app/torrent-status.pipe.ts +++ b/client/src/app/torrent-status.pipe.ts @@ -39,7 +39,7 @@ export class TorrentStatusPipe implements PipeTransform { if (allSpeeds > 0) { speed = this.pipe.transform(allSpeeds, 'filesize'); - return `Downloading (${progress.toFixed(2)}% - ${speed}/s)`; + return `Downloading file ${downloading.length}/${torrent.downloads.length} (${progress.toFixed(2)}% - ${speed}/s)`; } } @@ -57,7 +57,7 @@ export class TorrentStatusPipe implements PipeTransform { let allSpeeds = unpacking.sum((m) => m.speed) / unpacking.length; if (allSpeeds > 0) { - return `Extracting (${progress.toFixed(2)}%)`; + return `Extracting file ${unpacking.length}/${torrent.downloads.length} (${progress.toFixed(2)}%)`; } } diff --git a/package.json b/package.json index ebf6cb5..6a4ded9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdt-client", - "version": "1.9.0", + "version": "1.9.1", "description": "This is a web interface to manage your torrents on Real-Debrid.", "main": "index.js", "dependencies": { diff --git a/server/RdtClient.Data/Data/DownloadData.cs b/server/RdtClient.Data/Data/DownloadData.cs index 7521f43..cafc98a 100644 --- a/server/RdtClient.Data/Data/DownloadData.cs +++ b/server/RdtClient.Data/Data/DownloadData.cs @@ -35,6 +35,7 @@ namespace RdtClient.Data.Data public async Task Get(Guid torrentId, String path) { return await _dataContext.Downloads + .Include(m => m.Torrent) .AsNoTracking() .FirstOrDefaultAsync(m => m.TorrentId == torrentId && m.Path == path); } @@ -241,6 +242,40 @@ namespace RdtClient.Data.Data await TorrentData.VoidCache(); } + public async Task Download(Guid downloadId) + { + var dbDownload = await _dataContext.Downloads + .FirstOrDefaultAsync(m => m.DownloadId == downloadId); + + dbDownload.DownloadStarted = null; + dbDownload.DownloadFinished = null; + dbDownload.UnpackingQueued = null; + dbDownload.UnpackingStarted = null; + dbDownload.UnpackingFinished = null; + dbDownload.Completed = null; + dbDownload.Error = null; + + await _dataContext.SaveChangesAsync(); + + await TorrentData.VoidCache(); + } + + public async Task Unpack(Guid downloadId) + { + var dbDownload = await _dataContext.Downloads + .FirstOrDefaultAsync(m => m.DownloadId == downloadId); + + dbDownload.UnpackingQueued = DateTimeOffset.UtcNow; + dbDownload.UnpackingStarted = null; + dbDownload.UnpackingFinished = null; + dbDownload.Completed = null; + dbDownload.Error = null; + + await _dataContext.SaveChangesAsync(); + + await TorrentData.VoidCache(); + } + public async Task Reset(Guid downloadId) { var dbDownload = await _dataContext.Downloads diff --git a/server/RdtClient.Data/RdtClient.Data.csproj b/server/RdtClient.Data/RdtClient.Data.csproj index 7c5eef3..0b49acd 100644 --- a/server/RdtClient.Data/RdtClient.Data.csproj +++ b/server/RdtClient.Data/RdtClient.Data.csproj @@ -5,11 +5,11 @@ - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index 751ea3d..0309b5c 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using RdtClient.Service.Services; +using RdtClient.Service.Services.TorrentClients; namespace RdtClient.Service { @@ -11,6 +12,7 @@ namespace RdtClient.Service services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/server/RdtClient.Service/Helpers/FileHelper.cs b/server/RdtClient.Service/Helpers/FileHelper.cs new file mode 100644 index 0000000..830e73c --- /dev/null +++ b/server/RdtClient.Service/Helpers/FileHelper.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace RdtClient.Service.Helpers +{ + public static class FileHelper + { + public static async Task Delete(String path) + { + if (String.IsNullOrWhiteSpace(path)) + { + return; + } + + if (!File.Exists(path)) + { + return; + } + + var retry = 0; + + while (true) + { + try + { + File.Delete(path); + + break; + } + catch + { + if (retry >= 3) + { + throw; + } + + retry++; + + await Task.Delay(1000 * retry); + } + } + } + } +} diff --git a/server/RdtClient.Service/Helpers/Logger.cs b/server/RdtClient.Service/Helpers/Logger.cs new file mode 100644 index 0000000..5d963d6 --- /dev/null +++ b/server/RdtClient.Service/Helpers/Logger.cs @@ -0,0 +1,31 @@ +using System; +using System.Linq; +using System.Web; +using RdtClient.Data.Models.Data; + +namespace RdtClient.Service.Helpers +{ + public static class Logger + { + public static String ToLog(this Download download) + { + var fileName = download.Path; + + if (!String.IsNullOrWhiteSpace(download.Link)) + { + var uri = new Uri(download.Link); + fileName = uri.Segments.Last(); + fileName = HttpUtility.UrlDecode(fileName); + } + + var done = (Int32)((Double)download.BytesDone / download.BytesTotal) * 100; + + return $"for download {fileName} {done}% ({download.DownloadId})"; + } + + public static String ToLog(this Torrent torrent) + { + return $"for torrent {torrent.RdName} ({torrent.RdId} - {torrent.RdStatusRaw} {torrent.RdProgress}%) ({torrent.TorrentId})"; + } + } +} diff --git a/server/RdtClient.Service/Models/TorrentClient/TorrentClientAvailableFile.cs b/server/RdtClient.Service/Models/TorrentClient/TorrentClientAvailableFile.cs new file mode 100644 index 0000000..7e543b4 --- /dev/null +++ b/server/RdtClient.Service/Models/TorrentClient/TorrentClientAvailableFile.cs @@ -0,0 +1,11 @@ +using System; + +namespace RdtClient.Service.Models.TorrentClient +{ + public class TorrentClientAvailableFile + { + public String Filename { get; set; } + + public Int64 Filesize { get; set; } + } +} diff --git a/server/RdtClient.Service/Models/TorrentClient/TorrentClientTorrent.cs b/server/RdtClient.Service/Models/TorrentClient/TorrentClientTorrent.cs new file mode 100644 index 0000000..ce3c651 --- /dev/null +++ b/server/RdtClient.Service/Models/TorrentClient/TorrentClientTorrent.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; + +namespace RdtClient.Service.Models.TorrentClient +{ + public class TorrentClientTorrent + { + public String Id { get; set; } + public String Filename { get; set; } + public String OriginalFilename { get; set; } + public String Hash { get; set; } + public Int64 Bytes { get; set; } + public Int64 OriginalBytes { get; set; } + public String Host { get; set; } + public Int64 Split { get; set; } + public Int64 Progress { get; set; } + public String Status { get; set; } + public DateTimeOffset Added { get; set; } + public List Files { get; set; } + public List Links { get; set; } + public DateTimeOffset? Ended { get; set; } + public Int64? Speed { get; set; } + public Int64? Seeders { get; set; } + } + + public class TorrentClientTorrentFile + { + public Int64 Id { get; set; } + public String Path { get; set; } + public Int64 Bytes { get; set; } + public Boolean Selected { get; set; } + } +} diff --git a/server/RdtClient.Service/Models/TorrentClient/TorrentClientUser.cs b/server/RdtClient.Service/Models/TorrentClient/TorrentClientUser.cs new file mode 100644 index 0000000..5697035 --- /dev/null +++ b/server/RdtClient.Service/Models/TorrentClient/TorrentClientUser.cs @@ -0,0 +1,10 @@ +using System; + +namespace RdtClient.Service.Models.TorrentClient +{ + public class TorrentClientUser + { + public String Username { get; set; } + public DateTimeOffset Expiration { get; set; } + } +} diff --git a/server/RdtClient.Service/RdtClient.Service.csproj b/server/RdtClient.Service/RdtClient.Service.csproj index 9b441ed..0fb6877 100644 --- a/server/RdtClient.Service/RdtClient.Service.csproj +++ b/server/RdtClient.Service/RdtClient.Service.csproj @@ -6,11 +6,11 @@ - + - + diff --git a/server/RdtClient.Service/Services/DownloadClient.cs b/server/RdtClient.Service/Services/DownloadClient.cs index 6278375..128cb21 100644 --- a/server/RdtClient.Service/Services/DownloadClient.cs +++ b/server/RdtClient.Service/Services/DownloadClient.cs @@ -1,5 +1,4 @@ using System; -using System.IO; using System.Threading.Tasks; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.Internal; @@ -15,7 +14,7 @@ namespace RdtClient.Service.Services private readonly Download _download; private readonly Torrent _torrent; - private IDownloader _downloader; + public IDownloader Downloader; public DownloadClient(Download download, Torrent torrent, String destinationPath) { @@ -24,6 +23,8 @@ namespace RdtClient.Service.Services _destinationPath = destinationPath; } + public String Type { get; set; } + public Boolean Finished { get; private set; } public String Error { get; private set; } @@ -47,12 +48,11 @@ namespace RdtClient.Service.Services throw new Exception("Invalid download path"); } - if (File.Exists(filePath)) - { - File.Delete(filePath); - } + await FileHelper.Delete(filePath); - _downloader = settings.DownloadClient switch + Type = settings.DownloadClient; + + Downloader = settings.DownloadClient switch { "Simple" => new SimpleDownloader(_download.Link, filePath), "MultiPart" => new MultiDownloader(_download.Link, filePath, settings), @@ -60,20 +60,24 @@ namespace RdtClient.Service.Services _ => throw new Exception($"Unknown download client {settings.DownloadClient}") }; - _downloader.DownloadComplete += (_, args) => + Downloader.DownloadComplete += (_, args) => { Finished = true; Error = args.Error; }; - _downloader.DownloadProgress += (_, args) => + Downloader.DownloadProgress += (_, args) => { Speed = args.Speed; BytesDone = args.BytesDone; BytesTotal = args.BytesTotal; }; - return await _downloader.Download(); + var result = await Downloader.Download(); + + await Task.Delay(1000); + + return result; } catch (Exception ex) { @@ -86,29 +90,29 @@ namespace RdtClient.Service.Services public async Task Cancel() { - if (_downloader == null) + if (Downloader == null) { return; } - await _downloader.Cancel(); + await Downloader.Cancel(); } public async Task Pause() { - if (_downloader == null) + if (Downloader == null) { return; } - await _downloader.Pause(); + await Downloader.Pause(); } public async Task Resume() { - if (_downloader == null) + if (Downloader == null) { return; } - await _downloader.Resume(); + await Downloader.Resume(); } } } diff --git a/server/RdtClient.Service/Services/Downloaders/Aria2cDownloader.cs b/server/RdtClient.Service/Services/Downloaders/Aria2cDownloader.cs index 8002e55..3b6ad40 100644 --- a/server/RdtClient.Service/Services/Downloaders/Aria2cDownloader.cs +++ b/server/RdtClient.Service/Services/Downloaders/Aria2cDownloader.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using System.Timers; using Aria2NET; using RdtClient.Data.Models.Internal; @@ -20,8 +20,7 @@ namespace RdtClient.Service.Services.Downloaders private readonly String _filePath; private readonly Aria2NetClient _aria2NetClient; - private readonly Timer _timer; - + private String _gid; public Aria2cDownloader(String gid, String uri, String filePath, DbSettings settings) @@ -35,14 +34,7 @@ namespace RdtClient.Service.Services.Downloaders Timeout = TimeSpan.FromSeconds(10) }; - _aria2NetClient = new Aria2NetClient(settings.Aria2cUrl, settings.Aria2cSecret, httpClient); - - _timer = new Timer(); - - _timer.Elapsed += OnTimedEvent; - - _timer.Interval = 1000; - _timer.Enabled = false; + _aria2NetClient = new Aria2NetClient(settings.Aria2cUrl, settings.Aria2cSecret, httpClient, 10); } public async Task Download() @@ -50,16 +42,11 @@ namespace RdtClient.Service.Services.Downloaders var path = Path.GetDirectoryName(_filePath); var fileName = Path.GetFileName(_filePath); - if (_gid != null) + var isAlreadyAdded = await CheckIfAdded(); + + if (isAlreadyAdded) { - try - { - await _aria2NetClient.TellStatus(_gid); - } - catch - { - _gid = null; - } + throw new Exception($"The download link {_uri} has already been added to Aria2"); } var retryCount = 0; @@ -67,6 +54,20 @@ namespace RdtClient.Service.Services.Downloaders { try { + if (_gid != null) + { + try + { + await _aria2NetClient.TellStatus(_gid); + + return _gid; + } + catch + { + _gid = null; + } + } + _gid ??= await _aria2NetClient.AddUri(new List { _uri @@ -81,7 +82,9 @@ namespace RdtClient.Service.Services.Downloaders } }); - break; + await _aria2NetClient.TellStatus(_gid); + + return _gid; } catch { @@ -95,19 +98,10 @@ namespace RdtClient.Service.Services.Downloaders retryCount++; } } - - // Add a delay to prevent sending too many Add requests to Aria2 at the same time. - await Task.Delay(1000); - - _timer.Start(); - - return _gid; } public async Task Cancel() { - _timer.Stop(); - if (String.IsNullOrWhiteSpace(_gid)) { return; @@ -166,46 +160,82 @@ namespace RdtClient.Service.Services.Downloaders } } - private async void OnTimedEvent(Object source, ElapsedEventArgs e) + public async Task Update(IEnumerable allDownloads) { if (_gid == null) { return; } - try + var download = allDownloads.FirstOrDefault(m => m.Gid == _gid); + + if (download == null) { - var status = await _aria2NetClient.TellStatus(_gid); - - if (!String.IsNullOrWhiteSpace(status.ErrorMessage) || status.Status == "error") - { - await Cancel(); - DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs - { - Error = $"{status.ErrorCode}: {status.ErrorMessage}" - }); - return; - } - - if (status.Status == "complete" || status.Status == "removed") - { - await Cancel(); - DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs()); - return; - } - - DownloadProgress?.Invoke(this, new DownloadProgressEventArgs - { - BytesDone = status.CompletedLength, - BytesTotal = status.TotalLength, - Speed = status.DownloadSpeed - }); + return; } - catch + + if (!String.IsNullOrWhiteSpace(download.ErrorMessage) || download.Status == "error") { await Cancel(); - DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs()); + DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs + { + Error = $"{download.ErrorCode}: {download.ErrorMessage}" + }); + return; } + + if (download.Status == "complete" || download.Status == "removed") + { + await Cancel(); + + var retryCount = 0; + while (true) + { + if (retryCount >= 10) + { + DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs + { + Error = $"File not found at {_filePath}" + }); + break; + } + + if (File.Exists(_filePath)) + { + DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs()); + break; + } + + await Task.Delay(1000 * retryCount); + retryCount++; + } + return; + } + + DownloadProgress?.Invoke(this, new DownloadProgressEventArgs + { + BytesDone = download.CompletedLength, + BytesTotal = download.TotalLength, + Speed = download.DownloadSpeed + }); + } + + private async Task CheckIfAdded() + { + var allDownloads = await _aria2NetClient.TellAll(); + + foreach (var download in allDownloads.Where(m => m.Files != null)) + { + foreach (var file in download.Files.Where(m => m.Uris != null)) + { + if (file.Uris.Any(uri => uri.Uri == _uri)) + { + return true; + } + } + } + + return false; } } } diff --git a/server/RdtClient.Service/Services/Downloaders/SimpleDownloader.cs b/server/RdtClient.Service/Services/Downloaders/SimpleDownloader.cs index f0101e0..4268e00 100644 --- a/server/RdtClient.Service/Services/Downloaders/SimpleDownloader.cs +++ b/server/RdtClient.Service/Services/Downloaders/SimpleDownloader.cs @@ -28,9 +28,9 @@ namespace RdtClient.Service.Services.Downloaders _filePath = filePath; } - public async Task Download() + public Task Download() { - Task.Run(async () => + _ = Task.Run(async () => { await StartDownloadTask(); }); diff --git a/server/RdtClient.Service/Services/Downloads.cs b/server/RdtClient.Service/Services/Downloads.cs index 61007ac..6ccae14 100644 --- a/server/RdtClient.Service/Services/Downloads.cs +++ b/server/RdtClient.Service/Services/Downloads.cs @@ -90,6 +90,16 @@ namespace RdtClient.Service.Services await _downloadData.DeleteForTorrent(torrentId); } + public async Task UpdateDownload(Guid downloadId) + { + await _downloadData.Download(downloadId); + } + + public async Task UpdateUnpack(Guid downloadId) + { + await _downloadData.Unpack(downloadId); + } + public async Task Reset(Guid downloadId) { await _downloadData.Reset(downloadId); diff --git a/server/RdtClient.Service/Services/QBittorrent.cs b/server/RdtClient.Service/Services/QBittorrent.cs index fbb6be8..db267e5 100644 --- a/server/RdtClient.Service/Services/QBittorrent.cs +++ b/server/RdtClient.Service/Services/QBittorrent.cs @@ -293,25 +293,19 @@ namespace RdtClient.Service.Services Upspeed = speed }; - if (torrent.Completed.HasValue && torrent.Downloads.All(m => m.Completed.HasValue)) + if (torrent.Completed.HasValue) { - // Indicates completed torrent and not seeding anymore. - result.State = "pausedUP"; + var allDownloadsComplete = torrent.Downloads.All(m => m.Completed.HasValue); + var hasDownloadsWithErrors = torrent.Downloads.Any(m => m.Error != null); - if (torrent.Downloads.Count > 0) + if (torrent.Downloads.Count == 0 || hasDownloadsWithErrors || torrent.RdStatus == RealDebridStatus.Error) { - var error = torrent.Downloads.FirstOrDefault(m => m.Error != null); - - if (error != null) - { - result.State = "error"; - } + result.State = "error"; + } + else if (allDownloadsComplete) + { + result.State = "pausedUP"; } - } - - if (torrent.RdStatus == RealDebridStatus.Error) - { - result.State = "error"; } results.Add(result); @@ -538,7 +532,7 @@ namespace RdtClient.Service.Services { if (TorrentRunner.ActiveDownloadClients.TryGetValue(download.DownloadId, out var downloadClient)) { - downloadClient.Pause(); + await downloadClient.Pause(); } } } @@ -558,7 +552,7 @@ namespace RdtClient.Service.Services { if (TorrentRunner.ActiveDownloadClients.TryGetValue(download.DownloadId, out var downloadClient)) { - downloadClient.Resume(); + await downloadClient.Resume(); } } } diff --git a/server/RdtClient.Service/Services/Settings.cs b/server/RdtClient.Service/Services/Settings.cs index 1d503ad..2c80e0b 100644 --- a/server/RdtClient.Service/Services/Settings.cs +++ b/server/RdtClient.Service/Services/Settings.cs @@ -8,6 +8,7 @@ using Aria2NET; using RdtClient.Data.Data; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.Internal; +using RdtClient.Service.Helpers; namespace RdtClient.Service.Services { @@ -54,7 +55,8 @@ namespace RdtClient.Service.Services var testFile = $"{path}/test.txt"; await File.WriteAllTextAsync(testFile, "RealDebridClient Test File, you can remove this file."); - File.Delete(testFile); + + await FileHelper.Delete(testFile); } public async Task TestDownloadSpeed(CancellationToken cancellationToken) @@ -63,10 +65,7 @@ namespace RdtClient.Service.Services var testFilePath = Path.Combine(downloadPath, "testDefault.rar"); - if (File.Exists(testFilePath)) - { - File.Delete(testFilePath); - } + await FileHelper.Delete(testFilePath); var download = new Download { @@ -90,23 +89,20 @@ namespace RdtClient.Service.Services if (cancellationToken.IsCancellationRequested) { - downloadClient.Cancel(); + await downloadClient.Cancel(); } if (downloadClient.BytesDone > 1024 * 1024 * 50) { - downloadClient.Cancel(); + await downloadClient.Cancel(); break; } } - if (File.Exists(testFilePath)) - { - File.Delete(testFilePath); - } + await FileHelper.Delete(testFilePath); - Clean(); + await Clean(); return downloadClient.Speed; } @@ -117,10 +113,7 @@ namespace RdtClient.Service.Services var testFilePath = Path.Combine(downloadPath, "test.tmp"); - if (File.Exists(testFilePath)) - { - File.Delete(testFilePath); - } + await FileHelper.Delete(testFilePath); const Int32 testFileSize = 64 * 1024 * 1024; @@ -147,10 +140,7 @@ namespace RdtClient.Service.Services fileStream.Close(); - if (File.Exists(testFilePath)) - { - File.Delete(testFilePath); - } + await FileHelper.Delete(testFilePath); return writeSpeed; } @@ -162,7 +152,7 @@ namespace RdtClient.Service.Services return await client.GetVersion(); } - public void Clean() + public async Task Clean() { try { @@ -174,7 +164,7 @@ namespace RdtClient.Service.Services foreach (var file in files) { - File.Delete(file); + await FileHelper.Delete(file); } } } diff --git a/server/RdtClient.Service/Services/Startup.cs b/server/RdtClient.Service/Services/Startup.cs index 3d5fafa..8acd183 100644 --- a/server/RdtClient.Service/Services/Startup.cs +++ b/server/RdtClient.Service/Services/Startup.cs @@ -23,7 +23,7 @@ namespace RdtClient.Service.Services await settings.ResetCache(); - settings.Clean(); + await settings.Clean(); } public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; diff --git a/server/RdtClient.Service/Services/TaskRunner.cs b/server/RdtClient.Service/Services/TaskRunner.cs index 9919c3e..faa385e 100644 --- a/server/RdtClient.Service/Services/TaskRunner.cs +++ b/server/RdtClient.Service/Services/TaskRunner.cs @@ -33,18 +33,12 @@ namespace RdtClient.Service.Services { try { - await Torrents.TorrentResetLock.WaitAsync(stoppingToken); - await torrentRunner.Tick(); } catch (Exception ex) { _logger.LogError(ex, "Unexpected error occurred in TorrentDownloadManager.Tick"); } - finally - { - Torrents.TorrentResetLock.Release(); - } await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); } diff --git a/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs new file mode 100644 index 0000000..260910d --- /dev/null +++ b/server/RdtClient.Service/Services/TorrentClients/ITorrentClient.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using RdtClient.Service.Models.TorrentClient; + +namespace RdtClient.Service.Services.TorrentClients +{ + public interface ITorrentClient + { + Task> GetTorrents(); + Task GetUser(); + Task AddMagnet(String magnetLink); + Task AddFile(Byte[] bytes); + Task> GetAvailableFiles(String hash); + Task SelectFiles(String torrentId, IList fileIds); + Task GetInfo(String torrentId); + Task Delete(String torrentId); + Task Unrestrict(String link); + } +} diff --git a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs new file mode 100644 index 0000000..9fd3154 --- /dev/null +++ b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using RDNET; +using RdtClient.Service.Models.TorrentClient; + +namespace RdtClient.Service.Services.TorrentClients +{ + public class RealDebridTorrentClient : ITorrentClient + { + private readonly IHttpClientFactory _httpClientFactory; + + public RealDebridTorrentClient(IHttpClientFactory httpClientFactory) + { + _httpClientFactory = httpClientFactory; + } + + private RdNetClient GetRdNetClient() + { + var apiKey = Settings.Get.RealDebridApiKey; + + if (String.IsNullOrWhiteSpace(apiKey)) + { + throw new Exception("Real-Debrid API Key not set in the settings"); + } + + var httpClient = _httpClientFactory.CreateClient(); + httpClient.Timeout = TimeSpan.FromSeconds(10); + + var rdtNetClient = new RdNetClient(null, httpClient, 5); + rdtNetClient.UseApiAuthentication(apiKey); + + return rdtNetClient; + } + + private static TorrentClientTorrent Map(Torrent torrent) + { + return new TorrentClientTorrent + { + Id = torrent.Id, + Filename = torrent.Filename, + OriginalFilename = torrent.OriginalFilename, + Hash = torrent.Hash, + Bytes = torrent.Bytes, + OriginalBytes = torrent.OriginalBytes, + Host = torrent.Host, + Split = torrent.Split, + Progress = torrent.Progress, + Status = torrent.Status, + Added = torrent.Added, + Files = (torrent.Files ?? new List()).Select(m => new TorrentClientTorrentFile + { + Path = m.Path, + Bytes = m.Bytes, + Id = m.Id, + Selected = m.Selected + }).ToList(), + Links = torrent.Links, + Ended = torrent.Ended, + Speed = torrent.Speed, + Seeders = torrent.Seeders, + }; + } + + public async Task> GetTorrents() + { + var page = 0; + var results = new List(); + + while (true) + { + var pagedResults = await GetRdNetClient().Torrents.GetAsync(page, 100); + + results.AddRange(pagedResults); + + if (pagedResults.Count == 0) + { + break; + } + + page += 100; + } + + return results.Select(Map).ToList(); + } + + public async Task GetUser() + { + var user = await GetRdNetClient().User.GetAsync(); + + return new TorrentClientUser + { + Username = user.Username, + Expiration = user.Expiration + }; + } + + public async Task AddMagnet(String magnetLink) + { + var result = await GetRdNetClient().Torrents.AddMagnetAsync(magnetLink); + + return result.Id; + } + + public async Task AddFile(Byte[] bytes) + { + var result = await GetRdNetClient().Torrents.AddFileAsync(bytes); + + return result.Id; + } + + public async Task> GetAvailableFiles(String hash) + { + var result = await GetRdNetClient().Torrents.GetAvailableFiles(hash); + + var files = result.SelectMany(m => m.Value).SelectMany(m => m.Value).SelectMany(m => m.Values); + + var groups = files.GroupBy(m => $"{m.Filename}-{m.Filesize}"); + + var torrentClientAvailableFiles = groups.Select(m => new TorrentClientAvailableFile + { + Filename = m.First().Filename, + Filesize = m.First().Filesize + } ).ToList(); + + return torrentClientAvailableFiles; + } + + public async Task SelectFiles(String torrentId, IList fileIds) + { + await GetRdNetClient().Torrents.SelectFilesAsync(torrentId, fileIds.ToArray()); + } + + public async Task GetInfo(String torrentId) + { + var result = await GetRdNetClient().Torrents.GetInfoAsync(torrentId); + + return Map(result); + } + + public async Task Delete(String torrentId) + { + await GetRdNetClient().Torrents.DeleteAsync(torrentId); + } + + public async Task Unrestrict(String link) + { + var result = await GetRdNetClient().Unrestrict.LinkAsync(link); + + return result.Download; + } + } +} diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index d35217a..ae7a836 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -3,10 +3,17 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.IO; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; using System.Web; +using Aria2NET; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; using RdtClient.Data.Enums; -using Serilog; +using RdtClient.Data.Models.Data; +using RdtClient.Data.Models.Internal; +using RdtClient.Service.Helpers; +using RdtClient.Service.Services.Downloaders; namespace RdtClient.Service.Services { @@ -19,54 +26,75 @@ namespace RdtClient.Service.Services public static readonly ConcurrentDictionary ActiveDownloadClients = new(); public static readonly ConcurrentDictionary ActiveUnpackClients = new(); + + private readonly ILogger _logger; + private readonly Torrents _torrents; private readonly Downloads _downloads; private readonly RemoteService _remoteService; - - private readonly Torrents _torrents; + private readonly HttpClient _httpClient; - public TorrentRunner(Torrents torrents, Downloads downloads, RemoteService remoteService) + public TorrentRunner(ILogger logger, Torrents torrents, Downloads downloads, RemoteService remoteService) { + _logger = logger; _torrents = torrents; _downloads = downloads; _remoteService = remoteService; + + _httpClient = new HttpClient + { + Timeout = TimeSpan.FromSeconds(10) + }; } public async Task Initialize() { + Log("Initializing TorrentRunner"); + + var settingsCopy = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(Settings.Get)); + + if (settingsCopy != null) + { + settingsCopy.RealDebridApiKey = "*****"; + settingsCopy.Aria2cSecret = "*****"; + + Log(JsonConvert.SerializeObject(settingsCopy)); + } + // When starting up reset any pending downloads or unpackings so that they are restarted. var torrents = await _torrents.Get(); torrents = torrents.Where(m => m.Completed == null).ToList(); - - var downloads = torrents.SelectMany(m => m.Downloads) - .Where(m => m.DownloadQueued != null && m.DownloadStarted != null && m.DownloadFinished == null && m.Error == null) - .OrderBy(m => m.DownloadQueued); - foreach (var download in downloads) + Log($"Found {torrents.Count} not completed torrents"); + + foreach (var torrent in torrents) { - await _downloads.UpdateDownloadStarted(download.DownloadId, null); + foreach (var download in torrent.Downloads) + { + if (download.DownloadQueued != null && download.DownloadStarted != null && download.DownloadFinished == null && download.Error == null) + { + Log("Resetting download status", download, torrent); + + await _downloads.UpdateDownloadStarted(download.DownloadId, null); + } + + if (download.UnpackingQueued != null && download.UnpackingStarted != null && download.UnpackingFinished == null && download.Error == null) + { + Log("Resetting unpack status", download, torrent); + + await _downloads.UpdateUnpackingStarted(download.DownloadId, null); + } + } } - var unpacks = torrents.SelectMany(m => m.Downloads) - .Where(m => m.UnpackingQueued != null && m.UnpackingStarted != null && m.UnpackingFinished == null && m.Error == null) - .OrderBy(m => m.DownloadQueued); - - foreach (var download in unpacks) - { - await _downloads.UpdateUnpackingStarted(download.DownloadId, null); - } + Log("TorrentRunner Initialized"); } public async Task Tick() { - var sw = new Stopwatch(); - sw.Start(); - - Log.Debug("TorrentRunner Tick Start"); - if (String.IsNullOrWhiteSpace(Settings.Get.RealDebridApiKey)) { - Log.Debug($"No RealDebridApiKey set!"); + Log($"No RealDebridApiKey set in settings"); return; } @@ -85,304 +113,369 @@ namespace RdtClient.Service.Services var settingDownloadPath = Settings.Get.DownloadPath; if (String.IsNullOrWhiteSpace(settingDownloadPath)) { + _logger.LogError("No DownloadPath set in settings"); return; } - Log.Debug($"Currently {ActiveDownloadClients.Count} active downloads"); - Log.Debug($"Currently {ActiveUnpackClients.Count} active unpacks"); + var sw = new Stopwatch(); + sw.Start(); + + if (ActiveDownloadClients.Count > 0 || ActiveUnpackClients.Count > 0) + { + Log($"TorrentRunner Tick Start, {ActiveDownloadClients.Count} active downloads, {ActiveUnpackClients.Count} active unpacks"); + } + + if (ActiveDownloadClients.Any(m => m.Value.Type == "Aria2c")) + { + Log("Updating Aria2 status"); + + var aria2NetClient = new Aria2NetClient(Settings.Get.Aria2cUrl, Settings.Get.Aria2cSecret, _httpClient, 1); + + var allDownloads = await aria2NetClient.TellAll(); + + Log($"Found {allDownloads.Count} Aria2 downloads"); + + foreach (var activeDownload in ActiveDownloadClients) + { + if (activeDownload.Value.Downloader is Aria2cDownloader aria2Downloader) + { + await aria2Downloader.Update(allDownloads); + } + } + + Log("Finished updating Aria2 status"); + } // Check if any torrents are finished downloading to the host, remove them from the active download list. var completedActiveDownloads = ActiveDownloadClients.Where(m => m.Value.Finished).ToList(); - Log.Debug($"Processing {completedActiveDownloads.Count} completed downloads"); - - foreach (var (downloadId, downloadClient) in completedActiveDownloads) + if (completedActiveDownloads.Count > 0) { - if (downloadClient.Error != null) + Log($"Processing {completedActiveDownloads.Count} completed downloads"); + + foreach (var (downloadId, downloadClient) in completedActiveDownloads) { - // Retry the download - Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}"); var download = await _downloads.GetById(downloadId); if (download == null) { ActiveDownloadClients.TryRemove(downloadId, out _); - Log.Debug($"Download with ID {downloadId} not found! Removed from queue"); + Log($"Download with ID {downloadId} not found! Removed from download queue"); + continue; } - if (download.RetryCount < DownloadRetryCount) - { - Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}, download retry count {download.RetryCount}/{DownloadRetryCount}, torrent retry count {download.Torrent.RetryCount}/{TorrentRetryCount}, retrying download"); + Log("Processing download", download, download.Torrent); - await _downloads.UpdateRetryCount(downloadId, download.RetryCount + 1); - await _torrents.Download(downloadId); - } - else if (download.Torrent.RetryCount < TorrentRetryCount) + if (downloadClient.Error != null) { - Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}, download retry count {download.RetryCount}/{DownloadRetryCount}, torrent retry count {download.Torrent.RetryCount}/{TorrentRetryCount}, retrying torrent"); - - Task.Run(async () => + // Retry the download if an error is encountered. + Log($"Download reported an error: {downloadClient.Error}", download, download.Torrent); + Log($"Download retry count {download.RetryCount}/{DownloadRetryCount}, torrent retry count {download.Torrent.RetryCount}/{TorrentRetryCount}", download, download.Torrent); + + if (download.RetryCount < DownloadRetryCount) { - await _torrents.RetryTorrent(download.TorrentId); - }); + Log($"Retrying download", download, download.Torrent); + + await _downloads.UpdateRetryCount(downloadId, download.RetryCount + 1); + await _downloads.UpdateDownload(downloadId); + } + else if (download.Torrent.RetryCount < TorrentRetryCount) + { + Log($"Retrying torrent", download, download.Torrent); + + await _torrents.RetryTorrent(download.TorrentId, download.Torrent.RetryCount + 1); + + continue; + } + else + { + Log($"Not retrying", download, download.Torrent); + + await _downloads.UpdateError(downloadId, downloadClient.Error); + await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + } } else { - Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}, not retrying"); + Log($"Download finished successfully", download, download.Torrent); - await _downloads.UpdateError(downloadId, downloadClient.Error); - await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + await _downloads.UpdateDownloadFinished(downloadId, DateTimeOffset.UtcNow); + await _downloads.UpdateUnpackingQueued(downloadId, DateTimeOffset.UtcNow); } + + ActiveDownloadClients.TryRemove(downloadId, out _); + + Log($"Removed from ActiveDownloadClients", download, download.Torrent); } - else - { - Log.Debug($"Processing active download {downloadId}: finished succesfully"); - - await _downloads.UpdateDownloadFinished(downloadId, DateTimeOffset.UtcNow); - await _downloads.UpdateUnpackingQueued(downloadId, DateTimeOffset.UtcNow); - } - - ActiveDownloadClients.TryRemove(downloadId, out _); - - Log.Debug($"Removed active download {downloadId} from queue"); } - + // Check if any torrents are finished unpacking, remove them from the active unpack list. var completedUnpacks = ActiveUnpackClients.Where(m => m.Value.Finished).ToList(); - - Log.Debug($"Processing {completedUnpacks.Count} completed extractions"); - foreach (var (downloadId, unpackClient) in completedUnpacks) + if (completedUnpacks.Count > 0) { - if (unpackClient.Error != null) + Log($"Processing {completedUnpacks.Count} completed unpacks"); + + foreach (var (downloadId, unpackClient) in completedUnpacks) { - Log.Debug($"Processing active unpack {downloadId}: error {unpackClient.Error}"); + var download = await _downloads.GetById(downloadId); - await _downloads.UpdateError(downloadId, unpackClient.Error); - await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + if (download == null) + { + ActiveUnpackClients.TryRemove(downloadId, out _); + + Log($"Download with ID {downloadId} not found! Removed from unpack queue"); + + continue; + } + + if (unpackClient.Error != null) + { + Log($"Unpack reported an error: {unpackClient.Error}", download, download.Torrent); + + await _downloads.UpdateError(downloadId, unpackClient.Error); + await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + } + else + { + Log($"Unpack finished successfully", download, download.Torrent); + + await _downloads.UpdateUnpackingFinished(downloadId, DateTimeOffset.UtcNow); + await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + } + + ActiveUnpackClients.TryRemove(downloadId, out _); + + Log($"Removed from ActiveUnpackClients", download, download.Torrent); } - else - { - Log.Debug($"Processing active unpack {downloadId}: finished succesfully"); - - await _downloads.UpdateUnpackingFinished(downloadId, DateTimeOffset.UtcNow); - await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); - } - - ActiveUnpackClients.TryRemove(downloadId, out _); - - Log.Debug($"Removed active unpack {downloadId} from queue"); } - + var torrents = await _torrents.Get(); - Log.Debug($"Found {torrents.Count} torrents"); + torrents = torrents.Where(m => m.Completed == null).ToList(); // Only poll Real-Debrid every second when a hub is connected, otherwise every 30 seconds if (_nextUpdate < DateTime.UtcNow && torrents.Count > 0) { - Log.Debug($"Updating torrent info from Real-Debrid"); + Log($"Updating torrent info from Real-Debrid"); +#if DEBUG + var updateTime = 0; + + _nextUpdate = DateTime.UtcNow; +#else var updateTime = 30; if (RdtHub.HasConnections) { - updateTime = 1; + updateTime = 5; } + updateTime = 0; + _nextUpdate = DateTime.UtcNow.AddSeconds(updateTime); +#endif await _torrents.UpdateRdData(); // Re-get torrents to account for updated info torrents = await _torrents.Get(); + torrents = torrents.Where(m => m.Completed == null).ToList(); - Log.Debug($"Finished updating torrent info from Real-Debrid, next update in {updateTime} seconds"); + Log($"Finished updating torrent info from Real-Debrid, next update in {updateTime} seconds"); } - torrents = torrents.Where(m => m.Completed == null).ToList(); - - // Check if there are any downloads that are queued and can be started. - var queuedDownloads = torrents.SelectMany(m => m.Downloads) - .Where(m => m.Completed == null && m.DownloadQueued != null && m.DownloadStarted == null && m.Error == null) - .OrderBy(m => m.DownloadQueued) - .ToList(); - - Log.Debug($"Found {queuedDownloads.Count} torrents queued for download"); - - foreach (var download in queuedDownloads) + if (torrents.Count > 0) { - Log.Debug($"Starting download {download.DownloadId}"); - - var torrentDownload = torrents.First(m => m.TorrentId == download.TorrentId); - - if (TorrentRunner.ActiveDownloadClients.Count >= settingDownloadLimit) - { - Log.Debug($"Not starting download {download.DownloadId} because there are already the max number of downloads active"); - - continue; - } - - if (TorrentRunner.ActiveDownloadClients.ContainsKey(download.DownloadId)) - { - Log.Debug($"Not starting download {download.DownloadId} because this download is already active"); - - continue; - } - - try - { - var downloadLink = await _torrents.UnrestrictLink(download.DownloadId); - download.Link = downloadLink; - } - catch (Exception ex) - { - await _downloads.UpdateError(download.DownloadId, ex.Message); - await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); - download.Error = ex.Message; - download.Completed = DateTimeOffset.UtcNow; - - Log.Information($"Cannot unrestrict: {ex.Message}"); - continue; - } - - download.DownloadStarted = DateTime.UtcNow; - await _downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted); - - var downloadPath = settingDownloadPath; - - if (!String.IsNullOrWhiteSpace(torrentDownload.Category)) - { - downloadPath = Path.Combine(downloadPath, torrentDownload.Category); - } - - Log.Debug($"Setting download path for {download.DownloadId} to {downloadPath}"); - - // Start the download process - var downloadClient = new DownloadClient(download, torrentDownload, downloadPath); - - if (TorrentRunner.ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient)) - { - Log.Debug($"Added download {download.DownloadId} to active downloads"); - - var remoteId = await downloadClient.Start(Settings.Get); - - if (!String.IsNullOrWhiteSpace(remoteId) && download.RemoteId != remoteId) - { - Log.Debug($"Received ID {remoteId} for download {download.DownloadId}"); - - await _downloads.UpdateRemoteId(download.DownloadId, remoteId); - } - - Log.Debug($"Download {download.DownloadId} started"); - } - } - - // Check if there are any unpacks that are queued and can be started. - var queuedUnpacks = torrents.SelectMany(m => m.Downloads) - .Where(m => m.Completed == null && m.UnpackingQueued != null && m.UnpackingStarted == null && m.Error == null) - .OrderBy(m => m.DownloadQueued) - .ToList(); - - Log.Debug($"Found {queuedUnpacks.Count} torrents queued for unpacking"); - - foreach (var download in queuedUnpacks) - { - Log.Debug($"Starting unpack {download.DownloadId}"); - - var torrentDownload = torrents.First(m => m.TorrentId == download.TorrentId); - - if (download.Link == null) - { - await _downloads.UpdateError(download.DownloadId, "Download Link cannot be null"); - await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); - - continue; - } - - // Check if the unpacking process is even needed - var uri = new Uri(download.Link); - var fileName = uri.Segments.Last(); - - fileName = HttpUtility.UrlDecode(fileName); - - Log.Debug($"Found file name {fileName} for {download.DownloadId}"); - - var extension = Path.GetExtension(fileName); - - if (extension != ".rar") - { - Log.Debug($"No need to unpack {download.DownloadId}, setting it as unpacked"); - - download.UnpackingStarted = DateTimeOffset.UtcNow; - download.UnpackingFinished = DateTimeOffset.UtcNow; - download.Completed = DateTimeOffset.UtcNow; - - await _downloads.UpdateUnpackingStarted(download.DownloadId, download.UnpackingStarted); - await _downloads.UpdateUnpackingFinished(download.DownloadId, download.UnpackingFinished); - await _downloads.UpdateCompleted(download.DownloadId, download.Completed); - - continue; - } - - // Check if we have reached the download limit, if so queue the download, but don't start it. - if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit) - { - Log.Debug($"Not starting unpack {download.DownloadId} because there are already the max number of unpacks active"); - - continue; - } - - if (TorrentRunner.ActiveUnpackClients.ContainsKey(download.DownloadId)) - { - Log.Debug($"Not starting unpack {download.DownloadId} because this download is already active"); - - continue; - } - - download.UnpackingStarted = DateTimeOffset.UtcNow; - await _downloads.UpdateUnpackingStarted(download.DownloadId, download.UnpackingStarted); - - var downloadPath = settingDownloadPath; - - if (!String.IsNullOrWhiteSpace(torrentDownload.Category)) - { - downloadPath = Path.Combine(downloadPath, torrentDownload.Category); - } - - Log.Debug($"Setting unpack path for {download.DownloadId} to {downloadPath}"); - - // Start the unpacking process - var unpackClient = new UnpackClient(download, downloadPath); - - if (TorrentRunner.ActiveUnpackClients.TryAdd(download.DownloadId, unpackClient)) - { - Log.Debug($"Added unpack {download.DownloadId} to active unpacks"); - - unpackClient.Start(); - - Log.Debug($"Unpack {download.DownloadId} started"); - } + Log($"Processing {torrents.Count} torrents"); } foreach (var torrent in torrents) { - // If torrent is erroring out on the Real-Debrid side, skip processing this torrent. - if (torrent.RdStatus == RealDebridStatus.Error) - { - Log.Debug($"Torrent {torrent.RdId} has an error: {torrent.RdStatusRaw}, not processing further"); + // Check if there are any downloads that are queued and can be started. + var queuedDownloads = torrents.SelectMany(m => m.Downloads) + .Where(m => m.Completed == null && m.DownloadQueued != null && m.DownloadStarted == null && m.Error == null) + .OrderBy(m => m.DownloadQueued) + .ToList(); - continue; + foreach (var download in queuedDownloads) + { + Log($"Processing to download", download, torrent); + + if (ActiveDownloadClients.Count >= settingDownloadLimit) + { + Log($"Not starting download because there are already the max number of downloads active", download, torrent); + + continue; + } + + if (ActiveDownloadClients.ContainsKey(download.DownloadId)) + { + Log($"Not starting download because this download is already active", download, torrent); + + continue; + } + + try + { + Log($"Unrestricting links", download, torrent); + + var downloadLink = await _torrents.UnrestrictLink(download.DownloadId); + download.Link = downloadLink; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Cannot unrestrict link: {ex.Message}"); + + await _downloads.UpdateError(download.DownloadId, ex.Message); + await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); + download.Error = ex.Message; + download.Completed = DateTimeOffset.UtcNow; + + continue; + } + + Log($"Marking download as started", download, torrent); + + download.DownloadStarted = DateTime.UtcNow; + await _downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted); + + var downloadPath = settingDownloadPath; + + if (!String.IsNullOrWhiteSpace(torrent.Category)) + { + downloadPath = Path.Combine(downloadPath, torrent.Category); + } + + Log($"Setting download path to {downloadPath}", download, torrent); + + // Start the download process + var downloadClient = new DownloadClient(download, torrent, downloadPath); + + if (ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient)) + { + Log($"Starting download", download, torrent); + + var remoteId = await downloadClient.Start(Settings.Get); + + if (!String.IsNullOrWhiteSpace(remoteId) && download.RemoteId != remoteId) + { + Log($"Received ID {remoteId}", download, torrent); + + await _downloads.UpdateRemoteId(download.DownloadId, remoteId); + } + else + { + Log($"No ID received", download, torrent); + } + } } - // The files are selected but there are no downloads yet, check if Real-Debrid has generated links yet. - if (torrent.Downloads.Count == 0 && torrent.FilesSelected != null) - { - Log.Debug($"Torrent {torrent.RdId} checking for links"); + // Check if there are any unpacks that are queued and can be started. + var queuedUnpacks = torrents.SelectMany(m => m.Downloads) + .Where(m => m.Completed == null && m.UnpackingQueued != null && m.UnpackingStarted == null && m.Error == null) + .OrderBy(m => m.DownloadQueued) + .ToList(); - await _torrents.CheckForLinks(torrent.TorrentId); + foreach (var download in queuedUnpacks) + { + Log($"Starting unpack", download, torrent); + + if (download.Link == null) + { + Log($"No download link found", download, torrent); + + await _downloads.UpdateError(download.DownloadId, "Download Link cannot be null"); + await _downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow); + + continue; + } + + // Check if the unpacking process is even needed + var uri = new Uri(download.Link); + var fileName = uri.Segments.Last(); + + fileName = HttpUtility.UrlDecode(fileName); + + Log($"Found file name {fileName}", download, torrent); + + var extension = Path.GetExtension(fileName); + + if (extension != ".rar") + { + Log($"No need to unpack, setting it as unpacked", download, torrent); + + download.UnpackingStarted = DateTimeOffset.UtcNow; + download.UnpackingFinished = DateTimeOffset.UtcNow; + download.Completed = DateTimeOffset.UtcNow; + + await _downloads.UpdateUnpackingStarted(download.DownloadId, download.UnpackingStarted); + await _downloads.UpdateUnpackingFinished(download.DownloadId, download.UnpackingFinished); + await _downloads.UpdateCompleted(download.DownloadId, download.Completed); + + continue; + } + + // Check if we have reached the download limit, if so queue the download, but don't start it. + if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit) + { + Log($"Not starting unpack because there are already the max number of unpacks active", download, torrent); + + continue; + } + + if (TorrentRunner.ActiveUnpackClients.ContainsKey(download.DownloadId)) + { + Log($"Not starting unpack because this download is already active", download, torrent); + + continue; + } + + download.UnpackingStarted = DateTimeOffset.UtcNow; + await _downloads.UpdateUnpackingStarted(download.DownloadId, download.UnpackingStarted); + + var downloadPath = settingDownloadPath; + + if (!String.IsNullOrWhiteSpace(torrent.Category)) + { + downloadPath = Path.Combine(downloadPath, torrent.Category); + } + + Log($"Setting unpack path to {downloadPath}", download, torrent); + + // Start the unpacking process + var unpackClient = new UnpackClient(download, downloadPath); + + if (TorrentRunner.ActiveUnpackClients.TryAdd(download.DownloadId, unpackClient)) + { + Log($"Starting unpack", download, torrent); + + unpackClient.Start(); + } + } + + Log("Processing", torrent); + + // If torrent is erroring out on the Real-Debrid side. + if (torrent.RdStatus == RealDebridStatus.Error) + { + Log($"Torrent reported an error: {torrent.RdStatusRaw}", torrent); + Log($"Torrent retry count {torrent.RetryCount}/{TorrentRetryCount}", torrent); + + if (torrent.RetryCount < TorrentRetryCount) + { + Log($"Retrying torrent", torrent); + + await _torrents.RetryTorrent(torrent.TorrentId, torrent.RetryCount + 1); + continue; + } + + Log($"Received RealDebrid error: {torrent.RdStatusRaw}, not processing further", torrent); + await _torrents.UpdateComplete(torrent.TorrentId, DateTimeOffset.Now); + + continue; } // Real-Debrid is waiting for file selection, select which files to download. @@ -391,55 +484,55 @@ namespace RdtClient.Service.Services torrent.Downloads.Count == 0 && torrent.FilesSelected == null) { - Log.Debug($"Torrent {torrent.RdId} selecting files"); + Log($"Selecting files", torrent); var files = torrent.Files; if (torrent.DownloadAction == TorrentDownloadAction.DownloadAvailableFiles) { - Log.Debug($"Torrent {torrent.RdId} determining which files are already available"); + Log($"Determining which files are already available on RealDebrid", torrent); var availableFiles = await _torrents.GetAvailableFiles(torrent.Hash); - Log.Debug($"Found {files.Count}/{torrent.Files.Count} available files for torrent {torrent.RdId}"); + Log($"Found {files.Count}/{torrent.Files.Count} available files on RealDebrid", torrent); files = torrent.Files.Where(m => availableFiles.Any(f => m.Path.EndsWith(f.Filename))).ToList(); } else if (torrent.DownloadAction == TorrentDownloadAction.DownloadAll) { - Log.Debug("Selecting all files"); + Log("Selecting all files", torrent); files = torrent.Files.ToList(); } else if (torrent.DownloadAction == TorrentDownloadAction.DownloadManual) { - Log.Debug("Selecting manual selected files"); + Log("Selecting manual selected files", torrent); files = torrent.Files.Where(m => torrent.ManualFiles.Any(f => m.Path.EndsWith(f))).ToList(); } - Log.Debug($"Selecting {files.Count}/{torrent.Files.Count} files for torrent {torrent.RdId}"); + Log($"Selecting {files.Count}/{torrent.Files.Count} files", torrent); if (torrent.DownloadAction != TorrentDownloadAction.DownloadManual && torrent.DownloadMinSize > 0) { var minFileSize = torrent.DownloadMinSize * 1024 * 1024; - Log.Debug($"Torrent {torrent.RdId} determining which files are over {minFileSize} bytes"); + Log($"Determining which files are over {minFileSize} bytes", torrent); files = files.Where(m => m.Bytes > minFileSize) .ToList(); - Log.Debug($"Found {files.Count} files that match the minimum file size criterea for torrent {torrent.RdId}"); + Log($"Found {files.Count} files that match the minimum file size criterea", torrent); } if (files.Count == 0) { - Log.Debug($"Filtered all files out for {torrent.RdId}! Downloading ALL files!"); + Log($"Filtered all files out! Downloading ALL files instead!", torrent); files = torrent.Files; } var fileIds = files.Select(m => m.Id.ToString()).ToArray(); - Log.Debug($"Selecting files for torrent {torrent.RdId}: {String.Join(", ", fileIds)}"); + Log($"Selecting files:{Environment.NewLine}{String.Join(Environment.NewLine, files.Select(m => m.Path))}", torrent); await _torrents.SelectFiles(torrent.TorrentId, fileIds); @@ -449,6 +542,14 @@ namespace RdtClient.Service.Services // Real-Debrid finished downloading the torrent, process the file to host. if (torrent.RdStatus == RealDebridStatus.Finished) { + // The files are selected but there are no downloads yet, check if Real-Debrid has generated links yet. + if (torrent.Downloads.Count == 0 && torrent.FilesSelected != null) + { + Log($"Checking for links", torrent); + + await _torrents.CheckForLinks(torrent.TorrentId); + } + // If the torrent has any files that need starting to be downloaded, download them. var downloadsPending = torrent.Downloads .Where(m => m.Completed == null && @@ -458,18 +559,16 @@ namespace RdtClient.Service.Services .OrderBy(m => m.Added) .ToList(); - Log.Debug($"Torrent {torrent.RdId} found {downloadsPending.Count} downloads pending"); - if (downloadsPending.Count > 0) { + Log($"Found {downloadsPending.Count} downloads pending", torrent); + foreach (var download in downloadsPending) { - Log.Debug($"Torrent {torrent.RdId} starting download {download.DownloadId}"); + Log($"Marking to download", download, torrent); - await _torrents.Download(download.DownloadId); + await _downloads.UpdateDownload(download.DownloadId); } - - continue; } // If all files are finished downloading, move to the unpacking step. @@ -480,47 +579,52 @@ namespace RdtClient.Service.Services m.UnpackingFinished == null) .ToList(); - Log.Debug($"Torrent {torrent.RdId} found {unpackingPending.Count} unpacks pending"); - if (unpackingPending.Count > 0) { + Log($"Found {unpackingPending.Count} unpacks pending", torrent); + foreach (var download in unpackingPending) { - Log.Debug($"Torrent {torrent.RdId} starting unpack {download.DownloadId}"); + Log($"Marking to unpack", download, torrent); - await _torrents.Unpack(download.DownloadId); + await _downloads.UpdateUnpack(download.DownloadId); } - - continue; } } // Check if torrent is complete if (torrent.Downloads.Count > 0) { - var allComplete = torrent.Downloads.All(m => m.Completed != null); + var allComplete = torrent.Downloads.Count(m => m.Completed != null); - if (allComplete) + if (allComplete > 0) { - Log.Debug($"Torrent {torrent.RdId} all downloads complete"); + Log($"All downloads complete, marking torrent as complete", torrent); await _torrents.UpdateComplete(torrent.TorrentId, DateTimeOffset.UtcNow); switch (torrent.FinishedAction) { case TorrentFinishedAction.RemoveAllTorrents: - Log.Debug($"Torrent {torrent.RdId} removing torrents from Real-Debrid and Real-Debrid Client, no files"); + Log($"Removing torrents from Real-Debrid and Real-Debrid Client, no files", torrent); await _torrents.Delete(torrent.TorrentId, true, true, false); break; case TorrentFinishedAction.RemoveRealDebrid: - Log.Debug($"Torrent {torrent.RdId} removing torrents from Real-Debrid, no files"); + Log($"Removing torrents from Real-Debrid, no files", torrent); await _torrents.Delete(torrent.TorrentId, false, true, false); break; case TorrentFinishedAction.None: - Log.Debug($"Torrent {torrent.RdId} not removing torrents or files"); + Log($"Not removing torrents or files", torrent); + break; + default: + Log($"Invalid torrent FinishedAction {torrent.FinishedAction}", torrent); break; } } + else + { + Log($"Waiting for downloads to complete. {allComplete}/{torrent.Downloads.Count} complete", torrent); + } } } @@ -528,7 +632,35 @@ namespace RdtClient.Service.Services sw.Stop(); - Log.Debug($"TorrentRunner Tick End (took {sw.ElapsedMilliseconds}ms)"); + if (sw.ElapsedMilliseconds > 1000) + { + Log($"TorrentRunner Tick End (took {sw.ElapsedMilliseconds}ms)"); + } + } + + private void Log(String message, Download download, Torrent torrent) + { + if (download != null) + { + message = $"{message} {download.ToLog()}"; + } + + if (torrent != null) + { + message = $"{message} {torrent.ToLog()}"; + } + + _logger.LogDebug(message); + } + + private void Log(String message, Torrent torrent = null) + { + if (torrent != null) + { + message = $"{message} {torrent.ToLog()}"; + } + + _logger.LogDebug(message); } } } diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 8fcd8d8..35d0896 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -2,53 +2,41 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using MonoTorrent; using Newtonsoft.Json; -using RDNET; using RdtClient.Data.Data; using RdtClient.Data.Enums; using RdtClient.Service.Helpers; using RdtClient.Service.Models; +using RdtClient.Service.Models.TorrentClient; +using RdtClient.Service.Services.TorrentClients; using Torrent = RdtClient.Data.Models.Data.Torrent; namespace RdtClient.Service.Services { public class Torrents { - public static readonly SemaphoreSlim TorrentResetLock = new(1, 1); - private static readonly SemaphoreSlim RealDebridUpdateLock = new(1, 1); - private readonly Downloads _downloads; - private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; private readonly TorrentData _torrentData; + private readonly Downloads _downloads; + + private readonly ITorrentClient _torrentClient; - public Torrents(IHttpClientFactory httpClientFactory, TorrentData torrentData, Downloads downloads) + public Torrents(ILogger logger, + TorrentData torrentData, + Downloads downloads, + RealDebridTorrentClient realDebridTorrentClient) { - _httpClientFactory = httpClientFactory; + _logger = logger; _torrentData = torrentData; _downloads = downloads; - } - - private RdNetClient GetRdNetClient() - { - var apiKey = Settings.Get.RealDebridApiKey; - - if (String.IsNullOrWhiteSpace(apiKey)) - { - throw new Exception("Real-Debrid API Key not set in the settings"); - } - - var httpClient = _httpClientFactory.CreateClient(); - httpClient.Timeout = TimeSpan.FromSeconds(10); - - var rdtNetClient = new RdNetClient(null, httpClient); - rdtNetClient.UseApiAuthentication(apiKey); - - return rdtNetClient; + + _torrentClient = realDebridTorrentClient; } public async Task> Get() @@ -98,6 +86,8 @@ namespace RdtClient.Service.Services return; } + Log($"Update category to {category}", torrent); + await _torrentData.UpdateCategory(torrent.TorrentId, category); } @@ -117,12 +107,19 @@ namespace RdtClient.Service.Services } catch (Exception ex) { + _logger.LogError(ex, $"{ex.Message}, trying to parse {magnetLink}"); throw new Exception($"{ex.Message}, trying to parse {magnetLink}"); } - var rdTorrent = await GetRdNetClient().Torrents.AddMagnetAsync(magnetLink); + var id = await _torrentClient.AddMagnet(magnetLink); - return await Add(rdTorrent.Id, magnet.InfoHash.ToHex(), category, downloadAction, finishedAction, downloadMinSize, downloadManualFiles, magnetLink, false, priority); + var hash = magnet.InfoHash.ToHex(); + + var newTorrent = await Add(id, hash, category, downloadAction, finishedAction, downloadMinSize, downloadManualFiles, magnetLink, false, priority); + + Log($"Adding {hash} magnet link {magnetLink}", newTorrent); + + return newTorrent; } public async Task UploadFile(Byte[] bytes, @@ -133,33 +130,35 @@ namespace RdtClient.Service.Services String downloadManualFiles, Int32? priority) { - MonoTorrent.Torrent torrent; + MonoTorrent.Torrent monoTorrent; var fileAsBase64 = Convert.ToBase64String(bytes); try { - torrent = await MonoTorrent.Torrent.LoadAsync(bytes); + monoTorrent = await MonoTorrent.Torrent.LoadAsync(bytes); } catch (Exception ex) { throw new Exception($"{ex.Message}, trying to parse {fileAsBase64}"); } - var rdTorrent = await GetRdNetClient().Torrents.AddFileAsync(bytes); + var id = await _torrentClient.AddFile(bytes); - return await Add(rdTorrent.Id, torrent.InfoHash.ToHex(), category, downloadAction, finishedAction, downloadMinSize, downloadManualFiles, fileAsBase64, true, priority); + var hash = monoTorrent.InfoHash.ToHex(); + + var newTorrent = await Add(id, hash, category, downloadAction, finishedAction, downloadMinSize, downloadManualFiles, fileAsBase64, true, priority); + + Log($"Adding {hash} torrent file {fileAsBase64}", newTorrent); + + return newTorrent; } - public async Task> GetAvailableFiles(String hash) + public async Task> GetAvailableFiles(String hash) { - var result = await GetRdNetClient().Torrents.GetAvailableFiles(hash); + var result = await _torrentClient.GetAvailableFiles(hash); - var files = result.SelectMany(m => m.Value).SelectMany(m => m.Value).SelectMany(m => m.Values); - - var groups = files.GroupBy(m => $"{m.Filename}-{m.Filesize}"); - - return groups.Select(m => m.First()).ToList(); + return result; } public async Task SelectFiles(Guid torrentId, IList fileIds) @@ -171,7 +170,7 @@ namespace RdtClient.Service.Services return; } - await GetRdNetClient().Torrents.SelectFilesAsync(torrent.RdId, fileIds.ToArray()); + await _torrentClient.SelectFiles(torrent.RdId, fileIds); } public async Task CheckForLinks(Guid torrentId) @@ -183,10 +182,12 @@ namespace RdtClient.Service.Services return; } - var rdTorrent = await GetRdNetClient().Torrents.GetInfoAsync(torrent.RdId); + var rdTorrent = await _torrentClient.GetInfo(torrent.RdId); var torrentLinks = rdTorrent.Links.Where(m => !String.IsNullOrWhiteSpace(m)).ToList(); + Log($"Found {torrentLinks} links", torrent); + // Sometimes RD will give you 1 rar with all files, sometimes it will give you 1 link per file. if (torrent.Files.Count(m => m.Selected) != torrentLinks.Count && torrent.ManualFiles.Count != torrentLinks.Count && @@ -216,21 +217,33 @@ namespace RdtClient.Service.Services return; } + Log($"Deleting", torrent); + foreach (var download in torrent.Downloads) { - if (TorrentRunner.ActiveUnpackClients.TryRemove(download.DownloadId, out var activeUnpack)) + while (TorrentRunner.ActiveDownloadClients.TryGetValue(download.DownloadId, out var downloadClient)) { - activeUnpack.Cancel(); + Log($"Cancelling download", download, torrent); + + await downloadClient.Cancel(); + + await Task.Delay(100); } - if (TorrentRunner.ActiveDownloadClients.TryRemove(download.DownloadId, out var activeDownload)) + while (TorrentRunner.ActiveUnpackClients.TryGetValue(download.DownloadId, out var unpackClient)) { - activeDownload.Cancel(); + Log($"Cancelling unpack", download, torrent); + + unpackClient.Cancel(); + + await Task.Delay(100); } } if (deleteData) { + Log($"Deleting RdtClient data", torrent); + await _torrentData.UpdateComplete(torrent.TorrentId, DateTimeOffset.UtcNow); await _downloads.DeleteForTorrent(torrent.TorrentId); await _torrentData.Delete(torrentId); @@ -238,7 +251,16 @@ namespace RdtClient.Service.Services if (deleteRdTorrent) { - await GetRdNetClient().Torrents.DeleteAsync(torrent.RdId); + Log($"Deleting RealDebrid Torrent", torrent); + + try + { + await _torrentClient.Delete(torrent.RdId); + } + catch + { + // ignored + } } if (deleteLocalFiles) @@ -246,6 +268,8 @@ namespace RdtClient.Service.Services var downloadPath = DownloadPath(torrent); downloadPath = Path.Combine(downloadPath, torrent.RdName); + Log($"Deleting local files in {downloadPath}", torrent); + if (Directory.Exists(downloadPath)) { var retry = 0; @@ -274,58 +298,6 @@ namespace RdtClient.Service.Services } } - private async Task DeleteDownload(Guid torrentId, Guid downloadId) - { - var torrent = await GetById(torrentId); - - if (torrent == null) - { - return; - } - - var download = await _downloads.GetById(downloadId); - - if (download == null) - { - return; - } - - var downloadPath = DownloadPath(torrent); - - var filePath = DownloadHelper.GetDownloadPath(downloadPath, torrent, download); - - if (filePath == null) - { - return; - } - - if (File.Exists(filePath)) - { - var retry = 0; - - while (true) - { - try - { - File.Delete(filePath); - - break; - } - catch - { - retry++; - - if (retry >= 3) - { - throw; - } - - await Task.Delay(1000); - } - } - } - } - public async Task UnrestrictLink(Guid downloadId) { var download = await _downloads.GetById(downloadId); @@ -335,36 +307,18 @@ namespace RdtClient.Service.Services throw new Exception($"Download with ID {downloadId} not found"); } - var unrestrictedLink = await GetRdNetClient().Unrestrict.LinkAsync(download.Path); + Log($"Unrestricting link", download, download.Torrent); - await _downloads.UpdateUnrestrictedLink(downloadId, unrestrictedLink.Download); + var unrestrictedLink = await _torrentClient.Unrestrict(download.Path); - return unrestrictedLink.Download; - } + await _downloads.UpdateUnrestrictedLink(downloadId, unrestrictedLink); - public async Task Download(Guid downloadId) - { - await _downloads.UpdateDownloadStarted(downloadId, null); - await _downloads.UpdateDownloadFinished(downloadId, null); - await _downloads.UpdateUnpackingQueued(downloadId, null); - await _downloads.UpdateUnpackingStarted(downloadId, null); - await _downloads.UpdateUnpackingFinished(downloadId, null); - await _downloads.UpdateCompleted(downloadId, null); - await _downloads.UpdateError(downloadId, null); - } - - public async Task Unpack(Guid downloadId) - { - await _downloads.UpdateUnpackingQueued(downloadId, DateTimeOffset.UtcNow); - await _downloads.UpdateUnpackingStarted(downloadId, null); - await _downloads.UpdateUnpackingFinished(downloadId, null); - await _downloads.UpdateCompleted(downloadId, null); - await _downloads.UpdateError(downloadId, null); + return unrestrictedLink; } public async Task GetProfile() { - var user = await GetRdNetClient().User.GetAsync(); + var user = await _torrentClient.GetUser(); var profile = new Profile { @@ -383,7 +337,7 @@ namespace RdtClient.Service.Services try { - var rdTorrents = await GetRdNetClient().Torrents.GetAsync(0, 100); + var rdTorrents = await _torrentClient.GetTorrents(); foreach (var rdTorrent in rdTorrents) { @@ -396,17 +350,6 @@ namespace RdtClient.Service.Services await UpdateRdData(torrent); } - - foreach (var torrent in torrents) - { - var rdTorrent = rdTorrents.FirstOrDefault(m => m.Id == torrent.RdId); - - if (rdTorrent == null) - { - await _downloads.DeleteForTorrent(torrent.TorrentId); - await _torrentData.Delete(torrent.TorrentId); - } - } } finally { @@ -414,7 +357,7 @@ namespace RdtClient.Service.Services } } - public async Task RetryTorrent(Guid torrentId) + public async Task RetryTorrent(Guid torrentId, Int32 retryCount) { var torrent = await _torrentData.GetById(torrentId); @@ -423,13 +366,13 @@ namespace RdtClient.Service.Services return; } - var newRetryCount = torrent.RetryCount + 1; + Log($"Retrying Torrent", torrent); foreach (var download in torrent.Downloads) { while (TorrentRunner.ActiveDownloadClients.TryGetValue(download.DownloadId, out var downloadClient)) { - downloadClient.Cancel(); + await downloadClient.Cancel(); await Task.Delay(100); } @@ -449,35 +392,26 @@ namespace RdtClient.Service.Services throw new Exception($"Cannot re-add this torrent, original magnet or file not found"); } - await TorrentResetLock.WaitAsync(); + Torrent newTorrent; - try + if (torrent.IsFile) { - Torrent newTorrent; + var bytes = Convert.FromBase64String(torrent.FileOrMagnet); - if (torrent.IsFile) - { - var bytes = Convert.FromBase64String(torrent.FileOrMagnet); - - newTorrent = await UploadFile(bytes, torrent.Category, torrent.DownloadAction, torrent.FinishedAction, torrent.DownloadMinSize, torrent.DownloadManualFiles, torrent.Priority); - } - else - { - newTorrent = await UploadMagnet(torrent.FileOrMagnet, - torrent.Category, - torrent.DownloadAction, - torrent.FinishedAction, - torrent.DownloadMinSize, - torrent.DownloadManualFiles, - torrent.Priority); - } - - await _torrentData.UpdateRetryCount(newTorrent.TorrentId, newRetryCount); + newTorrent = await UploadFile(bytes, torrent.Category, torrent.DownloadAction, torrent.FinishedAction, torrent.DownloadMinSize, torrent.DownloadManualFiles, torrent.Priority); } - finally + else { - TorrentResetLock.Release(); + newTorrent = await UploadMagnet(torrent.FileOrMagnet, + torrent.Category, + torrent.DownloadAction, + torrent.FinishedAction, + torrent.DownloadMinSize, + torrent.DownloadManualFiles, + torrent.Priority); } + + await _torrentData.UpdateRetryCount(newTorrent.TorrentId, retryCount); } public async Task RetryDownload(Guid downloadId) @@ -489,9 +423,11 @@ namespace RdtClient.Service.Services return; } + Log($"Retrying Download", download, download.Torrent); + while (TorrentRunner.ActiveDownloadClients.TryGetValue(download.DownloadId, out var downloadClient)) { - downloadClient.Cancel(); + await downloadClient.Cancel(); await Task.Delay(100); } @@ -503,21 +439,19 @@ namespace RdtClient.Service.Services await Task.Delay(100); } - await TorrentResetLock.WaitAsync(); + var downloadPath = DownloadPath(download.Torrent); + + var filePath = DownloadHelper.GetDownloadPath(downloadPath, download.Torrent, download); - try - { + Log($"Deleting {filePath}", download, download.Torrent); + + await FileHelper.Delete(filePath); + + await _torrentData.UpdateComplete(download.TorrentId, null); - await DeleteDownload(download.TorrentId, download.DownloadId); + Log($"Resetting", download, download.Torrent); - await _torrentData.UpdateComplete(download.TorrentId, null); - - await _downloads.Reset(downloadId); - } - finally - { - TorrentResetLock.Release(); - } + await _downloads.Reset(downloadId); } public async Task UpdateComplete(Guid torrentId, DateTimeOffset datetime) @@ -640,57 +574,71 @@ namespace RdtClient.Service.Services ReferenceLoopHandling = ReferenceLoopHandling.Ignore }); - var rdTorrent = await GetRdNetClient().Torrents.GetInfoAsync(torrent.RdId); - - if (!String.IsNullOrWhiteSpace(rdTorrent.Filename)) + try { - torrent.RdName = rdTorrent.Filename; + var rdTorrent = await _torrentClient.GetInfo(torrent.RdId); + + if (!String.IsNullOrWhiteSpace(rdTorrent.Filename)) + { + torrent.RdName = rdTorrent.Filename; + } + + if (!String.IsNullOrWhiteSpace(rdTorrent.OriginalFilename)) + { + torrent.RdName = rdTorrent.OriginalFilename; + } + + if (rdTorrent.Bytes > 0) + { + torrent.RdSize = rdTorrent.Bytes; + } + else if (rdTorrent.OriginalBytes > 0) + { + torrent.RdSize = rdTorrent.OriginalBytes; + } + + if (rdTorrent.Files != null) + { + torrent.RdFiles = JsonConvert.SerializeObject(rdTorrent.Files); + } + + torrent.RdHost = rdTorrent.Host; + torrent.RdSplit = rdTorrent.Split; + torrent.RdProgress = rdTorrent.Progress; + torrent.RdAdded = rdTorrent.Added; + torrent.RdEnded = rdTorrent.Ended; + torrent.RdSpeed = rdTorrent.Speed; + torrent.RdSeeders = rdTorrent.Seeders; + torrent.RdStatusRaw = rdTorrent.Status; + + torrent.RdStatus = rdTorrent.Status switch + { + "magnet_error" => RealDebridStatus.Error, + "magnet_conversion" => RealDebridStatus.Processing, + "waiting_files_selection" => RealDebridStatus.WaitingForFileSelection, + "queued" => RealDebridStatus.Downloading, + "downloading" => RealDebridStatus.Downloading, + "downloaded" => RealDebridStatus.Finished, + "error" => RealDebridStatus.Error, + "virus" => RealDebridStatus.Error, + "compressing" => RealDebridStatus.Downloading, + "uploading" => RealDebridStatus.Downloading, + "dead" => RealDebridStatus.Error, + _ => RealDebridStatus.Error + }; } - - if (!String.IsNullOrWhiteSpace(rdTorrent.OriginalFilename)) + catch (Exception ex) { - torrent.RdName = rdTorrent.OriginalFilename; + if (ex.Message == "Resource not found") + { + torrent.RdStatusRaw = "deleted"; + } + else + { + throw; + } } - - if (rdTorrent.Bytes > 0) - { - torrent.RdSize = rdTorrent.Bytes; - } - else if (rdTorrent.OriginalBytes > 0) - { - torrent.RdSize = rdTorrent.OriginalBytes; - } - - if (rdTorrent.Files != null) - { - torrent.RdFiles = JsonConvert.SerializeObject(rdTorrent.Files); - } - - torrent.RdHost = rdTorrent.Host; - torrent.RdSplit = rdTorrent.Split; - torrent.RdProgress = rdTorrent.Progress; - torrent.RdAdded = rdTorrent.Added; - torrent.RdEnded = rdTorrent.Ended; - torrent.RdSpeed = rdTorrent.Speed; - torrent.RdSeeders = rdTorrent.Seeders; - torrent.RdStatusRaw = rdTorrent.Status; - - torrent.RdStatus = rdTorrent.Status switch - { - "magnet_error" => RealDebridStatus.Error, - "magnet_conversion" => RealDebridStatus.Processing, - "waiting_files_selection" => RealDebridStatus.WaitingForFileSelection, - "queued" => RealDebridStatus.Downloading, - "downloading" => RealDebridStatus.Downloading, - "downloaded" => RealDebridStatus.Finished, - "error" => RealDebridStatus.Error, - "virus" => RealDebridStatus.Error, - "compressing" => RealDebridStatus.Downloading, - "uploading" => RealDebridStatus.Finished, - "dead" => RealDebridStatus.Error, - _ => RealDebridStatus.Error - }; - + var newTorrent = JsonConvert.SerializeObject(torrent, new JsonSerializerSettings { @@ -702,5 +650,30 @@ namespace RdtClient.Service.Services await _torrentData.UpdateRdData(torrent); } } + + private void Log(String message, Data.Models.Data.Download download, Torrent torrent) + { + if (download != null) + { + message = $"{message} {download.ToLog()}"; + } + + if (torrent != null) + { + message = $"{message} {torrent.ToLog()}"; + } + + _logger.LogDebug(message); + } + + private void Log(String message, Torrent torrent = null) + { + if (torrent != null) + { + message = $"{message} {torrent.ToLog()}"; + } + + _logger.LogDebug(message); + } } } diff --git a/server/RdtClient.Service/Services/UnpackClient.cs b/server/RdtClient.Service/Services/UnpackClient.cs index 9dda73d..082deb9 100644 --- a/server/RdtClient.Service/Services/UnpackClient.cs +++ b/server/RdtClient.Service/Services/UnpackClient.cs @@ -122,20 +122,7 @@ namespace RdtClient.Service.Services } } - var retryCount = 0; - while (File.Exists(filePath) && retryCount < 10) - { - retryCount++; - - try - { - File.Delete(filePath); - } - catch - { - await Task.Delay(1000); - } - } + await FileHelper.Delete(filePath); } catch (Exception ex) { diff --git a/server/RdtClient.Web/Controllers/TorrentsController.cs b/server/RdtClient.Web/Controllers/TorrentsController.cs index 62b734c..a8f4b54 100644 --- a/server/RdtClient.Web/Controllers/TorrentsController.cs +++ b/server/RdtClient.Web/Controllers/TorrentsController.cs @@ -159,7 +159,7 @@ namespace RdtClient.Web.Controllers [Route("Retry/{torrentId:guid}")] public async Task Retry(Guid torrentId) { - await _torrents.RetryTorrent(torrentId); + await _torrents.RetryTorrent(torrentId, 0); return Ok(); } diff --git a/server/RdtClient.Web/Program.cs b/server/RdtClient.Web/Program.cs index 40781ea..64d6a22 100644 --- a/server/RdtClient.Web/Program.cs +++ b/server/RdtClient.Web/Program.cs @@ -88,9 +88,15 @@ namespace RdtClient.Web Log.Logger = new LoggerConfiguration() .Enrich.FromLogContext() .Enrich.WithExceptionDetails() - .WriteTo.File(appSettings.Logging.File.Path, rollOnFileSizeLimit: true, fileSizeLimitBytes: appSettings.Logging.File.FileSizeLimitBytes, retainedFileCountLimit: appSettings.Logging.File.MaxRollingFiles) + .WriteTo.File(appSettings.Logging.File.Path, + rollOnFileSizeLimit: true, + fileSizeLimitBytes: appSettings.Logging.File.FileSizeLimitBytes, + retainedFileCountLimit: appSettings.Logging.File.MaxRollingFiles, + outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext}: {Message:lj}{NewLine}{Exception}") .WriteTo.Console() .MinimumLevel.ControlledBy(LoggingLevelSwitch) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .MinimumLevel.Override("System.Net.Http", LogEventLevel.Warning) .CreateLogger(); Serilog.Debugging.SelfLog.Enable(msg => diff --git a/server/RdtClient.Web/RdtClient.Web.csproj b/server/RdtClient.Web/RdtClient.Web.csproj index 68f1edc..01919dc 100644 --- a/server/RdtClient.Web/RdtClient.Web.csproj +++ b/server/RdtClient.Web/RdtClient.Web.csproj @@ -4,7 +4,7 @@ net5.0 Exe 94c24cba-f03f-4453-a671-3640b517c573 - 1.9.0 + 1.9.1 @@ -26,16 +26,16 @@ - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - +