From 8114baa7b8ecedefc230beac515588e13f6d340a Mon Sep 17 00:00:00 2001 From: Roger Far Date: Fri, 5 Jan 2024 11:07:38 -0700 Subject: [PATCH] Switch the internal downloader (again). --- .../Models/Internal/DbSettings.cs | 4 + .../RdtClient.Service.csproj | 4 +- .../Downloaders/InternalDownloader.cs | 117 ++++++++++++++---- 3 files changed, 98 insertions(+), 27 deletions(-) diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 3520ca1..3866aa9 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -99,6 +99,10 @@ public class DbSettingsDownloadClient [Description("Maximum amount of parallel threads that are used to download a single file to your host. If set to 0 no parallel downloading will be done.")] public Int32 ParallelCount { get; set; } = 0; + [DisplayName("Chunk Count")] + [Description("Split the downloaded file in this amount of chunks. If left to 0, automatically deteremine the chunk size based on the file size.")] + public Int32 ChunkCount { get; set; } = 0; + [DisplayName("Connection Timeout (only used for the Internal Downloader)")] [Description("Timeout in milliseconds before the downloader times out.")] public Int32 Timeout { get; set; } = 5000; diff --git a/server/RdtClient.Service/RdtClient.Service.csproj b/server/RdtClient.Service/RdtClient.Service.csproj index f5b569a..e3f7748 100644 --- a/server/RdtClient.Service/RdtClient.Service.csproj +++ b/server/RdtClient.Service/RdtClient.Service.csproj @@ -11,13 +11,13 @@ - + - + diff --git a/server/RdtClient.Service/Services/Downloaders/InternalDownloader.cs b/server/RdtClient.Service/Services/Downloaders/InternalDownloader.cs index c3f011a..084f6d4 100644 --- a/server/RdtClient.Service/Services/Downloaders/InternalDownloader.cs +++ b/server/RdtClient.Service/Services/Downloaders/InternalDownloader.cs @@ -1,4 +1,5 @@ -using DownloaderNET; +using System.Net; +using Downloader; using Serilog; namespace RdtClient.Service.Services.Downloaders; @@ -8,16 +9,14 @@ public class InternalDownloader : IDownloader public event EventHandler? DownloadComplete; public event EventHandler? DownloadProgress; - private readonly Downloader _downloadService; - private readonly DownloaderNET.Settings _downloadConfiguration; + private readonly DownloadService _downloadService; + private readonly DownloadConfiguration _downloadConfiguration; private readonly String _filePath; private readonly String _uri; private readonly ILogger _logger; - private readonly CancellationTokenSource _cancellationToken = new(); - private Boolean _finished; public InternalDownloader(String uri, String filePath) @@ -26,16 +25,40 @@ public class InternalDownloader : IDownloader _uri = uri; _filePath = filePath; - - _downloadConfiguration = new DownloaderNET.Settings(); + + var settingProxyServer = Settings.Get.DownloadClient.ProxyServer; + + // For all options, see https://github.com/bezzad/Downloader + _downloadConfiguration = new DownloadConfiguration + { + BufferBlockSize = 1024 * 8, + MaxTryAgainOnFailover = 5, + RangeDownload = false, + ClearPackageOnCompletionWithFailure = false, + MinimumSizeOfChunking = 1024, + ReserveStorageSpaceBeforeStartingDownload = false, + CheckDiskSizeBeforeDownload = true, + RequestConfiguration = + { + Accept = "*/*", + UserAgent = $"rdt-client", + ProtocolVersion = HttpVersion.Version11, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + KeepAlive = true, + UseDefaultCredentials = false + } + }; SetSettings(); - _downloadService = new Downloader(_uri, _filePath, _downloadConfiguration); + if (!String.IsNullOrWhiteSpace(settingProxyServer)) + { + _downloadConfiguration.RequestConfiguration.Proxy = new WebProxy(new Uri(settingProxyServer), false); + } - //_downloadService.OnLog += message => Debug.WriteLine(message.Message); + _downloadService = new DownloadService(_downloadConfiguration); - _downloadService.OnProgress += (chunks, _) => + _downloadService.DownloadProgressChanged += (_, args) => { if (DownloadProgress == null) { @@ -45,41 +68,68 @@ public class InternalDownloader : IDownloader DownloadProgress.Invoke(this, new DownloadProgressEventArgs { - Speed = (Int64)chunks.Where(m => m.IsActive).Sum(m => m.Speed), - BytesDone = chunks.Sum(m => m.DownloadBytes), - BytesTotal = chunks.Sum(m => m.LengthBytes) + Speed = (Int64)args.BytesPerSecondSpeed, + BytesDone = args.ReceivedBytesSize, + BytesTotal = args.TotalBytesToReceive }); }; - _downloadService.OnComplete += (_, error) => + _downloadService.DownloadFileCompleted += (_, args) => { + String? error = null; + + if (args.Cancelled) + { + error = $"The download was cancelled"; + } + else if (args.Error != null) + { + error = args.Error.Message; + } + DownloadComplete?.Invoke(this, new DownloadCompleteEventArgs { - Error = error?.Message + Error = error }); _finished = true; - - return Task.CompletedTask; }; } - public Task Download() + public async Task Download() { _logger.Debug($"Starting download of {_uri}, writing to path: {_filePath}"); - Task.Run(async () => await _downloadService.Download(_cancellationToken.Token)); - Task.Run(StartTimer); + if (Settings.Get.DownloadClient.ChunkCount == 0) + { + var contentSize = await GetContentSize(); - return Task.FromResult(null); + var chunkSize = contentSize switch + { + <= 1024 * 1024 * 10 => 1024 * 1024 * 10, + <= 1024 * 1024 * 100 => 1024 * 1024 * 25, + _ => 1024 * 1024 * 50 + }; + + _downloadConfiguration.ChunkCount = (Int32) Math.Ceiling(contentSize / (Double) chunkSize); + } + + _ = Task.Run(async () => + { + await _downloadService.DownloadFileTaskAsync(_uri, _filePath); + }); + + _ = Task.Run(StartTimer); + + return null; } public Task Cancel() { _logger.Debug($"Cancelling download {_uri}"); - _cancellationToken.Cancel(false); + _downloadService.CancelAsync(); return Task.CompletedTask; } @@ -102,7 +152,7 @@ public class InternalDownloader : IDownloader { settingDownloadParallelCount = 1; } - + var settingDownloadMaxSpeed = Settings.Get.DownloadClient.MaxSpeed; if (settingDownloadMaxSpeed <= 0) @@ -119,10 +169,10 @@ public class InternalDownloader : IDownloader settingDownloadTimeout = 1000; } - _downloadConfiguration.Parallel = settingDownloadParallelCount; _downloadConfiguration.MaximumBytesPerSecond = settingDownloadMaxSpeed; + _downloadConfiguration.ParallelDownload = settingDownloadParallelCount > 1; + _downloadConfiguration.ParallelCount = settingDownloadParallelCount; _downloadConfiguration.Timeout = settingDownloadTimeout; - _downloadConfiguration.RetryCount = 5; } private async Task StartTimer() @@ -139,4 +189,21 @@ public class InternalDownloader : IDownloader SetSettings(); } } + + private async Task GetContentSize() + { + var httpClient = new HttpClient(); + var responseHeaders = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, _uri)); + + if (!responseHeaders.IsSuccessStatusCode) + { + var content = await responseHeaders.Content.ReadAsStringAsync(); + var ex = new Exception($"Unable to retrieve content size before downloading, received response: {responseHeaders.StatusCode} {content}"); + + throw ex; + } + + var result = responseHeaders.Content.Headers.ContentLength ?? -1; + return result; + } } \ No newline at end of file