diff --git a/client/src/app/models/rate-limit-status.model.ts b/client/src/app/models/rate-limit-status.model.ts new file mode 100644 index 0000000..c6009dc --- /dev/null +++ b/client/src/app/models/rate-limit-status.model.ts @@ -0,0 +1,4 @@ +export interface RateLimitStatus { + nextDequeueTime: Date | null; + secondsRemaining: number; +} diff --git a/client/src/app/torrent-table/torrent-table.component.html b/client/src/app/torrent-table/torrent-table.component.html index b4c0e79..d441002 100644 --- a/client/src/app/torrent-table/torrent-table.component.html +++ b/client/src/app/torrent-table/torrent-table.component.html @@ -13,6 +13,13 @@ Last check: {{ diskSpaceStatus.lastCheckTime | date: 'short' }} } +@if (rateLimitStatus?.nextDequeueTime) { +
+ Debrid provider rate limit reached +
+ New torrents will not be added until {{ rateLimitStatus.nextDequeueTime | date: 'medium' }} +
+}
diff --git a/client/src/app/torrent-table/torrent-table.component.ts b/client/src/app/torrent-table/torrent-table.component.ts index da7478f..5e011c1 100644 --- a/client/src/app/torrent-table/torrent-table.component.ts +++ b/client/src/app/torrent-table/torrent-table.component.ts @@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'; import { Router } from '@angular/router'; import { Torrent } from '../models/torrent.model'; import { DiskSpaceStatus } from '../models/disk-space-status.model'; +import { RateLimitStatus } from '../models/rate-limit-status.model'; import { TorrentService } from '../torrent.service'; import { forkJoin, Observable } from 'rxjs'; import { FormsModule } from '@angular/forms'; @@ -50,6 +51,7 @@ export class TorrentTableComponent implements OnInit { public updateSettingsTorrentLifetime: number; public diskSpaceStatus: DiskSpaceStatus | null = null; + public rateLimitStatus: RateLimitStatus | null = null; constructor( private router: Router, @@ -67,6 +69,16 @@ export class TorrentTableComponent implements OnInit { this.diskSpaceStatus = status; }); + this.torrentService.getRateLimitStatus().subscribe({ + next: (status) => { + this.rateLimitStatus = status; + }, + }); + + this.torrentService.rateLimitStatus$.subscribe((status) => { + this.rateLimitStatus = status; + }); + this.torrentService.update$.subscribe((result) => { this.torrents = result; }); diff --git a/client/src/app/torrent.service.ts b/client/src/app/torrent.service.ts index 061d724..d3f3398 100644 --- a/client/src/app/torrent.service.ts +++ b/client/src/app/torrent.service.ts @@ -4,6 +4,7 @@ import * as signalR from '@microsoft/signalr'; import { Observable, Subject } from 'rxjs'; import { Torrent, TorrentFileAvailability } from './models/torrent.model'; import { DiskSpaceStatus } from './models/disk-space-status.model'; +import { RateLimitStatus } from './models/rate-limit-status.model'; import { APP_BASE_HREF } from '@angular/common'; @Injectable({ @@ -12,6 +13,7 @@ import { APP_BASE_HREF } from '@angular/common'; export class TorrentService { public update$: Subject = new Subject(); public diskSpaceStatus$: Subject = new Subject(); + public rateLimitStatus$: Subject = new Subject(); private connection: signalR.HubConnection; @@ -40,6 +42,10 @@ export class TorrentService { this.diskSpaceStatus$.next(status); }); + this.connection.on('rateLimitStatus', (status: any) => { + this.rateLimitStatus$.next(status); + }); + this.connection.onreconnected(() => { this.getDiskSpaceStatus().subscribe({ next: (status) => { @@ -65,6 +71,10 @@ export class TorrentService { return this.http.get(`${this.baseHref}Api/Torrents/DiskSpaceStatus`); } + public getRateLimitStatus(): Observable { + return this.http.get(`${this.baseHref}Api/Torrents/RateLimitStatus`); + } + public uploadMagnet(magnetLink: string, torrent: Torrent): Observable { return this.http.post(`${this.baseHref}Api/Torrents/UploadMagnet`, { magnetLink, diff --git a/server/RdtClient.Data/Data/ITorrentData.cs b/server/RdtClient.Data/Data/ITorrentData.cs index 7b04b88..3ff6071 100644 --- a/server/RdtClient.Data/Data/ITorrentData.cs +++ b/server/RdtClient.Data/Data/ITorrentData.cs @@ -19,6 +19,7 @@ public interface ITorrentData Task UpdateRdData(Torrent torrent); Task UpdateRdId(Torrent torrent, String rdId); + Task UpdateHash(Torrent torrent, String hash); Task Update(Torrent torrent); Task UpdateCategory(Guid torrentId, String? category); Task UpdateComplete(Guid torrentId, String? error, DateTimeOffset? datetime, Boolean retry); diff --git a/server/RdtClient.Data/Models/Internal/RateLimitException.cs b/server/RdtClient.Data/Models/Internal/RateLimitException.cs new file mode 100644 index 0000000..3564975 --- /dev/null +++ b/server/RdtClient.Data/Models/Internal/RateLimitException.cs @@ -0,0 +1,6 @@ +namespace RdtClient.Data.Models.Internal; + +public class RateLimitException(String message, TimeSpan retryAfter) : Exception(message) +{ + public TimeSpan RetryAfter { get; } = retryAfter; +} diff --git a/server/RdtClient.Data/Models/Internal/RateLimitStatus.cs b/server/RdtClient.Data/Models/Internal/RateLimitStatus.cs new file mode 100644 index 0000000..dd4d7f1 --- /dev/null +++ b/server/RdtClient.Data/Models/Internal/RateLimitStatus.cs @@ -0,0 +1,7 @@ +namespace RdtClient.Data.Models.Internal; + +public class RateLimitStatus +{ + public DateTimeOffset? NextDequeueTime { get; set; } + public Double SecondsRemaining { get; set; } +} diff --git a/server/RdtClient.Service.Test/Helpers/RateLimitHandlerTest.cs b/server/RdtClient.Service.Test/Helpers/RateLimitHandlerTest.cs new file mode 100644 index 0000000..7ed3bda --- /dev/null +++ b/server/RdtClient.Service.Test/Helpers/RateLimitHandlerTest.cs @@ -0,0 +1,52 @@ +using System.Net; +using RdtClient.Data.Models.Internal; +using RdtClient.Service.Helpers; + +namespace RdtClient.Service.Test.Helpers; + +public class RateLimitHandlerTest +{ + [Fact] + public async Task SendAsync_ThrowsRateLimitException_On429WithRetryAfter() + { + // Arrange + var handler = new RateLimitHandler + { + InnerHandler = new MockHttpMessageHandler(HttpStatusCode.TooManyRequests, 3600) + }; + var client = new HttpClient(handler); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => client.GetAsync("http://example.com")); + Assert.Equal(TimeSpan.FromSeconds(3600), ex.RetryAfter); + Assert.Equal("TorBox rate limit exceeded", ex.Message); + } + + [Fact] + public async Task SendAsync_ThrowsRateLimitException_On429WithoutRetryAfter() + { + // Arrange + var handler = new RateLimitHandler + { + InnerHandler = new MockHttpMessageHandler(HttpStatusCode.TooManyRequests, null) + }; + var client = new HttpClient(handler); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => client.GetAsync("http://example.com")); + Assert.Equal(TimeSpan.FromMinutes(2), ex.RetryAfter); + } + + private class MockHttpMessageHandler(HttpStatusCode statusCode, Int32? retryAfterSeconds) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(statusCode); + if (retryAfterSeconds.HasValue) + { + response.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(retryAfterSeconds.Value)); + } + return Task.FromResult(response); + } + } +} diff --git a/server/RdtClient.Service.Test/RdtClient.Service.Test.csproj b/server/RdtClient.Service.Test/RdtClient.Service.Test.csproj index a105272..181fff8 100644 --- a/server/RdtClient.Service.Test/RdtClient.Service.Test.csproj +++ b/server/RdtClient.Service.Test/RdtClient.Service.Test.csproj @@ -20,7 +20,7 @@ - + all diff --git a/server/RdtClient.Service.Test/Services/TorrentClients/TorBoxDebridClientTest.cs b/server/RdtClient.Service.Test/Services/TorrentClients/TorBoxDebridClientTest.cs index f303bb5..f8fcb19 100644 --- a/server/RdtClient.Service.Test/Services/TorrentClients/TorBoxDebridClientTest.cs +++ b/server/RdtClient.Service.Test/Services/TorrentClients/TorBoxDebridClientTest.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json; using RdtClient.Data.Enums; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.DebridClient; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Services; using RdtClient.Service.Services.DebridClients; using TorBoxNET; @@ -269,16 +270,15 @@ public class TorBoxDebridClientTest torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object); clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); - usenetApiMock.Setup(m => m.AddFileAsync(bytes, -1, name, null, It.IsAny())) - .ReturnsAsync(new Response { Data = new() - { Hash = "new-hash" } }); + usenetApiMock.Setup(m => m.AddFileAsync(bytes, It.IsAny(), name, It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new Response { Data = new UsenetAddResult { Hash = "new-hash" } }); // Act var result = await clientMock.Object.AddNzbFile(bytes, name); // Assert Assert.Equal("new-hash", result); - usenetApiMock.Verify(m => m.AddFileAsync(bytes, -1, name, null, It.IsAny()), Times.Once); + usenetApiMock.Verify(m => m.AddFileAsync(bytes, It.IsAny(), name, It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } [Fact] public async Task GetDownloadInfos_GeneratesCorrectFakedlLinks_ForIndividualFiles() @@ -479,6 +479,198 @@ public class TorBoxDebridClientTest Assert.Equal("https://real-usenet-link", result); usenetApiMock.Verify(m => m.RequestDownloadAsync(98765, 4321, false, It.IsAny()), Times.Once); } + [Fact] + public async Task AddTorrentMagnet_ThrowsRateLimitException_OnActiveLimit() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:test"; + var torrentsApiMock = new Mock(); + var userApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object); + torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + userApiMock.Setup(m => m.GetAsync(true, It.IsAny())) + .ReturnsAsync(new Response { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } }); + + torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny(), It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new TorBoxException("active_limit", "Active limit reached")); + + // Act & Assert + await Assert.ThrowsAsync(() => clientMock.Object.AddTorrentMagnet(magnetLink)); + torrentsApiMock.Verify(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny(), It.IsAny(), false, It.IsAny()), Times.Once); + torrentsApiMock.Verify(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny(), It.IsAny(), true, It.IsAny()), Times.Never); + } + + [Fact] + public async Task AddTorrentMagnet_ThrowsRateLimitException_OnSlowDown() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:test"; + var torrentsApiMock = new Mock(); + var userApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object); + torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + userApiMock.Setup(m => m.GetAsync(true, It.IsAny())) + .ReturnsAsync(new Response { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } }); + + torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny(), It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new Exception("slow_down")); + + // Act & Assert + await Assert.ThrowsAsync(() => clientMock.Object.AddTorrentMagnet(magnetLink)); + } + + [Fact] + public async Task AddTorrentMagnet_ThrowsRateLimitException_OnRateLimitException() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:test"; + var torrentsApiMock = new Mock(); + var userApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object); + torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + userApiMock.Setup(m => m.GetAsync(true, It.IsAny())) + .ReturnsAsync(new Response { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } }); + + torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny(), It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new RateLimitException("TorBox rate limit exceeded", TimeSpan.FromMinutes(60))); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => clientMock.Object.AddTorrentMagnet(magnetLink)); + Assert.Equal(TimeSpan.FromMinutes(60), ex.RetryAfter); + } + + [Fact] + public async Task AddTorrentMagnet_ThrowsRateLimitException_OnWrappedRateLimitException() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:test"; + var torrentsApiMock = new Mock(); + var userApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object); + torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + userApiMock.Setup(m => m.GetAsync(true, It.IsAny())) + .ReturnsAsync(new Response { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } }); + + torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny(), It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new Exception("Wrapped", new RateLimitException("TorBox rate limit exceeded", TimeSpan.FromMinutes(60)))); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => clientMock.Object.AddTorrentMagnet(magnetLink)); + Assert.Equal(TimeSpan.FromMinutes(60), ex.RetryAfter); + } + + [Fact] + public async Task AddTorrentFile_ThrowsRateLimitException_OnActiveLimit() + { + // Arrange + var bytes = new Byte[] { 1, 2, 3 }; + var torrentsApiMock = new Mock(); + var userApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object); + torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + userApiMock.Setup(m => m.GetAsync(true, It.IsAny())) + .ReturnsAsync(new Response { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } }); + + torrentsApiMock.Setup(m => m.AddFileAsync(bytes, 5, It.IsAny(), It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new TorBoxException("active_limit", "Active limit reached")); + + // Act & Assert + await Assert.ThrowsAsync(() => clientMock.Object.AddTorrentFile(bytes)); + torrentsApiMock.Verify(m => m.AddFileAsync(bytes, 5, It.IsAny(), It.IsAny(), false, It.IsAny()), Times.Once); + torrentsApiMock.Verify(m => m.AddFileAsync(bytes, 5, It.IsAny(), It.IsAny(), true, It.IsAny()), Times.Never); + } + + [Fact] + public async Task AddNzbLink_ThrowsRateLimitException_OnActiveLimit() + { + // Arrange + var nzbLink = "https://example.com/test.nzb"; + var usenetApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + usenetApiMock.Setup(m => m.AddLinkAsync(nzbLink, It.IsAny(), It.IsAny(), It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new TorBoxException("active_limit", "Active limit reached")); + + // Act & Assert + await Assert.ThrowsAsync(() => clientMock.Object.AddNzbLink(nzbLink)); + usenetApiMock.Verify(m => m.AddLinkAsync(nzbLink, It.IsAny(), It.IsAny(), It.IsAny(), false, It.IsAny()), Times.Once); + usenetApiMock.Verify(m => m.AddLinkAsync(nzbLink, It.IsAny(), It.IsAny(), It.IsAny(), true, It.IsAny()), Times.Never); + } + + [Fact] + public async Task AddNzbFile_ThrowsRateLimitException_OnActiveLimit() + { + // Arrange + var bytes = new Byte[] { 1, 2, 3 }; + var name = "test.nzb"; + var usenetApiMock = new Mock(); + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object) { CallBase = true }; + var torBoxClientMock = new Mock(); + + torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object); + clientMock.Protected().Setup("GetClient").Returns(torBoxClientMock.Object); + + usenetApiMock.Setup(m => m.AddFileAsync(bytes, It.IsAny(), name, It.IsAny(), false, It.IsAny())) + .ThrowsAsync(new TorBoxException("active_limit", "Active limit reached")); + + // Act & Assert + await Assert.ThrowsAsync(() => clientMock.Object.AddNzbFile(bytes, name)); + usenetApiMock.Verify(m => m.AddFileAsync(bytes, It.IsAny(), name, It.IsAny(), false, It.IsAny()), Times.Once); + usenetApiMock.Verify(m => m.AddFileAsync(bytes, It.IsAny(), name, It.IsAny(), true, It.IsAny()), Times.Never); + } + + [Fact] + public async Task UpdateData_SetsErrorStatus_WhenTorBoxStatusStartsWithFailed() + { + // Arrange + var torrent = new Torrent + { + RdId = "test-rd-id", + RdStatus = TorrentStatus.Downloading + }; + var torrentClientTorrent = new DebridClientTorrent + { + Status = "failed (Aborted, cannot be completed - https://sabnzbd.org/not-complete)", + Filename = "test-file" + }; + + var clientMock = new Mock(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object); + + // Act + var result = await clientMock.Object.UpdateData(torrent, torrentClientTorrent); + + // Assert + Assert.Equal(TorrentStatus.Error, result.RdStatus); + } [Fact] public async Task UpdateData_LogsWarning_WhenTorBoxStatusIsUnmapped() diff --git a/server/RdtClient.Service.Test/Services/TorrentsTest.cs b/server/RdtClient.Service.Test/Services/TorrentsTest.cs index 1b2b5ab..d856866 100644 --- a/server/RdtClient.Service.Test/Services/TorrentsTest.cs +++ b/server/RdtClient.Service.Test/Services/TorrentsTest.cs @@ -14,7 +14,7 @@ using TorrentsService = RdtClient.Service.Services.Torrents; namespace RdtClient.Service.Test.Services; -internal class Mocks +class Mocks { public readonly Mock DownloadsMock; public readonly Mock EnricherMock; diff --git a/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs b/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs index f32b8cc..8ab529d 100644 --- a/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs +++ b/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using RdtClient.Data.Enums; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Services; namespace RdtClient.Service.BackgroundServices; @@ -19,7 +20,7 @@ public class ProviderUpdater(ILogger logger, IServiceProvider s using var scope = serviceProvider.CreateScope(); var torrentService = scope.ServiceProvider.GetRequiredService(); - + var torrentRunner = scope.ServiceProvider.GetRequiredService(); logger.LogInformation("ProviderUpdater started."); while (!stoppingToken.IsCancellationRequested) @@ -28,7 +29,7 @@ public class ProviderUpdater(ILogger logger, IServiceProvider s { var torrents = await torrentService.Get(); - if (_nextUpdate < DateTime.UtcNow && (Settings.Get.Provider.AutoImport || torrents.Any(t => t.RdStatus != TorrentStatus.Finished))) + if (_nextUpdate < DateTime.UtcNow && (Settings.Get.Provider.AutoImport || torrents.Any(t => t.RdStatus != TorrentStatus.Finished) || RdtHub.HasConnections)) { logger.LogDebug($"Updating torrent info from debrid provider"); @@ -43,9 +44,9 @@ public class ProviderUpdater(ILogger logger, IServiceProvider s { updateTime = Settings.Get.Provider.CheckInterval; - if (updateTime < 5) + if (updateTime < 1) { - updateTime = 5; + updateTime = 1; } } @@ -56,6 +57,11 @@ public class ProviderUpdater(ILogger logger, IServiceProvider s logger.LogDebug("Finished updating torrent info from debrid provider, next update in {updateTime} seconds", updateTime); } } + catch (RateLimitException ex) + { + await torrentRunner.SetRateLimit(ex.RetryAfter, ex.Message); + _nextUpdate = DateTime.UtcNow.Add(ex.RetryAfter); + } catch (Exception ex) { logger.LogError(ex, "Unexpected error occurred in ProviderUpdater: {ex.Message}", ex.Message); diff --git a/server/RdtClient.Service/BackgroundServices/UpdateChecker.cs b/server/RdtClient.Service/BackgroundServices/UpdateChecker.cs index 3ae9bfb..f76f9c7 100644 --- a/server/RdtClient.Service/BackgroundServices/UpdateChecker.cs +++ b/server/RdtClient.Service/BackgroundServices/UpdateChecker.cs @@ -5,7 +5,7 @@ using Newtonsoft.Json; namespace RdtClient.Service.BackgroundServices; -public class UpdateChecker(ILogger logger) : BackgroundService +public class UpdateChecker(ILogger logger, IHttpClientFactory httpClientFactory) : BackgroundService { private static readonly List KnownGhsaIds = []; public static String? CurrentVersion { get; private set; } @@ -79,13 +79,13 @@ public class UpdateChecker(ILogger logger) : BackgroundService logger.LogInformation("UpdateChecker stopped."); } - private static async Task GitHubRequest(String endpoint, CancellationToken cancellationToken) + private async Task GitHubRequest(String endpoint, CancellationToken cancellationToken) { - var httpClient = new HttpClient(); - httpClient.DefaultRequestHeaders.UserAgent.Add(new("RdtClient", CurrentVersion)); - var response = await httpClient.GetStringAsync($"https://api.github.com{endpoint}", cancellationToken); - - return JsonConvert.DeserializeObject(response); + var httpClient = httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.UserAgent.Add(new("RdtClient", CurrentVersion)); + var response = await httpClient.GetStringAsync($"https://api.github.com{endpoint}", cancellationToken); + + return JsonConvert.DeserializeObject(response); } } diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index a22a266..549d99d 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -4,8 +4,11 @@ using System.Reflection; using Microsoft.AspNetCore.Authorization; using Microsoft.Extensions.DependencyInjection; using Polly; -using Polly.Extensions.Http; +using Polly.Timeout; +using RateLimitHeaders.Polly; +using RdtClient.Data.Models.Internal; using RdtClient.Service.BackgroundServices; +using RdtClient.Service.Helpers; using RdtClient.Service.Middleware; using RdtClient.Service.Services; using RdtClient.Service.Services.DebridClients; @@ -16,6 +19,7 @@ namespace RdtClient.Service; public static class DiConfig { public const String RD_CLIENT = "RdClient"; + public const String TORBOX_CLIENT = "TorBoxClient"; public static readonly String UserAgent = $"rdt-client {Assembly.GetEntryAssembly()?.GetName().Version}"; public static void RegisterRdtServices(this IServiceCollection services) @@ -60,13 +64,7 @@ public static class DiConfig public static void RegisterHttpClients(this IServiceCollection services) { - var retryPolicy = HttpPolicyExtensions - .HandleTransientHttpError() - .OrResult(r => r.StatusCode == HttpStatusCode.TooManyRequests) - .WaitAndRetryAsync(5, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))); - services.AddHttpClient(); - services.ConfigureHttpClientDefaults(builder => { builder.ConfigureHttpClient(httpClient => @@ -75,7 +73,64 @@ public static class DiConfig }); }); + services.AddTransient(); + services.AddHttpClient(RD_CLIENT) - .AddPolicyHandler(retryPolicy); + .AddHttpMessageHandler() + .AddResilienceHandler("rd_client_handler", ConfigureResiliencePipeline); + + // This likely works for most providers, but should be verified and then the providers changed + // to this HTTP client for added resilience. + services.AddHttpClient(TORBOX_CLIENT) + .AddHttpMessageHandler() + .AddResilienceHandler("torbox_client_handler", ConfigureResiliencePipeline); + } + + private static void ConfigureResiliencePipeline(ResiliencePipelineBuilder builder) + { + builder.AddTimeout(new TimeoutStrategyOptions + { + TimeoutGenerator = _ => new ValueTask(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout)) + }); + builder.AddRateLimitHeaders(options => + { + options.EnableProactiveThrottling = true; + }); + builder.AddRetry(new() + { + ShouldHandle = args => args.Outcome switch + { + { Exception: HttpRequestException } => PredicateResult.True(), + { Result.StatusCode: HttpStatusCode.RequestTimeout } => PredicateResult.True(), + { Result.StatusCode: HttpStatusCode.TooManyRequests } => PredicateResult.True(), + _ => PredicateResult.False() + }, + MaxRetryAttempts = 2, + BackoffType = DelayBackoffType.Exponential, + Delay = TimeSpan.FromSeconds(2), + UseJitter = true, + DelayGenerator = args => + { + if (args.Outcome.Result is { StatusCode: HttpStatusCode.TooManyRequests } response) + { + var retryAfter = response.Headers.RetryAfter; + var delay = retryAfter?.Delta ?? (retryAfter?.Date.HasValue == true ? retryAfter.Date.Value - DateTimeOffset.UtcNow : TimeSpan.FromMinutes(2)); + + if (delay < TimeSpan.Zero) + { + delay = TimeSpan.FromMinutes(2); + } + + if (delay >= TimeSpan.FromSeconds(Settings.Get.Provider.Timeout)) + { + throw new RateLimitException("TorBox rate limit exceeded", delay); + } + + return new ValueTask(delay); + } + + return new ValueTask((TimeSpan?)null); + } + }); } } diff --git a/server/RdtClient.Service/Helpers/RateLimitHandler.cs b/server/RdtClient.Service/Helpers/RateLimitHandler.cs new file mode 100644 index 0000000..6ddf292 --- /dev/null +++ b/server/RdtClient.Service/Helpers/RateLimitHandler.cs @@ -0,0 +1,35 @@ +using System.Net; +using Polly.Timeout; +using RdtClient.Data.Models.Internal; + +namespace RdtClient.Service.Helpers; + +public class RateLimitHandler : DelegatingHandler +{ + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + try + { + var response = await base.SendAsync(request, cancellationToken); + + if (response.StatusCode == HttpStatusCode.TooManyRequests) + { + var retryAfter = response.Headers.RetryAfter; + var delay = retryAfter?.Delta ?? (retryAfter?.Date.HasValue == true ? retryAfter.Date.Value - DateTimeOffset.UtcNow : TimeSpan.FromMinutes(2)); + + if (delay < TimeSpan.Zero) + { + delay = TimeSpan.FromMinutes(2); + } + + throw new RateLimitException("TorBox rate limit exceeded", delay); + } + + return response; + } + catch (Exception ex) when (ex is TimeoutRejectedException or TaskCanceledException) + { + throw new RateLimitException("TorBox rate limit exceeded (timeout)", TimeSpan.FromMinutes(2)); + } + } +} diff --git a/server/RdtClient.Service/RdtClient.Service.csproj b/server/RdtClient.Service/RdtClient.Service.csproj index 00172d1..701730b 100644 --- a/server/RdtClient.Service/RdtClient.Service.csproj +++ b/server/RdtClient.Service/RdtClient.Service.csproj @@ -18,6 +18,7 @@ + @@ -25,7 +26,7 @@ - + diff --git a/server/RdtClient.Service/Services/DebridClients/AllDebridDebridClient.cs b/server/RdtClient.Service/Services/DebridClients/AllDebridDebridClient.cs index bb6dedf..f8d5102 100644 --- a/server/RdtClient.Service/Services/DebridClients/AllDebridDebridClient.cs +++ b/server/RdtClient.Service/Services/DebridClients/AllDebridDebridClient.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json; using RdtClient.Data.Enums; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.DebridClient; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Helpers; using File = AllDebridNET.File; using Torrent = RdtClient.Data.Models.Data.Torrent; @@ -126,30 +127,46 @@ public class AllDebridDebridClient(ILogger logger, IAllDe public async Task AddTorrentMagnet(String magnetLink) { - var result = await allDebridNetClientFactory.GetClient().Magnet.UploadMagnetAsync(magnetLink); - - if (result?.Id == null) + try { - throw new("Unable to add magnet link"); + var result = await allDebridNetClientFactory.GetClient().Magnet.UploadMagnetAsync(magnetLink); + + if (result?.Id == null) + { + throw new("Unable to add magnet link"); + } + + var resultId = result.Id.ToString() ?? throw new($"Invalid responseID {result.Id}"); + + return resultId; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); } - - var resultId = result.Id.ToString() ?? throw new($"Invalid responseID {result.Id}"); - - return resultId; } public async Task AddTorrentFile(Byte[] bytes) { - var result = await allDebridNetClientFactory.GetClient().Magnet.UploadFileAsync(bytes); - - if (result?.Id == null) + try { - throw new("Unable to add torrent file"); + var result = await allDebridNetClientFactory.GetClient().Magnet.UploadFileAsync(bytes); + + if (result?.Id == null) + { + throw new("Unable to add torrent file"); + } + + var resultId = result.Id.ToString() ?? throw new($"Invalid responseID {result.Id}"); + + return resultId; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); } - - var resultId = result.Id.ToString() ?? throw new($"Invalid responseID {result.Id}"); - - return resultId; } public Task AddNzbLink(String nzbLink) diff --git a/server/RdtClient.Service/Services/DebridClients/DebridLinkTorrentClient.cs b/server/RdtClient.Service/Services/DebridClients/DebridLinkTorrentClient.cs index a18ee6a..e75eb12 100644 --- a/server/RdtClient.Service/Services/DebridClients/DebridLinkTorrentClient.cs +++ b/server/RdtClient.Service/Services/DebridClients/DebridLinkTorrentClient.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json; using RdtClient.Data.Enums; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.DebridClient; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Helpers; using Download = RdtClient.Data.Models.Data.Download; using Torrent = DebridLinkFrNET.Models.Torrent; @@ -120,16 +121,32 @@ public class DebridLinkClient(ILogger logger, IHttpClientFacto public async Task AddTorrentMagnet(String magnetLink) { - var result = await GetClient().Seedbox.AddTorrentAsync(magnetLink); + try + { + var result = await GetClient().Seedbox.AddTorrentAsync(magnetLink); - return result.Id ?? ""; + return result.Id ?? ""; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } } public async Task AddTorrentFile(Byte[] bytes) { - var result = await GetClient().Seedbox.AddTorrentByFileAsync(bytes); + try + { + var result = await GetClient().Seedbox.AddTorrentByFileAsync(bytes); - return result.Id ?? ""; + return result.Id ?? ""; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } } public Task AddNzbLink(String nzbLink) diff --git a/server/RdtClient.Service/Services/DebridClients/PremiumizeDebridClient.cs b/server/RdtClient.Service/Services/DebridClients/PremiumizeDebridClient.cs index a95297d..a52d251 100644 --- a/server/RdtClient.Service/Services/DebridClients/PremiumizeDebridClient.cs +++ b/server/RdtClient.Service/Services/DebridClients/PremiumizeDebridClient.cs @@ -5,6 +5,7 @@ using PremiumizeNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.DebridClient; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Helpers; using Torrent = RdtClient.Data.Models.Data.Torrent; @@ -92,30 +93,46 @@ public class PremiumizeDebridClient(ILogger logger, IHtt public async Task AddTorrentMagnet(String magnetLink) { - var result = await GetClient().Transfers.CreateAsync(magnetLink, ""); - - if (result?.Id == null) + try { - throw new("Unable to add magnet link"); + var result = await GetClient().Transfers.CreateAsync(magnetLink, ""); + + if (result?.Id == null) + { + throw new("Unable to add magnet link"); + } + + var resultId = result.Id ?? throw new($"Invalid responseID {result.Id}"); + + return resultId; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); } - - var resultId = result.Id ?? throw new($"Invalid responseID {result.Id}"); - - return resultId; } public async Task AddTorrentFile(Byte[] bytes) { - var result = await GetClient().Transfers.CreateAsync(bytes, ""); - - if (result?.Id == null) + try { - throw new("Unable to add torrent file"); + var result = await GetClient().Transfers.CreateAsync(bytes, ""); + + if (result?.Id == null) + { + throw new("Unable to add torrent file"); + } + + var resultId = result.Id ?? throw new($"Invalid responseID {result.Id}"); + + return resultId; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); } - - var resultId = result.Id ?? throw new($"Invalid responseID {result.Id}"); - - return resultId; } public Task AddNzbLink(String nzbLink) diff --git a/server/RdtClient.Service/Services/DebridClients/RealDebridDebridClient.cs b/server/RdtClient.Service/Services/DebridClients/RealDebridDebridClient.cs index 5c24607..184f64d 100644 --- a/server/RdtClient.Service/Services/DebridClients/RealDebridDebridClient.cs +++ b/server/RdtClient.Service/Services/DebridClients/RealDebridDebridClient.cs @@ -5,6 +5,7 @@ using RDNET; using RdtClient.Data.Enums; using RdtClient.Data.Models.Data; using RdtClient.Data.Models.DebridClient; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Helpers; using Download = RdtClient.Data.Models.Data.Download; using Torrent = RDNET.Torrent; @@ -128,20 +129,36 @@ public class RealDebridDebridClient(ILogger logger, IHtt public async Task AddTorrentMagnet(String magnetLink) { - var timeoutCancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout)); + try + { + var timeoutCancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout)); - var result = await GetClient().Torrents.AddMagnetAsync(magnetLink, timeoutCancellationToken.Token); + var result = await GetClient().Torrents.AddMagnetAsync(magnetLink, timeoutCancellationToken.Token); - return result.Id; + return result.Id; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } } public async Task AddTorrentFile(Byte[] bytes) { - var timeoutCancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout)); + try + { + var timeoutCancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout)); - var result = await GetClient().Torrents.AddFileAsync(bytes, timeoutCancellationToken.Token); + var result = await GetClient().Torrents.AddFileAsync(bytes, timeoutCancellationToken.Token); - return result.Id; + return result.Id; + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase) || + ex.Message.Contains("rate limit exceeded", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } } public Task AddNzbLink(String nzbLink) diff --git a/server/RdtClient.Service/Services/DebridClients/TorBoxDebridClient.cs b/server/RdtClient.Service/Services/DebridClients/TorBoxDebridClient.cs index 53ccb7b..3515278 100644 --- a/server/RdtClient.Service/Services/DebridClients/TorBoxDebridClient.cs +++ b/server/RdtClient.Service/Services/DebridClients/TorBoxDebridClient.cs @@ -1,10 +1,10 @@ using System.Diagnostics; using Microsoft.Extensions.Logging; -using MonoTorrent; using Newtonsoft.Json; using RdtClient.Data.Enums; using RdtClient.Data.Models.DebridClient; using RdtClient.Data.Models.Data; +using RdtClient.Data.Models.Internal; using RdtClient.Service.Helpers; using TorBoxNET; using Torrent = RdtClient.Data.Models.Data.Torrent; @@ -25,10 +25,8 @@ public class TorBoxDebridClient(ILogger logger, IHttpClientF throw new("TorBox API Key not set in the settings"); } - var httpClient = httpClientFactory.CreateClient(); - httpClient.Timeout = TimeSpan.FromSeconds(Settings.Get.Provider.Timeout); - - var torBoxNetClient = new TorBoxNetClient(null, httpClient, 5); + var httpClient = httpClientFactory.CreateClient(DiConfig.TORBOX_CLIENT); + var torBoxNetClient = new TorBoxNetClient(null, httpClient, 1); torBoxNetClient.UseApiAuthentication(apiKey); // Get the server time to fix up the timezones on results @@ -195,52 +193,90 @@ public class TorBoxDebridClient(ILogger logger, IHttpClientF }; } + private async Task HandleAddTorrentErrors(Func> action) + { + try + { + return await action(false); + } + catch (RateLimitException) + { + throw; + } + catch (Exception ex) when (ex.InnerException is RateLimitException rateLimitException) + { + throw rateLimitException; + } + catch (TorBoxException ex) when (ex.Error.Equals("active_limit", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } + } + + private async Task HandleAddUsenetErrors(Func> action) + { + try + { + return await action(false); + } + catch (RateLimitException) + { + throw; + } + catch (Exception ex) when (ex.InnerException is RateLimitException rateLimitException) + { + throw rateLimitException; + } + catch (TorBoxException ex) when (ex.Error.Equals("active_limit", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } + catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase)) + { + throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2)); + } + } + public async Task AddTorrentMagnet(String magnetLink) { - var user = await GetClient().User.GetAsync(true); - - var result = await GetClient().Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3); - - if (result.Error == "ACTIVE_LIMIT") + return await HandleAddTorrentErrors(async asQueued => { - var magnetLinkInfo = MagnetLink.Parse(magnetLink); - - return magnetLinkInfo.InfoHashes.V1!.ToHex().ToLowerInvariant(); - } - - return result.Data!.Hash!; + var user = await GetClient().User.GetAsync(true); + var result = await GetClient().Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued); + return result.Data!.Hash!; + }); } public async Task AddTorrentFile(Byte[] bytes) { - var user = await GetClient().User.GetAsync(true); - - var result = await GetClient().Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3); - - if (result.Error == "ACTIVE_LIMIT") + return await HandleAddTorrentErrors(async asQueued => { - using var stream = new MemoryStream(bytes); - - var torrent = await MonoTorrent.Torrent.LoadAsync(stream); - - return torrent.InfoHashes.V1!.ToHex().ToLowerInvariant(); - } - - return result.Data!.Hash!; + var user = await GetClient().User.GetAsync(true); + var result = await GetClient().Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued); + return result.Data!.Hash!; + }); } public async Task AddNzbLink(String nzbLink) { - var result = await GetClient().Usenet.AddLinkAsync(nzbLink); - - return result.Data!.Hash!; + return await HandleAddUsenetErrors(async asQueued => + { + var result = await GetClient().Usenet.AddLinkAsync(nzbLink, as_queued: asQueued); + return result.Data!.Hash!; + }); } public virtual async Task AddNzbFile(Byte[] bytes, String? name) { - var result = await GetClient().Usenet.AddFileAsync(bytes, name: name); - - return result.Data!.Hash!; + return await HandleAddUsenetErrors(async asQueued => + { + var result = await GetClient().Usenet.AddFileAsync(bytes, name: name, as_queued: asQueued); + return result.Data!.Hash!; + }); } public async Task> GetAvailableFiles(String hash) diff --git a/server/RdtClient.Service/Services/RemoteService.cs b/server/RdtClient.Service/Services/RemoteService.cs index a985c9b..460b81b 100644 --- a/server/RdtClient.Service/Services/RemoteService.cs +++ b/server/RdtClient.Service/Services/RemoteService.cs @@ -79,4 +79,9 @@ public class RemoteService(IHubContext hub, Torrents torrents) { await hub.Clients.All.SendCoreAsync("diskSpaceStatus", [status]); } + + public async Task UpdateRateLimitStatus(RateLimitStatus status) + { + await hub.Clients.All.SendCoreAsync("rateLimitStatus", [status]); + } } diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index 4c40837..ab14a4b 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -11,18 +11,11 @@ using RdtClient.Service.Services.Downloaders; namespace RdtClient.Service.Services; -public class TorrentRunner(ILogger logger, Torrents torrents, Downloads downloads) +public class TorrentRunner(ILogger logger, Torrents torrents, Downloads downloads, RemoteService remoteService, IHttpClientFactory httpClientFactory) { public static readonly ConcurrentDictionary ActiveDownloadClients = new(); public static readonly ConcurrentDictionary ActiveUnpackClients = new(); - private readonly HttpClient _httpClient = new() - { - Timeout = TimeSpan.FromSeconds(10) - }; - - public static Boolean IsPausedForLowDiskSpace { get; set; } - public static (Int64 Speed, Int64 BytesTotal, Int64 BytesDone) GetStats(Guid downloadId) { if (ActiveDownloadClients.TryGetValue(downloadId, out var downloadClient)) @@ -37,7 +30,11 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow return (0, 0, 0); } - + + public static Boolean IsPausedForLowDiskSpace { get; set; } + + public static DateTimeOffset NextDequeueTime { get; private set; } = DateTimeOffset.MinValue; + public async Task Initialize() { Log("Initializing TorrentRunner"); @@ -127,7 +124,10 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow { Log("Updating Aria2 status"); - var aria2NetClient = new Aria2NetClient(Settings.Get.DownloadClient.Aria2cUrl, Settings.Get.DownloadClient.Aria2cSecret, _httpClient, 1); + var httpClient = httpClientFactory.CreateClient(); + httpClient.Timeout = TimeSpan.FromSeconds(10); + + var aria2NetClient = new Aria2NetClient(Settings.Get.DownloadClient.Aria2cUrl, Settings.Get.DownloadClient.Aria2cSecret, httpClient, 1); var allDownloads = await aria2NetClient.TellAllAsync(); @@ -349,27 +349,51 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow if (torrentsToAddToProvider.Count != 0) { - var downloadingTorrentsCount = allTorrents.Count(m => m.RdStatus is not (TorrentStatus.Queued or TorrentStatus.Finished or TorrentStatus.Error)); - - var maxParallelDownloads = Settings.Get.Provider.MaxParallelDownloads; - - logger.LogDebug("Currently downloading {downloadingTorrentCount}/{maxParallelDownloads} torrents, {queuedCount} queued.", - downloadingTorrentsCount, - maxParallelDownloads, - torrentsToAddToProvider.Count); - - var dequeueCount = maxParallelDownloads == 0 ? torrentsToAddToProvider.Count : maxParallelDownloads - downloadingTorrentsCount; - - foreach (var torrent in torrentsToAddToProvider.Take(dequeueCount)) + if (DateTimeOffset.Now < NextDequeueTime) { - try + logger.LogDebug($"Dequeuing torrents is paused until {NextDequeueTime}, {NextDequeueTime - DateTimeOffset.Now} remaining"); + } + else + { + if (NextDequeueTime != DateTimeOffset.MinValue) { - await torrents.DequeueFromDebridQueue(torrent); + NextDequeueTime = DateTimeOffset.MinValue; + + await remoteService.UpdateRateLimitStatus(new RateLimitStatus + { + NextDequeueTime = null, + SecondsRemaining = 0 + }); } - catch (Exception ex) + + var downloadingTorrentsCount = allTorrents.Count(m => m.RdStatus is not (TorrentStatus.Queued or TorrentStatus.Finished or TorrentStatus.Error)); + + var maxParallelDownloads = Settings.Get.Provider.MaxParallelDownloads; + + logger.LogDebug("Currently downloading {downloadingTorrentCount}/{maxParallelDownloads} torrents, {queuedCount} queued.", + downloadingTorrentsCount, + maxParallelDownloads, + torrentsToAddToProvider.Count); + + var dequeueCount = maxParallelDownloads == 0 ? torrentsToAddToProvider.Count : maxParallelDownloads - downloadingTorrentsCount; + + foreach (var torrent in torrentsToAddToProvider.Take(dequeueCount)) { - await torrents.UpdateComplete(torrent.TorrentId, $"Could not add to provider: {ex.Message}", DateTimeOffset.Now, true); - logger.LogWarning(ex, "Could not dequeue torrent {torrentId}", torrent.TorrentId); + try + { + await torrents.DequeueFromDebridQueue(torrent); + } + catch (RateLimitException ex) + { + await SetRateLimit(ex.RetryAfter, ex.Message); + + break; + } + catch (Exception ex) + { + await torrents.UpdateComplete(torrent.TorrentId, $"Could not add to provider: {ex.Message}", DateTimeOffset.Now, true); + logger.LogWarning(ex, "Could not dequeue torrent {torrentId}", torrent.TorrentId); + } } } } @@ -724,6 +748,19 @@ public class TorrentRunner(ILogger logger, Torrents torrents, Dow } } + public async Task SetRateLimit(TimeSpan retryAfter, String message) + { + NextDequeueTime = DateTimeOffset.Now.Add(retryAfter); + + Log($"Rate-limit reached, pausing dequeuing for {retryAfter.TotalMinutes} minutes (until {NextDequeueTime}): {message}"); + + await remoteService.UpdateRateLimitStatus(new RateLimitStatus + { + NextDequeueTime = NextDequeueTime, + SecondsRemaining = retryAfter.TotalSeconds + }); + } + private void Log(String message, Download? download, Torrent? torrent) { if (download != null) diff --git a/server/RdtClient.Web/Controllers/QBittorrentController.cs b/server/RdtClient.Web/Controllers/QBittorrentController.cs index b491eb2..1692978 100644 --- a/server/RdtClient.Web/Controllers/QBittorrentController.cs +++ b/server/RdtClient.Web/Controllers/QBittorrentController.cs @@ -14,7 +14,7 @@ namespace RdtClient.Web.Controllers; [ApiController] [Route("api/v2")] [Route("qbittorrent/api/v2")] -public class QBittorrentController(ILogger logger, QBittorrent qBittorrent) : Controller +public class QBittorrentController(ILogger logger, QBittorrent qBittorrent, IHttpClientFactory httpClientFactory) : Controller { [AllowAnonymous] [Route("auth/login")] @@ -338,7 +338,7 @@ public class QBittorrentController(ILogger logger, QBitto } else if (url.StartsWith("http")) { - var httpClient = new HttpClient(); + var httpClient = httpClientFactory.CreateClient(); var result = await httpClient.GetByteArrayAsync(url); await qBittorrent.TorrentsAddFile(result, request.Category, null); } diff --git a/server/RdtClient.Web/Controllers/TorrentsController.cs b/server/RdtClient.Web/Controllers/TorrentsController.cs index 37a815a..d1dce19 100644 --- a/server/RdtClient.Web/Controllers/TorrentsController.cs +++ b/server/RdtClient.Web/Controllers/TorrentsController.cs @@ -173,6 +173,28 @@ public class TorrentsController(ILogger logger, Torrents tor return Ok(status); } + [HttpGet] + [Route("RateLimitStatus")] + public ActionResult GetRateLimitStatus() + { + var nextDequeueTime = TorrentRunner.NextDequeueTime; + + if (nextDequeueTime < DateTimeOffset.Now) + { + return Ok(new RateLimitStatus + { + NextDequeueTime = null, + SecondsRemaining = 0 + }); + } + + return Ok(new RateLimitStatus + { + NextDequeueTime = nextDequeueTime, + SecondsRemaining = (nextDequeueTime - DateTimeOffset.Now).TotalSeconds + }); + } + /// /// Used for debugging only. Force a tick. ///