danvics/rdt-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #945 from omgbeez/upstream/even-better-ratelimitng

Browse source 
feat: Improved rate limit handling
This commit is contained in:
Roger Far 2026-03-14 13:21:43 -06:00 committed by GitHub
commit a4706a5830
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 647 additions and 476 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
client/src/app/torrent-table/torrent-table.component.html
View file

@ -17,7 +17,10 @@
<div class="notification is-warning">
<strong>Debrid provider rate limit reached</strong>
<br />
New torrents will not be added until {{ rateLimitStatus.nextDequeueTime | date: 'medium' }}
Processing paused until {{ rateLimitStatus.nextDequeueTime | date: 'medium' }}
@if (rateLimitStatus.secondsRemaining > 0) {
({{ rateLimitStatus.secondsRemaining | number: '1.0-0' }} seconds remaining)
}
</div>
}
<div class="table-container">

68
server/RdtClient.Service.Test/Helpers/RateLimitCoordinatorTest.cs Normal file
View file

@ -0,0 +1,68 @@
using RdtClient.Service.Helpers;
namespace RdtClient.Service.Test.Helpers;
public class RateLimitCoordinatorTest
{
[Fact]
public void UpdateCooldown_SetsCooldown()
{
// Arrange
var coordinator = new RateLimitCoordinator();
var key = "test.host";
var delay = TimeSpan.FromMinutes(5);
// Act
coordinator.UpdateCooldown(key, delay);
var remaining = coordinator.GetRemainingCooldown(key);
// Assert
Assert.True(remaining > TimeSpan.FromMinutes(4) && remaining <= TimeSpan.FromMinutes(5));
}
[Fact]
public void GetRemainingCooldown_ReturnsZero_WhenNoCooldown()
{
// Arrange
var coordinator = new RateLimitCoordinator();
var key = "test.host";
// Act
var remaining = coordinator.GetRemainingCooldown(key);
// Assert
Assert.Equal(TimeSpan.Zero, remaining);
}
[Fact]
public void GetMaxNextAllowedAt_ReturnsMax()
{
// Arrange
var coordinator = new RateLimitCoordinator();
var now = DateTimeOffset.UtcNow;
coordinator.UpdateCooldown("host1", TimeSpan.FromMinutes(10));
coordinator.UpdateCooldown("host2", TimeSpan.FromMinutes(20));
// Act
var maxNext = coordinator.GetMaxNextAllowedAt();
// Assert
Assert.NotNull(maxNext);
Assert.True(maxNext > now.AddMinutes(19));
Assert.True(maxNext < now.AddMinutes(21));
}
[Fact]
public void GetMaxNextAllowedAt_ReturnsNull_WhenAllExpired()
{
// Arrange
var coordinator = new RateLimitCoordinator();
coordinator.UpdateCooldown("host1", TimeSpan.FromSeconds(-10));
// Act
var maxNext = coordinator.GetMaxNextAllowedAt();
// Assert
Assert.Null(maxNext);
}
}

50
server/RdtClient.Service.Test/Helpers/RateLimitHandlerTest.cs
View file

@ -1,4 +1,5 @@
using System.Net;
using Moq;
using RdtClient.Data.Models.Internal;
using RdtClient.Service.Helpers;
@ -6,11 +7,13 @@ namespace RdtClient.Service.Test.Helpers;
public class RateLimitHandlerTest
{
private readonly Mock<IRateLimitCoordinator> _coordinatorMock = new();
[Fact]
public async Task SendAsync_ThrowsRateLimitException_On429WithRetryAfter()
{
// Arrange
var handler = new RateLimitHandler
var handler = new RateLimitHandler(_coordinatorMock.Object)
{
InnerHandler = new MockHttpMessageHandler(HttpStatusCode.TooManyRequests, 3600)
};
@ -20,14 +23,16 @@ public class RateLimitHandlerTest
// Act & Assert
var ex = await Assert.ThrowsAsync<RateLimitException>(() => client.GetAsync("http://example.com"));
Assert.Equal(TimeSpan.FromSeconds(3600), ex.RetryAfter);
Assert.Equal("TorBox rate limit exceeded", ex.Message);
Assert.Contains("rate limit exceeded", ex.Message);
_coordinatorMock.Verify(m => m.UpdateCooldown("example.com", TimeSpan.FromSeconds(3600)), Times.Once);
}
[Fact]
public async Task SendAsync_ThrowsRateLimitException_On429WithoutRetryAfter()
{
// Arrange
var handler = new RateLimitHandler
var handler = new RateLimitHandler(_coordinatorMock.Object)
{
InnerHandler = new MockHttpMessageHandler(HttpStatusCode.TooManyRequests, null)
};
@ -39,6 +44,37 @@ public class RateLimitHandlerTest
Assert.Equal(TimeSpan.FromMinutes(2), ex.RetryAfter);
}
[Fact]
public async Task SendAsync_ThrowsRateLimitException_WhenCooldownActive()
{
// Arrange
_coordinatorMock.Setup(m => m.GetRemainingCooldown("example.com")).Returns(TimeSpan.FromMinutes(5));
var handler = new RateLimitHandler(_coordinatorMock.Object)
{
InnerHandler = new MockHttpMessageHandler(HttpStatusCode.OK, null)
};
var client = new HttpClient(handler);
// Act & Assert
var ex = await Assert.ThrowsAsync<RateLimitException>(() => client.GetAsync("http://example.com"));
Assert.Equal(TimeSpan.FromMinutes(5), ex.RetryAfter);
Assert.Contains("cooldown active", ex.Message);
}
[Fact]
public async Task SendAsync_DoesNotCatchTimeoutException()
{
// Arrange
var handler = new RateLimitHandler(_coordinatorMock.Object)
{
InnerHandler = new MockExceptionHandler(new TimeoutException())
};
var client = new HttpClient(handler);
// Act & Assert
await Assert.ThrowsAsync<TimeoutException>(() => client.GetAsync("http://example.com"));
}
private class MockHttpMessageHandler(HttpStatusCode statusCode, Int32? retryAfterSeconds) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
@ -53,4 +89,12 @@ public class RateLimitHandlerTest
return Task.FromResult(response);
}
}
private class MockExceptionHandler(Exception ex) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
throw ex;
}
}
}

264
server/RdtClient.Service.Test/Services/TorrentClients/TorBoxDebridClientTest.cs
View file

@ -6,6 +6,7 @@ using RdtClient.Data.Enums;
using RdtClient.Data.Models.Data;
using RdtClient.Data.Models.DebridClient;
using RdtClient.Data.Models.Internal;
using RdtClient.Service.Helpers;
using RdtClient.Service.Services;
using RdtClient.Service.Services.DebridClients;
using TorBoxNET;
@ -17,15 +18,17 @@ namespace RdtClient.Service.Test.Services.TorrentClients;
public class TorBoxDebridClientTest
{
private readonly Mock<IDownloadableFileFilter> _fileFilterMock;
private readonly Mock<IHttpClientFactory> _httpClientFactoryMock;
private readonly Mock<ILogger<TorBoxDebridClient>> _loggerMock;
private readonly Mock<IHttpClientFactory> _httpClientFactoryMock;
private readonly Mock<IRateLimitCoordinator> _coordinatorMock;
public TorBoxDebridClientTest()
{
_loggerMock = new();
_httpClientFactoryMock = new();
_fileFilterMock = new();
_coordinatorMock = new();
var httpClient = new HttpClient();
_httpClientFactoryMock.Setup(m => m.CreateClient(It.IsAny<String>())).Returns(httpClient);
@ -61,7 +64,7 @@ public class TorBoxDebridClientTest
}
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
clientMock.Protected().Setup<Task<IEnumerable<TorrentInfoResult>?>>("GetCurrentTorrents").ReturnsAsync(torrents);
clientMock.Protected().Setup<Task<IEnumerable<TorrentInfoResult>?>>("GetQueuedTorrents").ReturnsAsync(new List<TorrentInfoResult>());
clientMock.Protected().Setup<Task<IEnumerable<UsenetInfoResult>?>>("GetCurrentUsenet").ReturnsAsync(nzbs);
@ -111,7 +114,7 @@ public class TorBoxDebridClientTest
}
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
clientMock.Protected().Setup<Task<Response<List<AvailableTorrent?>>>>("GetTorrentAvailability", hash).ReturnsAsync(availability);
// Act
@ -154,7 +157,7 @@ public class TorBoxDebridClientTest
}
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
clientMock.Protected().Setup<Task<Response<List<AvailableTorrent?>>>>("GetTorrentAvailability", hash).ReturnsAsync(torrentAvailability);
clientMock.Protected().Setup<Task<Response<List<AvailableUsenet?>>>>("GetUsenetAvailability", hash).ReturnsAsync(usenetAvailability);
@ -183,7 +186,7 @@ public class TorBoxDebridClientTest
Data = new()
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
clientMock.Protected().Setup<Task<Response<List<AvailableTorrent?>>>>("GetTorrentAvailability", hash).ReturnsAsync(torrentAvailability);
clientMock.Protected().Setup<Task<Response<List<AvailableUsenet?>>>>("GetUsenetAvailability", hash).ReturnsAsync(usenetAvailability);
@ -205,11 +208,11 @@ public class TorBoxDebridClientTest
};
var torrentsApiMock = new Mock<ITorrentsApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
// Act
await clientMock.Object.Delete(torrent);
@ -229,11 +232,11 @@ public class TorBoxDebridClientTest
};
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
// Act
await clientMock.Object.Delete(torrent);
@ -255,12 +258,12 @@ public class TorBoxDebridClientTest
var link = "https://torbox.app/d/123/456";
var torrentsApiMock = new Mock<ITorrentsApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
torrentsApiMock.Setup(m => m.RequestDownloadAsync(123, 456, false, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<String>
{
@ -288,12 +291,12 @@ public class TorBoxDebridClientTest
var link = "https://torbox.app/d/123/456";
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
usenetApiMock.Setup(m => m.RequestDownloadAsync(123, 456, false, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<String>
{
@ -319,25 +322,15 @@ public class TorBoxDebridClientTest
var name = "test-nzb";
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
usenetApiMock.Setup(m => m.AddFileAsync(bytes, It.IsAny<Int32>(), name, It.IsAny<String?>(), It.IsAny<Boolean>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<UsenetAddResult>
{
Data = new()
{
Hash = "new-hash"
}
});
.ReturnsAsync(new Response<UsenetAddResult> { Data = new UsenetAddResult { Hash = "new-hash" } });
// Act
var result = await clientMock.Object.AddNzbFile(bytes, name);
@ -374,12 +367,12 @@ public class TorBoxDebridClientTest
};
var torrentsApiMock = new Mock<ITorrentsApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
torrentsApiMock.Setup(m => m.GetHashInfoAsync("test-hash", true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new TorrentInfoResult
{
@ -425,12 +418,12 @@ public class TorBoxDebridClientTest
Settings.Get.Provider.PreferZippedDownloads = true;
var torrentsApiMock = new Mock<ITorrentsApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
torrentsApiMock.Setup(m => m.GetHashInfoAsync("test-hash", true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new TorrentInfoResult
{
@ -461,12 +454,12 @@ public class TorBoxDebridClientTest
var link = "https://torbox.app/fakedl/12345/6789";
var torrentsApiMock = new Mock<ITorrentsApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
torrentsApiMock.Setup(m => m.RequestDownloadAsync(12345, 6789, false, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<String>
{
@ -493,12 +486,12 @@ public class TorBoxDebridClientTest
var link = "https://torbox.app/fakedl/12345/zip";
var torrentsApiMock = new Mock<ITorrentsApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
torrentsApiMock.Setup(m => m.RequestDownloadAsync(12345, 0, true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<String>
{
@ -535,12 +528,12 @@ public class TorBoxDebridClientTest
};
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
usenetApiMock.Setup(m => m.GetCurrentAsync(true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new List<UsenetInfoResult>
{
@ -576,12 +569,12 @@ public class TorBoxDebridClientTest
var link = "https://torbox.app/fakedl/98765/4321";
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
usenetApiMock.Setup(m => m.RequestDownloadAsync(98765, 4321, false, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<String>
{
@ -603,33 +596,20 @@ public class TorBoxDebridClientTest
var magnetLink = "magnet:?xt=urn:btih:test";
var torrentsApiMock = new Mock<ITorrentsApi>();
var userApiMock = new Mock<IUserApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
userApiMock.Setup(m => m.GetAsync(true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<User>
{
Data = new()
{
Settings = new()
{
SeedTorrents = 5
}
}
});
.ReturnsAsync(new Response<User> { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } });
torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny<Boolean>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new TorBoxException("active_limit", "Active limit reached"));
// Act & Assert
await Assert.ThrowsAsync<RateLimitException>(() => clientMock.Object.AddTorrentMagnet(magnetLink));
torrentsApiMock.Verify(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny<Boolean>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()), Times.Once);
@ -643,32 +623,19 @@ public class TorBoxDebridClientTest
var magnetLink = "magnet:?xt=urn:btih:test";
var torrentsApiMock = new Mock<ITorrentsApi>();
var userApiMock = new Mock<IUserApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
userApiMock.Setup(m => m.GetAsync(true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<User>
{
Data = new()
{
Settings = new()
{
SeedTorrents = 5
}
}
});
.ReturnsAsync(new Response<User> { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } });
torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny<Boolean>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new("slow_down"));
.ThrowsAsync(new Exception("slow_down"));
// Act & Assert
await Assert.ThrowsAsync<RateLimitException>(() => clientMock.Object.AddTorrentMagnet(magnetLink));
@ -681,29 +648,16 @@ public class TorBoxDebridClientTest
var magnetLink = "magnet:?xt=urn:btih:test";
var torrentsApiMock = new Mock<ITorrentsApi>();
var userApiMock = new Mock<IUserApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
userApiMock.Setup(m => m.GetAsync(true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<User>
{
Data = new()
{
Settings = new()
{
SeedTorrents = 5
}
}
});
.ReturnsAsync(new Response<User> { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } });
torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny<Boolean>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new RateLimitException("TorBox rate limit exceeded", TimeSpan.FromMinutes(60)));
@ -720,32 +674,19 @@ public class TorBoxDebridClientTest
var magnetLink = "magnet:?xt=urn:btih:test";
var torrentsApiMock = new Mock<ITorrentsApi>();
var userApiMock = new Mock<IUserApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
userApiMock.Setup(m => m.GetAsync(true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<User>
{
Data = new()
{
Settings = new()
{
SeedTorrents = 5
}
}
});
.ReturnsAsync(new Response<User> { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } });
torrentsApiMock.Setup(m => m.AddMagnetAsync(magnetLink, 5, It.IsAny<Boolean>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new("Wrapped", new RateLimitException("TorBox rate limit exceeded", TimeSpan.FromMinutes(60))));
.ThrowsAsync(new Exception("Wrapped", new RateLimitException("TorBox rate limit exceeded", TimeSpan.FromMinutes(60))));
// Act & Assert
var ex = await Assert.ThrowsAsync<RateLimitException>(() => clientMock.Object.AddTorrentMagnet(magnetLink));
@ -756,36 +697,18 @@ public class TorBoxDebridClientTest
public async Task AddTorrentFile_ThrowsRateLimitException_OnActiveLimit()
{
// Arrange
var bytes = new Byte[]
{
1, 2, 3
};
var bytes = new Byte[] { 1, 2, 3 };
var torrentsApiMock = new Mock<ITorrentsApi>();
var userApiMock = new Mock<IUserApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Torrents).Returns(torrentsApiMock.Object);
torBoxClientMock.Setup(m => m.User).Returns(userApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
userApiMock.Setup(m => m.GetAsync(true, It.IsAny<CancellationToken>()))
.ReturnsAsync(new Response<User>
{
Data = new()
{
Settings = new()
{
SeedTorrents = 5
}
}
});
.ReturnsAsync(new Response<User> { Data = new User { Settings = new UserSettings { SeedTorrents = 5 } } });
torrentsApiMock.Setup(m => m.AddFileAsync(bytes, 5, It.IsAny<Boolean>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new TorBoxException("active_limit", "Active limit reached"));
@ -802,17 +725,12 @@ public class TorBoxDebridClientTest
// Arrange
var nzbLink = "https://example.com/test.nzb";
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
usenetApiMock.Setup(m => m.AddLinkAsync(nzbLink, It.IsAny<Int32>(), It.IsAny<String?>(), It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new TorBoxException("active_limit", "Active limit reached"));
@ -826,24 +744,15 @@ public class TorBoxDebridClientTest
public async Task AddNzbFile_ThrowsRateLimitException_OnActiveLimit()
{
// Arrange
var bytes = new Byte[]
{
1, 2, 3
};
var bytes = new Byte[] { 1, 2, 3 };
var name = "test.nzb";
var usenetApiMock = new Mock<IUsenetApi>();
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object)
{
CallBase = true
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object) { CallBase = true };
var torBoxClientMock = new Mock<ITorBoxNetClient>();
torBoxClientMock.Setup(m => m.Usenet).Returns(usenetApiMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient").Returns(torBoxClientMock.Object);
clientMock.Protected().Setup<ITorBoxNetClient>("GetClient", ItExpr.IsAny<String>()).Returns(torBoxClientMock.Object);
usenetApiMock.Setup(m => m.AddFileAsync(bytes, It.IsAny<Int32>(), name, It.IsAny<String?>(), false, It.IsAny<CancellationToken>()))
.ThrowsAsync(new TorBoxException("active_limit", "Active limit reached"));
@ -862,14 +771,13 @@ public class TorBoxDebridClientTest
RdId = "test-rd-id",
RdStatus = TorrentStatus.Downloading
};
var torrentClientTorrent = new DebridClientTorrent
{
Status = "failed (Aborted, cannot be completed - https://sabnzbd.org/not-complete)",
Filename = "test-file"
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
// Act
var result = await clientMock.Object.UpdateData(torrent, torrentClientTorrent);
@ -895,7 +803,7 @@ public class TorBoxDebridClientTest
Filename = "test-torrent"
};
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object);
var clientMock = new Mock<TorBoxDebridClient>(_loggerMock.Object, _httpClientFactoryMock.Object, _fileFilterMock.Object, _coordinatorMock.Object);
// Act
await clientMock.Object.UpdateData(torrent, torrentClientTorrent);

34
server/RdtClient.Service/DiConfig.cs
View file

@ -6,7 +6,6 @@ using Microsoft.Extensions.DependencyInjection;
using Polly;
using Polly.Timeout;
using RateLimitHeaders.Polly;
using RdtClient.Data.Models.Internal;
using RdtClient.Service.BackgroundServices;
using RdtClient.Service.Helpers;
using RdtClient.Service.Middleware;
@ -20,6 +19,7 @@ public static class DiConfig
{
public const String RD_CLIENT = "RdClient";
public const String TORBOX_CLIENT = "TorBoxClient";
public const String TORBOX_CLIENT_SLOW = "TorBoxClientSlow";
public static readonly String UserAgent = $"rdt-client {Assembly.GetEntryAssembly()?.GetName().Version}";
public static void RegisterRdtServices(this IServiceCollection services)
@ -29,6 +29,7 @@ public static class DiConfig
services.AddSingleton<IAllDebridNetClientFactory, AllDebridNetClientFactory>();
services.AddScoped<AllDebridDebridClient>();
services.AddSingleton<IRateLimitCoordinator, RateLimitCoordinator>();
services.AddSingleton<IProcessFactory, ProcessFactory>();
services.AddSingleton<IFileSystem, FileSystem>();
@ -65,7 +66,6 @@ public static class DiConfig
public static void RegisterHttpClients(this IServiceCollection services)
{
services.AddHttpClient();
services.ConfigureHttpClientDefaults(builder =>
{
builder.ConfigureHttpClient(httpClient =>
@ -75,7 +75,7 @@ public static class DiConfig
});
services.AddTransient<RateLimitHandler>();
services.AddHttpClient(RD_CLIENT)
.AddHttpMessageHandler<RateLimitHandler>()
.AddResilienceHandler("rd_client_handler", ConfigureResiliencePipeline);
@ -83,17 +83,15 @@ public static class DiConfig
// This likely works for most providers, but should be verified and then the providers changed
// to this HTTP client for added resilience.
services.AddHttpClient(TORBOX_CLIENT)
.AddHttpMessageHandler<RateLimitHandler>()
.AddResilienceHandler("torbox_client_handler", ConfigureResiliencePipeline);
services.AddHttpClient(TORBOX_CLIENT_SLOW)
.AddHttpMessageHandler<RateLimitHandler>()
.AddResilienceHandler("torbox_client_handler_slow", ConfigureResiliencePipeline);
}
private static void ConfigureResiliencePipeline(ResiliencePipelineBuilder<HttpResponseMessage> builder)
{
builder.AddTimeout(new TimeoutStrategyOptions
{
TimeoutGenerator = _ => new(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout))
});
builder.AddRateLimitHeaders(options =>
{
options.EnableProactiveThrottling = true;
@ -116,17 +114,12 @@ public static class DiConfig
{
if (args.Outcome.Result is { StatusCode: HttpStatusCode.TooManyRequests } response)
{
var retryAfter = response.Headers.RetryAfter;
var delay = retryAfter?.Delta ?? (retryAfter?.Date.HasValue == true ? retryAfter.Date.Value - DateTimeOffset.UtcNow : TimeSpan.FromMinutes(2));
var delay = RateLimitHandler.GetRetryAfterDelay(response);
var timeout = TimeSpan.FromSeconds(Settings.Get.Provider.Timeout);
if (delay < TimeSpan.Zero)
if (delay >= timeout)
{
delay = TimeSpan.FromMinutes(2);
}
if (delay >= TimeSpan.FromSeconds(Settings.Get.Provider.Timeout))
{
throw new RateLimitException("Provider rate limit exceeded", delay);
return new ValueTask<TimeSpan?>((TimeSpan?)null);
}
return new(delay);
@ -135,5 +128,10 @@ public static class DiConfig
return new((TimeSpan?)null);
}
});
builder.AddTimeout(new TimeoutStrategyOptions
{
TimeoutGenerator = _ => new(TimeSpan.FromSeconds(Settings.Get.Provider.Timeout))
});
}
}

10
server/RdtClient.Service/Helpers/IRateLimitCoordinator.cs Normal file
View file

@ -0,0 +1,10 @@
namespace RdtClient.Service.Helpers;
public interface IRateLimitCoordinator
{
TimeSpan GetRemainingCooldown(String key);
TimeSpan GetMaxRemainingCooldown();
DateTimeOffset? GetMaxNextAllowedAt();
void UpdateCooldown(String key, TimeSpan retryAfter);
DateTimeOffset? GetNextAllowedAt(String key);
}

84
server/RdtClient.Service/Helpers/RateLimitCoordinator.cs Normal file
View file

@ -0,0 +1,84 @@
using System.Collections.Concurrent;
namespace RdtClient.Service.Helpers;
public class RateLimitCoordinator : IRateLimitCoordinator
{
private readonly ConcurrentDictionary<String, DateTimeOffset> _cooldowns = new();
public TimeSpan GetRemainingCooldown(String key)
{
if (_cooldowns.TryGetValue(key, out var nextAllowedAt))
{
var remaining = nextAllowedAt - DateTimeOffset.UtcNow;
if (remaining > TimeSpan.Zero)
{
return remaining;
}
_cooldowns.TryRemove(key, out _);
}
return TimeSpan.Zero;
}
public TimeSpan GetMaxRemainingCooldown()
{
var now = DateTimeOffset.UtcNow;
var max = TimeSpan.Zero;
foreach (var (key, nextAllowedAt) in _cooldowns)
{
var remaining = nextAllowedAt - now;
if (remaining > TimeSpan.Zero)
{
if (remaining > max)
{
max = remaining;
}
}
else
{
_cooldowns.TryRemove(key, out _);
}
}
return max;
}
public DateTimeOffset? GetMaxNextAllowedAt()
{
var now = DateTimeOffset.UtcNow;
DateTimeOffset? max = null;
foreach (var (key, nextAllowedAt) in _cooldowns)
{
if (nextAllowedAt > now)
{
if (max == null || nextAllowedAt > max)
{
max = nextAllowedAt;
}
}
else
{
_cooldowns.TryRemove(key, out _);
}
}
return max;
}
public void UpdateCooldown(String key, TimeSpan retryAfter)
{
var nextAllowedAt = DateTimeOffset.UtcNow.Add(retryAfter);
_cooldowns.AddOrUpdate(key, nextAllowedAt, (_, existing) => nextAllowedAt > existing ? nextAllowedAt : existing);
}
public DateTimeOffset? GetNextAllowedAt(String key)
{
if (_cooldowns.TryGetValue(key, out var nextAllowedAt))
{
if (nextAllowedAt > DateTimeOffset.UtcNow)
{
return nextAllowedAt;
}
_cooldowns.TryRemove(key, out _);
}
return null;
}
}

67
server/RdtClient.Service/Helpers/RateLimitHandler.cs
View file

@ -1,37 +1,58 @@
using System.Net;
using Polly.Timeout;
using Polly;
using RdtClient.Data.Models.Internal;
namespace RdtClient.Service.Helpers;
public class RateLimitHandler : DelegatingHandler
public class RateLimitHandler(IRateLimitCoordinator coordinator) : DelegatingHandler
{
public static readonly ResiliencePropertyKey<DateTimeOffset> StartTimeKey = new("StartTime");
public static TimeSpan GetRetryAfterDelay(HttpResponseMessage response)
{
var retryAfter = response.Headers.RetryAfter;
var delay = retryAfter?.Delta ?? (retryAfter?.Date.HasValue == true ? retryAfter.Date.Value - DateTimeOffset.UtcNow : TimeSpan.FromMinutes(2));
if (delay < TimeSpan.Zero)
{
delay = TimeSpan.FromMinutes(2);
}
return delay;
}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
try
var host = request.RequestUri?.Host ?? "unknown";
var cooldown = coordinator.GetRemainingCooldown(host);
if (cooldown > TimeSpan.Zero)
{
var response = await base.SendAsync(request, cancellationToken);
if (response.StatusCode == HttpStatusCode.TooManyRequests)
{
var retryAfter = response.Headers.RetryAfter;
var delay = retryAfter?.Delta ?? (retryAfter?.Date.HasValue == true ? retryAfter.Date.Value - DateTimeOffset.UtcNow : TimeSpan.FromMinutes(2));
if (delay < TimeSpan.Zero)
{
delay = TimeSpan.FromMinutes(2);
}
response.Dispose();
throw new RateLimitException("TorBox rate limit exceeded", delay);
}
return response;
throw new RateLimitException($"Rate limit cooldown active for {host}", cooldown);
}
catch (Exception ex) when (ex is TimeoutRejectedException or TaskCanceledException)
var context = request.GetResilienceContext();
if (context == null)
{
throw new RateLimitException("Provider rate limit exceeded (timeout)", TimeSpan.FromMinutes(2));
context = ResilienceContextPool.Shared.Get(cancellationToken);
request.SetResilienceContext(context);
}
if (!context.Properties.TryGetValue(StartTimeKey, out _))
{
context.Properties.Set(StartTimeKey, DateTimeOffset.UtcNow);
}
var response = await base.SendAsync(request, cancellationToken);
if (response.StatusCode == HttpStatusCode.TooManyRequests)
{
var delay = GetRetryAfterDelay(response);
coordinator.UpdateCooldown(host, delay);
response.Dispose();
throw new RateLimitException($"Provider {host} rate limit exceeded", delay);
}
return response;
}
}

462
server/RdtClient.Service/Services/DebridClients/TorBoxDebridClient.cs
View file

@ -2,8 +2,8 @@ using System.Diagnostics;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using RdtClient.Data.Enums;
using RdtClient.Data.Models.Data;
using RdtClient.Data.Models.DebridClient;
using RdtClient.Data.Models.Data;
using RdtClient.Data.Models.Internal;
using RdtClient.Service.Helpers;
using TorBoxNET;
@ -11,10 +11,149 @@ using Torrent = RdtClient.Data.Models.Data.Torrent;
namespace RdtClient.Service.Services.DebridClients;
public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientFactory httpClientFactory, IDownloadableFileFilter fileFilter) : IDebridClient
public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientFactory httpClientFactory, IDownloadableFileFilter fileFilter, IRateLimitCoordinator coordinator) : IDebridClient
{
private const String TorBoxApiHost = "api.torbox.app";
private TimeSpan? _offset;
protected virtual ITorBoxNetClient GetClient(String clientId = DiConfig.TORBOX_CLIENT)
{
try
{
var apiKey = Settings.Get.Provider.ApiKey;
if (String.IsNullOrWhiteSpace(apiKey))
{
throw new("TorBox API Key not set in the settings");
}
var httpClient = httpClientFactory.CreateClient(clientId);
var torBoxNetClient = new TorBoxNetClient(null, httpClient, 1);
torBoxNetClient.UseApiAuthentication(apiKey);
// Get the server time to fix up the timezones on results
if (_offset == null)
{
var serverTime = DateTimeOffset.UtcNow;
_offset = serverTime.Offset;
}
return torBoxNetClient;
}
catch (AggregateException ae)
{
foreach (var inner in ae.InnerExceptions)
{
logger.LogError(inner, $"The connection to TorBox has failed: {inner.Message}");
}
throw;
}
catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException)
{
logger.LogError(ex, $"The connection to TorBox has timed out: {ex.Message}");
throw;
}
catch (TaskCanceledException ex)
{
logger.LogError(ex, $"The connection to TorBox has timed out: {ex.Message}");
throw;
}
}
protected virtual async Task<IEnumerable<TorrentInfoResult>?> GetCurrentTorrents()
{
return await HandleErrors(() => GetClient().Torrents.GetCurrentAsync(true));
}
protected virtual async Task<IEnumerable<TorrentInfoResult>?> GetQueuedTorrents()
{
return await HandleErrors(() => GetClient().Torrents.GetQueuedAsync(true));
}
protected virtual async Task<IEnumerable<UsenetInfoResult>?> GetCurrentUsenet()
{
return await HandleErrors(() => GetClient().Usenet.GetCurrentAsync(true));
}
protected virtual async Task<IEnumerable<UsenetInfoResult>?> GetQueuedUsenet()
{
return await HandleErrors(() => GetClient().Usenet.GetQueuedAsync(true));
}
protected virtual async Task<Response<List<AvailableTorrent?>>> GetTorrentAvailability(String hash)
{
return await HandleErrors(() => GetClient().Torrents.GetAvailabilityAsync(hash, listFiles: true));
}
protected virtual async Task<Response<List<AvailableUsenet?>>> GetUsenetAvailability(String hash)
{
return await HandleErrors(() => GetClient().Usenet.GetAvailabilityAsync(hash, listFiles: true));
}
private DebridClientTorrent Map(TorrentInfoResult torrent)
{
return new()
{
Id = torrent.Hash,
Filename = torrent.Name,
OriginalFilename = torrent.Name,
Hash = torrent.Hash,
Bytes = torrent.Size,
OriginalBytes = torrent.Size,
Host = torrent.DownloadPresent.ToString(),
Split = 0,
Progress = (Int64)(torrent.Progress * 100.0),
Status = torrent.DownloadState,
Type = DownloadType.Torrent,
Added = ChangeTimeZone(torrent.CreatedAt)!.Value,
Files = (torrent.Files ?? []).Select(m => new DebridClientFile
{
Path = String.Join("/", m.Name.Split('/').Skip(1)),
Bytes = m.Size,
Id = m.Id,
Selected = true
}).ToList(),
Links = [],
Ended = ChangeTimeZone(torrent.UpdatedAt),
Speed = torrent.DownloadSpeed,
Seeders = torrent.Seeds,
};
}
private DebridClientTorrent Map(UsenetInfoResult usenet)
{
return new()
{
Id = usenet.Hash,
Filename = usenet.Name,
OriginalFilename = usenet.Name,
Hash = usenet.Hash,
Bytes = usenet.Size,
OriginalBytes = usenet.Size,
Host = usenet.DownloadPresent.ToString(),
Split = 0,
Progress = (Int64)(usenet.Progress * 100.0),
Status = usenet.DownloadState,
Type = DownloadType.Nzb,
Added = ChangeTimeZone(usenet.CreatedAt)!.Value,
Files = (usenet.Files ?? []).Select(m => new DebridClientFile
{
Path = String.Join("/", m.Name.Split('/').Skip(1)),
Bytes = m.Size,
Id = m.Id,
Selected = true
}).ToList(),
Links = [],
Ended = ChangeTimeZone(usenet.UpdatedAt),
Speed = usenet.DownloadSpeed,
Seeders = 0,
};
}
public async Task<IList<DebridClientTorrent>> GetDownloads()
{
var results = new List<DebridClientTorrent>();
@ -52,7 +191,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
public async Task<DebridClientUser> GetUser()
{
var user = await GetClient().User.GetAsync(false);
var user = await HandleErrors(() => GetClient().User.GetAsync(false));
return new()
{
@ -61,13 +200,74 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
};
}
private async Task<T> HandleErrors<T>(Func<Task<T>> action)
{
try
{
return await action();
}
catch (RateLimitException)
{
throw;
}
catch (Exception ex) when (ex.InnerException is RateLimitException rateLimitException)
{
throw rateLimitException;
}
catch (TorBoxException ex) when ("active_limit".Equals(ex.Error, StringComparison.OrdinalIgnoreCase))
{
coordinator.UpdateCooldown(TorBoxApiHost, TimeSpan.FromMinutes(2));
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase))
{
coordinator.UpdateCooldown(TorBoxApiHost, TimeSpan.FromMinutes(2));
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
}
private async Task HandleErrors(Func<Task> action)
{
try
{
await action();
}
catch (RateLimitException)
{
throw;
}
catch (Exception ex) when (ex.InnerException is RateLimitException rateLimitException)
{
throw rateLimitException;
}
catch (TorBoxException ex) when ("active_limit".Equals(ex.Error, StringComparison.OrdinalIgnoreCase))
{
coordinator.UpdateCooldown(TorBoxApiHost, TimeSpan.FromMinutes(2));
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase))
{
coordinator.UpdateCooldown(TorBoxApiHost, TimeSpan.FromMinutes(2));
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
}
private async Task<String> HandleAddTorrentErrors(Func<Boolean, Task<String>> action)
{
return await HandleErrors(() => action(false));
}
private async Task<String> HandleAddUsenetErrors(Func<Boolean, Task<String>> action)
{
return await HandleErrors(() => action(false));
}
public async Task<String> AddTorrentMagnet(String magnetLink)
{
return await HandleAddTorrentErrors(async asQueued =>
{
var user = await GetClient().User.GetAsync(true);
var result = await GetClient().Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued);
var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued);
return result.Data!.Hash!;
});
}
@ -77,8 +277,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
return await HandleAddTorrentErrors(async asQueued =>
{
var user = await GetClient().User.GetAsync(true);
var result = await GetClient().Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued);
var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued);
return result.Data!.Hash!;
});
}
@ -87,8 +286,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
{
return await HandleAddUsenetErrors(async asQueued =>
{
var result = await GetClient().Usenet.AddLinkAsync(nzbLink, as_queued: asQueued);
var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Usenet.AddLinkAsync(nzbLink, as_queued: asQueued);
return result.Data!.Hash!;
});
}
@ -97,8 +295,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
{
return await HandleAddUsenetErrors(async asQueued =>
{
var result = await GetClient().Usenet.AddFileAsync(bytes, name: name, as_queued: asQueued);
var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Usenet.AddFileAsync(bytes, name: name, as_queued: asQueued);
return result.Data!.Hash!;
});
}
@ -145,14 +342,17 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
return;
}
if (torrent.Type == DownloadType.Nzb)
await HandleErrors(async () =>
{
await GetClient().Usenet.ControlAsync(torrent.RdId, "delete");
}
else
{
await GetClient().Torrents.ControlAsync(torrent.RdId, "delete");
}
if (torrent.Type == DownloadType.Nzb)
{
await GetClient().Usenet.ControlAsync(torrent.RdId, "delete");
}
else
{
await GetClient().Torrents.ControlAsync(torrent.RdId, "delete");
}
});
}
public async Task<String> Unrestrict(Torrent torrent, String link)
@ -186,11 +386,11 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
if (torrent.Type == DownloadType.Nzb)
{
result = await GetClient().Usenet.RequestDownloadAsync(torrentId, fileId, zipped);
result = await HandleErrors(() => GetClient().Usenet.RequestDownloadAsync(torrentId, fileId, zipped));
}
else
{
result = await GetClient().Torrents.RequestDownloadAsync(torrentId, fileId, zipped);
result = await HandleErrors(() => GetClient().Torrents.RequestDownloadAsync(torrentId, fileId, zipped));
}
if (result.Error != null)
@ -304,13 +504,13 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
return null;
}
var usenets = await GetClient().Usenet.GetCurrentAsync(true);
var usenets = await HandleErrors(() => GetClient().Usenet.GetCurrentAsync(true));
var usenet = usenets?.FirstOrDefault(m => m.Hash == torrent.RdId);
id = (Int32?)usenet?.Id;
}
else
{
var torrentId = await GetClient().Torrents.GetHashInfoAsync(torrent.Hash, true);
var torrentId = await HandleErrors(() => GetClient().Torrents.GetHashInfoAsync(torrent.Hash, skipCache: true));
id = torrentId?.Id;
}
@ -354,192 +554,6 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
return Task.FromResult(download.FileName);
}
protected virtual ITorBoxNetClient GetClient()
{
try
{
var apiKey = Settings.Get.Provider.ApiKey;
if (String.IsNullOrWhiteSpace(apiKey))
{
throw new("TorBox API Key not set in the settings");
}
var httpClient = httpClientFactory.CreateClient(DiConfig.TORBOX_CLIENT);
var torBoxNetClient = new TorBoxNetClient(null, httpClient);
torBoxNetClient.UseApiAuthentication(apiKey);
// Get the server time to fix up the timezones on results
if (_offset == null)
{
var serverTime = DateTimeOffset.UtcNow;
_offset = serverTime.Offset;
}
return torBoxNetClient;
}
catch (AggregateException ae)
{
foreach (var inner in ae.InnerExceptions)
{
logger.LogError(inner, $"The connection to TorBox has failed: {inner.Message}");
}
throw;
}
catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException)
{
logger.LogError(ex, $"The connection to TorBox has timed out: {ex.Message}");
throw;
}
catch (TaskCanceledException ex)
{
logger.LogError(ex, $"The connection to TorBox has timed out: {ex.Message}");
throw;
}
}
protected virtual async Task<IEnumerable<TorrentInfoResult>?> GetCurrentTorrents()
{
return await GetClient().Torrents.GetCurrentAsync(true);
}
protected virtual async Task<IEnumerable<TorrentInfoResult>?> GetQueuedTorrents()
{
return await GetClient().Torrents.GetQueuedAsync(true);
}
protected virtual async Task<IEnumerable<UsenetInfoResult>?> GetCurrentUsenet()
{
return await GetClient().Usenet.GetCurrentAsync(true);
}
protected virtual async Task<IEnumerable<UsenetInfoResult>?> GetQueuedUsenet()
{
return await GetClient().Usenet.GetQueuedAsync(true);
}
protected virtual async Task<Response<List<AvailableTorrent?>>> GetTorrentAvailability(String hash)
{
return await GetClient().Torrents.GetAvailabilityAsync(hash, true);
}
protected virtual async Task<Response<List<AvailableUsenet?>>> GetUsenetAvailability(String hash)
{
return await GetClient().Usenet.GetAvailabilityAsync(hash, true);
}
private DebridClientTorrent Map(TorrentInfoResult torrent)
{
return new()
{
Id = torrent.Hash,
Filename = torrent.Name,
OriginalFilename = torrent.Name,
Hash = torrent.Hash,
Bytes = torrent.Size,
OriginalBytes = torrent.Size,
Host = torrent.DownloadPresent.ToString(),
Split = 0,
Progress = (Int64)(torrent.Progress * 100.0),
Status = torrent.DownloadState,
Type = DownloadType.Torrent,
Added = ChangeTimeZone(torrent.CreatedAt)!.Value,
Files = (torrent.Files ?? []).Select(m => new DebridClientFile
{
Path = String.Join("/", m.Name.Split('/').Skip(1)),
Bytes = m.Size,
Id = m.Id,
Selected = true
})
.ToList(),
Links = [],
Ended = ChangeTimeZone(torrent.UpdatedAt),
Speed = torrent.DownloadSpeed,
Seeders = torrent.Seeds
};
}
private DebridClientTorrent Map(UsenetInfoResult usenet)
{
return new()
{
Id = usenet.Hash,
Filename = usenet.Name,
OriginalFilename = usenet.Name,
Hash = usenet.Hash,
Bytes = usenet.Size,
OriginalBytes = usenet.Size,
Host = usenet.DownloadPresent.ToString(),
Split = 0,
Progress = (Int64)(usenet.Progress * 100.0),
Status = usenet.DownloadState,
Type = DownloadType.Nzb,
Added = ChangeTimeZone(usenet.CreatedAt)!.Value,
Files = (usenet.Files ?? []).Select(m => new DebridClientFile
{
Path = String.Join("/", m.Name.Split('/').Skip(1)),
Bytes = m.Size,
Id = m.Id,
Selected = true
})
.ToList(),
Links = [],
Ended = ChangeTimeZone(usenet.UpdatedAt),
Speed = usenet.DownloadSpeed,
Seeders = 0
};
}
private async Task<String> HandleAddTorrentErrors(Func<Boolean, Task<String>> action)
{
try
{
return await action(false);
}
catch (RateLimitException)
{
throw;
}
catch (Exception ex) when (ex.InnerException is RateLimitException rateLimitException)
{
throw rateLimitException;
}
catch (TorBoxException ex) when (ex.Error.Equals("active_limit", StringComparison.OrdinalIgnoreCase))
{
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase))
{
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
}
private async Task<String> HandleAddUsenetErrors(Func<Boolean, Task<String>> action)
{
try
{
return await action(false);
}
catch (RateLimitException)
{
throw;
}
catch (Exception ex) when (ex.InnerException is RateLimitException rateLimitException)
{
throw rateLimitException;
}
catch (TorBoxException ex) when (ex.Error.Equals("active_limit", StringComparison.OrdinalIgnoreCase))
{
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
catch (Exception ex) when (ex.Message.Contains("slow_down", StringComparison.OrdinalIgnoreCase))
{
throw new RateLimitException(ex.Message, TimeSpan.FromMinutes(2));
}
}
private DateTimeOffset? ChangeTimeZone(DateTimeOffset? dateTimeOffset)
{
@ -553,26 +567,28 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
private async Task<DebridClientTorrent?> GetInfo(String id, DownloadType type)
{
if (type == DownloadType.Nzb)
return await HandleErrors(async () =>
{
var usenet = await GetClient().Usenet.GetHashInfoAsync(id, true);
if (usenet != null)
if (type == DownloadType.Nzb)
{
return Map(usenet);
var usenet = await GetClient().Usenet.GetHashInfoAsync(id, skipCache: true);
if (usenet != null)
{
return Map(usenet);
}
}
}
else
{
var result = await GetClient().Torrents.GetHashInfoAsync(id, true);
if (result != null)
else
{
return Map(result);
}
}
var result = await GetClient().Torrents.GetHashInfoAsync(id, skipCache: true);
return null;
if (result != null)
{
return Map(result);
}
}
return null;
});
}
public static void MoveHashDirContents(String extractPath, Torrent torrent)

59
server/RdtClient.Service/Services/TorrentRunner.cs
View file

@ -11,15 +11,13 @@ using RdtClient.Service.Services.Downloaders;
namespace RdtClient.Service.Services;
public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Downloads downloads, RemoteService remoteService, IHttpClientFactory httpClientFactory)
public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Downloads downloads, RemoteService remoteService, IHttpClientFactory httpClientFactory, IRateLimitCoordinator coordinator)
{
private DateTimeOffset? _lastNextAllowedAt;
public static readonly ConcurrentDictionary<Guid, DownloadClient> ActiveDownloadClients = new();
public static readonly ConcurrentDictionary<Guid, UnpackClient> ActiveUnpackClients = new();
public static Boolean IsPausedForLowDiskSpace { get; set; }
public static DateTimeOffset NextDequeueTime { get; private set; } = DateTimeOffset.MinValue;
public static (Int64 Speed, Int64 BytesTotal, Int64 BytesDone) GetStats(Guid downloadId)
{
if (ActiveDownloadClients.TryGetValue(downloadId, out var downloadClient))
@ -35,6 +33,8 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
return (0, 0, 0);
}
public static Boolean IsPausedForLowDiskSpace { get; set; }
public async Task Initialize()
{
Log("Initializing TorrentRunner");
@ -115,6 +115,26 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
var sw = new Stopwatch();
sw.Start();
var currentNextAllowedAt = coordinator.GetMaxNextAllowedAt();
if (currentNextAllowedAt != _lastNextAllowedAt)
{
if (currentNextAllowedAt == null || currentNextAllowedAt <= DateTimeOffset.UtcNow)
{
if (_lastNextAllowedAt > DateTimeOffset.UtcNow)
{
Log("Rate-limit cooldown expired, resuming dequeuing");
await remoteService.UpdateRateLimitStatus(new()
{
NextDequeueTime = null,
SecondsRemaining = 0
});
}
}
_lastNextAllowedAt = currentNextAllowedAt;
}
if (!ActiveDownloadClients.IsEmpty || !ActiveUnpackClients.IsEmpty)
{
Log($"TorrentRunner Tick Start, {ActiveDownloadClients.Count} active downloads, {ActiveUnpackClients.Count} active unpacks");
@ -349,23 +369,13 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
if (torrentsToAddToProvider.Count != 0)
{
if (DateTimeOffset.Now < NextDequeueTime)
var nextAllowedAt = coordinator.GetMaxNextAllowedAt();
if (nextAllowedAt > DateTimeOffset.UtcNow)
{
logger.LogDebug($"Dequeuing torrents is paused until {NextDequeueTime}, {NextDequeueTime - DateTimeOffset.Now} remaining");
logger.LogDebug($"Dequeuing torrents is paused until {nextAllowedAt}, {nextAllowedAt - DateTimeOffset.Now} remaining");
}
else
{
if (NextDequeueTime != DateTimeOffset.MinValue)
{
NextDequeueTime = DateTimeOffset.MinValue;
await remoteService.UpdateRateLimitStatus(new()
{
NextDequeueTime = null,
SecondsRemaining = 0
});
}
var downloadingTorrentsCount = allTorrents.Count(m => m.RdStatus is not (TorrentStatus.Queued or TorrentStatus.Finished or TorrentStatus.Error));
var maxParallelDownloads = Settings.Get.Provider.MaxParallelDownloads;
@ -752,14 +762,19 @@ public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Dow
public async Task SetRateLimit(TimeSpan retryAfter, String message)
{
NextDequeueTime = DateTimeOffset.Now.Add(retryAfter);
coordinator.UpdateCooldown("General", retryAfter);
var nextDequeueTime = coordinator.GetMaxNextAllowedAt();
var now = DateTimeOffset.UtcNow;
var secondsRemaining = nextDequeueTime.HasValue ? (nextDequeueTime.Value - now).TotalSeconds : 0;
Log($"Rate-limit reached, pausing dequeuing for {retryAfter.TotalMinutes} minutes (until {NextDequeueTime}): {message}");
Log($"Rate-limit reached, pausing dequeuing for {retryAfter.TotalMinutes} minutes (until {nextDequeueTime}): {message}");
_lastNextAllowedAt = nextDequeueTime;
await remoteService.UpdateRateLimitStatus(new()
{
NextDequeueTime = NextDequeueTime,
SecondsRemaining = retryAfter.TotalSeconds
NextDequeueTime = nextDequeueTime,
SecondsRemaining = secondsRemaining
});
}

9
server/RdtClient.Web.Test/Controllers/TorrentsControllerNzbTest.cs
View file

@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Moq;
using RdtClient.Service.Helpers;
using RdtClient.Service.Services;
using RdtClient.Web.Controllers;
@ -9,15 +10,17 @@ namespace RdtClient.Web.Test.Controllers;
public class TorrentsControllerNzbTest
{
private readonly TorrentsController _controller;
private readonly Mock<ILogger<TorrentsController>> _loggerMock;
private readonly Mock<Torrents> _torrentsMock;
private readonly Mock<ILogger<TorrentsController>> _loggerMock;
private readonly Mock<IRateLimitCoordinator> _coordinatorMock;
private readonly TorrentsController _controller;
public TorrentsControllerNzbTest()
{
_torrentsMock = new(null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!);
_loggerMock = new();
_controller = new(_loggerMock.Object, _torrentsMock.Object, null!);
_coordinatorMock = new();
_controller = new(_loggerMock.Object, _torrentsMock.Object, null!, _coordinatorMock.Object);
}
[Fact]

8
server/RdtClient.Web/Controllers/TorrentsController.cs
View file

@ -13,7 +13,7 @@ namespace RdtClient.Web.Controllers;
[Authorize(Policy = "AuthSetting")]
[Route("Api/Torrents")]
public class TorrentsController(ILogger<TorrentsController> logger, Torrents torrents, TorrentRunner torrentRunner) : Controller
public class TorrentsController(ILogger<TorrentsController> logger, Torrents torrents, TorrentRunner torrentRunner, IRateLimitCoordinator coordinator) : Controller
{
[HttpGet]
[Route("")]
@ -191,9 +191,9 @@ public class TorrentsController(ILogger<TorrentsController> logger, Torrents tor
[Route("RateLimitStatus")]
public ActionResult<RateLimitStatus> GetRateLimitStatus()
{
var nextDequeueTime = TorrentRunner.NextDequeueTime;
var nextDequeueTime = coordinator.GetMaxNextAllowedAt();
if (nextDequeueTime < DateTimeOffset.Now)
if (nextDequeueTime == null || nextDequeueTime < DateTimeOffset.Now)
{
return Ok(new RateLimitStatus
{
@ -205,7 +205,7 @@ public class TorrentsController(ILogger<TorrentsController> logger, Torrents tor
return Ok(new RateLimitStatus
{
NextDequeueTime = nextDequeueTime,
SecondsRemaining = (nextDequeueTime - DateTimeOffset.Now).TotalSeconds
SecondsRemaining = (nextDequeueTime.Value - DateTimeOffset.Now).TotalSeconds
});
}

3
server/RdtClient.Web/Program.cs
View file

@ -52,7 +52,8 @@ if (appSettings.Logging?.File?.Path != null)
.WriteTo.Console()
.MinimumLevel.ControlledBy(Settings.LoggingLevelSwitch)
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.MinimumLevel.Override("System.Net.Http", LogEventLevel.Warning));
.MinimumLevel.Override("System.Net.Http", LogEventLevel.Warning)
.MinimumLevel.Override("Polly", LogEventLevel.Warning));
}
SelfLog.Enable(msg =>