danvics/rdt-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add rate limiting that respect retry-after and also TorBox specific limits

Browse source
This commit is contained in:
Colin Donaldson 2025-10-28 14:37:12 +00:00
parent c51f8cb503
commit 33225e323b
2 changed files with 140 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

149
server/RdtClient.Service/DiConfig.cs
View file

@ -1,10 +1,16 @@
using System.IO.Abstractions;
using System.Net;
using System.Reflection;
using System.Threading.RateLimiting;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http.Resilience;
using Polly;
using Polly.Extensions.Http;
using Polly.Retry;
using Polly.RateLimiting;
using RdtClient.Service.BackgroundServices;
using RdtClient.Service.Middleware;
using RdtClient.Service.Services;
@ -16,8 +22,43 @@ namespace RdtClient.Service;
public static class DiConfig
{
public const String RD_CLIENT = "RdClient";
public const String TORBOX_CLIENT = "TorboxClient";
public const String TORBOX_CLIENT_CREATETORRENT = "TorboxClientCreateTorrent";
public static readonly String UserAgent = $"rdt-client {Assembly.GetEntryAssembly()?.GetName().Version}";
private static readonly SlidingWindowRateLimiter TorboxPerSecondLimiter =
new(new SlidingWindowRateLimiterOptions
{
PermitLimit = 5,
Window = TimeSpan.FromSeconds(1),
SegmentsPerWindow = 4,
QueueLimit = 5,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
AutoReplenishment = true
});
private static readonly SlidingWindowRateLimiter TorboxCreateTorrentPerMinuteLimiter =
new(new SlidingWindowRateLimiterOptions
{
PermitLimit = 10,
Window = TimeSpan.FromMinutes(1),
SegmentsPerWindow = 4,
QueueLimit = 5,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
AutoReplenishment = true
});
private static readonly SlidingWindowRateLimiter TorboxCreateTorrentPerHourLimiter =
new(new SlidingWindowRateLimiterOptions
{
PermitLimit = 60,
Window = TimeSpan.FromHours(1),
SegmentsPerWindow = 60,
QueueLimit = 5,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
AutoReplenishment = true
});
public static void RegisterRdtServices(this IServiceCollection services)
{
services.AddMemoryCache();
@ -57,21 +98,103 @@ public static class DiConfig
public static void RegisterHttpClients(this IServiceCollection services)
{
var retryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.OrResult(r => r.StatusCode == HttpStatusCode.TooManyRequests)
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)));
services.AddHttpClient();
services.ConfigureHttpClientDefaults(builder =>
var retryStrategy = new RetryStrategyOptions<HttpResponseMessage>
{
builder.ConfigureHttpClient(httpClient =>
// Transient failures to handle (network errors, 5xx, 408, 429)
ShouldHandle = static args =>
{
httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgent);
});
if (args.Outcome.Exception is HttpRequestException)
{
return ValueTask.FromResult(true);
}
if (args.Outcome.Result is HttpResponseMessage r &&
(((int)r.StatusCode >= 500) ||
r.StatusCode == HttpStatusCode.RequestTimeout ||
r.StatusCode == HttpStatusCode.TooManyRequests))
{
return ValueTask.FromResult(true);
}
return ValueTask.FromResult(false);
},
// Default backoff when Retry-After is not provided
MaxRetryAttempts = 5,
Delay = TimeSpan.FromSeconds(1),
BackoffType = DelayBackoffType.Exponential,
UseJitter = true,
// Prefer server-provided delay; otherwise use the internal backoff
DelayGenerator = static args =>
{
if (args.Outcome.Result is HttpResponseMessage resp && resp.Headers.RetryAfter is { } ra)
{
if (ra.Delta is TimeSpan delta)
{
return new ValueTask<TimeSpan?>(delta);
}
if (ra.Date is DateTimeOffset when)
{
var delay = when - DateTimeOffset.UtcNow;
return new ValueTask<TimeSpan?>(delay > TimeSpan.Zero ? delay : TimeSpan.Zero);
}
}
// null => use the configured backoff (Delay/BackoffType/UseJitter)
return new ValueTask<TimeSpan?>((TimeSpan?)null);
}
};
services.AddHttpClient(RD_CLIENT, httpClient =>
{
httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgent);
})
.AddResilienceHandler("DefaultRetryPolicy", builder =>
{
builder.AddRetry(retryStrategy);
});
services.AddHttpClient(RD_CLIENT)
.AddPolicyHandler(retryPolicy);
services.AddHttpClient(TORBOX_CLIENT, httpClient =>
{
httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgent);
})
.AddResilienceHandler("TorBox-Limits", (builder, ctx) =>
{
builder.AddRateLimiter(new RateLimiterStrategyOptions
{
// Acquire 1 permit; throw RateLimiterRejectedException if unavailable
RateLimiter = args => TorboxPerSecondLimiter.AcquireAsync(1, args.Context.CancellationToken),
// Optional notification just before rejection is thrown
OnRejected = _ => default
});
// Keep your existing retry here so transient failures still retry
builder.AddRetry(retryStrategy);
});
services.AddHttpClient(TORBOX_CLIENT_CREATETORRENT, httpClient =>
{
httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgent);
})
.AddResilienceHandler("TorBox-CreateTorrent-Limits", (builder, ctx) =>
{
// First limiter: 10 per minute
builder.AddRateLimiter(new RateLimiterStrategyOptions
{
RateLimiter = args => TorboxCreateTorrentPerMinuteLimiter.AcquireAsync(1, args.Context.CancellationToken),
OnRejected = _ => default
});
// Second limiter: 60 per hour
builder.AddRateLimiter(new RateLimiterStrategyOptions
{
RateLimiter = args => TorboxCreateTorrentPerHourLimiter.AcquireAsync(1, args.Context.CancellationToken),
OnRejected = _ => default
});
builder.AddRetry(retryStrategy);
});
}
}

8
server/RdtClient.Service/Services/TorrentClients/TorBoxTorrentClient.cs
View file

@ -12,7 +12,7 @@ namespace RdtClient.Service.Services.TorrentClients;
public class TorBoxTorrentClient(ILogger<TorBoxTorrentClient> logger, IHttpClientFactory httpClientFactory, IDownloadableFileFilter fileFilter) : ITorrentClient
{
private TimeSpan? _offset;
private TorBoxNetClient GetClient()
private TorBoxNetClient GetClient(String? client = null)
{
try
{
@ -23,7 +23,7 @@ public class TorBoxTorrentClient(ILogger<TorBoxTorrentClient> logger, IHttpClien
throw new("TorBox API Key not set in the settings");
}
var httpClient = httpClientFactory.CreateClient();
var httpClient = httpClientFactory.CreateClient(client ?? DiConfig.TORBOX_CLIENT);
httpClient.Timeout = TimeSpan.FromSeconds(Settings.Get.Provider.Timeout);
var torBoxNetClient = new TorBoxNetClient(null, httpClient, 5);
@ -124,7 +124,7 @@ public class TorBoxTorrentClient(ILogger<TorBoxTorrentClient> logger, IHttpClien
{
var user = await GetClient().User.GetAsync(true);
var result = await GetClient().Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3, false);
var result = await GetClient(DiConfig.TORBOX_CLIENT_CREATETORRENT).Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3, false);
if (result.Error == "ACTIVE_LIMIT")
{
@ -139,7 +139,7 @@ public class TorBoxTorrentClient(ILogger<TorBoxTorrentClient> logger, IHttpClien
{
var user = await GetClient().User.GetAsync(true);
var result = await GetClient().Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3);
var result = await GetClient(DiConfig.TORBOX_CLIENT_CREATETORRENT).Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3);
if (result.Error == "ACTIVE_LIMIT")
{
using var stream = new MemoryStream(bytes);