handle rate limiting and database errors by delaying downloads

.gitignore

@@ -8,3 +8,4 @@ server/RdtClient.Web/appsettings.Development.json
 data
 Dockerfile.dev
 test.bat
+.DS_Store

server/RdtClient.Data/Data/DownloadData.cs

@@ -273,7 +273,7 @@ public class DownloadData(DataContext dataContext, ILogger<DownloadData>? logger
         await dataContext.SaveChangesAsync();
     }
 
-    public async Task Reset(Guid downloadId)
+    public async Task Reset(Guid downloadId, DateTimeOffset? downloadQueued = null)
     {
         var dbDownload = await dataContext.Downloads
                                           .FirstOrDefaultAsync(m => m.DownloadId == downloadId)
@@ -282,7 +282,7 @@ public class DownloadData(DataContext dataContext, ILogger<DownloadData>? logger
         dbDownload.RetryCount = 0;
         dbDownload.Link = null;
         dbDownload.Added = DateTimeOffset.UtcNow;
-        dbDownload.DownloadQueued = DateTimeOffset.UtcNow;
+        dbDownload.DownloadQueued = downloadQueued ?? DateTimeOffset.UtcNow;
         dbDownload.DownloadStarted = null;
         dbDownload.DownloadFinished = null;
         dbDownload.UnpackingQueued = null;

server/RdtClient.Service/Services/DebridClients/TorBoxDebridClient.cs

@@ -69,7 +69,10 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
         return await HandleAddTorrentErrors(async asQueued =>
         {
             var user = await GetClient().User.GetAsync(true);
-            var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Torrents.AddMagnetAsync(magnetLink, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued);
+            var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Torrents.AddMagnetAsync(magnetLink,
+                                                                                               user.Data?.Settings?.SeedTorrents ?? 3,
+                                                                                               allowZip: Settings.Get.Provider.PreferZippedDownloads,
+                                                                                               as_queued: asQueued);
 
             return result.Data!.Hash!;
         });
@@ -80,7 +83,10 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
         return await HandleAddTorrentErrors(async asQueued =>
         {
             var user = await GetClient().User.GetAsync(true);
-            var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Torrents.AddFileAsync(bytes, user.Data?.Settings?.SeedTorrents ?? 3, as_queued: asQueued);
+            var result = await GetClient(DiConfig.TORBOX_CLIENT_SLOW).Torrents.AddFileAsync(bytes,
+                                                                                             user.Data?.Settings?.SeedTorrents ?? 3,
+                                                                                             allowZip: Settings.Get.Provider.PreferZippedDownloads,
+                                                                                             as_queued: asQueued);
 
             return result.Data!.Hash!;
         });
@@ -372,7 +378,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
             }
 
             var httpClient = httpClientFactory.CreateClient(clientId);
-            var torBoxNetClient = new TorBoxNetClient(null, httpClient);
+            var torBoxNetClient = new TorBoxNetClient(null, httpClient, retryCount: 5);
             torBoxNetClient.UseApiAuthentication(apiKey);
 
             // Get the server time to fix up the timezones on results
@@ -513,7 +519,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
         {
             throw rateLimitException;
         }
-        catch (TorBoxException ex) when ("active_limit".Equals(ex.Error, StringComparison.OrdinalIgnoreCase))
+        catch (TorBoxException ex) when (IsRateLimit(ex))
         {
             coordinator.UpdateCooldown(TorBoxApiHost, TimeSpan.FromMinutes(2));
 
@@ -541,7 +547,7 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
         {
             throw rateLimitException;
         }
-        catch (TorBoxException ex) when ("active_limit".Equals(ex.Error, StringComparison.OrdinalIgnoreCase))
+        catch (TorBoxException ex) when (IsRateLimit(ex))
         {
             coordinator.UpdateCooldown(TorBoxApiHost, TimeSpan.FromMinutes(2));
 
@@ -560,6 +566,12 @@ public class TorBoxDebridClient(ILogger<TorBoxDebridClient> logger, IHttpClientF
         return await HandleErrors(() => action(false));
     }
 
+    private static Boolean IsRateLimit(TorBoxException exception)
+    {
+        return exception.Error.Equals("RATE_LIMIT", StringComparison.OrdinalIgnoreCase)
+               || exception.Error.Equals("ACTIVE_LIMIT", StringComparison.OrdinalIgnoreCase);
+    }
+
     private async Task<String> HandleAddUsenetErrors(Func<Boolean, Task<String>> action)
     {
         return await HandleErrors(() => action(false));

server/RdtClient.Service/Services/Downloads.cs

@@ -86,8 +86,8 @@ public class Downloads(DownloadData downloadData) : IDownloads
         await downloadData.DeleteForTorrent(torrentId);
     }
 
-    public async Task Reset(Guid downloadId)
+    public async Task Reset(Guid downloadId, DateTimeOffset? downloadQueued = null)
     {
-        await downloadData.Reset(downloadId);
+        await downloadData.Reset(downloadId, downloadQueued);
     }
 }

server/RdtClient.Service/Services/IDownloads.cs

@@ -21,5 +21,5 @@ public interface IDownloads
     Task UpdateRetryCount(Guid downloadId, Int32 retryCount);
     Task UpdateRemoteId(Guid downloadId, String remoteId);
     Task DeleteForTorrent(Guid torrentId);
-    Task Reset(Guid downloadId);
+    Task Reset(Guid downloadId, DateTimeOffset? downloadQueued = null);
 }

server/RdtClient.Service/Services/TorrentRunner.cs

@@ -477,7 +477,11 @@ public class TorrentRunner(
             {
                 // Check if there are any downloads that are queued and can be started.
                 var queuedDownloads = torrent.Downloads
-                                             .Where(m => m.Completed == null && m.DownloadQueued != null && m.DownloadStarted == null && m.Error == null)
+                                             .Where(m => m.Completed == null
+                                                         && m.DownloadQueued != null
+                                                         && m.DownloadQueued <= DateTimeOffset.UtcNow
+                                                         && m.DownloadStarted == null
+                                                         && m.Error == null)
                                              .OrderBy(m => m.DownloadQueued)
                                              .ToList();
 
@@ -528,10 +532,24 @@ public class TorrentRunner(
                     {
                         logger.LogError(ex, "Cannot unrestrict link: {ex.Message}", ex.Message);
 
+                        if (download.RetryCount < torrent.DownloadRetryAttempts)
+                        {
+                            var retryCount = download.RetryCount + 1;
+                            var retryDelay = GetDownloadLinkRetryDelay(retryCount);
+                            var retryAt = DateTimeOffset.UtcNow.Add(retryDelay);
+
+                            Log($"Retrying download link generation {retryCount}/{torrent.DownloadRetryAttempts} at {retryAt:u}", download, torrent);
+
+                            await downloads.Reset(download.DownloadId, retryAt);
+                            await downloads.UpdateRetryCount(download.DownloadId, retryCount);
+                        }
+                        else
+                        {
                             await downloads.UpdateError(download.DownloadId, ex.Message);
                             await downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow);
                             download.Error = ex.Message;
                             download.Completed = DateTimeOffset.UtcNow;
+                        }
 
                         return;
                     }
@@ -782,6 +800,19 @@ public class TorrentRunner(
         });
     }
 
+    private static TimeSpan GetDownloadLinkRetryDelay(Int32 retryCount)
+    {
+        var seconds = retryCount switch
+        {
+            <= 1 => 15,
+            2 => 30,
+            3 => 60,
+            _ => 120
+        };
+
+        return TimeSpan.FromSeconds(seconds);
+    }
+
     private void Log(String message, Download? download, Torrent? torrent)
     {
         if (download != null)