From 1764bdc803635437a7d01c6a97df68b455b89bf3 Mon Sep 17 00:00:00 2001 From: Roger Far Date: Thu, 1 Apr 2021 15:45:46 -0600 Subject: [PATCH] Fixed bug where downloads sometimes got added twice. Added a retry download functionality. --- server/RdtClient.Data/Data/DownloadData.cs | 22 +- ...11556_Downloads_Add_RetryCount.Designer.cs | 421 ++++++++++++++++++ ...20210401211556_Downloads_Add_RetryCount.cs | 24 + .../Migrations/DataContextModelSnapshot.cs | 3 + server/RdtClient.Data/Models/Data/Download.cs | 2 + .../RdtClient.Service/Services/Downloads.cs | 33 +- .../Services/TorrentRunner.cs | 24 +- server/RdtClient.Service/Services/Torrents.cs | 9 +- 8 files changed, 513 insertions(+), 25 deletions(-) create mode 100644 server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.Designer.cs create mode 100644 server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.cs diff --git a/server/RdtClient.Data/Data/DownloadData.cs b/server/RdtClient.Data/Data/DownloadData.cs index d8b3c7f..11aa605 100644 --- a/server/RdtClient.Data/Data/DownloadData.cs +++ b/server/RdtClient.Data/Data/DownloadData.cs @@ -9,9 +9,8 @@ namespace RdtClient.Data.Data { public interface IDownloadData { - Task> Get(); - Task> GetForTorrent(Guid torrentId); Task GetById(Guid downloadId); + Task Get(Guid torrentId, String path); Task Add(Guid torrentId, String path); Task UpdateUnrestrictedLink(Guid downloadId, String unrestrictedLink); Task UpdateDownloadStarted(Guid downloadId, DateTimeOffset? dateTime); @@ -21,6 +20,7 @@ namespace RdtClient.Data.Data Task UpdateUnpackingFinished(Guid downloadId, DateTimeOffset? dateTime); Task UpdateCompleted(Guid downloadId, DateTimeOffset? dateTime); Task UpdateError(Guid downloadId, String error); + Task UpdateRetryCount(Guid downloadId, Int32 retryCount); Task DeleteForTorrent(Guid torrentId); } @@ -57,6 +57,11 @@ namespace RdtClient.Data.Data .FirstOrDefaultAsync(m => m.DownloadId == downloadId); } + public async Task Get(Guid torrentId, String path) + { + return await _dataContext.Downloads.FirstOrDefaultAsync(m => m.TorrentId == torrentId && m.Path == path); + } + public async Task Add(Guid torrentId, String path) { var download = new Download @@ -65,7 +70,8 @@ namespace RdtClient.Data.Data TorrentId = torrentId, Path = path, Added = DateTimeOffset.UtcNow, - DownloadQueued = DateTimeOffset.UtcNow + DownloadQueued = DateTimeOffset.UtcNow, + RetryCount = 0 }; await _dataContext.Downloads.AddAsync(download); @@ -160,6 +166,16 @@ namespace RdtClient.Data.Data await _dataContext.SaveChangesAsync(); } + public async Task UpdateRetryCount(Guid downloadId, Int32 retryCount) + { + var dbDownload = await _dataContext.Downloads + .FirstOrDefaultAsync(m => m.DownloadId == downloadId); + + dbDownload.RetryCount = retryCount; + + await _dataContext.SaveChangesAsync(); + } + public async Task DeleteForTorrent(Guid torrentId) { var downloads = await _dataContext.Downloads diff --git a/server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.Designer.cs b/server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.Designer.cs new file mode 100644 index 0000000..2421fae --- /dev/null +++ b/server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.Designer.cs @@ -0,0 +1,421 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using RdtClient.Data.Data; + +namespace RdtClient.Data.Migrations +{ + [DbContext(typeof(DataContext))] + [Migration("20210401211556_Downloads_Add_RetryCount")] + partial class Downloads_Add_RetryCount + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "5.0.3"); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("TEXT"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("AspNetRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClaimType") + .HasColumnType("TEXT"); + + b.Property("ClaimValue") + .HasColumnType("TEXT"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUser", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AccessFailedCount") + .HasColumnType("INTEGER"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("TEXT"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("EmailConfirmed") + .HasColumnType("INTEGER"); + + b.Property("LockoutEnabled") + .HasColumnType("INTEGER"); + + b.Property("LockoutEnd") + .HasColumnType("TEXT"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("PasswordHash") + .HasColumnType("TEXT"); + + b.Property("PhoneNumber") + .HasColumnType("TEXT"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("INTEGER"); + + b.Property("SecurityStamp") + .HasColumnType("TEXT"); + + b.Property("TwoFactorEnabled") + .HasColumnType("INTEGER"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("AspNetUsers"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClaimType") + .HasColumnType("TEXT"); + + b.Property("ClaimValue") + .HasColumnType("TEXT"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("TEXT"); + + b.Property("ProviderKey") + .HasColumnType("TEXT"); + + b.Property("ProviderDisplayName") + .HasColumnType("TEXT"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("TEXT"); + + b.Property("RoleId") + .HasColumnType("TEXT"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("TEXT"); + + b.Property("LoginProvider") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens"); + }); + + modelBuilder.Entity("RdtClient.Data.Models.Data.Download", b => + { + b.Property("DownloadId") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Added") + .HasColumnType("TEXT"); + + b.Property("Completed") + .HasColumnType("TEXT"); + + b.Property("DownloadFinished") + .HasColumnType("TEXT"); + + b.Property("DownloadQueued") + .HasColumnType("TEXT"); + + b.Property("DownloadStarted") + .HasColumnType("TEXT"); + + b.Property("Error") + .HasColumnType("TEXT"); + + b.Property("Link") + .HasColumnType("TEXT"); + + b.Property("Path") + .HasColumnType("TEXT"); + + b.Property("RetryCount") + .HasColumnType("INTEGER"); + + b.Property("TorrentId") + .HasColumnType("TEXT"); + + b.Property("UnpackingFinished") + .HasColumnType("TEXT"); + + b.Property("UnpackingQueued") + .HasColumnType("TEXT"); + + b.Property("UnpackingStarted") + .HasColumnType("TEXT"); + + b.HasKey("DownloadId"); + + b.HasIndex("TorrentId"); + + b.ToTable("Downloads"); + }); + + modelBuilder.Entity("RdtClient.Data.Models.Data.Setting", b => + { + b.Property("SettingId") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("SettingId"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("RdtClient.Data.Models.Data.Torrent", b => + { + b.Property("TorrentId") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Added") + .HasColumnType("TEXT"); + + b.Property("AutoDelete") + .HasColumnType("INTEGER"); + + b.Property("Category") + .HasColumnType("TEXT"); + + b.Property("Completed") + .HasColumnType("TEXT"); + + b.Property("FileOrMagnet") + .HasColumnType("TEXT"); + + b.Property("Hash") + .HasColumnType("TEXT"); + + b.Property("IsFile") + .HasColumnType("INTEGER"); + + b.Property("RdAdded") + .HasColumnType("TEXT"); + + b.Property("RdEnded") + .HasColumnType("TEXT"); + + b.Property("RdFiles") + .HasColumnType("TEXT"); + + b.Property("RdHost") + .HasColumnType("TEXT"); + + b.Property("RdId") + .HasColumnType("TEXT"); + + b.Property("RdName") + .HasColumnType("TEXT"); + + b.Property("RdProgress") + .HasColumnType("INTEGER"); + + b.Property("RdSeeders") + .HasColumnType("INTEGER"); + + b.Property("RdSize") + .HasColumnType("INTEGER"); + + b.Property("RdSpeed") + .HasColumnType("INTEGER"); + + b.Property("RdSplit") + .HasColumnType("INTEGER"); + + b.Property("RdStatus") + .HasColumnType("INTEGER"); + + b.Property("RdStatusRaw") + .HasColumnType("TEXT"); + + b.HasKey("TorrentId"); + + b.ToTable("Torrents"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("RdtClient.Data.Models.Data.Download", b => + { + b.HasOne("RdtClient.Data.Models.Data.Torrent", "Torrent") + .WithMany("Downloads") + .HasForeignKey("TorrentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Torrent"); + }); + + modelBuilder.Entity("RdtClient.Data.Models.Data.Torrent", b => + { + b.Navigation("Downloads"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.cs b/server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.cs new file mode 100644 index 0000000..08ad93f --- /dev/null +++ b/server/RdtClient.Data/Migrations/20210401211556_Downloads_Add_RetryCount.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace RdtClient.Data.Migrations +{ + public partial class Downloads_Add_RetryCount : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "RetryCount", + table: "Downloads", + type: "INTEGER", + nullable: false, + defaultValue: 0); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "RetryCount", + table: "Downloads"); + } + } +} diff --git a/server/RdtClient.Data/Migrations/DataContextModelSnapshot.cs b/server/RdtClient.Data/Migrations/DataContextModelSnapshot.cs index ac3dcac..b614ada 100644 --- a/server/RdtClient.Data/Migrations/DataContextModelSnapshot.cs +++ b/server/RdtClient.Data/Migrations/DataContextModelSnapshot.cs @@ -238,6 +238,9 @@ namespace RdtClient.Data.Migrations b.Property("Path") .HasColumnType("TEXT"); + b.Property("RetryCount") + .HasColumnType("INTEGER"); + b.Property("TorrentId") .HasColumnType("TEXT"); diff --git a/server/RdtClient.Data/Models/Data/Download.cs b/server/RdtClient.Data/Models/Data/Download.cs index 252d59f..9efed0e 100644 --- a/server/RdtClient.Data/Models/Data/Download.cs +++ b/server/RdtClient.Data/Models/Data/Download.cs @@ -30,6 +30,8 @@ namespace RdtClient.Data.Models.Data public DateTimeOffset? UnpackingFinished { get; set; } public DateTimeOffset? Completed { get; set; } + + public Int32 RetryCount { get; set; } public String Error { get; set; } diff --git a/server/RdtClient.Service/Services/Downloads.cs b/server/RdtClient.Service/Services/Downloads.cs index 19bf0c5..245a526 100644 --- a/server/RdtClient.Service/Services/Downloads.cs +++ b/server/RdtClient.Service/Services/Downloads.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Threading.Tasks; using RdtClient.Data.Data; using Download = RdtClient.Data.Models.Data.Download; @@ -8,10 +7,9 @@ namespace RdtClient.Service.Services { public interface IDownloads { - Task> Get(); - Task> GetForTorrent(Guid torrentId); Task GetById(Guid downloadId); - Task Add(Guid torrentId, String link); + Task Get(Guid torrentId, String path); + Task Add(Guid torrentId, String path); Task UpdateUnrestrictedLink(Guid downloadId, String unrestrictedLink); Task UpdateDownloadStarted(Guid downloadId, DateTimeOffset? dateTime); Task UpdateDownloadFinished(Guid downloadId, DateTimeOffset? dateTime); @@ -20,6 +18,7 @@ namespace RdtClient.Service.Services Task UpdateUnpackingFinished(Guid downloadId, DateTimeOffset? dateTime); Task UpdateCompleted(Guid downloadId, DateTimeOffset? dateTime); Task UpdateError(Guid downloadId, String error); + Task UpdateRetryCount(Guid downloadId, Int32 retryCount); Task DeleteForTorrent(Guid torrentId); } @@ -31,25 +30,20 @@ namespace RdtClient.Service.Services { _downloadData = downloadData; } - - public async Task> Get() - { - return await _downloadData.Get(); - } - - public async Task> GetForTorrent(Guid torrentId) - { - return await _downloadData.GetForTorrent(torrentId); - } - + public async Task GetById(Guid downloadId) { return await _downloadData.GetById(downloadId); } - public async Task Add(Guid torrentId, String link) + public async Task Get(Guid torrentId, String path) { - return await _downloadData.Add(torrentId, link); + return await _downloadData.Get(torrentId, path); + } + + public async Task Add(Guid torrentId, String path) + { + return await _downloadData.Add(torrentId, path); } public async Task UpdateUnrestrictedLink(Guid downloadId, String unrestrictedLink) @@ -92,6 +86,11 @@ namespace RdtClient.Service.Services await _downloadData.UpdateError(downloadId, error); } + public async Task UpdateRetryCount(Guid downloadId, Int32 retryCount) + { + await _downloadData.UpdateRetryCount(downloadId, retryCount); + } + public async Task DeleteForTorrent(Guid torrentId) { await _downloadData.DeleteForTorrent(torrentId); diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index 31efb81..435bc1a 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -19,6 +19,8 @@ namespace RdtClient.Service.Services public class TorrentRunner : ITorrentRunner { + private const Int32 RetryCount = 3; + private static DateTime _nextUpdate = DateTime.UtcNow; public static readonly ConcurrentDictionary ActiveDownloadClients = new ConcurrentDictionary(); @@ -120,10 +122,24 @@ namespace RdtClient.Service.Services { if (downloadClient.Error != null) { + // Retry the download Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}"); + var download = await _downloads.GetById(downloadId); - await _downloads.UpdateError(downloadId, downloadClient.Error); - await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + if (download.RetryCount < RetryCount) + { + Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}, retry count {download.RetryCount}/{RetryCount}"); + + await _downloads.UpdateRetryCount(downloadId, download.RetryCount + 1); + await _torrents.Download(downloadId); + } + else + { + Log.Debug($"Processing active download {downloadId}: error {downloadClient.Error}, not retrying"); + + await _downloads.UpdateError(downloadId, downloadClient.Error); + await _downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow); + } } else { @@ -335,8 +351,8 @@ namespace RdtClient.Service.Services } // RealDebrid is waiting for file selection, select which files to download. - if (torrent.RdStatus == RealDebridStatus.WaitingForFileSelection || - (torrent.RdStatus == RealDebridStatus.Finished && torrent.Downloads.Count == 0)) + if ((torrent.RdStatus == RealDebridStatus.WaitingForFileSelection || torrent.RdStatus == RealDebridStatus.Finished) && + torrent.Downloads.Count == 0) { Log.Debug($"Torrent {torrent.RdId} selecting files"); diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 9d49db4..95561fb 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -176,7 +176,13 @@ namespace RdtClient.Service.Services foreach (var file in rdTorrent.Links) { - await _downloads.Add(torrent.TorrentId, file); + // Make sure downloads don't get added multiple times + var downloadExists = await _downloads.Get(torrent.TorrentId, file); + + if (downloadExists == null) + { + await _downloads.Add(torrent.TorrentId, file); + } } } @@ -201,6 +207,7 @@ namespace RdtClient.Service.Services if (deleteData) { + await _torrentData.UpdateComplete(torrent.TorrentId, DateTimeOffset.UtcNow); await _downloads.DeleteForTorrent(torrent.TorrentId); await _torrentData.Delete(torrentId); }