From 3fe9680e3508212ad7a48652cc1d59054ee9c3ac Mon Sep 17 00:00:00 2001 From: Roger Far Date: Sat, 30 Apr 2022 12:18:30 -0600 Subject: [PATCH] Add watch folder monitor. --- .../src/app/settings/settings.component.html | 70 +++++++++++++++++- client/src/app/settings/settings.component.ts | 12 +++ server/RdtClient.Data/Data/DataContext.cs | 12 +++ server/RdtClient.Data/Data/SettingData.cs | 2 + .../Models/Internal/DbSettings.cs | 2 + .../BackgroundServices/ProviderUpdater.cs | 74 +++++++++++++++++++ .../Startup.cs | 3 +- .../TaskRunner.cs | 5 +- .../UpdateChecker.cs | 6 +- server/RdtClient.Service/DiConfig.cs | 2 + .../TorrentClients/AllDebridTorrentClient.cs | 33 ++++++--- .../TorrentClients/RealDebridTorrentClient.cs | 55 ++++++++++---- .../Services/TorrentRunner.cs | 40 ++-------- server/RdtClient.Service/Services/Torrents.cs | 1 + 14 files changed, 253 insertions(+), 64 deletions(-) create mode 100644 server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs rename server/RdtClient.Service/{Services => BackgroundServices}/Startup.cs (95%) rename server/RdtClient.Service/{Services => BackgroundServices}/TaskRunner.cs (86%) rename server/RdtClient.Service/{Services => BackgroundServices}/UpdateChecker.cs (93%) diff --git a/client/src/app/settings/settings.component.html b/client/src/app/settings/settings.component.html index 8d9cd03..7657d21 100644 --- a/client/src/app/settings/settings.component.html +++ b/client/src/app/settings/settings.component.html @@ -12,6 +12,9 @@
  • Radarr/Sonarr
    • +
  • + Watch +
  • Tests
    • @@ -70,7 +73,7 @@
    - +
    @@ -87,6 +90,27 @@
    +
    + +
    + +
    +

    + The timeout to use to connect to the provider in seconds, with a minimum of 5 seconds. Increase if you experience + timeout errors in the log. This setting is only used when there is a user connected to the web interface or if + AutoImport/AutoDelete has been enabled. +

    +
    +
    + +
    + +
    +

    + This interval is used to check Real-Debrid or AllDebrid for updates. This setting is only used when there are + active downloads. If there are no active downloads it will use this setting * 3, with a minimum of 30 seconds. +

    +
    @@ -417,6 +441,50 @@
    +
    +
    + +
    + +
    +

    + Watch this path for .torrent or .magnet files. When a file is found it will be automatically imported. +

    +
    +
    +
    + +
    + When selected, cancel and delete downloads that have been removed from Real-Debrid or AllDebrid. +
    +
    +
    +
    + +
    + +
    +

    + The timeout to use to connect to the provider in seconds, with a minimum of 5 seconds. Increase if you experience + timeout errors in the log. This setting is only used when there is a user connected to the web interface or if + AutoImport/AutoDelete has been enabled. +

    +
    +
    + +
    + +
    +

    + This interval is used to check Real-Debrid or AllDebrid for updates. This setting is only used when there are + active downloads. If there are no active downloads it will use this setting * 3, with a minimum of 30 seconds. +

    +
    +
    +
    Error saving settings: {{ error }}
    diff --git a/client/src/app/settings/settings.component.ts b/client/src/app/settings/settings.component.ts index 129c113..22b36ee 100644 --- a/client/src/app/settings/settings.component.ts +++ b/client/src/app/settings/settings.component.ts @@ -30,6 +30,8 @@ export class SettingsComponent implements OnInit { public settingProviderAutoImport: boolean; public settingProviderAutoImportCategory: string; public settingProviderAutoDelete: boolean; + public settingProviderTimeout: number; + public settingProviderCheckInterval: number; public settingRealDebridApiKey: string; public settingDownloadPath: string; public settingMappedPath: string; @@ -67,6 +69,8 @@ export class SettingsComponent implements OnInit { this.settingProviderAutoImport = this.getSetting(results, 'ProviderAutoImport') === '1'; this.settingProviderAutoImportCategory = this.getSetting(results, 'ProviderAutoImportCategory'); this.settingProviderAutoDelete = this.getSetting(results, 'ProviderAutoDelete') === '1'; + this.settingProviderTimeout = parseInt(this.getSetting(results, 'ProviderTimeout'), 10); + this.settingProviderCheckInterval = parseInt(this.getSetting(results, 'ProviderCheckInterval'), 10); this.settingRealDebridApiKey = this.getSetting(results, 'RealDebridApiKey'); this.settingLogLevel = this.getSetting(results, 'LogLevel'); this.settingDownloadPath = this.getSetting(results, 'DownloadPath'); @@ -116,6 +120,14 @@ export class SettingsComponent implements OnInit { settingId: 'ProviderAutoDelete', value: this.settingProviderAutoDelete ? '1' : '0', }, + { + settingId: 'ProviderTimeout', + value: (this.settingProviderTimeout ?? 10).toString(), + }, + { + settingId: 'ProviderCheckInterval', + value: (this.settingProviderCheckInterval ?? 10).toString(), + }, { settingId: 'RealDebridApiKey', value: this.settingRealDebridApiKey, diff --git a/server/RdtClient.Data/Data/DataContext.cs b/server/RdtClient.Data/Data/DataContext.cs index f6af667..41cf1d0 100644 --- a/server/RdtClient.Data/Data/DataContext.cs +++ b/server/RdtClient.Data/Data/DataContext.cs @@ -62,6 +62,18 @@ public class DataContext : IdentityDbContext Value = "0" }, new Setting + { + SettingId = "ProviderTimeout", + Type = "Int32", + Value = "10" + }, + new Setting + { + SettingId = "ProviderCheckInterval", + Type = "Int32", + Value = "10" + }, + new Setting { SettingId = "DeleteOnError", Type = "Int32", diff --git a/server/RdtClient.Data/Data/SettingData.cs b/server/RdtClient.Data/Data/SettingData.cs index 722c4c8..f1745d9 100644 --- a/server/RdtClient.Data/Data/SettingData.cs +++ b/server/RdtClient.Data/Data/SettingData.cs @@ -47,6 +47,8 @@ public class SettingData ProviderAutoImport = GetInt32("ProviderAutoImport"), ProviderAutoImportCategory = GetString("ProviderAutoImportCategory"), ProviderAutoDelete = GetInt32("ProviderAutoDelete"), + ProviderTimeout = GetInt32("ProviderTimeout"), + ProviderCheckInterval = GetInt32("ProviderCheckInterval"), RealDebridApiKey = GetString("RealDebridApiKey"), DownloadPath = GetString("DownloadPath"), DownloadClient = GetString("DownloadClient"), diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index fadb390..7a87b8b 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -6,6 +6,8 @@ public class DbSettings public Int32 ProviderAutoImport { get; set; } public String ProviderAutoImportCategory { get; set; } public Int32 ProviderAutoDelete { get; set; } + public Int32 ProviderTimeout { get; set; } + public Int32 ProviderCheckInterval { get; set; } public String RealDebridApiKey { get; set; } public String DownloadPath { get; set; } public String DownloadClient { get; set; } diff --git a/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs b/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs new file mode 100644 index 0000000..b8b9a9d --- /dev/null +++ b/server/RdtClient.Service/BackgroundServices/ProviderUpdater.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using RdtClient.Service.Services; + +namespace RdtClient.Service.BackgroundServices; + +public class ProviderUpdater : BackgroundService +{ + private readonly ILogger _logger; + private readonly IServiceProvider _serviceProvider; + + private static DateTime _nextUpdate = DateTime.UtcNow; + + public ProviderUpdater(ILogger logger, IServiceProvider serviceProvider) + { + _logger = logger; + _serviceProvider = serviceProvider; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); + + using var scope = _serviceProvider.CreateScope(); + var torrentService = scope.ServiceProvider.GetRequiredService(); + + _logger.LogInformation("ProviderUpdater started."); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var torrents = await torrentService.Get(); + + if (_nextUpdate < DateTime.UtcNow && ((torrents.Count > 0 && Settings.Get.ProviderAutoImport == 0) || Settings.Get.ProviderAutoImport == 1)) + { + _logger.LogDebug($"Updating torrent info from Real-Debrid"); + + var updateTime = Settings.Get.ProviderCheckInterval * 3; + + if (updateTime < 30) + { + updateTime = 30; + } + + if (RdtHub.HasConnections) + { + updateTime = Settings.Get.ProviderCheckInterval; + + if (updateTime < 5) + { + updateTime = 5; + } + } + + _nextUpdate = DateTime.UtcNow.AddSeconds(updateTime); + + await torrentService.UpdateRdData(); + + _logger.LogDebug($"Finished updating torrent info from Real-Debrid, next update in {updateTime} seconds"); + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Unexpected error occurred in ProviderUpdater: {ex.Message}"); + } + + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + } + + _logger.LogInformation("ProviderUpdater stopped."); + } +} \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Startup.cs b/server/RdtClient.Service/BackgroundServices/Startup.cs similarity index 95% rename from server/RdtClient.Service/Services/Startup.cs rename to server/RdtClient.Service/BackgroundServices/Startup.cs index fe06155..4bffd7f 100644 --- a/server/RdtClient.Service/Services/Startup.cs +++ b/server/RdtClient.Service/BackgroundServices/Startup.cs @@ -3,10 +3,11 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using RdtClient.Data.Data; +using RdtClient.Service.Services; using Serilog; using Serilog.Events; -namespace RdtClient.Service.Services; +namespace RdtClient.Service.BackgroundServices; public class Startup : IHostedService { diff --git a/server/RdtClient.Service/Services/TaskRunner.cs b/server/RdtClient.Service/BackgroundServices/TaskRunner.cs similarity index 86% rename from server/RdtClient.Service/Services/TaskRunner.cs rename to server/RdtClient.Service/BackgroundServices/TaskRunner.cs index 41dc6b4..8bb361b 100644 --- a/server/RdtClient.Service/Services/TaskRunner.cs +++ b/server/RdtClient.Service/BackgroundServices/TaskRunner.cs @@ -1,8 +1,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using RdtClient.Service.Services; -namespace RdtClient.Service.Services; +namespace RdtClient.Service.BackgroundServices; public class TaskRunner : BackgroundService { @@ -34,7 +35,7 @@ public class TaskRunner : BackgroundService } catch (Exception ex) { - _logger.LogError(ex, "Unexpected error occurred in TorrentDownloadManager.Tick"); + _logger.LogError(ex, $"Unexpected error occurred in TorrentDownloadManager.Tick: {ex.Message}"); } await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); diff --git a/server/RdtClient.Service/Services/UpdateChecker.cs b/server/RdtClient.Service/BackgroundServices/UpdateChecker.cs similarity index 93% rename from server/RdtClient.Service/Services/UpdateChecker.cs rename to server/RdtClient.Service/BackgroundServices/UpdateChecker.cs index c28f26e..12eff0a 100644 --- a/server/RdtClient.Service/Services/UpdateChecker.cs +++ b/server/RdtClient.Service/BackgroundServices/UpdateChecker.cs @@ -4,16 +4,16 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -namespace RdtClient.Service.Services; +namespace RdtClient.Service.BackgroundServices; public class UpdateChecker : BackgroundService { public static String CurrentVersion { get; private set; } public static String LatestVersion { get; private set; } - private readonly ILogger _logger; + private readonly ILogger _logger; - public UpdateChecker(ILogger logger) + public UpdateChecker(ILogger logger) { _logger = logger; } diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index 7b2bf41..8fbab83 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using RdtClient.Service.BackgroundServices; using RdtClient.Service.Services; using RdtClient.Service.Services.TorrentClients; @@ -18,6 +19,7 @@ public static class DiConfig services.AddScoped(); services.AddScoped(); + services.AddHostedService(); services.AddHostedService(); services.AddHostedService(); services.AddHostedService(); diff --git a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs index 058eca7..6ecb511 100644 --- a/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/AllDebridTorrentClient.cs @@ -24,19 +24,34 @@ public class AllDebridTorrentClient : ITorrentClient private AllDebridNETClient GetClient() { - var apiKey = Settings.Get.RealDebridApiKey; - - if (String.IsNullOrWhiteSpace(apiKey)) + try { - throw new Exception("All-Debrid API Key not set in the settings"); + var apiKey = Settings.Get.RealDebridApiKey; + + if (String.IsNullOrWhiteSpace(apiKey)) + { + throw new Exception("All-Debrid API Key not set in the settings"); + } + + var httpClient = _httpClientFactory.CreateClient(); + httpClient.Timeout = TimeSpan.FromSeconds(10); + + var allDebridNetClient = new AllDebridNETClient("RealDebridClient", apiKey); + + return allDebridNetClient; } + catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException) + { + _logger.LogError(ex, $"The connection to RealDebrid has timed out: {ex.Message}"); - var httpClient = _httpClientFactory.CreateClient(); - httpClient.Timeout = TimeSpan.FromSeconds(10); + throw; + } + catch (TaskCanceledException ex) + { + _logger.LogError(ex, $"The connection to RealDebrid has timed out: {ex.Message}"); - var allDebridNetClient = new AllDebridNETClient("RealDebridClient", apiKey); - - return allDebridNetClient; + throw; + } } private static TorrentClientTorrent Map(Magnet torrent) diff --git a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs index d1aefef..50bb2bf 100644 --- a/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs +++ b/server/RdtClient.Service/Services/TorrentClients/RealDebridTorrentClient.cs @@ -11,7 +11,7 @@ public class RealDebridTorrentClient : ITorrentClient { private readonly ILogger _logger; private readonly IHttpClientFactory _httpClientFactory; - private TimeSpan _offset; + private TimeSpan? _offset = null; public RealDebridTorrentClient(ILogger logger, IHttpClientFactory httpClientFactory) { @@ -21,24 +21,42 @@ public class RealDebridTorrentClient : ITorrentClient private RdNetClient GetClient() { - var apiKey = Settings.Get.RealDebridApiKey; - - if (String.IsNullOrWhiteSpace(apiKey)) + try { - throw new Exception("Real-Debrid API Key not set in the settings"); + var apiKey = Settings.Get.RealDebridApiKey; + + if (String.IsNullOrWhiteSpace(apiKey)) + { + throw new Exception("Real-Debrid API Key not set in the settings"); + } + + var httpClient = _httpClientFactory.CreateClient(); + httpClient.Timeout = TimeSpan.FromSeconds(Settings.Get.ProviderTimeout); + + var rdtNetClient = new RdNetClient(null, httpClient, 5); + rdtNetClient.UseApiAuthentication(apiKey); + + // Get the server time to fix up the timezones on results + if (_offset == null) + { + var serverTime = rdtNetClient.Api.GetIsoTimeAsync().Result; + _offset = serverTime.Offset; + } + + return rdtNetClient; } + catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException) + { + _logger.LogError(ex, $"The connection to RealDebrid has timed out: {ex.Message}"); - var httpClient = _httpClientFactory.CreateClient(); - httpClient.Timeout = TimeSpan.FromSeconds(10); + throw; + } + catch (TaskCanceledException ex) + { + _logger.LogError(ex, $"The connection to RealDebrid has timed out: {ex.Message}"); - var rdtNetClient = new RdNetClient(null, httpClient, 5); - rdtNetClient.UseApiAuthentication(apiKey); - - // Get the server time to fix up the timezones on results - var serverTime = rdtNetClient.Api.GetIsoTimeAsync().Result; - _offset = serverTime.Offset; - - return rdtNetClient; + throw; + } } private TorrentClientTorrent Map(Torrent torrent) @@ -334,7 +352,12 @@ public class RealDebridTorrentClient : ITorrentClient private DateTimeOffset? ChangeTimeZone(DateTimeOffset? dateTimeOffset) { - return dateTimeOffset?.Subtract(_offset).ToOffset(_offset); + if (_offset == null) + { + return dateTimeOffset; + } + + return dateTimeOffset?.Subtract(_offset.Value).ToOffset(_offset.Value); } private void Log(String message, Data.Models.Data.Torrent torrent = null) diff --git a/server/RdtClient.Service/Services/TorrentRunner.cs b/server/RdtClient.Service/Services/TorrentRunner.cs index f58d7db..65f6ddc 100644 --- a/server/RdtClient.Service/Services/TorrentRunner.cs +++ b/server/RdtClient.Service/Services/TorrentRunner.cs @@ -14,8 +14,6 @@ namespace RdtClient.Service.Services; public class TorrentRunner { - private static DateTime _nextUpdate = DateTime.UtcNow; - public static readonly ConcurrentDictionary ActiveDownloadClients = new(); public static readonly ConcurrentDictionary ActiveUnpackClients = new(); @@ -297,35 +295,6 @@ public class TorrentRunner torrents = torrents.Where(m => m.Completed == null).ToList(); - // Only poll Real-Debrid every second when a hub is connected, otherwise every 30 seconds - if (_nextUpdate < DateTime.UtcNow && ((torrents.Count > 0 && Settings.Get.ProviderAutoImport == 0) || Settings.Get.ProviderAutoImport == 1)) - { - Log($"Updating torrent info from Real-Debrid"); - -#if DEBUG - var updateTime = 0; - - _nextUpdate = DateTime.UtcNow; -#else - var updateTime = 30; - - if (RdtHub.HasConnections) - { - updateTime = 10; - } - - _nextUpdate = DateTime.UtcNow.AddSeconds(updateTime); -#endif - - await _torrents.UpdateRdData(); - - // Re-get torrents to account for updated info - torrents = await _torrents.Get(); - torrents = torrents.Where(m => m.Completed == null).ToList(); - - Log($"Finished updating torrent info from Real-Debrid, next update in {updateTime} seconds"); - } - if (torrents.Count > 0) { Log($"Processing {torrents.Count} torrents"); @@ -569,7 +538,14 @@ public class TorrentRunner break; } - await _torrents.RunTorrentComplete(torrent.TorrentId); + try + { + await _torrents.RunTorrentComplete(torrent.TorrentId); + } + catch (Exception ex) + { + _logger.LogError(ex.Message, "Unable to run post process: {Message}", ex.Message); + } } else { diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 9b0f06a..c1c14e4 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -9,6 +9,7 @@ using RdtClient.Data.Data; using RdtClient.Data.Enums; using RdtClient.Data.Models.Internal; using RdtClient.Data.Models.TorrentClient; +using RdtClient.Service.BackgroundServices; using RdtClient.Service.Helpers; using RdtClient.Service.Services.TorrentClients; using Torrent = RdtClient.Data.Models.Data.Torrent;