rdt-client/server/RdtClient.Service/Services/TorrentRunner.cs

660 lines
No EOL
26 KiB
C#

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Text.Json;
using System.Web;
using Aria2NET;
using Microsoft.Extensions.Logging;
using RdtClient.Data.Enums;
using RdtClient.Data.Models.Data;
using RdtClient.Data.Models.Internal;
using RdtClient.Service.Helpers;
using RdtClient.Service.Services.Downloaders;
namespace RdtClient.Service.Services;
public class TorrentRunner(ILogger<TorrentRunner> logger, Torrents torrents, Downloads downloads, RemoteService remoteService)
{
public static readonly ConcurrentDictionary<Guid, DownloadClient> ActiveDownloadClients = new();
public static readonly ConcurrentDictionary<Guid, UnpackClient> ActiveUnpackClients = new();
private readonly HttpClient _httpClient = new()
{
Timeout = TimeSpan.FromSeconds(10)
};
public async Task Initialize()
{
Log("Initializing TorrentRunner");
var settingsCopy = JsonSerializer.Deserialize<DbSettings>(JsonSerializer.Serialize(Settings.Get));
if (settingsCopy != null)
{
settingsCopy.Provider.ApiKey = "*****";
settingsCopy.DownloadClient.Aria2cSecret = "*****";
Log(JsonSerializer.Serialize(settingsCopy));
}
// When starting up reset any pending downloads or unpackings so that they are restarted.
var torrents1 = await torrents.Get();
torrents1 = torrents1.Where(m => m.Completed == null).ToList();
Log($"Found {torrents1.Count} not completed torrents");
foreach (var torrent in torrents1)
{
foreach (var download in torrent.Downloads)
{
if (download.DownloadQueued != null && download.DownloadStarted != null && download.DownloadFinished == null && download.Error == null)
{
Log("Resetting download status", download, torrent);
await downloads.UpdateDownloadStarted(download.DownloadId, null);
}
if (download.UnpackingQueued != null && download.UnpackingStarted != null && download.UnpackingFinished == null && download.Error == null)
{
Log("Resetting unpack status", download, torrent);
await downloads.UpdateUnpackingStarted(download.DownloadId, null);
}
}
}
Log("TorrentRunner Initialized");
}
public async Task Tick()
{
if (String.IsNullOrWhiteSpace(Settings.Get.Provider.ApiKey))
{
Log($"No RealDebridApiKey set in settings");
return;
}
var settingDownloadLimit = Settings.Get.General.DownloadLimit;
if (settingDownloadLimit < 1)
{
settingDownloadLimit = 1;
}
var settingUnpackLimit = Settings.Get.General.UnpackLimit;
if (settingUnpackLimit < 1)
{
settingUnpackLimit = 1;
}
var settingDownloadPath = Settings.Get.DownloadClient.DownloadPath;
if (String.IsNullOrWhiteSpace(settingDownloadPath))
{
logger.LogError("No DownloadPath set in settings");
return;
}
var sw = new Stopwatch();
sw.Start();
if (!ActiveDownloadClients.IsEmpty || !ActiveUnpackClients.IsEmpty)
{
Log($"TorrentRunner Tick Start, {ActiveDownloadClients.Count} active downloads, {ActiveUnpackClients.Count} active unpacks");
}
if (ActiveDownloadClients.Any(m => m.Value.Type == Data.Enums.DownloadClient.Aria2c))
{
Log("Updating Aria2 status");
var aria2NetClient = new Aria2NetClient(Settings.Get.DownloadClient.Aria2cUrl, Settings.Get.DownloadClient.Aria2cSecret, _httpClient, 1);
var allDownloads = await aria2NetClient.TellAllAsync();
Log($"Found {allDownloads.Count} Aria2 downloads");
foreach (var activeDownload in ActiveDownloadClients)
{
if (activeDownload.Value.Downloader is Aria2cDownloader aria2Downloader)
{
await aria2Downloader.Update(allDownloads);
}
}
Log("Finished updating Aria2 status");
}
// Check if any torrents are finished downloading to the host, remove them from the active download list.
var completedActiveDownloads = ActiveDownloadClients.Where(m => m.Value.Finished).ToList();
if (completedActiveDownloads.Count > 0)
{
Log($"Processing {completedActiveDownloads.Count} completed downloads");
foreach (var (downloadId, downloadClient) in completedActiveDownloads)
{
var download = await downloads.GetById(downloadId);
if (download == null)
{
ActiveDownloadClients.TryRemove(downloadId, out _);
Log($"Download with ID {downloadId} not found! Removed from download queue");
continue;
}
Log("Processing download", download, download.Torrent);
if (!String.IsNullOrWhiteSpace(downloadClient.Error))
{
// Retry the download if an error is encountered.
LogError($"Download reported an error: {downloadClient.Error}", download, download.Torrent);
Log($"Download retry count {download.RetryCount}/{download.Torrent!.DownloadRetryAttempts}, torrent retry count {download.Torrent.RetryCount}/{download.Torrent.TorrentRetryAttempts}", download, download.Torrent);
if (download.RetryCount < download.Torrent.DownloadRetryAttempts)
{
Log($"Retrying download", download, download.Torrent);
await downloads.Reset(downloadId);
await downloads.UpdateRetryCount(downloadId, download.RetryCount + 1);
}
else
{
Log($"Not retrying download", download, download.Torrent);
await downloads.UpdateError(downloadId, downloadClient.Error);
await downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow);
}
}
else
{
Log($"Download finished successfully", download, download.Torrent);
await downloads.UpdateDownloadFinished(downloadId, DateTimeOffset.UtcNow);
await downloads.UpdateUnpackingQueued(downloadId, DateTimeOffset.UtcNow);
}
ActiveDownloadClients.TryRemove(downloadId, out _);
Log($"Removed from ActiveDownloadClients", download, download.Torrent);
}
}
// Check if any torrents are finished unpacking, remove them from the active unpack list.
var completedUnpacks = ActiveUnpackClients.Where(m => m.Value.Finished).ToList();
if (completedUnpacks.Count > 0)
{
Log($"Processing {completedUnpacks.Count} completed unpacks");
foreach (var (downloadId, unpackClient) in completedUnpacks)
{
var download = await downloads.GetById(downloadId);
if (download == null)
{
ActiveUnpackClients.TryRemove(downloadId, out _);
Log($"Download with ID {downloadId} not found! Removed from unpack queue");
continue;
}
if (unpackClient.Error != null)
{
Log($"Unpack reported an error: {unpackClient.Error}", download, download.Torrent);
await downloads.UpdateError(downloadId, unpackClient.Error);
await downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow);
}
else
{
Log($"Unpack finished successfully", download, download.Torrent);
await downloads.UpdateUnpackingFinished(downloadId, DateTimeOffset.UtcNow);
await downloads.UpdateCompleted(downloadId, DateTimeOffset.UtcNow);
}
ActiveUnpackClients.TryRemove(downloadId, out _);
Log($"Removed from ActiveUnpackClients", download, download.Torrent);
}
}
var torrents1 = await torrents.Get();
// Check for deleted torrents that are stuck in the ActiveDownloads or ActiveUnpacks
foreach (var activeDownload in ActiveDownloadClients)
{
var download = torrents1.SelectMany(m => m.Downloads).FirstOrDefault(m => m.DownloadId == activeDownload.Key);
if (download == null)
{
await activeDownload.Value.Cancel();
ActiveDownloadClients.TryRemove(activeDownload.Key, out _);
break;
}
}
foreach (var activeUnpacks in ActiveUnpackClients)
{
var download = torrents1.SelectMany(m => m.Downloads).FirstOrDefault(m => m.DownloadId == activeUnpacks.Key);
if (download == null)
{
activeUnpacks.Value.Cancel();
ActiveUnpackClients.TryRemove(activeUnpacks.Key, out _);
break;
}
}
// Process torrent retries
foreach (var torrent in torrents1.Where(m => m.Retry != null))
{
try
{
Log($"Retrying torrent {torrent.RetryCount}/{torrent.TorrentRetryAttempts}", torrent);
if (torrent.RetryCount > torrent.TorrentRetryAttempts)
{
await torrents.UpdateRetry(torrent.TorrentId, null, torrent.RetryCount);
Log($"Torrent reach max retry count");
continue;
}
await torrents.RetryTorrent(torrent.TorrentId, torrent.RetryCount);
}
catch (Exception ex)
{
await torrents.UpdateRetry(torrent.TorrentId, null, torrent.RetryCount);
await torrents.UpdateError(torrent.TorrentId, ex.Message);
}
}
// Process torrent errors
foreach (var torrent in torrents1.Where(m => m.Error != null && m.DeleteOnError > 0))
{
if (torrent.Completed == null)
{
continue;
}
if (torrent.Completed.Value.AddMinutes(torrent.DeleteOnError) > DateTime.UtcNow)
{
continue;
}
Log($"Removing torrent because it has been {torrent.DeleteOnError} minutes in the error state", torrent);
await torrents.Delete(torrent.TorrentId, true, true, true);
}
// Process torrent lifetime
foreach (var torrent in torrents1.Where(m => m.Downloads.Count == 0 && m.Completed == null && m.Lifetime > 0))
{
if (torrent.Added.AddMinutes(torrent.Lifetime) > DateTime.UtcNow)
{
continue;
}
Log($"Torrent has reached its {torrent.Lifetime} minutes lifetime, marking as error", torrent);
await torrents.UpdateRetry(torrent.TorrentId, null, torrent.TorrentRetryAttempts);
await torrents.UpdateComplete(torrent.TorrentId, $"Torrent lifetime of {torrent.Lifetime} minutes reached", DateTimeOffset.UtcNow, false);
}
torrents1 = await torrents.Get();
torrents1 = torrents1.Where(m => m.Completed == null).ToList();
if (torrents1.Count > 0)
{
Log($"Processing {torrents1.Count} torrents");
}
foreach (var torrent in torrents1)
{
try
{
// Check if there are any downloads that are queued and can be started.
var queuedDownloads = torrent.Downloads
.Where(m => m.Completed == null && m.DownloadQueued != null && m.DownloadStarted == null && m.Error == null)
.OrderBy(m => m.DownloadQueued)
.ToList();
Log($"Currently {queuedDownloads.Count} queued downloads and {ActiveDownloadClients.Count} total active downloads", torrent);
foreach (var download in queuedDownloads)
{
Log($"Processing to download", download, torrent);
if (ActiveDownloadClients.Count >= settingDownloadLimit)
{
Log($"Not starting download because there are already the max number of downloads active", download, torrent);
continue;
}
if (ActiveDownloadClients.ContainsKey(download.DownloadId))
{
Log($"Not starting download because this download is already active", download, torrent);
continue;
}
try
{
Log($"Unrestricting links", download, torrent);
var downloadLink = await torrents.UnrestrictLink(download.DownloadId);
download.Link = downloadLink;
}
catch (Exception ex)
{
logger.LogError(ex, "Cannot unrestrict link: {ex.Message}", ex.Message);
await downloads.UpdateError(download.DownloadId, ex.Message);
await downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow);
download.Error = ex.Message;
download.Completed = DateTimeOffset.UtcNow;
continue;
}
var downloadPath = settingDownloadPath;
if (!String.IsNullOrWhiteSpace(torrent.Category))
{
downloadPath = Path.Combine(downloadPath, torrent.Category);
}
Log($"Setting download path to {downloadPath}", download, torrent);
// Start the download process
var downloadClient = new DownloadClient(download, torrent, downloadPath);
Log($"Starting download", download, torrent);
var remoteId = await downloadClient.Start();
if (String.IsNullOrWhiteSpace(remoteId))
{
Log($"No ID received", download, torrent);
continue;
}
Log($"Received ID {remoteId}", download, torrent);
download.RemoteId = remoteId;
await downloads.UpdateRemoteId(download.DownloadId, remoteId);
Log($"Marking download as started", download, torrent);
download.DownloadStarted = DateTime.UtcNow;
await downloads.UpdateDownloadStarted(download.DownloadId, download.DownloadStarted);
ActiveDownloadClients.TryAdd(download.DownloadId, downloadClient);
}
// Check if there are any unpacks that are queued and can be started.
var queuedUnpacks = torrent.Downloads
.Where(m => m.Completed == null && m.UnpackingQueued != null && m.UnpackingStarted == null && m.Error == null)
.OrderBy(m => m.DownloadQueued)
.ToList();
foreach (var download in queuedUnpacks)
{
Log($"Starting unpack", download, torrent);
if (download.Link == null)
{
Log($"No download link found", download, torrent);
await downloads.UpdateError(download.DownloadId, "Download Link cannot be null");
await downloads.UpdateCompleted(download.DownloadId, DateTimeOffset.UtcNow);
continue;
}
// Check if the unpacking process is even needed
var uri = new Uri(download.Link);
var fileName = uri.Segments.Last();
fileName = HttpUtility.UrlDecode(fileName);
Log($"Found file name {fileName}", download, torrent);
var extension = Path.GetExtension(fileName);
if (extension != ".rar" && extension != ".zip")
{
Log($"No need to unpack, setting it as unpacked", download, torrent);
download.UnpackingStarted = DateTimeOffset.UtcNow;
download.UnpackingFinished = DateTimeOffset.UtcNow;
download.Completed = DateTimeOffset.UtcNow;
await downloads.UpdateUnpackingStarted(download.DownloadId, download.UnpackingStarted);
await downloads.UpdateUnpackingFinished(download.DownloadId, download.UnpackingFinished);
await downloads.UpdateCompleted(download.DownloadId, download.Completed);
continue;
}
// Check if we have reached the download limit, if so queue the download, but don't start it.
if (TorrentRunner.ActiveUnpackClients.Count >= settingUnpackLimit)
{
Log($"Not starting unpack because there are already the max number of unpacks active", download, torrent);
continue;
}
if (TorrentRunner.ActiveUnpackClients.ContainsKey(download.DownloadId))
{
Log($"Not starting unpack because this download is already active", download, torrent);
continue;
}
download.UnpackingStarted = DateTimeOffset.UtcNow;
await downloads.UpdateUnpackingStarted(download.DownloadId, download.UnpackingStarted);
var downloadPath = settingDownloadPath;
if (!String.IsNullOrWhiteSpace(torrent.Category))
{
downloadPath = Path.Combine(downloadPath, torrent.Category);
}
Log($"Setting unpack path to {downloadPath}", download, torrent);
// Start the unpacking process
var unpackClient = new UnpackClient(download, downloadPath);
if (TorrentRunner.ActiveUnpackClients.TryAdd(download.DownloadId, unpackClient))
{
Log($"Starting unpack", download, torrent);
unpackClient.Start();
}
}
Log("Processing", torrent);
// If torrent is erroring out on the debrid side.
if (torrent.RdStatus == TorrentStatus.Error)
{
Log($"Torrent reported an error: {torrent.RdStatusRaw}", torrent);
Log($"Torrent retry count {torrent.RetryCount}/{torrent.TorrentRetryAttempts}", torrent);
Log($"Received RealDebrid error: {torrent.RdStatusRaw}, not processing further", torrent);
await torrents.UpdateComplete(torrent.TorrentId, $"Received RealDebrid error: {torrent.RdStatusRaw}.", DateTimeOffset.UtcNow, true);
continue;
}
// Debrid provider is waiting for file selection, select which files to download.
if ((torrent.RdStatus == TorrentStatus.WaitingForFileSelection || torrent.RdStatus == TorrentStatus.Finished) &&
torrent.FilesSelected == null &&
torrent.Downloads.Count == 0)
{
Log($"Selecting files", torrent);
await torrents.SelectFiles(torrent.TorrentId);
await torrents.UpdateFilesSelected(torrent.TorrentId, DateTime.UtcNow);
}
// Debrid provider finished downloading the torrent, process the file to host.
if (torrent.RdStatus == TorrentStatus.Finished)
{
// The files are selected but there are no downloads yet, check if debrid provider has generated links yet.
if (torrent.Downloads.Count == 0 && torrent.FilesSelected != null)
{
Log($"Creating downloads", torrent);
if (torrent.HostDownloadAction == TorrentHostDownloadAction.DownloadAll)
{
await torrents.CreateDownloads(torrent.TorrentId);
}
}
}
// Check if torrent is complete, or if we don't want to download any files to the host.
if ((torrent.Downloads.Count > 0) ||
torrent.RdStatus == TorrentStatus.Finished && torrent.HostDownloadAction == TorrentHostDownloadAction.DownloadNone)
{
var completeCount = torrent.Downloads.Count(m => m.Completed != null);
var completePerc = 0;
var totalDownloadBytes = torrent.Downloads.Sum(m => m.BytesTotal);
var totalDoneBytes = torrent.Downloads.Sum(m => m.BytesDone);
if (totalDownloadBytes > 0)
{
completePerc = (Int32)((Double)totalDoneBytes / totalDownloadBytes * 100);
}
if (completeCount == torrent.Downloads.Count)
{
Log($"All downloads complete, marking torrent as complete", torrent);
await torrents.UpdateComplete(torrent.TorrentId, null, DateTimeOffset.UtcNow, true);
try
{
await torrents.RunTorrentComplete(torrent.TorrentId);
}
catch (Exception ex)
{
logger.LogError(ex.Message, "Unable to run post process: {Message}", ex.Message);
}
if (torrent.DownloadClient == Data.Enums.DownloadClient.Symlink)
{
switch (torrent.FinishedAction)
{
case TorrentFinishedAction.RemoveAllTorrents:
Log($"Force setting FinishedAction to RemoveClient as download client is Symlink and FinishedAction is RemoveAllTorrents", torrent);
torrent.FinishedAction = TorrentFinishedAction.RemoveClient;
break;
case TorrentFinishedAction.RemoveRealDebrid:
Log($"Force setting FinishedAction to TorrentFinishedAction.None as download client is Symlink and FinishedAction is RemoveRealDebrid", torrent);
torrent.FinishedAction = TorrentFinishedAction.None;
break;
}
}
switch (torrent.FinishedAction)
{
case TorrentFinishedAction.RemoveAllTorrents:
Log($"Removing torrents from debrid provider and RDT-Client, no files", torrent);
await torrents.Delete(torrent.TorrentId, true, true, false);
break;
case TorrentFinishedAction.RemoveRealDebrid:
Log($"Removing torrents from debrid provider, no files", torrent);
await torrents.Delete(torrent.TorrentId, false, true, false);
break;
case TorrentFinishedAction.RemoveClient:
Log($"Removing torrents from client, no files", torrent);
await torrents.Delete(torrent.TorrentId, true, false, false);
break;
case TorrentFinishedAction.None:
Log($"Not removing torrents or files", torrent);
break;
default:
Log($"Invalid torrent FinishedAction {torrent.FinishedAction}", torrent);
break;
}
}
else
{
Log($"Waiting for downloads to complete. {completeCount}/{torrent.Downloads.Count} complete ({completePerc}%)", torrent);
}
}
}
catch (Exception ex)
{
logger.LogError(ex.Message, "Torrent processing result in an unexpected exception: {Message}", ex.Message);
await torrents.UpdateComplete(torrent.TorrentId, ex.Message, DateTimeOffset.UtcNow, true);
}
}
await remoteService.Update();
sw.Stop();
if (sw.ElapsedMilliseconds > 1000)
{
Log($"TorrentRunner Tick End (took {sw.ElapsedMilliseconds}ms)");
}
}
private void Log(String message, Download? download, Torrent? torrent)
{
if (download != null)
{
message = $"{message} {download.ToLog()}";
}
if (torrent != null)
{
message = $"{message} {torrent.ToLog()}";
}
logger.LogDebug(message);
}
private void Log(String message, Torrent? torrent = null)
{
if (torrent != null)
{
message = $"{message} {torrent.ToLog()}";
}
logger.LogDebug(message);
}
private void LogError(String message, Download? download, Torrent? torrent)
{
if (download != null)
{
message = $"{message} {download.ToLog()}";
}
if (torrent != null)
{
message = $"{message} {torrent.ToLog()}";
}
logger.LogError(message);
}
}