From e6a22f6780a1ba6bd88737c953211869b5c03770 Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Mon, 12 May 2025 01:23:13 +1000 Subject: [PATCH 1/9] Magnet Enrichment Add magnet link enrichment using configurable tracker lists from ngosang/trackerslist --- .../RdtClient.Data/Enums/MagnetTrackerList.cs | 36 ++++++++++++ .../Models/Internal/DbSettings.cs | 6 +- server/RdtClient.Service/Services/Torrents.cs | 56 +++++++++++++++---- 3 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 server/RdtClient.Data/Enums/MagnetTrackerList.cs diff --git a/server/RdtClient.Data/Enums/MagnetTrackerList.cs b/server/RdtClient.Data/Enums/MagnetTrackerList.cs new file mode 100644 index 0000000..5a0dab2 --- /dev/null +++ b/server/RdtClient.Data/Enums/MagnetTrackerList.cs @@ -0,0 +1,36 @@ +using System.ComponentModel; + +namespace RdtClient.Data.Enums; + +public enum MagnetTrackerEnrichment +{ + [Description("None (do not modify magnet links)")] + None, + + [Description("Best trackers")] + trackers_best, + + [Description("All trackers")] + trackers_all, + + [Description("All UDP trackers")] + trackers_all_udp, + + [Description("All HTTP trackers")] + trackers_all_http, + + [Description("All HTTPS trackers")] + trackers_all_https, + + [Description("All WebSocket (WS) trackers")] + trackers_all_ws, + + [Description("Best IP-only trackers")] + trackers_best_ip, + + [Description("All I2P trackers")] + trackers_all_i2p, + + [Description("All IP-only trackers")] + trackers_all_ip +} diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 14c8606..136b6b3 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -74,7 +74,11 @@ Supports the following parameters: [DisplayName("Copy added torrent files")] [Description("When a torrent file or magnet is added, create a copy in this directory.")] public String? CopyAddedTorrents { get; set; } = null; - + + [DisplayName("Magnet tracker enrichment")] + [Description(@"Choose which tracker list to append to magnet links, see this README for more info.")] + public MagnetTrackerEnrichment MagnetTrackerEnrichment { get; set; } = MagnetTrackerEnrichment.None; + [DisplayName("Disable update notifications")] [Description("Ignore update notifications. You will still be notified if the version you are running has a security vulnerability.")] public Boolean DisableUpdateNotifications { get; set; } = false; diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 1e200bf..ec18b0e 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -3,6 +3,7 @@ using System.IO.Abstractions; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Web; using Microsoft.Extensions.Logging; using MonoTorrent; using RdtClient.Data.Data; @@ -31,7 +32,6 @@ public class Torrents( TorBoxTorrentClient torBoxTorrentClient) { private static readonly SemaphoreSlim RealDebridUpdateLock = new(1, 1); - private static readonly JsonSerializerOptions JsonSerializerOptions = new() { ReferenceHandler = ReferenceHandler.IgnoreCycles @@ -107,13 +107,51 @@ public class Torrents( await torrentData.UpdateCategory(torrent.TorrentId, category); } - public async Task AddMagnetToDebridQueue(String magnetLink, Torrent torrent) + public static async Task EnrichMagnetLink(string magnetLink) { - MagnetLink magnet; + var enrichment = Settings.Get.General.MagnetTrackerEnrichment; + if (enrichment == MagnetTrackerEnrichment.None) { return magnetLink; } + var url = $"https://github.com/ngosang/trackerslist/raw/refs/heads/master/{enrichment}.txt"; + var newTrackers = (await new HttpClient().GetStringAsync(url)).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + + var originalUri = new Uri(magnetLink); + var decodedParams = HttpUtility.ParseQueryString(originalUri.Query); + var rawXt = decodedParams["xt"]!; + decodedParams.Remove("xt"); + + var existingTrackers = decodedParams.GetValues("tr") ?? Array.Empty(); + decodedParams.Remove("tr"); + var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); + + var pieces = new List { $"xt={rawXt}" }; + var allKeys = decodedParams.AllKeys; + if (allKeys != null) + { + foreach (string? key in allKeys) + { + if (key == null) continue; + foreach (var val in decodedParams.GetValues(key)!) + { + pieces.Add($"{key}={HttpUtility.UrlEncode(val)}"); + } + } + } + foreach (var tr in allTrackers) { pieces.Add($"tr={HttpUtility.UrlEncode(tr)}"); } + + var basePart = originalUri.GetLeftPart(UriPartial.Path); + var enriched = $"{basePart}?{string.Join("&", pieces)}"; + + return enriched; + } + + public async Task AddMagnetToDebridQueue(string magnetLink, Torrent torrent) + { + var enriched = await EnrichMagnetLink(magnetLink); + MagnetLink magnet; try { - magnet = MagnetLink.Parse(magnetLink); + magnet = MagnetLink.Parse(enriched); } catch (Exception ex) { @@ -125,12 +163,10 @@ public class Torrents( torrent.RdName = magnet.Name; var hash = magnet.InfoHashes.V1OrV2.ToHex(); - - var newTorrent = await AddQueued(hash, magnetLink, false, torrent); + var newTorrent = await AddQueued(hash, enriched, false, torrent); Log($"Adding {hash} (magnet link) to queue", newTorrent); - - await CopyAddedTorrent(magnet.Name!, magnetLink); + await CopyAddedTorrent(magnet.Name!, enriched); return newTorrent; } @@ -224,7 +260,7 @@ public class Torrents( { throw new("Torrent has no torrent file or magnet link"); } - + logger.LogDebug("Adding {hash} to debrid provider {torrentInfo}", torrent.Hash, torrent.ToLog()); var id = torrent.IsFile @@ -426,7 +462,7 @@ public class Torrents( { var download = await downloads.GetById(downloadId) ?? throw new($"Download with ID {downloadId} not found"); - Log($"Unrestricting link", download, download.Torrent); + Log("Unrestricting link", download, download.Torrent); var unrestrictedLink = await TorrentClient.Unrestrict(download.Path); From ae66b40a3d4d40edf7fc928ecb97a5186515c580 Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Fri, 16 May 2025 07:57:48 +1000 Subject: [PATCH 2/9] Allow user to define tracker list --- .../RdtClient.Data/Enums/MagnetTrackerList.cs | 36 ------------ .../Models/Internal/DbSettings.cs | 6 +- .../Services/TorrentsTest.cs | 8 ++- server/RdtClient.Service/DiConfig.cs | 1 + .../Services/ITrackerListGrabber.cs | 6 ++ server/RdtClient.Service/Services/Torrents.cs | 56 +++++++++---------- .../Services/TrackerListGrabber.cs | 34 +++++++++++ 7 files changed, 76 insertions(+), 71 deletions(-) delete mode 100644 server/RdtClient.Data/Enums/MagnetTrackerList.cs create mode 100644 server/RdtClient.Service/Services/ITrackerListGrabber.cs create mode 100644 server/RdtClient.Service/Services/TrackerListGrabber.cs diff --git a/server/RdtClient.Data/Enums/MagnetTrackerList.cs b/server/RdtClient.Data/Enums/MagnetTrackerList.cs deleted file mode 100644 index 5a0dab2..0000000 --- a/server/RdtClient.Data/Enums/MagnetTrackerList.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.ComponentModel; - -namespace RdtClient.Data.Enums; - -public enum MagnetTrackerEnrichment -{ - [Description("None (do not modify magnet links)")] - None, - - [Description("Best trackers")] - trackers_best, - - [Description("All trackers")] - trackers_all, - - [Description("All UDP trackers")] - trackers_all_udp, - - [Description("All HTTP trackers")] - trackers_all_http, - - [Description("All HTTPS trackers")] - trackers_all_https, - - [Description("All WebSocket (WS) trackers")] - trackers_all_ws, - - [Description("Best IP-only trackers")] - trackers_best_ip, - - [Description("All I2P trackers")] - trackers_all_i2p, - - [Description("All IP-only trackers")] - trackers_all_ip -} diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 136b6b3..0d6e693 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -75,9 +75,9 @@ Supports the following parameters: [Description("When a torrent file or magnet is added, create a copy in this directory.")] public String? CopyAddedTorrents { get; set; } = null; - [DisplayName("Magnet tracker enrichment")] - [Description(@"Choose which tracker list to append to magnet links, see this README for more info.")] - public MagnetTrackerEnrichment MagnetTrackerEnrichment { get; set; } = MagnetTrackerEnrichment.None; + [DisplayName("Magnet enrichment list")] + [Description("Optional. Specify the URL of a tracker list file to be appended to magnet links.")] + public String? MagnetTrackerEnrichment { get; set; } = null; [DisplayName("Disable update notifications")] [Description("Ignore update notifications. You will still be notified if the version you are running has a security vulnerability.")] diff --git a/server/RdtClient.Service.Test/Services/TorrentsTest.cs b/server/RdtClient.Service.Test/Services/TorrentsTest.cs index addd45d..90fe94d 100644 --- a/server/RdtClient.Service.Test/Services/TorrentsTest.cs +++ b/server/RdtClient.Service.Test/Services/TorrentsTest.cs @@ -20,11 +20,13 @@ class Mocks public readonly Mock> TorrentsLoggerMock; public readonly Mock DownloadsMock; public readonly Mock TorrentDataMock; + public readonly Mock TrackerListGrabberMock; public Mocks() { TorrentDataMock = new(); DownloadsMock = new(); + TrackerListGrabberMock = new(); TorrentsLoggerMock = new(); @@ -58,7 +60,7 @@ public class TorrentsTest } ]; - return new () + return new() { { torrent, @@ -102,6 +104,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.TrackerListGrabberMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -167,6 +170,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.TrackerListGrabberMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -214,6 +218,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.TrackerListGrabberMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -280,6 +285,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.TrackerListGrabberMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index fd944e6..a62986d 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -38,6 +38,7 @@ public static class DiConfig services.AddScoped(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); diff --git a/server/RdtClient.Service/Services/ITrackerListGrabber.cs b/server/RdtClient.Service/Services/ITrackerListGrabber.cs new file mode 100644 index 0000000..d3ccc08 --- /dev/null +++ b/server/RdtClient.Service/Services/ITrackerListGrabber.cs @@ -0,0 +1,6 @@ +namespace RdtClient.Service.Services; + +public interface ITrackerListGrabber +{ + Task GetTrackers(); +} \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index ec18b0e..7fe79ad 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -25,6 +25,7 @@ public class Torrents( IDownloads downloads, IProcessFactory processFactory, IFileSystem fileSystem, + ITrackerListGrabber trackerListGrabber, AllDebridTorrentClient allDebridTorrentClient, PremiumizeTorrentClient premiumizeTorrentClient, RealDebridTorrentClient realDebridTorrentClient, @@ -32,6 +33,7 @@ public class Torrents( TorBoxTorrentClient torBoxTorrentClient) { private static readonly SemaphoreSlim RealDebridUpdateLock = new(1, 1); + private readonly ITrackerListGrabber _trackerListGrabber = trackerListGrabber; private static readonly JsonSerializerOptions JsonSerializerOptions = new() { ReferenceHandler = ReferenceHandler.IgnoreCycles @@ -107,45 +109,37 @@ public class Torrents( await torrentData.UpdateCategory(torrent.TorrentId, category); } - public static async Task EnrichMagnetLink(string magnetLink) + private async Task EnrichMagnetLink(String magnetLink) { - var enrichment = Settings.Get.General.MagnetTrackerEnrichment; - if (enrichment == MagnetTrackerEnrichment.None) { return magnetLink; } + if (string.IsNullOrWhiteSpace(Settings.Get.General.MagnetTrackerEnrichment)) return magnetLink; - var url = $"https://github.com/ngosang/trackerslist/raw/refs/heads/master/{enrichment}.txt"; - var newTrackers = (await new HttpClient().GetStringAsync(url)).Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); - - var originalUri = new Uri(magnetLink); - var decodedParams = HttpUtility.ParseQueryString(originalUri.Query); - var rawXt = decodedParams["xt"]!; - decodedParams.Remove("xt"); - - var existingTrackers = decodedParams.GetValues("tr") ?? Array.Empty(); - decodedParams.Remove("tr"); - var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); - - var pieces = new List { $"xt={rawXt}" }; - var allKeys = decodedParams.AllKeys; - if (allKeys != null) + try { - foreach (string? key in allKeys) + var newTrackers = await _trackerListGrabber.GetTrackers(); + + var uri = new Uri(magnetLink); + var query = HttpUtility.ParseQueryString(uri.Query); + var existingTrackers = query.GetValues("tr") ?? []; + var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); + + var trackerQuery = string.Join("&tr=", allTrackers.Select(Uri.EscapeDataString)); + if (!string.IsNullOrEmpty(trackerQuery)) { - if (key == null) continue; - foreach (var val in decodedParams.GetValues(key)!) - { - pieces.Add($"{key}={HttpUtility.UrlEncode(val)}"); - } + trackerQuery = "&tr=" + trackerQuery; } + + var baseWithoutTrackers = magnetLink.Split(["&tr="], StringSplitOptions.None)[0]; + var separator = baseWithoutTrackers.Contains('?') ? "&" : "?"; + return baseWithoutTrackers + separator + trackerQuery.TrimStart('&'); + } + catch (Exception ex) + { + logger.LogError(ex, "{Message}, trying to enrich {Magnet}", ex.Message, magnetLink); + return magnetLink; } - foreach (var tr in allTrackers) { pieces.Add($"tr={HttpUtility.UrlEncode(tr)}"); } - - var basePart = originalUri.GetLeftPart(UriPartial.Path); - var enriched = $"{basePart}?{string.Join("&", pieces)}"; - - return enriched; } - public async Task AddMagnetToDebridQueue(string magnetLink, Torrent torrent) + public async Task AddMagnetToDebridQueue(String magnetLink, Torrent torrent) { var enriched = await EnrichMagnetLink(magnetLink); MagnetLink magnet; diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs new file mode 100644 index 0000000..5e034a9 --- /dev/null +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -0,0 +1,34 @@ +namespace RdtClient.Service.Services; + +public class TrackerListGrabber : ITrackerListGrabber +{ + private readonly IHttpClientFactory _httpClientFactory; + + public TrackerListGrabber(IHttpClientFactory httpClientFactory) + { + _httpClientFactory = httpClientFactory; + } + + public async Task GetTrackers() + { + var trackerUrlList = Settings.Get.General.MagnetTrackerEnrichment; + if (string.IsNullOrWhiteSpace(trackerUrlList)) + { + return Array.Empty(); + } + + try + { + var httpClient = _httpClientFactory.CreateClient(); + var result = await httpClient.GetStringAsync(trackerUrlList); + return result + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + catch + { + return Array.Empty(); + } + } +} \ No newline at end of file From 5de0bac9d5519b3eea46b46092a60c1145e1de03 Mon Sep 17 00:00:00 2001 From: Cucumberrbob <128094686+Cucumberrbob@users.noreply.github.com> Date: Thu, 15 May 2025 23:37:17 +0100 Subject: [PATCH 3/9] reformat `EnrichMagnetLink` --- server/RdtClient.Service/Services/Torrents.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 7fe79ad..aa59d59 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -111,7 +111,10 @@ public class Torrents( private async Task EnrichMagnetLink(String magnetLink) { - if (string.IsNullOrWhiteSpace(Settings.Get.General.MagnetTrackerEnrichment)) return magnetLink; + if (String.IsNullOrWhiteSpace(Settings.Get.General.MagnetTrackerEnrichment)) + { + return magnetLink; + } try { @@ -122,19 +125,22 @@ public class Torrents( var existingTrackers = query.GetValues("tr") ?? []; var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); - var trackerQuery = string.Join("&tr=", allTrackers.Select(Uri.EscapeDataString)); - if (!string.IsNullOrEmpty(trackerQuery)) + var trackerQuery = String.Join("&tr=", allTrackers.Select(Uri.EscapeDataString)); + + if (!String.IsNullOrEmpty(trackerQuery)) { trackerQuery = "&tr=" + trackerQuery; } var baseWithoutTrackers = magnetLink.Split(["&tr="], StringSplitOptions.None)[0]; var separator = baseWithoutTrackers.Contains('?') ? "&" : "?"; + return baseWithoutTrackers + separator + trackerQuery.TrimStart('&'); } catch (Exception ex) { logger.LogError(ex, "{Message}, trying to enrich {Magnet}", ex.Message, magnetLink); + return magnetLink; } } From 80a8a1020caad562578af3899510ad397d0ce804 Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Tue, 20 May 2025 03:27:41 +1000 Subject: [PATCH 4/9] Expand enrichment to torrent files --- .../Models/Internal/DbSettings.cs | 4 +- .../Services/EnricherTest.cs | 134 ++++++++++++++++++ .../Services/TorrentsTest.cs | 12 +- server/RdtClient.Service/DiConfig.cs | 1 + server/RdtClient.Service/Services/Enricher.cs | 111 +++++++++++++++ server/RdtClient.Service/Services/Torrents.cs | 49 ++----- .../Services/TrackerListGrabber.cs | 8 +- 7 files changed, 266 insertions(+), 53 deletions(-) create mode 100644 server/RdtClient.Service.Test/Services/EnricherTest.cs create mode 100644 server/RdtClient.Service/Services/Enricher.cs diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 0d6e693..24db43f 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -75,8 +75,8 @@ Supports the following parameters: [Description("When a torrent file or magnet is added, create a copy in this directory.")] public String? CopyAddedTorrents { get; set; } = null; - [DisplayName("Magnet enrichment list")] - [Description("Optional. Specify the URL of a tracker list file to be appended to magnet links.")] + [DisplayName("Tracker enrichment list")] + [Description("Optional. Specify the URL of a tracker list file to be appended to magnet links and torrent files.")] public String? MagnetTrackerEnrichment { get; set; } = null; [DisplayName("Disable update notifications")] diff --git a/server/RdtClient.Service.Test/Services/EnricherTest.cs b/server/RdtClient.Service.Test/Services/EnricherTest.cs new file mode 100644 index 0000000..4c7ceb7 --- /dev/null +++ b/server/RdtClient.Service.Test/Services/EnricherTest.cs @@ -0,0 +1,134 @@ +using Microsoft.Extensions.Logging; +using Moq; +using RdtClient.Service.Services; +using MonoTorrent.BEncoding; + +namespace RdtClient.Service.Test.Services; + +public class EnricherTest : IDisposable +{ + private readonly MockRepository _mockRepository; + private readonly Mock> _loggerMock; + private readonly Mock _trackerListGrabberMock; + + public EnricherTest() + { + _mockRepository = new MockRepository(MockBehavior.Strict); + _loggerMock = _mockRepository.Create>(MockBehavior.Loose); + _trackerListGrabberMock = _mockRepository.Create(); + } + + public void Dispose() + { + _mockRepository.VerifyAll(); + } + + private const String TestMagnetLink = "magnet:?xt=urn:btih:1234567890123456789012345678901234567890&dn=TestFile&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce"; + + [Fact] + public async Task EnrichMagnetLink_AddsNoTrackers_WhenNoTrackersFromTrackerGrabber() + { + // Arrange + SetupTrackerListGrabber([]); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enriched = await enricher.EnrichMagnetLink(TestMagnetLink); + + // Assert + Assert.Equal(TestMagnetLink, enriched); + } + + [Fact] + public async Task EnrichMagnetLink_AddsTrackers_WhenTrackersFromTrackerGrabber() + { + // Arrange + SetupTrackerListGrabber(["http://new-tracker.com/announce"]); + + var Enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enriched = await Enricher.EnrichMagnetLink(TestMagnetLink); + + // Assert + Assert.Equal(TestMagnetLink + $"&tr={Uri.EscapeDataString("http://new-tracker.com/announce")}", enriched); + } + + [Fact] + public async Task EnrichMagnetLink_DoesNotAddDuplicateTrackers_WhenTrackersFromTrackerGrabberAlreadyPresent() + { + // Arrange + SetupTrackerListGrabber(["http://new-tracker.com/announce", "http://tracker1.com/announce"]); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enriched = await enricher.EnrichMagnetLink(TestMagnetLink); + + // Assert + Assert.Equal(TestMagnetLink + $"&tr={Uri.EscapeDataString("http://new-tracker.com/announce")}", enriched); + } + + [Fact] + public async Task EnrichMagnetLink_Throws_WhenTrackerGrabberThrows() + { + // Arrange + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ThrowsAsync(new InvalidOperationException("Unable to fetch tracker list for enrichment.")); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => enricher.EnrichMagnetLink(TestMagnetLink)); + } + + + [Fact] + public async Task EnrichTorrentBytes_AddsTrackers_WhenTrackersFromTrackerGrabber() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(originalTracker), + ["announce-list"] = new BEncodedList + { + new BEncodedList { new BEncodedString(originalTracker) } + } + }; + + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber(new[] { newTracker }); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce")); + Assert.True(enrichedDict.ContainsKey("announce-list")); + + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + + Assert.Contains(originalTracker, flattened); + Assert.Contains(newTracker, flattened); + + var announce = ((BEncodedString)enrichedDict["announce"]).Text; + Assert.Equal(flattened.First(), announce); + } + + private void SetupTrackerListGrabber(String[] trackerList) + { + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ReturnsAsync(trackerList) + .Verifiable(); + } +} \ No newline at end of file diff --git a/server/RdtClient.Service.Test/Services/TorrentsTest.cs b/server/RdtClient.Service.Test/Services/TorrentsTest.cs index 90fe94d..054a62f 100644 --- a/server/RdtClient.Service.Test/Services/TorrentsTest.cs +++ b/server/RdtClient.Service.Test/Services/TorrentsTest.cs @@ -20,13 +20,13 @@ class Mocks public readonly Mock> TorrentsLoggerMock; public readonly Mock DownloadsMock; public readonly Mock TorrentDataMock; - public readonly Mock TrackerListGrabberMock; + public readonly Mock EnricherMock; public Mocks() { TorrentDataMock = new(); DownloadsMock = new(); - TrackerListGrabberMock = new(); + EnricherMock = new(); TorrentsLoggerMock = new(); @@ -104,7 +104,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, - mocks.TrackerListGrabberMock.Object, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -170,7 +170,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, - mocks.TrackerListGrabberMock.Object, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -218,7 +218,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, - mocks.TrackerListGrabberMock.Object, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -285,7 +285,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, - mocks.TrackerListGrabberMock.Object, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index a62986d..fe3edbf 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -39,6 +39,7 @@ public static class DiConfig services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); diff --git a/server/RdtClient.Service/Services/Enricher.cs b/server/RdtClient.Service/Services/Enricher.cs new file mode 100644 index 0000000..140d07a --- /dev/null +++ b/server/RdtClient.Service/Services/Enricher.cs @@ -0,0 +1,111 @@ +using System.Web; +using Microsoft.Extensions.Logging; +using MonoTorrent.BEncoding; + +namespace RdtClient.Service.Services; + +public interface IEnricher +{ + Task EnrichMagnetLink(String magnetLink); + Task EnrichTorrentBytes(Byte[] torrentBytes); +} + +/// +/// Enriches magnet links and torrents by adding trackers from the tracker list grabber. +/// +public class Enricher(ILogger logger, ITrackerListGrabber trackerListGrabber) : IEnricher +{ + /// + /// Add trackers from the tracker list grabber to the magnet link. + /// + /// Magnet link to add trackres to. Is not modified + /// Magnet link with additional trackers + public async Task EnrichMagnetLink(String magnetLink) + { + var newTrackers = await trackerListGrabber.GetTrackers(); + + var uri = new Uri(magnetLink); + var query = HttpUtility.ParseQueryString(uri.Query); + + var existingTrackers = query.GetValues("tr") ?? []; + var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); + + var trackerQuery = String.Join("&tr=", allTrackers.Select(Uri.EscapeDataString)); + + if (!String.IsNullOrEmpty(trackerQuery)) + { + trackerQuery = "&tr=" + trackerQuery; + } + + var baseWithoutTrackers = magnetLink.Split("&tr=")[0]; + + var separator = baseWithoutTrackers.Contains('?') ? "&" : "?"; + + var newUri = baseWithoutTrackers + separator + trackerQuery.TrimStart('&'); + try + { + _ = new Uri(newUri); + } + catch (UriFormatException ex) + { + logger.LogWarning(ex, "Failed to enrich magnet link: {newUri}", newUri); + throw new InvalidOperationException($"Failed to enrich magnet link: {newUri}", ex); + } + return newUri; + } + + /// + /// Add trackers from the tracker list grabber to the .torrent file bytes. + /// + /// Torrent file bytes to add trackers to. Is not modified + /// Torrent file bytes with additional trackers + public async Task EnrichTorrentBytes(Byte[] torrentBytes) + { + var newTrackers = await trackerListGrabber.GetTrackers(); + + if (torrentBytes == null) throw new ArgumentNullException(nameof(torrentBytes)); + var torrentDict = BEncodedValue.Decode(torrentBytes); + + + if (!torrentDict.TryGetValue("announce-list", out var announceListValue) || announceListValue is not BEncodedList announceList) + { + announceList = new BEncodedList(); + if (torrentDict.TryGetValue("announce", out var announceValue) && announceValue is BEncodedString announceStr) + { + announceList.Add(new BEncodedList { announceStr }); + } + } + + var existingTrackers = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var tier in announceList) + { + if (tier is BEncodedList tierList) + { + foreach (var tracker in tierList) + { + if (tracker is BEncodedString trackerStr) + { + existingTrackers.Add(trackerStr.Text); + } + } + } + } + + foreach (var tracker in newTrackers) + { + if (!existingTrackers.Contains(tracker)) + { + announceList.Add(new BEncodedList { new BEncodedString(tracker) }); + existingTrackers.Add(tracker); + } + } + + torrentDict["announce-list"] = announceList; + if (announceList.Count > 0 && announceList[0] is BEncodedList firstTier && firstTier.Count > 0 && firstTier[0] is BEncodedString firstTracker) + { + torrentDict["announce"] = firstTracker; + } + + return torrentDict.Encode(); + } +} \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index aa59d59..3ff2e69 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -25,7 +25,7 @@ public class Torrents( IDownloads downloads, IProcessFactory processFactory, IFileSystem fileSystem, - ITrackerListGrabber trackerListGrabber, + IEnricher enricher, AllDebridTorrentClient allDebridTorrentClient, PremiumizeTorrentClient premiumizeTorrentClient, RealDebridTorrentClient realDebridTorrentClient, @@ -33,7 +33,7 @@ public class Torrents( TorBoxTorrentClient torBoxTorrentClient) { private static readonly SemaphoreSlim RealDebridUpdateLock = new(1, 1); - private readonly ITrackerListGrabber _trackerListGrabber = trackerListGrabber; + private static readonly JsonSerializerOptions JsonSerializerOptions = new() { ReferenceHandler = ReferenceHandler.IgnoreCycles @@ -109,45 +109,9 @@ public class Torrents( await torrentData.UpdateCategory(torrent.TorrentId, category); } - private async Task EnrichMagnetLink(String magnetLink) - { - if (String.IsNullOrWhiteSpace(Settings.Get.General.MagnetTrackerEnrichment)) - { - return magnetLink; - } - - try - { - var newTrackers = await _trackerListGrabber.GetTrackers(); - - var uri = new Uri(magnetLink); - var query = HttpUtility.ParseQueryString(uri.Query); - var existingTrackers = query.GetValues("tr") ?? []; - var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); - - var trackerQuery = String.Join("&tr=", allTrackers.Select(Uri.EscapeDataString)); - - if (!String.IsNullOrEmpty(trackerQuery)) - { - trackerQuery = "&tr=" + trackerQuery; - } - - var baseWithoutTrackers = magnetLink.Split(["&tr="], StringSplitOptions.None)[0]; - var separator = baseWithoutTrackers.Contains('?') ? "&" : "?"; - - return baseWithoutTrackers + separator + trackerQuery.TrimStart('&'); - } - catch (Exception ex) - { - logger.LogError(ex, "{Message}, trying to enrich {Magnet}", ex.Message, magnetLink); - - return magnetLink; - } - } - public async Task AddMagnetToDebridQueue(String magnetLink, Torrent torrent) { - var enriched = await EnrichMagnetLink(magnetLink); + var enriched = await enricher.EnrichMagnetLink(magnetLink); MagnetLink magnet; try { @@ -166,7 +130,7 @@ public class Torrents( var newTorrent = await AddQueued(hash, enriched, false, torrent); Log($"Adding {hash} (magnet link) to queue", newTorrent); - await CopyAddedTorrent(magnet.Name!, enriched); + await CopyAddedTorrent(magnet.Name!, magnetLink); return newTorrent; } @@ -178,9 +142,12 @@ public class Torrents( var fileAsBase64 = Convert.ToBase64String(bytes); logger.LogDebug($"bytes {bytes}"); + var enriched = await enricher.EnrichTorrentBytes(bytes); + logger.LogDebug($"enriched bytes {enriched}"); + try { - monoTorrent = await MonoTorrent.Torrent.LoadAsync(bytes); + monoTorrent = await MonoTorrent.Torrent.LoadAsync(enriched); } catch (Exception ex) { diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs index 5e034a9..d82fb07 100644 --- a/server/RdtClient.Service/Services/TrackerListGrabber.cs +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -12,9 +12,9 @@ public class TrackerListGrabber : ITrackerListGrabber public async Task GetTrackers() { var trackerUrlList = Settings.Get.General.MagnetTrackerEnrichment; - if (string.IsNullOrWhiteSpace(trackerUrlList)) + if (String.IsNullOrWhiteSpace(trackerUrlList)) { - return Array.Empty(); + return []; } try @@ -26,9 +26,9 @@ public class TrackerListGrabber : ITrackerListGrabber .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); } - catch + catch (Exception ex) { - return Array.Empty(); + throw new InvalidOperationException("Unable to fetch tracker list for enrichment.", ex); } } } \ No newline at end of file From 408c475af54e1ebbbbdc601e2139e80a9e8aec78 Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Wed, 21 May 2025 00:45:14 +1000 Subject: [PATCH 5/9] Tracker list caching with configurable expiration Add new option to general settings that allows the user to set the time in minutes to cache the tracker list. 0 disables caching. Default value is 60. --- .../Models/Internal/DbSettings.cs | 6 +- server/RdtClient.Service/DiConfig.cs | 2 + .../Services/TrackerListGrabber.cs | 85 +++++++++++++++---- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 24db43f..5eb800f 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -77,7 +77,11 @@ Supports the following parameters: [DisplayName("Tracker enrichment list")] [Description("Optional. Specify the URL of a tracker list file to be appended to magnet links and torrent files.")] - public String? MagnetTrackerEnrichment { get; set; } = null; + public String? TrackerEnrichmentList { get; set; } = null; + + [DisplayName("Tracker enrichment cache expiration")] + [Description("The time in minutes to cache the tracker list. Set to 0 to disable caching.")] + public Int32 TrackerEnrichmentCacheExpiration { get; set; } = 60; [DisplayName("Disable update notifications")] [Description("Ignore update notifications. You will still be notified if the version you are running has a security vulnerability.")] diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index fe3edbf..abe8af7 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -18,6 +18,8 @@ public static class DiConfig public static void RegisterRdtServices(this IServiceCollection services) { + services.AddMemoryCache(); + services.AddSingleton(); services.AddScoped(); diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs index d82fb07..f9f64d9 100644 --- a/server/RdtClient.Service/Services/TrackerListGrabber.cs +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -1,34 +1,89 @@ +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; + namespace RdtClient.Service.Services; -public class TrackerListGrabber : ITrackerListGrabber +public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCache memoryCache, ILogger logger) : ITrackerListGrabber { - private readonly IHttpClientFactory _httpClientFactory; + private const String CacheKey = "TrackerList"; - public TrackerListGrabber(IHttpClientFactory httpClientFactory) - { - _httpClientFactory = httpClientFactory; - } + private Int32? _lastExpirationMinutes; + + private static readonly SemaphoreSlim Semaphore = new(1, 1); public async Task GetTrackers() { - var trackerUrlList = Settings.Get.General.MagnetTrackerEnrichment; - if (String.IsNullOrWhiteSpace(trackerUrlList)) + var currentExpiration = Settings.Get.General.TrackerEnrichmentCacheExpiration; + var useCache = currentExpiration > 0; + + if (!useCache) { - return []; + memoryCache.Remove(CacheKey); + _lastExpirationMinutes = null; } + else + { + if (_lastExpirationMinutes is not null && currentExpiration != _lastExpirationMinutes) + { + logger.LogDebug("Tracker list cache timeout changed, invalidating cache."); + memoryCache.Remove(CacheKey); + } + + _lastExpirationMinutes = currentExpiration; + + if (memoryCache.TryGetValue(CacheKey, out String[]? cachedTrackers) && cachedTrackers is { Length: > 0 }) + { + logger.LogDebug("Using cached tracker list."); + + return cachedTrackers; + } + } + + await Semaphore.WaitAsync(); try { - var httpClient = _httpClientFactory.CreateClient(); - var result = await httpClient.GetStringAsync(trackerUrlList); - return result - .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); + logger.LogDebug("Tracker cache miss or cache disabled. Fetching tracker list."); + var trackerUrlList = Settings.Get.General.TrackerEnrichmentList; + + if (String.IsNullOrWhiteSpace(trackerUrlList)) + { + return []; + } + + var httpClient = httpClientFactory.CreateClient(); + var response = await httpClient.GetAsync(trackerUrlList); + + response.EnsureSuccessStatusCode(); + + var result = await response.Content.ReadAsStringAsync(); + + var trackers = result + .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (useCache) + { + memoryCache.Set(CacheKey, + trackers, + new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(currentExpiration) + }); + } + + return trackers; } catch (Exception ex) { + logger.LogError(ex, "Unable to fetch tracker list."); + throw new InvalidOperationException("Unable to fetch tracker list for enrichment.", ex); } + finally + { + Semaphore.Release(); + } } } \ No newline at end of file From 452d8cc1cf5193599606dedaae1b23c3be112580 Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Thu, 22 May 2025 22:23:49 +1000 Subject: [PATCH 6/9] Split tracker grabbing and parsing logic. Split grab and parse logic, recheck cache after semaphore lock, timeouts, add URL validation check, add torrent file tests, replace test hash with valid SHA1 format, tracker URL validation, fix token disposal issue --- .../Services/EnricherTest.cs | 149 +++++++++++++++++- server/RdtClient.Service/Services/Enricher.cs | 10 +- .../Services/TrackerListGrabber.cs | 87 ++++++++-- 3 files changed, 220 insertions(+), 26 deletions(-) diff --git a/server/RdtClient.Service.Test/Services/EnricherTest.cs b/server/RdtClient.Service.Test/Services/EnricherTest.cs index 4c7ceb7..41ae6a2 100644 --- a/server/RdtClient.Service.Test/Services/EnricherTest.cs +++ b/server/RdtClient.Service.Test/Services/EnricherTest.cs @@ -23,7 +23,8 @@ public class EnricherTest : IDisposable _mockRepository.VerifyAll(); } - private const String TestMagnetLink = "magnet:?xt=urn:btih:1234567890123456789012345678901234567890&dn=TestFile&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce"; + private const String TestMagnetLink = + "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=TestFile&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce"; [Fact] public async Task EnrichMagnetLink_AddsNoTrackers_WhenNoTrackersFromTrackerGrabber() @@ -46,10 +47,10 @@ public class EnricherTest : IDisposable // Arrange SetupTrackerListGrabber(["http://new-tracker.com/announce"]); - var Enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); // Act - var enriched = await Enricher.EnrichMagnetLink(TestMagnetLink); + var enriched = await enricher.EnrichMagnetLink(TestMagnetLink); // Assert Assert.Equal(TestMagnetLink + $"&tr={Uri.EscapeDataString("http://new-tracker.com/announce")}", enriched); @@ -84,7 +85,6 @@ public class EnricherTest : IDisposable await Assert.ThrowsAsync(() => enricher.EnrichMagnetLink(TestMagnetLink)); } - [Fact] public async Task EnrichTorrentBytes_AddsTrackers_WhenTrackersFromTrackerGrabber() { @@ -96,14 +96,17 @@ public class EnricherTest : IDisposable { ["announce"] = new BEncodedString(originalTracker), ["announce-list"] = new BEncodedList + { + new BEncodedList { - new BEncodedList { new BEncodedString(originalTracker) } + new BEncodedString(originalTracker) } + } }; var originalTorrentBytes = torrentDict.Encode(); - SetupTrackerListGrabber(new[] { newTracker }); + SetupTrackerListGrabber([newTracker]); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); // Act @@ -131,4 +134,138 @@ public class EnricherTest : IDisposable .ReturnsAsync(trackerList) .Verifiable(); } + + [Fact] + public async Task EnrichTorrentBytes_DoesNotAddTrackers_WhenNoTrackersFromTrackerGrabber() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(originalTracker), + ["announce-list"] = new BEncodedList + { + new BEncodedList + { + new BEncodedString(originalTracker) + } + } + }; + + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce")); + Assert.True(enrichedDict.ContainsKey("announce-list")); + + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + + Assert.Single(flattened); + Assert.Contains(originalTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_DoesNotAddDuplicateTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var duplicateTracker = "http://tracker1.com/announce"; + + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(originalTracker), + ["announce-list"] = new BEncodedList + { + new BEncodedList + { + new BEncodedString(originalTracker) + } + } + }; + + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([duplicateTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + + Assert.Single(flattened); + Assert.Contains(originalTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_AddsTrackers_WhenNoAnnounceListPresent() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(originalTracker) + + // No "announce-list" + }; + + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + + Assert.Contains(originalTracker, flattened); + Assert.Contains(newTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_Throws_WhenTrackerGrabberThrows() + { + // Arrange + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ThrowsAsync(new InvalidOperationException("Unable to fetch tracker list for enrichment.")); + + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString("http://tracker1.com/announce"), + ["announce-list"] = new BEncodedList + { + new BEncodedList + { + new BEncodedString("http://tracker1.com/announce") + } + } + }; + + var originalTorrentBytes = torrentDict.Encode(); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(originalTorrentBytes)); + } } \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Enricher.cs b/server/RdtClient.Service/Services/Enricher.cs index 140d07a..f0e1d12 100644 --- a/server/RdtClient.Service/Services/Enricher.cs +++ b/server/RdtClient.Service/Services/Enricher.cs @@ -11,14 +11,14 @@ public interface IEnricher } /// -/// Enriches magnet links and torrents by adding trackers from the tracker list grabber. +/// Enriches magnet links and torrents by adding trackers from the tracker list grabber. /// public class Enricher(ILogger logger, ITrackerListGrabber trackerListGrabber) : IEnricher { /// /// Add trackers from the tracker list grabber to the magnet link. /// - /// Magnet link to add trackres to. Is not modified + /// Magnet link to add trackers to. Is not modified /// Magnet link with additional trackers public async Task EnrichMagnetLink(String magnetLink) { @@ -63,7 +63,11 @@ public class Enricher(ILogger logger, ITrackerListGrabber trackerListG { var newTrackers = await trackerListGrabber.GetTrackers(); - if (torrentBytes == null) throw new ArgumentNullException(nameof(torrentBytes)); + if (torrentBytes == null) + { + throw new ArgumentNullException(nameof(torrentBytes)); + } + var torrentDict = BEncodedValue.Decode(torrentBytes); diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs index f9f64d9..f7a34c5 100644 --- a/server/RdtClient.Service/Services/TrackerListGrabber.cs +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -13,6 +13,21 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac public async Task GetTrackers() { + var trackerUrlList = Settings.Get.General.TrackerEnrichmentList; + + if (String.IsNullOrWhiteSpace(trackerUrlList)) + { + return []; + } + + if (!Uri.TryCreate(trackerUrlList, UriKind.Absolute, out var trackerUri) || + (trackerUri.Scheme != Uri.UriSchemeHttp && trackerUri.Scheme != Uri.UriSchemeHttps)) + { + logger.LogError("Invalid tracker list URL: {Url}", trackerUrlList); + + throw new InvalidOperationException("Invalid tracker list URL."); + } + var currentExpiration = Settings.Get.General.TrackerEnrichmentCacheExpiration; var useCache = currentExpiration > 0; @@ -39,29 +54,23 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac } } - await Semaphore.WaitAsync(); + await Semaphore.WaitAsync().ConfigureAwait(false); try { - logger.LogDebug("Tracker cache miss or cache disabled. Fetching tracker list."); - var trackerUrlList = Settings.Get.General.TrackerEnrichmentList; - - if (String.IsNullOrWhiteSpace(trackerUrlList)) + if (useCache) { - return []; + if (memoryCache.TryGetValue(CacheKey, out String[]? cachedTrackers) && cachedTrackers is { Length: > 0 }) + { + logger.LogDebug("Using cached tracker list (after lock)."); + + return cachedTrackers; + } } - var httpClient = httpClientFactory.CreateClient(); - var response = await httpClient.GetAsync(trackerUrlList); + logger.LogDebug("Tracker cache miss or cache disabled. Fetching tracker list."); - response.EnsureSuccessStatusCode(); - - var result = await response.Content.ReadAsStringAsync(); - - var trackers = result - .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); + var trackers = await FetchAndParseTrackersAsync(trackerUri).ConfigureAwait(false); if (useCache) { @@ -75,6 +84,12 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac return trackers; } + catch (TaskCanceledException ex) + { + logger.LogError(ex, "Fetching tracker list was canceled (timeout or cancellation)."); + + throw new InvalidOperationException("Fetching tracker list was canceled due to timeout or cancellation.", ex); + } catch (Exception ex) { logger.LogError(ex, "Unable to fetch tracker list."); @@ -86,4 +101,42 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac Semaphore.Release(); } } -} \ No newline at end of file + + private async Task FetchAndParseTrackersAsync(Uri trackerUri) + { + logger.LogDebug("Fetching tracker list from URL: {TrackerUrl}", trackerUri); + + var httpClient = httpClientFactory.CreateClient(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + var token = cts.Token; + using var response = await httpClient.GetAsync(trackerUri, token).ConfigureAwait(false); + + response.EnsureSuccessStatusCode(); + + await using var contentStream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false); + using var reader = new StreamReader(contentStream); + var result = await reader.ReadToEndAsync(token).ConfigureAwait(false); + + String[] trackers; + + try + { + trackers = result + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => !String.IsNullOrWhiteSpace(line)) + .Select(t => t.EndsWith("/") ? t.TrimEnd('/') : t) + .Where(t => Uri.TryCreate(t, UriKind.Absolute, out var uri) && + (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + catch (Exception ex) + { + logger.LogError(ex, "Error parsing tracker list response."); + + throw new InvalidOperationException("Failed to parse tracker list response.", ex); + } + + return trackers; + } +} From 4be3280c3a6ac04f29859d7bdab01e178835deea Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Sat, 24 May 2025 02:15:53 +1000 Subject: [PATCH 7/9] Enricher code refactoring with small tweaks. Added additional tests. Changed enricher class to sealed. Changed logging. Changed how trackers are added. Fixed some issues around edge cases. --- .../Services/EnricherTest.cs | 231 +++++++++++++++--- server/RdtClient.Service/Services/Enricher.cs | 206 ++++++++++++---- .../Services/TrackerListGrabber.cs | 81 +++--- 3 files changed, 404 insertions(+), 114 deletions(-) diff --git a/server/RdtClient.Service.Test/Services/EnricherTest.cs b/server/RdtClient.Service.Test/Services/EnricherTest.cs index 41ae6a2..ebdd84e 100644 --- a/server/RdtClient.Service.Test/Services/EnricherTest.cs +++ b/server/RdtClient.Service.Test/Services/EnricherTest.cs @@ -135,44 +135,6 @@ public class EnricherTest : IDisposable .Verifiable(); } - [Fact] - public async Task EnrichTorrentBytes_DoesNotAddTrackers_WhenNoTrackersFromTrackerGrabber() - { - // Arrange - var originalTracker = "http://tracker1.com/announce"; - - var torrentDict = new BEncodedDictionary - { - ["announce"] = new BEncodedString(originalTracker), - ["announce-list"] = new BEncodedList - { - new BEncodedList - { - new BEncodedString(originalTracker) - } - } - }; - - var originalTorrentBytes = torrentDict.Encode(); - - SetupTrackerListGrabber([]); - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - // Act - var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); - var enrichedDict = BEncodedValue.Decode(enrichedBytes); - - // Assert - Assert.True(enrichedDict.ContainsKey("announce")); - Assert.True(enrichedDict.ContainsKey("announce-list")); - - var announceList = (BEncodedList)enrichedDict["announce-list"]; - var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); - - Assert.Single(flattened); - Assert.Contains(originalTracker, flattened); - } - [Fact] public async Task EnrichTorrentBytes_DoesNotAddDuplicateTrackers() { @@ -268,4 +230,197 @@ public class EnricherTest : IDisposable // Act & Assert await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(originalTorrentBytes)); } + + [Fact] + public async Task EnrichTorrentBytes_ThrowsArgumentException_OnNullOrEmptyInput() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(null!)); + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes([])); + } + + [Fact] + public async Task EnrichTorrentBytes_ThrowsInvalidOperationException_OnNonTorrentBytes() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + var notTorrent = new Byte[] + { + 1, 2, 3, 4, 5 + }; + + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(notTorrent)); + } + + [Fact] + public async Task EnrichMagnetLink_WithExistingTrackersAndOtherParameters() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:HASH&dn=MyFile&tr=udp%3A%2F%2Fexisting"; + + var newTrackers = new[] + { + "udp://new1", "udp://existing" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var result = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + var queryParams = System.Web.HttpUtility.ParseQueryString(new Uri(result).Query); + var trValues = queryParams.GetValues("tr")?.ToList() ?? new List(); + + Assert.Equal("urn:btih:HASH", queryParams["xt"]); + Assert.Equal("MyFile", queryParams["dn"]); + Assert.Contains("udp://existing", trValues); + Assert.Contains("udp://new1", trValues); + Assert.Equal(2, trValues.Count); + } + + [Fact] + public async Task EnrichMagnetLink_BasicMagnetWithHashOnly() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:HASH"; + + var newTrackers = new[] + { + "udp://tracker1" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var result = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + Assert.Equal($"{magnetLink}&tr={Uri.EscapeDataString("udp://tracker1")}", result); + } + + [Fact] + public async Task EnrichMagnetLink_MagnetEndsWithQuestionMarkOnly() + { + // Arrange + var magnetLink = "magnet:?"; + + var newTrackers = new[] + { + "udp://tracker1" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var result = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + Assert.Equal($"{magnetLink}tr={Uri.EscapeDataString("udp://tracker1")}", result); + } + + [Fact] + public async Task EnrichMagnetLink_GetTrackersReturnsEmptyArray_Unchanged() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Fexisting"; + SetupTrackerListGrabber([]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var result = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + Assert.Equal(magnetLink, result); + } + + [Fact] + public async Task EnrichTorrentBytes_NoExistingAnnounceOrAnnounceList() + { + // Arrange + var torrentDict = new BEncodedDictionary + { + ["info"] = new BEncodedDictionary() + }; + + var originalBytes = torrentDict.Encode(); + + var newTrackers = new[] + { + "udp://tracker1", "udp://tracker2" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.Equal(newTrackers[0], ((BEncodedString)enrichedDict["announce"]).Text); + var announceList = (BEncodedList)enrichedDict["announce-list"]; + Assert.Equal(2, announceList.Count); + Assert.Equal(newTrackers[0], ((BEncodedString)((BEncodedList)announceList[0])[0]).Text); + Assert.Equal(newTrackers[1], ((BEncodedString)((BEncodedList)announceList[1])[0]).Text); + } + + [Fact] + public async Task EnrichTorrentBytes_WithExistingAnnounceAndAnnounceList() + { + // Arrange + var existingTrackers = new List + { + "http://tier1.com", + "http://existing.com/announce" + }; + + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(existingTrackers[1]), + ["announce-list"] = new BEncodedList + { + new BEncodedList + { + new BEncodedString(existingTrackers[0]) + }, + new BEncodedList + { + new BEncodedString(existingTrackers[1]) + } + }, + ["info"] = new BEncodedDictionary() + }; + + var originalBytes = torrentDict.Encode(); + + var newTrackersFromGrabber = new[] + { + "udp://new1", "http://existing.com/announce" + }; + + SetupTrackerListGrabber(newTrackersFromGrabber); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var finalAnnounceList = ((BEncodedList)enrichedDict["announce-list"]) + .Cast() + .SelectMany(tier => tier.Cast().Select(t => t.Text)) + .ToList(); + + Assert.Contains("http://tier1.com", finalAnnounceList); + Assert.Contains("http://existing.com/announce", finalAnnounceList); + Assert.Contains("udp://new1", finalAnnounceList); + Assert.Equal(3, finalAnnounceList.Count); + Assert.Contains(((BEncodedString)enrichedDict["announce"]).Text, finalAnnounceList); + } } \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Enricher.cs b/server/RdtClient.Service/Services/Enricher.cs index f0e1d12..222cdca 100644 --- a/server/RdtClient.Service/Services/Enricher.cs +++ b/server/RdtClient.Service/Services/Enricher.cs @@ -1,3 +1,5 @@ + +using System.Text; using System.Web; using Microsoft.Extensions.Logging; using MonoTorrent.BEncoding; @@ -13,7 +15,7 @@ public interface IEnricher /// /// Enriches magnet links and torrents by adding trackers from the tracker list grabber. /// -public class Enricher(ILogger logger, ITrackerListGrabber trackerListGrabber) : IEnricher +public sealed class Enricher(ILogger logger, ITrackerListGrabber trackerListGrabber) : IEnricher { /// /// Add trackers from the tracker list grabber to the magnet link. @@ -22,36 +24,107 @@ public class Enricher(ILogger logger, ITrackerListGrabber trackerListG /// Magnet link with additional trackers public async Task EnrichMagnetLink(String magnetLink) { - var newTrackers = await trackerListGrabber.GetTrackers(); + var newTrackers = await trackerListGrabber.GetTrackers().ConfigureAwait(false); - var uri = new Uri(magnetLink); - var query = HttpUtility.ParseQueryString(uri.Query); - - var existingTrackers = query.GetValues("tr") ?? []; - var allTrackers = existingTrackers.Concat(newTrackers).Distinct(StringComparer.OrdinalIgnoreCase); - - var trackerQuery = String.Join("&tr=", allTrackers.Select(Uri.EscapeDataString)); - - if (!String.IsNullOrEmpty(trackerQuery)) + if (newTrackers.Length == 0) { - trackerQuery = "&tr=" + trackerQuery; + logger.LogWarning("No new trackers were retrieved."); + + return magnetLink; } - var baseWithoutTrackers = magnetLink.Split("&tr=")[0]; + var qmIdx = magnetLink.IndexOf('?'); - var separator = baseWithoutTrackers.Contains('?') ? "&" : "?"; + if (qmIdx == -1 || qmIdx == magnetLink.Length - 1) + { + var sb = new StringBuilder(magnetLink); - var newUri = baseWithoutTrackers + separator + trackerQuery.TrimStart('&'); - try - { - _ = new Uri(newUri); + if (qmIdx == -1) + { + sb.Append('?'); + } + + for (var i = 0; i < newTrackers.Length; i++) + { + if (i > 0 || (qmIdx != magnetLink.Length - 1 && qmIdx != -1)) + { + sb.Append('&'); + } + + sb.Append("tr=").Append(Uri.EscapeDataString(newTrackers[i])); + } + + logger.LogInformation("Added {NewTrackersCount} new trackers to a magnet link with no initial query string. Total trackers: {TotalTrackersCount}.", + newTrackers.Length, + newTrackers.Length); + + return sb.ToString(); } - catch (UriFormatException ex) + + var schemePart = magnetLink[..qmIdx]; + var queryPart = magnetLink[(qmIdx + 1)..]; + + var queryKVs = HttpUtility.ParseQueryString(queryPart); + + var paramDict = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (String key in queryKVs) { - logger.LogWarning(ex, "Failed to enrich magnet link: {newUri}", newUri); - throw new InvalidOperationException($"Failed to enrich magnet link: {newUri}", ex); + if (key == null) + { + continue; + } + + if (!paramDict.ContainsKey(key)) + { + paramDict[key] = new List(); + } + + paramDict[key].AddRange(queryKVs.GetValues(key) ?? []); } - return newUri; + + var existingTrackers = paramDict.TryGetValue("tr", out var value) + ? new HashSet(value, StringComparer.OrdinalIgnoreCase) + : new HashSet(StringComparer.OrdinalIgnoreCase); + + paramDict.Remove("tr"); + + var newUniqueTrackers = newTrackers.Where(t => !existingTrackers.Contains(t)).ToList(); + + foreach (var tr in newUniqueTrackers) + { + existingTrackers.Add(tr); + } + + var outParams = new List(); + + foreach (var kv in paramDict) + { + foreach (var v in kv.Value) + { + if (kv.Key == "xt" || kv.Key == "dn") + { + outParams.Add($"{kv.Key}={v}"); + } + else + { + outParams.Add($"{kv.Key}={HttpUtility.UrlEncode(v)}"); + } + } + } + + foreach (var tr in existingTrackers) + { + outParams.Add($"tr={Uri.EscapeDataString(tr)}"); + } + + var finalMagnet = schemePart + "?" + String.Join("&", outParams); + + logger.LogInformation("Added {NewTrackersCount} new trackers to the magnet link. Total trackers: {TotalTrackersCount}.", + newUniqueTrackers.Count, + existingTrackers.Count); + + return finalMagnet; } /// @@ -61,55 +134,94 @@ public class Enricher(ILogger logger, ITrackerListGrabber trackerListG /// Torrent file bytes with additional trackers public async Task EnrichTorrentBytes(Byte[] torrentBytes) { - var newTrackers = await trackerListGrabber.GetTrackers(); - - if (torrentBytes == null) + if (torrentBytes == null || torrentBytes.Length == 0) { - throw new ArgumentNullException(nameof(torrentBytes)); + throw new ArgumentException("Torrent bytes cannot be null or empty.", nameof(torrentBytes)); } - var torrentDict = BEncodedValue.Decode(torrentBytes); + BEncodedDictionary torrentDict; - - if (!torrentDict.TryGetValue("announce-list", out var announceListValue) || announceListValue is not BEncodedList announceList) + try { - announceList = new BEncodedList(); - if (torrentDict.TryGetValue("announce", out var announceValue) && announceValue is BEncodedString announceStr) - { - announceList.Add(new BEncodedList { announceStr }); - } + torrentDict = BEncodedValue.Decode(torrentBytes); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to decode torrent bytes."); + + throw new InvalidOperationException("Invalid torrent file format.", ex); } - var existingTrackers = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var tier in announceList) + var newTrackers = await trackerListGrabber.GetTrackers().ConfigureAwait(false); + + if (newTrackers.Length == 0) { - if (tier is BEncodedList tierList) + logger.LogWarning("No new trackers were retrieved."); + + return torrentBytes; + } + + var seenTrackers = new HashSet(StringComparer.OrdinalIgnoreCase); + var allTrackers = new List(); + + if (torrentDict.TryGetValue("announce-list", out var alc) && alc is BEncodedList alList) + { + foreach (var tier in alList.OfType()) { - foreach (var tracker in tierList) + foreach (var s in tier.OfType()) { - if (tracker is BEncodedString trackerStr) + if (seenTrackers.Add(s.Text)) { - existingTrackers.Add(trackerStr.Text); + allTrackers.Add(s.Text); } } } } - foreach (var tracker in newTrackers) + if (torrentDict.TryGetValue("announce", out var announceValue) && announceValue is BEncodedString announceStr) { - if (!existingTrackers.Contains(tracker)) + if (seenTrackers.Add(announceStr.Text)) { - announceList.Add(new BEncodedList { new BEncodedString(tracker) }); - existingTrackers.Add(tracker); + allTrackers.Add(announceStr.Text); } } - torrentDict["announce-list"] = announceList; - if (announceList.Count > 0 && announceList[0] is BEncodedList firstTier && firstTier.Count > 0 && firstTier[0] is BEncodedString firstTracker) + var addedTrackersCount = 0; + + foreach (var tracker in newTrackers) { - torrentDict["announce"] = firstTracker; + if (seenTrackers.Add(tracker)) + { + allTrackers.Add(tracker); + addedTrackersCount++; + } } + var dedupedAnnounceList = new BEncodedList(); + + foreach (var tracker in allTrackers) + { + dedupedAnnounceList.Add(new BEncodedList + { + new BEncodedString(tracker) + }); + } + + torrentDict["announce-list"] = dedupedAnnounceList; + + if (allTrackers.Count > 0) + { + torrentDict["announce"] = new BEncodedString(allTrackers[0]); + } + else + { + return torrentBytes; + } + + logger.LogInformation("Added {NewTrackersCount} new trackers to the torrent. Total trackers: {TotalTrackersCount}.", + addedTrackersCount, + allTrackers.Count); + return torrentDict.Encode(); } } \ No newline at end of file diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs index f7a34c5..3ccc049 100644 --- a/server/RdtClient.Service/Services/TrackerListGrabber.cs +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -23,9 +23,9 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac if (!Uri.TryCreate(trackerUrlList, UriKind.Absolute, out var trackerUri) || (trackerUri.Scheme != Uri.UriSchemeHttp && trackerUri.Scheme != Uri.UriSchemeHttps)) { - logger.LogError("Invalid tracker list URL: {Url}", trackerUrlList); + logger.LogWarning("Invalid tracker list URL format: {Url}", trackerUrlList); - throw new InvalidOperationException("Invalid tracker list URL."); + return []; } var currentExpiration = Settings.Get.General.TrackerEnrichmentCacheExpiration; @@ -88,13 +88,13 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac { logger.LogError(ex, "Fetching tracker list was canceled (timeout or cancellation)."); - throw new InvalidOperationException("Fetching tracker list was canceled due to timeout or cancellation.", ex); + throw new TaskCanceledException("Fetching tracker list was canceled due to timeout or cancellation.", ex); } catch (Exception ex) { logger.LogError(ex, "Unable to fetch tracker list."); - throw new InvalidOperationException("Unable to fetch tracker list for enrichment.", ex); + throw new Exception("Unable to fetch tracker list for enrichment.", ex); } finally { @@ -109,34 +109,57 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac var httpClient = httpClientFactory.CreateClient(); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); var token = cts.Token; - using var response = await httpClient.GetAsync(trackerUri, token).ConfigureAwait(false); + var response = await httpClient.GetAsync(trackerUri, token).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - await using var contentStream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false); - using var reader = new StreamReader(contentStream); - var result = await reader.ReadToEndAsync(token).ConfigureAwait(false); - - String[] trackers; - - try + using (response) { - trackers = result - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Where(line => !String.IsNullOrWhiteSpace(line)) - .Select(t => t.EndsWith("/") ? t.TrimEnd('/') : t) - .Where(t => Uri.TryCreate(t, UriKind.Absolute, out var uri) && - (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - catch (Exception ex) - { - logger.LogError(ex, "Error parsing tracker list response."); + response.EnsureSuccessStatusCode(); - throw new InvalidOperationException("Failed to parse tracker list response.", ex); - } + await using var contentStream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false); + using var reader = new StreamReader(contentStream); + var result = await reader.ReadToEndAsync(token).ConfigureAwait(false); - return trackers; + String[] trackers; + + try + { + trackers = result + .Split([ + "\r\n", "\n" + ], + StringSplitOptions.RemoveEmptyEntries) + .Where(line => !String.IsNullOrWhiteSpace(line) && !line.TrimStart().StartsWith('#')) + .Select(t => t.EndsWith("/") ? t.TrimEnd('/') : t) + .Select(t => t.Trim()) + .Where(t => + { + if (!Uri.TryCreate(t, UriKind.Absolute, out var uri)) + { + return false; + } + + return (uri.Scheme == Uri.UriSchemeHttp || + uri.Scheme == Uri.UriSchemeHttps || + uri.Scheme.Equals("udp", StringComparison.OrdinalIgnoreCase) || + uri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase)) && + !t.Contains("..") && + !t.Contains("\\") && + !t.Any(Char.IsControl) && + uri.Host.All(c => Char.IsLetterOrDigit(c) || c == '.' || c == '-' || c == '_' || c == ':') && + uri.Host.Length > 0; + }) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + catch (Exception ex) + + { + logger.LogError(ex, "Error parsing tracker list response."); + + throw new InvalidOperationException("Failed to parse tracker list response.", ex); + } + + return trackers; + } } } From 98042f80404107bfa24dfc21e84a70eb85b8119a Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Sun, 25 May 2025 21:50:26 +1000 Subject: [PATCH 8/9] Improved enrichment validation, logging, and test coverage. Strengthened URL Validation. Improved HTTP Client Configuration. Logged total rejected tracker count at the information level. Logged individual rejected tracker details at the debug level. Debug logs for bytes and enriched bytes are now conditional. Converted some tests from Fact to Theory. Used Assert.Collection in EnricherTest.cs. Standardized URL Encoding in Enricher.cs. --- .../Services/EnricherTest.cs | 89 +++++++++++++------ server/RdtClient.Service/Services/Enricher.cs | 3 +- server/RdtClient.Service/Services/Torrents.cs | 21 +++-- .../Services/TrackerListGrabber.cs | 47 +++++++--- 4 files changed, 114 insertions(+), 46 deletions(-) diff --git a/server/RdtClient.Service.Test/Services/EnricherTest.cs b/server/RdtClient.Service.Test/Services/EnricherTest.cs index ebdd84e..dfd1933 100644 --- a/server/RdtClient.Service.Test/Services/EnricherTest.cs +++ b/server/RdtClient.Service.Test/Services/EnricherTest.cs @@ -41,19 +41,30 @@ public class EnricherTest : IDisposable Assert.Equal(TestMagnetLink, enriched); } - [Fact] - public async Task EnrichMagnetLink_AddsTrackers_WhenTrackersFromTrackerGrabber() + [Theory] + [InlineData("magnet:?xt=urn:btih:HASH", + new[] + { + "http://new-tracker.com/announce" + }, + "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Fnew-tracker.com%2Fannounce")] + [InlineData("magnet:?xt=urn:btih:HASH", + new[] + { + "http://tracker1.com/announce", "http://tracker2.com/announce" + }, + "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce")] + public async Task EnrichMagnetLink_AddsTrackers_WhenTrackersFromTrackerGrabber_Theory(String magnetLink, String[] newTrackers, String expectedResult) { // Arrange - SetupTrackerListGrabber(["http://new-tracker.com/announce"]); - + SetupTrackerListGrabber(newTrackers); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); // Act - var enriched = await enricher.EnrichMagnetLink(TestMagnetLink); + var enriched = await enricher.EnrichMagnetLink(magnetLink); // Assert - Assert.Equal(TestMagnetLink + $"&tr={Uri.EscapeDataString("http://new-tracker.com/announce")}", enriched); + Assert.Equal(expectedResult, enriched); } [Fact] @@ -281,17 +292,27 @@ public class EnricherTest : IDisposable Assert.Equal(2, trValues.Count); } - [Fact] - public async Task EnrichMagnetLink_BasicMagnetWithHashOnly() + [Theory] + [InlineData("magnet:?xt=urn:btih:HASH", + new[] + { + "udp://tracker1" + }, + "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1")] + [InlineData("magnet:?xt=urn:btih:HASH", + new[] + { + "udp://tracker1", "udp://tracker2" + }, + "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2")] + [InlineData("magnet:?xt=urn:btih:HASH", + new String[] + { + }, + "magnet:?xt=urn:btih:HASH")] + public async Task EnrichMagnetLink_BasicMagnetWithHashOnly_Theory(String magnetLink, String[] newTrackers, String expectedResult) { // Arrange - var magnetLink = "magnet:?xt=urn:btih:HASH"; - - var newTrackers = new[] - { - "udp://tracker1" - }; - SetupTrackerListGrabber(newTrackers); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); @@ -299,20 +320,30 @@ public class EnricherTest : IDisposable var result = await enricher.EnrichMagnetLink(magnetLink); // Assert - Assert.Equal($"{magnetLink}&tr={Uri.EscapeDataString("udp://tracker1")}", result); + Assert.Equal(expectedResult, result); } - [Fact] - public async Task EnrichMagnetLink_MagnetEndsWithQuestionMarkOnly() + [Theory] + [InlineData("magnet:?", + new[] + { + "udp://tracker1" + }, + "magnet:?tr=udp%3A%2F%2Ftracker1")] + [InlineData("magnet:?", + new[] + { + "udp://tracker1", "udp://tracker2" + }, + "magnet:?tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2")] + [InlineData("magnet:?", + new String[] + { + }, + "magnet:?")] + public async Task EnrichMagnetLink_MagnetEndsWithQuestionMarkOnly_Theory(String magnetLink, String[] newTrackers, String expectedResult) { // Arrange - var magnetLink = "magnet:?"; - - var newTrackers = new[] - { - "udp://tracker1" - }; - SetupTrackerListGrabber(newTrackers); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); @@ -320,7 +351,7 @@ public class EnricherTest : IDisposable var result = await enricher.EnrichMagnetLink(magnetLink); // Assert - Assert.Equal($"{magnetLink}tr={Uri.EscapeDataString("udp://tracker1")}", result); + Assert.Equal(expectedResult, result); } [Fact] @@ -365,8 +396,10 @@ public class EnricherTest : IDisposable Assert.Equal(newTrackers[0], ((BEncodedString)enrichedDict["announce"]).Text); var announceList = (BEncodedList)enrichedDict["announce-list"]; Assert.Equal(2, announceList.Count); - Assert.Equal(newTrackers[0], ((BEncodedString)((BEncodedList)announceList[0])[0]).Text); - Assert.Equal(newTrackers[1], ((BEncodedString)((BEncodedList)announceList[1])[0]).Text); + + Assert.Collection(announceList, + tier => Assert.Equal(newTrackers[0], ((BEncodedString)((BEncodedList)tier)[0]).Text), + tier => Assert.Equal(newTrackers[1], ((BEncodedString)((BEncodedList)tier)[0]).Text)); } [Fact] diff --git a/server/RdtClient.Service/Services/Enricher.cs b/server/RdtClient.Service/Services/Enricher.cs index 222cdca..64a2965 100644 --- a/server/RdtClient.Service/Services/Enricher.cs +++ b/server/RdtClient.Service/Services/Enricher.cs @@ -1,4 +1,3 @@ - using System.Text; using System.Web; using Microsoft.Extensions.Logging; @@ -108,7 +107,7 @@ public sealed class Enricher(ILogger logger, ITrackerListGrabber track } else { - outParams.Add($"{kv.Key}={HttpUtility.UrlEncode(v)}"); + outParams.Add($"{kv.Key}={Uri.EscapeDataString(v)}"); } } } diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 3ff2e69..7b77350 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -115,7 +115,7 @@ public class Torrents( MagnetLink magnet; try { - magnet = MagnetLink.Parse(enriched); + magnet = MagnetLink.Parse(magnetLink); } catch (Exception ex) { @@ -139,15 +139,24 @@ public class Torrents( { MonoTorrent.Torrent monoTorrent; - var fileAsBase64 = Convert.ToBase64String(bytes); - logger.LogDebug($"bytes {bytes}"); - var enriched = await enricher.EnrichTorrentBytes(bytes); - logger.LogDebug($"enriched bytes {enriched}"); + + String fileAsBase64; + + if (enriched.SequenceEqual(bytes)) + { + fileAsBase64 = Convert.ToBase64String(bytes); + logger.LogDebug($"bytes {bytes}"); + } + else + { + fileAsBase64 = Convert.ToBase64String(enriched); + logger.LogDebug($"enriched bytes {enriched}"); + } try { - monoTorrent = await MonoTorrent.Torrent.LoadAsync(enriched); + monoTorrent = await MonoTorrent.Torrent.LoadAsync(bytes); } catch (Exception ex) { diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs index 3ccc049..7e49b32 100644 --- a/server/RdtClient.Service/Services/TrackerListGrabber.cs +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -1,5 +1,7 @@ +using System.Net.Http.Headers; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; +using System.Reflection; namespace RdtClient.Service.Services; @@ -107,9 +109,17 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac logger.LogDebug("Fetching tracker list from URL: {TrackerUrl}", trackerUri); var httpClient = httpClientFactory.CreateClient(); + + var version = Assembly.GetEntryAssembly()?.GetName().Version?.ToString(); + + var currentVersion = version != null && version.LastIndexOf('.') > 0 + ? $"v{version[..version.LastIndexOf('.')]}" + : ""; + + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("RdtClient", currentVersion)); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); var token = cts.Token; - var response = await httpClient.GetAsync(trackerUri, token).ConfigureAwait(false); + var response = await httpClient.GetAsync(trackerUri, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false); using (response) { @@ -123,6 +133,8 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac try { + var trackerRejectionCount = 0; + trackers = result .Split([ "\r\n", "\n" @@ -135,21 +147,36 @@ public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCac { if (!Uri.TryCreate(t, UriKind.Absolute, out var uri)) { + logger.LogDebug("Rejected tracker: {TrackerUrl} - Reason: Invalid format or unsupported scheme.", t); + trackerRejectionCount++; return false; } - return (uri.Scheme == Uri.UriSchemeHttp || - uri.Scheme == Uri.UriSchemeHttps || - uri.Scheme.Equals("udp", StringComparison.OrdinalIgnoreCase) || - uri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase)) && - !t.Contains("..") && - !t.Contains("\\") && - !t.Any(Char.IsControl) && - uri.Host.All(c => Char.IsLetterOrDigit(c) || c == '.' || c == '-' || c == '_' || c == ':') && - uri.Host.Length > 0; + var isIpv6 = uri.Host.StartsWith('[') && uri.Host.Contains(']'); + + var valid = ((isIpv6 && uri.Host.All(c => Char.IsLetterOrDigit(c) || c == '.' || c == ':' || c == '[' || c == ']')) || + (!isIpv6 && uri.Host.All(c => Char.IsLetterOrDigit(c) || c == '.' || c == '-' || c == '_'))) && + (uri.Scheme == Uri.UriSchemeHttp || + uri.Scheme == Uri.UriSchemeHttps || + uri.Scheme.Equals("udp", StringComparison.OrdinalIgnoreCase) || + uri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase)) && + !t.Contains("..") && + !t.Contains("\\") && + !t.Any(Char.IsControl) && + uri.Host.Length > 0; + + if (!valid) + { + logger.LogDebug("Enrichment tracker rejected: {TrackerUrl} - Reason: Invalid format or unsupported scheme.", t); + trackerRejectionCount++; + } + + return valid; }) .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); + + logger.LogInformation("{TrackerRejectionCount} trackers were rejected during enrichment.", trackerRejectionCount); } catch (Exception ex) From 60263ca685ce3d5093be5b3e1c8592832ad28cdc Mon Sep 17 00:00:00 2001 From: MentalBlank Date: Mon, 26 May 2025 00:39:47 +1000 Subject: [PATCH 9/9] Tracker enrichment test changes --- .../Services/EnricherTest.cs | 665 +++++++++++------- 1 file changed, 414 insertions(+), 251 deletions(-) diff --git a/server/RdtClient.Service.Test/Services/EnricherTest.cs b/server/RdtClient.Service.Test/Services/EnricherTest.cs index dfd1933..95a36a8 100644 --- a/server/RdtClient.Service.Test/Services/EnricherTest.cs +++ b/server/RdtClient.Service.Test/Services/EnricherTest.cs @@ -26,8 +26,103 @@ public class EnricherTest : IDisposable private const String TestMagnetLink = "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=TestFile&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce"; + // Helper methods for creating BEncodedDictionary objects for torrents + private static BEncodedDictionary CreateStandardTorrentDict(String announceUrl, List? announceListTier = null) + { + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(announceUrl) + }; + + var finalAnnounceList = new BEncodedList(); + + if (announceListTier != null) + { + var tier = new BEncodedList(); + + foreach (var tracker in announceListTier) + { + tier.Add(new BEncodedString(tracker)); + } + + finalAnnounceList.Add(tier); + } + else + { + finalAnnounceList.Add(new BEncodedList + { + new BEncodedString(announceUrl) + }); + } + + torrentDict["announce-list"] = finalAnnounceList; + + if (!torrentDict.ContainsKey("info")) + { + torrentDict["info"] = new BEncodedDictionary(); + } + + return torrentDict; + } + + private static BEncodedDictionary CreateTorrentDictWithOnlyAnnounce(String announceUrl) + { + return new BEncodedDictionary + { + ["announce"] = new BEncodedString(announceUrl), + ["info"] = new BEncodedDictionary() + }; + } + + private static BEncodedDictionary CreateTorrentDictWithNoTrackers() + { + return new BEncodedDictionary + { + ["info"] = new BEncodedDictionary() + }; + } + + private static BEncodedDictionary CreateTorrentDictWithComplexAnnounceList(String primaryAnnounceUrl, List> announceListTiers) + { + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(primaryAnnounceUrl) + }; + + var bencodedAnnounceList = new BEncodedList(); + + foreach (var tier in announceListTiers) + { + var bencodedTier = new BEncodedList(); + + foreach (var tracker in tier) + { + bencodedTier.Add(new BEncodedString(tracker)); + } + + bencodedAnnounceList.Add(bencodedTier); + } + + torrentDict["announce-list"] = bencodedAnnounceList; + + if (!torrentDict.ContainsKey("info")) + { + torrentDict["info"] = new BEncodedDictionary(); + } + + return torrentDict; + } + + private void SetupTrackerListGrabber(String[] trackerList) + { + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ReturnsAsync(trackerList) + .Verifiable(); + } + [Fact] - public async Task EnrichMagnetLink_AddsNoTrackers_WhenNoTrackersFromTrackerGrabber() + public async Task EnrichMagnetLink_WhenNoTrackers_ReturnsOriginal() { // Arrange SetupTrackerListGrabber([]); @@ -43,18 +138,25 @@ public class EnricherTest : IDisposable [Theory] [InlineData("magnet:?xt=urn:btih:HASH", - new[] - { - "http://new-tracker.com/announce" - }, - "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Fnew-tracker.com%2Fannounce")] + "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Fnew-tracker.com%2Fannounce", + "http://new-tracker.com/announce")] [InlineData("magnet:?xt=urn:btih:HASH", - new[] - { - "http://tracker1.com/announce", "http://tracker2.com/announce" - }, - "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce")] - public async Task EnrichMagnetLink_AddsTrackers_WhenTrackersFromTrackerGrabber_Theory(String magnetLink, String[] newTrackers, String expectedResult) + "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce", + "http://tracker1.com/announce", + "http://tracker2.com/announce")] + [InlineData("magnet:?xt=urn:btih:HASH", "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1", "udp://tracker1")] + [InlineData("magnet:?xt=urn:btih:HASH", "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2", "udp://tracker1", "udp://tracker2")] + [InlineData("magnet:?xt=urn:btih:HASH", "magnet:?xt=urn:btih:HASH")] + [InlineData("magnet:?", + "magnet:?tr=udp%3A%2F%2Ftracker1", + "udp://tracker1")] + [InlineData("magnet:?", + "magnet:?tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2", + "udp://tracker1", + "udp://tracker2")] + [InlineData("magnet:?", + "magnet:?")] + public async Task EnrichMagnetLink_WhenTrackersProvided_AddsTrackers(String magnetLink, String expectedResult, params String[] newTrackers) { // Arrange SetupTrackerListGrabber(newTrackers); @@ -68,7 +170,7 @@ public class EnricherTest : IDisposable } [Fact] - public async Task EnrichMagnetLink_DoesNotAddDuplicateTrackers_WhenTrackersFromTrackerGrabberAlreadyPresent() + public async Task EnrichMagnetLink_WhenDuplicatesProvided_AddsUniqueTrackers() { // Arrange SetupTrackerListGrabber(["http://new-tracker.com/announce", "http://tracker1.com/announce"]); @@ -83,7 +185,7 @@ public class EnricherTest : IDisposable } [Fact] - public async Task EnrichMagnetLink_Throws_WhenTrackerGrabberThrows() + public async Task EnrichMagnetLink_WhenGrabberFails_ThrowsException() { // Arrange _trackerListGrabberMock @@ -97,175 +199,30 @@ public class EnricherTest : IDisposable } [Fact] - public async Task EnrichTorrentBytes_AddsTrackers_WhenTrackersFromTrackerGrabber() + public async Task EnrichMagnetLink_WhenTrackersAdded_PreservesParameters() { // Arrange - var originalTracker = "http://tracker1.com/announce"; - var newTracker = "http://new-tracker.com/announce"; + var magnetLink = "magnet:?xt=urn:btih:HASH&dn=MyFile&tr=udp%3A%2F%2Fexisting"; - var torrentDict = new BEncodedDictionary + var newTrackers = new[] { - ["announce"] = new BEncodedString(originalTracker), - ["announce-list"] = new BEncodedList - { - new BEncodedList - { - new BEncodedString(originalTracker) - } - } + "udp://new1", "udp://existing" }; - var originalTorrentBytes = torrentDict.Encode(); - - SetupTrackerListGrabber([newTracker]); + SetupTrackerListGrabber(newTrackers); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); // Act - var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); - var enrichedDict = BEncodedValue.Decode(enrichedBytes); + var result = await enricher.EnrichMagnetLink(magnetLink); // Assert - Assert.True(enrichedDict.ContainsKey("announce")); - Assert.True(enrichedDict.ContainsKey("announce-list")); - - var announceList = (BEncodedList)enrichedDict["announce-list"]; - var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); - - Assert.Contains(originalTracker, flattened); - Assert.Contains(newTracker, flattened); - - var announce = ((BEncodedString)enrichedDict["announce"]).Text; - Assert.Equal(flattened.First(), announce); - } - - private void SetupTrackerListGrabber(String[] trackerList) - { - _trackerListGrabberMock - .Setup(t => t.GetTrackers()) - .ReturnsAsync(trackerList) - .Verifiable(); + var queryParams = System.Web.HttpUtility.ParseQueryString(new Uri(result).Query); + Assert.Equal("urn:btih:HASH", queryParams["xt"]); + Assert.Equal("MyFile", queryParams["dn"]); } [Fact] - public async Task EnrichTorrentBytes_DoesNotAddDuplicateTrackers() - { - // Arrange - var originalTracker = "http://tracker1.com/announce"; - var duplicateTracker = "http://tracker1.com/announce"; - - var torrentDict = new BEncodedDictionary - { - ["announce"] = new BEncodedString(originalTracker), - ["announce-list"] = new BEncodedList - { - new BEncodedList - { - new BEncodedString(originalTracker) - } - } - }; - - var originalTorrentBytes = torrentDict.Encode(); - - SetupTrackerListGrabber([duplicateTracker]); - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - // Act - var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); - var enrichedDict = BEncodedValue.Decode(enrichedBytes); - - // Assert - var announceList = (BEncodedList)enrichedDict["announce-list"]; - var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); - - Assert.Single(flattened); - Assert.Contains(originalTracker, flattened); - } - - [Fact] - public async Task EnrichTorrentBytes_AddsTrackers_WhenNoAnnounceListPresent() - { - // Arrange - var originalTracker = "http://tracker1.com/announce"; - var newTracker = "http://new-tracker.com/announce"; - - var torrentDict = new BEncodedDictionary - { - ["announce"] = new BEncodedString(originalTracker) - - // No "announce-list" - }; - - var originalTorrentBytes = torrentDict.Encode(); - - SetupTrackerListGrabber([newTracker]); - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - // Act - var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); - var enrichedDict = BEncodedValue.Decode(enrichedBytes); - - // Assert - Assert.True(enrichedDict.ContainsKey("announce-list")); - var announceList = (BEncodedList)enrichedDict["announce-list"]; - var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); - - Assert.Contains(originalTracker, flattened); - Assert.Contains(newTracker, flattened); - } - - [Fact] - public async Task EnrichTorrentBytes_Throws_WhenTrackerGrabberThrows() - { - // Arrange - _trackerListGrabberMock - .Setup(t => t.GetTrackers()) - .ThrowsAsync(new InvalidOperationException("Unable to fetch tracker list for enrichment.")); - - var torrentDict = new BEncodedDictionary - { - ["announce"] = new BEncodedString("http://tracker1.com/announce"), - ["announce-list"] = new BEncodedList - { - new BEncodedList - { - new BEncodedString("http://tracker1.com/announce") - } - } - }; - - var originalTorrentBytes = torrentDict.Encode(); - - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - // Act & Assert - await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(originalTorrentBytes)); - } - - [Fact] - public async Task EnrichTorrentBytes_ThrowsArgumentException_OnNullOrEmptyInput() - { - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(null!)); - await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes([])); - } - - [Fact] - public async Task EnrichTorrentBytes_ThrowsInvalidOperationException_OnNonTorrentBytes() - { - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - var notTorrent = new Byte[] - { - 1, 2, 3, 4, 5 - }; - - await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(notTorrent)); - } - - [Fact] - public async Task EnrichMagnetLink_WithExistingTrackersAndOtherParameters() + public async Task EnrichMagnetLink_WhenTrackersAdded_IncludesAllTrackers() { // Arrange var magnetLink = "magnet:?xt=urn:btih:HASH&dn=MyFile&tr=udp%3A%2F%2Fexisting"; @@ -285,99 +242,267 @@ public class EnricherTest : IDisposable var queryParams = System.Web.HttpUtility.ParseQueryString(new Uri(result).Query); var trValues = queryParams.GetValues("tr")?.ToList() ?? new List(); - Assert.Equal("urn:btih:HASH", queryParams["xt"]); - Assert.Equal("MyFile", queryParams["dn"]); Assert.Contains("udp://existing", trValues); Assert.Contains("udp://new1", trValues); Assert.Equal(2, trValues.Count); } - [Theory] - [InlineData("magnet:?xt=urn:btih:HASH", - new[] - { - "udp://tracker1" - }, - "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1")] - [InlineData("magnet:?xt=urn:btih:HASH", - new[] - { - "udp://tracker1", "udp://tracker2" - }, - "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2")] - [InlineData("magnet:?xt=urn:btih:HASH", - new String[] - { - }, - "magnet:?xt=urn:btih:HASH")] - public async Task EnrichMagnetLink_BasicMagnetWithHashOnly_Theory(String magnetLink, String[] newTrackers, String expectedResult) + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_CreatesAnnounceKey() { // Arrange - SetupTrackerListGrabber(newTrackers); + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); // Act - var result = await enricher.EnrichMagnetLink(magnetLink); + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); // Assert - Assert.Equal(expectedResult, result); - } - - [Theory] - [InlineData("magnet:?", - new[] - { - "udp://tracker1" - }, - "magnet:?tr=udp%3A%2F%2Ftracker1")] - [InlineData("magnet:?", - new[] - { - "udp://tracker1", "udp://tracker2" - }, - "magnet:?tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2")] - [InlineData("magnet:?", - new String[] - { - }, - "magnet:?")] - public async Task EnrichMagnetLink_MagnetEndsWithQuestionMarkOnly_Theory(String magnetLink, String[] newTrackers, String expectedResult) - { - // Arrange - SetupTrackerListGrabber(newTrackers); - var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); - - // Act - var result = await enricher.EnrichMagnetLink(magnetLink); - - // Assert - Assert.Equal(expectedResult, result); + Assert.True(enrichedDict.ContainsKey("announce")); } [Fact] - public async Task EnrichMagnetLink_GetTrackersReturnsEmptyArray_Unchanged() + public async Task EnrichTorrentBytes_WhenTrackersAdded_CreatesAnnounceListKey() { // Arrange - var magnetLink = "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Fexisting"; + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_IncludesAllTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); // Grabber returns newTracker + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + Assert.Contains(originalTracker, flattened); + Assert.Contains(newTracker, flattened); + Assert.Equal(2, flattened.Count); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_SetsPrimaryAnnounce() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + var announce = ((BEncodedString)enrichedDict["announce"]).Text; + Assert.Equal(flattened.First(), announce); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenDuplicatesProvided_AddsUniqueTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var duplicateTracker = "http://tracker1.com/announce"; + + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([duplicateTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + + Assert.Single(flattened); + Assert.Contains(originalTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceList_CreatesAnnounceListKey() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateTorrentDictWithOnlyAnnounce(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceList_AddsOriginalTracker() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateTorrentDictWithOnlyAnnounce(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + Assert.Contains(originalTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceList_AddsGrabbedTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateTorrentDictWithOnlyAnnounce(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + Assert.Contains(newTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenGrabberFails_ThrowsException() + { + // Arrange + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ThrowsAsync(new InvalidOperationException("Unable to fetch tracker list for enrichment.")); + + var torrentDict = CreateStandardTorrentDict("http://tracker1.com/announce"); + var originalTorrentBytes = torrentDict.Encode(); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(originalTorrentBytes)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoNewTrackers_ReturnsOriginalBytes() + { + // Arrange + var torrentDict = CreateStandardTorrentDict("http://existing.tracker/announce"); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([]); var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); // Act - var result = await enricher.EnrichMagnetLink(magnetLink); + var resultBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); // Assert - Assert.Equal(magnetLink, result); + Assert.Equal(originalTorrentBytes, resultBytes); } [Fact] - public async Task EnrichTorrentBytes_NoExistingAnnounceOrAnnounceList() + public async Task EnrichTorrentBytes_WhenNoTrackers_ReturnsOriginalBytes() { // Arrange - var torrentDict = new BEncodedDictionary + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([]); // Grabber returns no new trackers + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var resultBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + + // Assert + Assert.Equal(originalTorrentBytes, resultBytes); + + var decodedResult = BEncodedValue.Decode(resultBytes); + Assert.False(decodedResult.ContainsKey("announce")); + Assert.False(decodedResult.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNullInput_ThrowsArgumentException() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(null!)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenEmptyInput_ThrowsArgumentException() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes([])); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenInvalidInput_ThrowsInvalidOperationException() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + var notTorrent = new Byte[] { - ["info"] = new BEncodedDictionary() + 1, 2, 3, 4, 5 }; + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(notTorrent)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceFields_CreatesAnnounceField() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); var originalBytes = torrentDict.Encode(); var newTrackers = new[] @@ -393,9 +518,55 @@ public class EnricherTest : IDisposable var enrichedDict = BEncodedValue.Decode(enrichedBytes); // Assert + Assert.True(enrichedDict.ContainsKey("announce")); Assert.Equal(newTrackers[0], ((BEncodedString)enrichedDict["announce"]).Text); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceFields_CreatesAnnounceListKey() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalBytes = torrentDict.Encode(); + + var newTrackers = new[] + { + "udp://tracker1", "udp://tracker2" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceFields_AddsGrabbedTrackers() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalBytes = torrentDict.Encode(); + + var newTrackers = new[] + { + "udp://tracker1", "udp://tracker2" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert var announceList = (BEncodedList)enrichedDict["announce-list"]; - Assert.Equal(2, announceList.Count); + Assert.Equal(newTrackers.Length, announceList.Count); Assert.Collection(announceList, tier => Assert.Equal(newTrackers[0], ((BEncodedString)((BEncodedList)tier)[0]).Text), @@ -403,32 +574,24 @@ public class EnricherTest : IDisposable } [Fact] - public async Task EnrichTorrentBytes_WithExistingAnnounceAndAnnounceList() + public async Task EnrichTorrentBytes_WhenExistingTrackers_MergesNewTrackers() { // Arrange - var existingTrackers = new List - { - "http://tier1.com", - "http://existing.com/announce" - }; + var primaryAnnounceUrl = "http://existing.com/announce"; - var torrentDict = new BEncodedDictionary + var announceListTiers = new List> { - ["announce"] = new BEncodedString(existingTrackers[1]), - ["announce-list"] = new BEncodedList + new() { - new BEncodedList - { - new BEncodedString(existingTrackers[0]) - }, - new BEncodedList - { - new BEncodedString(existingTrackers[1]) - } + "http://tier1.com" }, - ["info"] = new BEncodedDictionary() + new() + { + primaryAnnounceUrl + } }; + var torrentDict = CreateTorrentDictWithComplexAnnounceList(primaryAnnounceUrl, announceListTiers); var originalBytes = torrentDict.Encode(); var newTrackersFromGrabber = new[] @@ -456,4 +619,4 @@ public class EnricherTest : IDisposable Assert.Equal(3, finalAnnounceList.Count); Assert.Contains(((BEncodedString)enrichedDict["announce"]).Text, finalAnnounceList); } -} \ No newline at end of file +}