diff --git a/server/RdtClient.Data/Models/Internal/DbSettings.cs b/server/RdtClient.Data/Models/Internal/DbSettings.cs index 8a67e9d..dae9217 100644 --- a/server/RdtClient.Data/Models/Internal/DbSettings.cs +++ b/server/RdtClient.Data/Models/Internal/DbSettings.cs @@ -74,7 +74,15 @@ Supports the following parameters: [DisplayName("Copy added torrent files")] [Description("When a torrent file or magnet is added, create a copy in this directory.")] public String? CopyAddedTorrents { get; set; } = null; - + + [DisplayName("Tracker enrichment list")] + [Description("Optional. Specify the URL of a tracker list file to be appended to magnet links and torrent files.")] + public String? TrackerEnrichmentList { get; set; } = null; + + [DisplayName("Tracker enrichment cache expiration")] + [Description("The time in minutes to cache the tracker list. Set to 0 to disable caching.")] + public Int32 TrackerEnrichmentCacheExpiration { get; set; } = 60; + [DisplayName("Disable update notifications")] [Description("Ignore update notifications. You will still be notified if the version you are running has a security vulnerability.")] public Boolean DisableUpdateNotifications { get; set; } = false; diff --git a/server/RdtClient.Service.Test/Services/EnricherTest.cs b/server/RdtClient.Service.Test/Services/EnricherTest.cs new file mode 100644 index 0000000..95a36a8 --- /dev/null +++ b/server/RdtClient.Service.Test/Services/EnricherTest.cs @@ -0,0 +1,622 @@ +using Microsoft.Extensions.Logging; +using Moq; +using RdtClient.Service.Services; +using MonoTorrent.BEncoding; + +namespace RdtClient.Service.Test.Services; + +public class EnricherTest : IDisposable +{ + private readonly MockRepository _mockRepository; + private readonly Mock> _loggerMock; + private readonly Mock _trackerListGrabberMock; + + public EnricherTest() + { + _mockRepository = new MockRepository(MockBehavior.Strict); + _loggerMock = _mockRepository.Create>(MockBehavior.Loose); + _trackerListGrabberMock = _mockRepository.Create(); + } + + public void Dispose() + { + _mockRepository.VerifyAll(); + } + + private const String TestMagnetLink = + "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=TestFile&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce"; + + // Helper methods for creating BEncodedDictionary objects for torrents + private static BEncodedDictionary CreateStandardTorrentDict(String announceUrl, List? announceListTier = null) + { + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(announceUrl) + }; + + var finalAnnounceList = new BEncodedList(); + + if (announceListTier != null) + { + var tier = new BEncodedList(); + + foreach (var tracker in announceListTier) + { + tier.Add(new BEncodedString(tracker)); + } + + finalAnnounceList.Add(tier); + } + else + { + finalAnnounceList.Add(new BEncodedList + { + new BEncodedString(announceUrl) + }); + } + + torrentDict["announce-list"] = finalAnnounceList; + + if (!torrentDict.ContainsKey("info")) + { + torrentDict["info"] = new BEncodedDictionary(); + } + + return torrentDict; + } + + private static BEncodedDictionary CreateTorrentDictWithOnlyAnnounce(String announceUrl) + { + return new BEncodedDictionary + { + ["announce"] = new BEncodedString(announceUrl), + ["info"] = new BEncodedDictionary() + }; + } + + private static BEncodedDictionary CreateTorrentDictWithNoTrackers() + { + return new BEncodedDictionary + { + ["info"] = new BEncodedDictionary() + }; + } + + private static BEncodedDictionary CreateTorrentDictWithComplexAnnounceList(String primaryAnnounceUrl, List> announceListTiers) + { + var torrentDict = new BEncodedDictionary + { + ["announce"] = new BEncodedString(primaryAnnounceUrl) + }; + + var bencodedAnnounceList = new BEncodedList(); + + foreach (var tier in announceListTiers) + { + var bencodedTier = new BEncodedList(); + + foreach (var tracker in tier) + { + bencodedTier.Add(new BEncodedString(tracker)); + } + + bencodedAnnounceList.Add(bencodedTier); + } + + torrentDict["announce-list"] = bencodedAnnounceList; + + if (!torrentDict.ContainsKey("info")) + { + torrentDict["info"] = new BEncodedDictionary(); + } + + return torrentDict; + } + + private void SetupTrackerListGrabber(String[] trackerList) + { + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ReturnsAsync(trackerList) + .Verifiable(); + } + + [Fact] + public async Task EnrichMagnetLink_WhenNoTrackers_ReturnsOriginal() + { + // Arrange + SetupTrackerListGrabber([]); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enriched = await enricher.EnrichMagnetLink(TestMagnetLink); + + // Assert + Assert.Equal(TestMagnetLink, enriched); + } + + [Theory] + [InlineData("magnet:?xt=urn:btih:HASH", + "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Fnew-tracker.com%2Fannounce", + "http://new-tracker.com/announce")] + [InlineData("magnet:?xt=urn:btih:HASH", + "magnet:?xt=urn:btih:HASH&tr=http%3A%2F%2Ftracker1.com%2Fannounce&tr=http%3A%2F%2Ftracker2.com%2Fannounce", + "http://tracker1.com/announce", + "http://tracker2.com/announce")] + [InlineData("magnet:?xt=urn:btih:HASH", "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1", "udp://tracker1")] + [InlineData("magnet:?xt=urn:btih:HASH", "magnet:?xt=urn:btih:HASH&tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2", "udp://tracker1", "udp://tracker2")] + [InlineData("magnet:?xt=urn:btih:HASH", "magnet:?xt=urn:btih:HASH")] + [InlineData("magnet:?", + "magnet:?tr=udp%3A%2F%2Ftracker1", + "udp://tracker1")] + [InlineData("magnet:?", + "magnet:?tr=udp%3A%2F%2Ftracker1&tr=udp%3A%2F%2Ftracker2", + "udp://tracker1", + "udp://tracker2")] + [InlineData("magnet:?", + "magnet:?")] + public async Task EnrichMagnetLink_WhenTrackersProvided_AddsTrackers(String magnetLink, String expectedResult, params String[] newTrackers) + { + // Arrange + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enriched = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + Assert.Equal(expectedResult, enriched); + } + + [Fact] + public async Task EnrichMagnetLink_WhenDuplicatesProvided_AddsUniqueTrackers() + { + // Arrange + SetupTrackerListGrabber(["http://new-tracker.com/announce", "http://tracker1.com/announce"]); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enriched = await enricher.EnrichMagnetLink(TestMagnetLink); + + // Assert + Assert.Equal(TestMagnetLink + $"&tr={Uri.EscapeDataString("http://new-tracker.com/announce")}", enriched); + } + + [Fact] + public async Task EnrichMagnetLink_WhenGrabberFails_ThrowsException() + { + // Arrange + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ThrowsAsync(new InvalidOperationException("Unable to fetch tracker list for enrichment.")); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => enricher.EnrichMagnetLink(TestMagnetLink)); + } + + [Fact] + public async Task EnrichMagnetLink_WhenTrackersAdded_PreservesParameters() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:HASH&dn=MyFile&tr=udp%3A%2F%2Fexisting"; + + var newTrackers = new[] + { + "udp://new1", "udp://existing" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var result = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + var queryParams = System.Web.HttpUtility.ParseQueryString(new Uri(result).Query); + Assert.Equal("urn:btih:HASH", queryParams["xt"]); + Assert.Equal("MyFile", queryParams["dn"]); + } + + [Fact] + public async Task EnrichMagnetLink_WhenTrackersAdded_IncludesAllTrackers() + { + // Arrange + var magnetLink = "magnet:?xt=urn:btih:HASH&dn=MyFile&tr=udp%3A%2F%2Fexisting"; + + var newTrackers = new[] + { + "udp://new1", "udp://existing" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var result = await enricher.EnrichMagnetLink(magnetLink); + + // Assert + var queryParams = System.Web.HttpUtility.ParseQueryString(new Uri(result).Query); + var trValues = queryParams.GetValues("tr")?.ToList() ?? new List(); + + Assert.Contains("udp://existing", trValues); + Assert.Contains("udp://new1", trValues); + Assert.Equal(2, trValues.Count); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_CreatesAnnounceKey() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_CreatesAnnounceListKey() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_IncludesAllTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); // Grabber returns newTracker + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + Assert.Contains(originalTracker, flattened); + Assert.Contains(newTracker, flattened); + Assert.Equal(2, flattened.Count); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenTrackersAdded_SetsPrimaryAnnounce() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + var announce = ((BEncodedString)enrichedDict["announce"]).Text; + Assert.Equal(flattened.First(), announce); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenDuplicatesProvided_AddsUniqueTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var duplicateTracker = "http://tracker1.com/announce"; + + var torrentDict = CreateStandardTorrentDict(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([duplicateTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + + Assert.Single(flattened); + Assert.Contains(originalTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceList_CreatesAnnounceListKey() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateTorrentDictWithOnlyAnnounce(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceList_AddsOriginalTracker() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateTorrentDictWithOnlyAnnounce(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + Assert.Contains(originalTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceList_AddsGrabbedTrackers() + { + // Arrange + var originalTracker = "http://tracker1.com/announce"; + var newTracker = "http://new-tracker.com/announce"; + var torrentDict = CreateTorrentDictWithOnlyAnnounce(originalTracker); + var originalTorrentBytes = torrentDict.Encode(); + SetupTrackerListGrabber([newTracker]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + var flattened = announceList.Cast().SelectMany(l => l.Cast().Select(s => s.Text)).ToList(); + Assert.Contains(newTracker, flattened); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenGrabberFails_ThrowsException() + { + // Arrange + _trackerListGrabberMock + .Setup(t => t.GetTrackers()) + .ThrowsAsync(new InvalidOperationException("Unable to fetch tracker list for enrichment.")); + + var torrentDict = CreateStandardTorrentDict("http://tracker1.com/announce"); + var originalTorrentBytes = torrentDict.Encode(); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(originalTorrentBytes)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoNewTrackers_ReturnsOriginalBytes() + { + // Arrange + var torrentDict = CreateStandardTorrentDict("http://existing.tracker/announce"); + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([]); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var resultBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + + // Assert + Assert.Equal(originalTorrentBytes, resultBytes); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoTrackers_ReturnsOriginalBytes() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalTorrentBytes = torrentDict.Encode(); + + SetupTrackerListGrabber([]); // Grabber returns no new trackers + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var resultBytes = await enricher.EnrichTorrentBytes(originalTorrentBytes); + + // Assert + Assert.Equal(originalTorrentBytes, resultBytes); + + var decodedResult = BEncodedValue.Decode(resultBytes); + Assert.False(decodedResult.ContainsKey("announce")); + Assert.False(decodedResult.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNullInput_ThrowsArgumentException() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(null!)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenEmptyInput_ThrowsArgumentException() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes([])); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenInvalidInput_ThrowsInvalidOperationException() + { + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + var notTorrent = new Byte[] + { + 1, 2, 3, 4, 5 + }; + + await Assert.ThrowsAsync(() => enricher.EnrichTorrentBytes(notTorrent)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceFields_CreatesAnnounceField() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalBytes = torrentDict.Encode(); + + var newTrackers = new[] + { + "udp://tracker1", "udp://tracker2" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce")); + Assert.Equal(newTrackers[0], ((BEncodedString)enrichedDict["announce"]).Text); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceFields_CreatesAnnounceListKey() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalBytes = torrentDict.Encode(); + + var newTrackers = new[] + { + "udp://tracker1", "udp://tracker2" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + Assert.True(enrichedDict.ContainsKey("announce-list")); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenNoAnnounceFields_AddsGrabbedTrackers() + { + // Arrange + var torrentDict = CreateTorrentDictWithNoTrackers(); + var originalBytes = torrentDict.Encode(); + + var newTrackers = new[] + { + "udp://tracker1", "udp://tracker2" + }; + + SetupTrackerListGrabber(newTrackers); + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var announceList = (BEncodedList)enrichedDict["announce-list"]; + Assert.Equal(newTrackers.Length, announceList.Count); + + Assert.Collection(announceList, + tier => Assert.Equal(newTrackers[0], ((BEncodedString)((BEncodedList)tier)[0]).Text), + tier => Assert.Equal(newTrackers[1], ((BEncodedString)((BEncodedList)tier)[0]).Text)); + } + + [Fact] + public async Task EnrichTorrentBytes_WhenExistingTrackers_MergesNewTrackers() + { + // Arrange + var primaryAnnounceUrl = "http://existing.com/announce"; + + var announceListTiers = new List> + { + new() + { + "http://tier1.com" + }, + new() + { + primaryAnnounceUrl + } + }; + + var torrentDict = CreateTorrentDictWithComplexAnnounceList(primaryAnnounceUrl, announceListTiers); + var originalBytes = torrentDict.Encode(); + + var newTrackersFromGrabber = new[] + { + "udp://new1", "http://existing.com/announce" + }; + + SetupTrackerListGrabber(newTrackersFromGrabber); + + var enricher = new Enricher(_loggerMock.Object, _trackerListGrabberMock.Object); + + // Act + var enrichedBytes = await enricher.EnrichTorrentBytes(originalBytes); + var enrichedDict = BEncodedValue.Decode(enrichedBytes); + + // Assert + var finalAnnounceList = ((BEncodedList)enrichedDict["announce-list"]) + .Cast() + .SelectMany(tier => tier.Cast().Select(t => t.Text)) + .ToList(); + + Assert.Contains("http://tier1.com", finalAnnounceList); + Assert.Contains("http://existing.com/announce", finalAnnounceList); + Assert.Contains("udp://new1", finalAnnounceList); + Assert.Equal(3, finalAnnounceList.Count); + Assert.Contains(((BEncodedString)enrichedDict["announce"]).Text, finalAnnounceList); + } +} diff --git a/server/RdtClient.Service.Test/Services/TorrentsTest.cs b/server/RdtClient.Service.Test/Services/TorrentsTest.cs index addd45d..054a62f 100644 --- a/server/RdtClient.Service.Test/Services/TorrentsTest.cs +++ b/server/RdtClient.Service.Test/Services/TorrentsTest.cs @@ -20,11 +20,13 @@ class Mocks public readonly Mock> TorrentsLoggerMock; public readonly Mock DownloadsMock; public readonly Mock TorrentDataMock; + public readonly Mock EnricherMock; public Mocks() { TorrentDataMock = new(); DownloadsMock = new(); + EnricherMock = new(); TorrentsLoggerMock = new(); @@ -58,7 +60,7 @@ public class TorrentsTest } ]; - return new () + return new() { { torrent, @@ -102,6 +104,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -167,6 +170,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -214,6 +218,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, @@ -280,6 +285,7 @@ public class TorrentsTest mocks.DownloadsMock.Object, mocks.ProcessFactoryMock.Object, fileSystemMock, + mocks.EnricherMock.Object, null!, // Torrent Clients are not used by `RunTorrentComplete`, this is fine null!, null!, diff --git a/server/RdtClient.Service/DiConfig.cs b/server/RdtClient.Service/DiConfig.cs index fd944e6..abe8af7 100644 --- a/server/RdtClient.Service/DiConfig.cs +++ b/server/RdtClient.Service/DiConfig.cs @@ -18,6 +18,8 @@ public static class DiConfig public static void RegisterRdtServices(this IServiceCollection services) { + services.AddMemoryCache(); + services.AddSingleton(); services.AddScoped(); @@ -38,6 +40,8 @@ public static class DiConfig services.AddScoped(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); diff --git a/server/RdtClient.Service/Services/Enricher.cs b/server/RdtClient.Service/Services/Enricher.cs new file mode 100644 index 0000000..64a2965 --- /dev/null +++ b/server/RdtClient.Service/Services/Enricher.cs @@ -0,0 +1,226 @@ +using System.Text; +using System.Web; +using Microsoft.Extensions.Logging; +using MonoTorrent.BEncoding; + +namespace RdtClient.Service.Services; + +public interface IEnricher +{ + Task EnrichMagnetLink(String magnetLink); + Task EnrichTorrentBytes(Byte[] torrentBytes); +} + +/// +/// Enriches magnet links and torrents by adding trackers from the tracker list grabber. +/// +public sealed class Enricher(ILogger logger, ITrackerListGrabber trackerListGrabber) : IEnricher +{ + /// + /// Add trackers from the tracker list grabber to the magnet link. + /// + /// Magnet link to add trackers to. Is not modified + /// Magnet link with additional trackers + public async Task EnrichMagnetLink(String magnetLink) + { + var newTrackers = await trackerListGrabber.GetTrackers().ConfigureAwait(false); + + if (newTrackers.Length == 0) + { + logger.LogWarning("No new trackers were retrieved."); + + return magnetLink; + } + + var qmIdx = magnetLink.IndexOf('?'); + + if (qmIdx == -1 || qmIdx == magnetLink.Length - 1) + { + var sb = new StringBuilder(magnetLink); + + if (qmIdx == -1) + { + sb.Append('?'); + } + + for (var i = 0; i < newTrackers.Length; i++) + { + if (i > 0 || (qmIdx != magnetLink.Length - 1 && qmIdx != -1)) + { + sb.Append('&'); + } + + sb.Append("tr=").Append(Uri.EscapeDataString(newTrackers[i])); + } + + logger.LogInformation("Added {NewTrackersCount} new trackers to a magnet link with no initial query string. Total trackers: {TotalTrackersCount}.", + newTrackers.Length, + newTrackers.Length); + + return sb.ToString(); + } + + var schemePart = magnetLink[..qmIdx]; + var queryPart = magnetLink[(qmIdx + 1)..]; + + var queryKVs = HttpUtility.ParseQueryString(queryPart); + + var paramDict = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (String key in queryKVs) + { + if (key == null) + { + continue; + } + + if (!paramDict.ContainsKey(key)) + { + paramDict[key] = new List(); + } + + paramDict[key].AddRange(queryKVs.GetValues(key) ?? []); + } + + var existingTrackers = paramDict.TryGetValue("tr", out var value) + ? new HashSet(value, StringComparer.OrdinalIgnoreCase) + : new HashSet(StringComparer.OrdinalIgnoreCase); + + paramDict.Remove("tr"); + + var newUniqueTrackers = newTrackers.Where(t => !existingTrackers.Contains(t)).ToList(); + + foreach (var tr in newUniqueTrackers) + { + existingTrackers.Add(tr); + } + + var outParams = new List(); + + foreach (var kv in paramDict) + { + foreach (var v in kv.Value) + { + if (kv.Key == "xt" || kv.Key == "dn") + { + outParams.Add($"{kv.Key}={v}"); + } + else + { + outParams.Add($"{kv.Key}={Uri.EscapeDataString(v)}"); + } + } + } + + foreach (var tr in existingTrackers) + { + outParams.Add($"tr={Uri.EscapeDataString(tr)}"); + } + + var finalMagnet = schemePart + "?" + String.Join("&", outParams); + + logger.LogInformation("Added {NewTrackersCount} new trackers to the magnet link. Total trackers: {TotalTrackersCount}.", + newUniqueTrackers.Count, + existingTrackers.Count); + + return finalMagnet; + } + + /// + /// Add trackers from the tracker list grabber to the .torrent file bytes. + /// + /// Torrent file bytes to add trackers to. Is not modified + /// Torrent file bytes with additional trackers + public async Task EnrichTorrentBytes(Byte[] torrentBytes) + { + if (torrentBytes == null || torrentBytes.Length == 0) + { + throw new ArgumentException("Torrent bytes cannot be null or empty.", nameof(torrentBytes)); + } + + BEncodedDictionary torrentDict; + + try + { + torrentDict = BEncodedValue.Decode(torrentBytes); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to decode torrent bytes."); + + throw new InvalidOperationException("Invalid torrent file format.", ex); + } + + var newTrackers = await trackerListGrabber.GetTrackers().ConfigureAwait(false); + + if (newTrackers.Length == 0) + { + logger.LogWarning("No new trackers were retrieved."); + + return torrentBytes; + } + + var seenTrackers = new HashSet(StringComparer.OrdinalIgnoreCase); + var allTrackers = new List(); + + if (torrentDict.TryGetValue("announce-list", out var alc) && alc is BEncodedList alList) + { + foreach (var tier in alList.OfType()) + { + foreach (var s in tier.OfType()) + { + if (seenTrackers.Add(s.Text)) + { + allTrackers.Add(s.Text); + } + } + } + } + + if (torrentDict.TryGetValue("announce", out var announceValue) && announceValue is BEncodedString announceStr) + { + if (seenTrackers.Add(announceStr.Text)) + { + allTrackers.Add(announceStr.Text); + } + } + + var addedTrackersCount = 0; + + foreach (var tracker in newTrackers) + { + if (seenTrackers.Add(tracker)) + { + allTrackers.Add(tracker); + addedTrackersCount++; + } + } + + var dedupedAnnounceList = new BEncodedList(); + + foreach (var tracker in allTrackers) + { + dedupedAnnounceList.Add(new BEncodedList + { + new BEncodedString(tracker) + }); + } + + torrentDict["announce-list"] = dedupedAnnounceList; + + if (allTrackers.Count > 0) + { + torrentDict["announce"] = new BEncodedString(allTrackers[0]); + } + else + { + return torrentBytes; + } + + logger.LogInformation("Added {NewTrackersCount} new trackers to the torrent. Total trackers: {TotalTrackersCount}.", + addedTrackersCount, + allTrackers.Count); + + return torrentDict.Encode(); + } +} \ No newline at end of file diff --git a/server/RdtClient.Service/Services/ITrackerListGrabber.cs b/server/RdtClient.Service/Services/ITrackerListGrabber.cs new file mode 100644 index 0000000..d3ccc08 --- /dev/null +++ b/server/RdtClient.Service/Services/ITrackerListGrabber.cs @@ -0,0 +1,6 @@ +namespace RdtClient.Service.Services; + +public interface ITrackerListGrabber +{ + Task GetTrackers(); +} \ No newline at end of file diff --git a/server/RdtClient.Service/Services/Torrents.cs b/server/RdtClient.Service/Services/Torrents.cs index 59ce8e8..8d4be9b 100644 --- a/server/RdtClient.Service/Services/Torrents.cs +++ b/server/RdtClient.Service/Services/Torrents.cs @@ -3,6 +3,7 @@ using System.IO.Abstractions; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Web; using Microsoft.Extensions.Logging; using MonoTorrent; using RdtClient.Data.Data; @@ -24,6 +25,7 @@ public class Torrents( IDownloads downloads, IProcessFactory processFactory, IFileSystem fileSystem, + IEnricher enricher, AllDebridTorrentClient allDebridTorrentClient, PremiumizeTorrentClient premiumizeTorrentClient, RealDebridTorrentClient realDebridTorrentClient, @@ -109,8 +111,8 @@ public class Torrents( public async Task AddMagnetToDebridQueue(String magnetLink, Torrent torrent) { + var enriched = await enricher.EnrichMagnetLink(magnetLink); MagnetLink magnet; - try { magnet = MagnetLink.Parse(magnetLink); @@ -125,11 +127,9 @@ public class Torrents( torrent.RdName = magnet.Name; var hash = magnet.InfoHashes.V1OrV2.ToHex(); - - var newTorrent = await AddQueued(hash, magnetLink, false, torrent); + var newTorrent = await AddQueued(hash, enriched, false, torrent); Log($"Adding {hash} (magnet link) to queue", newTorrent); - await CopyAddedTorrent(magnet.Name!, magnetLink); return newTorrent; @@ -139,8 +139,20 @@ public class Torrents( { MonoTorrent.Torrent monoTorrent; - var fileAsBase64 = Convert.ToBase64String(bytes); - logger.LogDebug($"bytes {bytes}"); + var enriched = await enricher.EnrichTorrentBytes(bytes); + + String fileAsBase64; + + if (enriched.SequenceEqual(bytes)) + { + fileAsBase64 = Convert.ToBase64String(bytes); + logger.LogDebug($"bytes {bytes}"); + } + else + { + fileAsBase64 = Convert.ToBase64String(enriched); + logger.LogDebug($"enriched bytes {enriched}"); + } try { @@ -224,7 +236,7 @@ public class Torrents( { throw new("Torrent has no torrent file or magnet link"); } - + logger.LogDebug("Adding {hash} to debrid provider {torrentInfo}", torrent.Hash, torrent.ToLog()); await RealDebridUpdateLock.WaitAsync(); @@ -435,7 +447,7 @@ public class Torrents( { var download = await downloads.GetById(downloadId) ?? throw new($"Download with ID {downloadId} not found"); - Log($"Unrestricting link", download, download.Torrent); + Log("Unrestricting link", download, download.Torrent); var unrestrictedLink = await TorrentClient.Unrestrict(download.Path); diff --git a/server/RdtClient.Service/Services/TrackerListGrabber.cs b/server/RdtClient.Service/Services/TrackerListGrabber.cs new file mode 100644 index 0000000..7e49b32 --- /dev/null +++ b/server/RdtClient.Service/Services/TrackerListGrabber.cs @@ -0,0 +1,192 @@ +using System.Net.Http.Headers; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using System.Reflection; + +namespace RdtClient.Service.Services; + +public class TrackerListGrabber(IHttpClientFactory httpClientFactory, IMemoryCache memoryCache, ILogger logger) : ITrackerListGrabber +{ + private const String CacheKey = "TrackerList"; + + private Int32? _lastExpirationMinutes; + + private static readonly SemaphoreSlim Semaphore = new(1, 1); + + public async Task GetTrackers() + { + var trackerUrlList = Settings.Get.General.TrackerEnrichmentList; + + if (String.IsNullOrWhiteSpace(trackerUrlList)) + { + return []; + } + + if (!Uri.TryCreate(trackerUrlList, UriKind.Absolute, out var trackerUri) || + (trackerUri.Scheme != Uri.UriSchemeHttp && trackerUri.Scheme != Uri.UriSchemeHttps)) + { + logger.LogWarning("Invalid tracker list URL format: {Url}", trackerUrlList); + + return []; + } + + var currentExpiration = Settings.Get.General.TrackerEnrichmentCacheExpiration; + var useCache = currentExpiration > 0; + + if (!useCache) + { + memoryCache.Remove(CacheKey); + _lastExpirationMinutes = null; + } + else + { + if (_lastExpirationMinutes is not null && currentExpiration != _lastExpirationMinutes) + { + logger.LogDebug("Tracker list cache timeout changed, invalidating cache."); + memoryCache.Remove(CacheKey); + } + + _lastExpirationMinutes = currentExpiration; + + if (memoryCache.TryGetValue(CacheKey, out String[]? cachedTrackers) && cachedTrackers is { Length: > 0 }) + { + logger.LogDebug("Using cached tracker list."); + + return cachedTrackers; + } + } + + await Semaphore.WaitAsync().ConfigureAwait(false); + + try + { + if (useCache) + { + if (memoryCache.TryGetValue(CacheKey, out String[]? cachedTrackers) && cachedTrackers is { Length: > 0 }) + { + logger.LogDebug("Using cached tracker list (after lock)."); + + return cachedTrackers; + } + } + + logger.LogDebug("Tracker cache miss or cache disabled. Fetching tracker list."); + + var trackers = await FetchAndParseTrackersAsync(trackerUri).ConfigureAwait(false); + + if (useCache) + { + memoryCache.Set(CacheKey, + trackers, + new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(currentExpiration) + }); + } + + return trackers; + } + catch (TaskCanceledException ex) + { + logger.LogError(ex, "Fetching tracker list was canceled (timeout or cancellation)."); + + throw new TaskCanceledException("Fetching tracker list was canceled due to timeout or cancellation.", ex); + } + catch (Exception ex) + { + logger.LogError(ex, "Unable to fetch tracker list."); + + throw new Exception("Unable to fetch tracker list for enrichment.", ex); + } + finally + { + Semaphore.Release(); + } + } + + private async Task FetchAndParseTrackersAsync(Uri trackerUri) + { + logger.LogDebug("Fetching tracker list from URL: {TrackerUrl}", trackerUri); + + var httpClient = httpClientFactory.CreateClient(); + + var version = Assembly.GetEntryAssembly()?.GetName().Version?.ToString(); + + var currentVersion = version != null && version.LastIndexOf('.') > 0 + ? $"v{version[..version.LastIndexOf('.')]}" + : ""; + + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("RdtClient", currentVersion)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + var token = cts.Token; + var response = await httpClient.GetAsync(trackerUri, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false); + + using (response) + { + response.EnsureSuccessStatusCode(); + + await using var contentStream = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false); + using var reader = new StreamReader(contentStream); + var result = await reader.ReadToEndAsync(token).ConfigureAwait(false); + + String[] trackers; + + try + { + var trackerRejectionCount = 0; + + trackers = result + .Split([ + "\r\n", "\n" + ], + StringSplitOptions.RemoveEmptyEntries) + .Where(line => !String.IsNullOrWhiteSpace(line) && !line.TrimStart().StartsWith('#')) + .Select(t => t.EndsWith("/") ? t.TrimEnd('/') : t) + .Select(t => t.Trim()) + .Where(t => + { + if (!Uri.TryCreate(t, UriKind.Absolute, out var uri)) + { + logger.LogDebug("Rejected tracker: {TrackerUrl} - Reason: Invalid format or unsupported scheme.", t); + trackerRejectionCount++; + return false; + } + + var isIpv6 = uri.Host.StartsWith('[') && uri.Host.Contains(']'); + + var valid = ((isIpv6 && uri.Host.All(c => Char.IsLetterOrDigit(c) || c == '.' || c == ':' || c == '[' || c == ']')) || + (!isIpv6 && uri.Host.All(c => Char.IsLetterOrDigit(c) || c == '.' || c == '-' || c == '_'))) && + (uri.Scheme == Uri.UriSchemeHttp || + uri.Scheme == Uri.UriSchemeHttps || + uri.Scheme.Equals("udp", StringComparison.OrdinalIgnoreCase) || + uri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase)) && + !t.Contains("..") && + !t.Contains("\\") && + !t.Any(Char.IsControl) && + uri.Host.Length > 0; + + if (!valid) + { + logger.LogDebug("Enrichment tracker rejected: {TrackerUrl} - Reason: Invalid format or unsupported scheme.", t); + trackerRejectionCount++; + } + + return valid; + }) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + logger.LogInformation("{TrackerRejectionCount} trackers were rejected during enrichment.", trackerRejectionCount); + } + catch (Exception ex) + + { + logger.LogError(ex, "Error parsing tracker list response."); + + throw new InvalidOperationException("Failed to parse tracker list response.", ex); + } + + return trackers; + } + } +}