From a080bf1c6ccd6edafb054033c64d23ae1a59ee6c Mon Sep 17 00:00:00 2001 From: Qstick Date: Mon, 15 Feb 2021 23:33:13 -0500 Subject: [PATCH] Proxy Nzb/Torrent Downloads thru Prowlarr --- .../Authentication/ProtectionService.cs | 101 ++++++++++++++++++ src/NzbDrone.Core/History/HistoryService.cs | 18 +++- .../IndexerSearch/NzbSearchService.cs | 1 + .../Indexers/DownloadMappingService.cs | 52 +++++++++ src/NzbDrone.Core/Indexers/DownloadService.cs | 86 +++++++++++++++ .../Indexers/Events/IndexerDownloadEvent.cs | 21 ++++ .../{ => Events}/IndexerQueryEvent.cs | 2 +- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 28 +++++ src/NzbDrone.Core/Indexers/IIndexer.cs | 5 +- src/NzbDrone.Core/Indexers/IndexerBase.cs | 2 + src/NzbDrone.Core/Prowlarr.Core.csproj | 1 + src/Prowlarr.Api.V1/Indexers/IndexerModule.cs | 66 +++++++++++- .../Extensions/RequestExtensions.cs | 29 +++++ .../Extensions/ResponseExtensions.cs | 29 +++++ 14 files changed, 435 insertions(+), 6 deletions(-) create mode 100644 src/NzbDrone.Core/Authentication/ProtectionService.cs create mode 100644 src/NzbDrone.Core/Indexers/DownloadMappingService.cs create mode 100644 src/NzbDrone.Core/Indexers/DownloadService.cs create mode 100644 src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs rename src/NzbDrone.Core/Indexers/{ => Events}/IndexerQueryEvent.cs (94%) create mode 100644 src/Prowlarr.Http/Extensions/ResponseExtensions.cs diff --git a/src/NzbDrone.Core/Authentication/ProtectionService.cs b/src/NzbDrone.Core/Authentication/ProtectionService.cs new file mode 100644 index 000000000..5fa5d6f2d --- /dev/null +++ b/src/NzbDrone.Core/Authentication/ProtectionService.cs @@ -0,0 +1,101 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; + +namespace NzbDrone.Core.Authentication +{ + public interface IProtectionService + { + string Protect(string plainText); + string UnProtect(string plainText); + } + + public class ProtectionService : IProtectionService + { + private readonly IConfigFileProvider _configService; + + public ProtectionService(IConfigFileProvider configService) + { + _configService = configService; + } + + public string Protect(string text) + { + var key = Encoding.UTF8.GetBytes(_configService.ApiKey); + + using (var aesAlg = Aes.Create()) + { + using (var encryptor = aesAlg.CreateEncryptor(key, aesAlg.IV)) + { + using (var msEncrypt = new MemoryStream()) + { + using (var csEncrypt = new CryptoStream(msEncrypt, encryptor, CryptoStreamMode.Write)) + using (var swEncrypt = new StreamWriter(csEncrypt)) + { + swEncrypt.Write(text); + } + + var iv = aesAlg.IV; + + var decryptedContent = msEncrypt.ToArray(); + + var result = new byte[iv.Length + decryptedContent.Length]; + + Buffer.BlockCopy(iv, 0, result, 0, iv.Length); + Buffer.BlockCopy(decryptedContent, 0, result, iv.Length, decryptedContent.Length); + + return Convert.ToBase64String(result); + } + } + } + } + + public string UnProtect(string value) + { + if (value.IsNullOrWhiteSpace()) + { + return value; + } + + try + { + value = value.Replace(" ", "+"); + var fullCipher = Convert.FromBase64String(value); + + var iv = new byte[16]; + var cipher = new byte[fullCipher.Length - iv.Length]; + + Buffer.BlockCopy(fullCipher, 0, iv, 0, iv.Length); + Buffer.BlockCopy(fullCipher, iv.Length, cipher, 0, fullCipher.Length - iv.Length); + var key = Encoding.UTF8.GetBytes(_configService.ApiKey); + + using (var aesAlg = Aes.Create()) + { + using (var decryptor = aesAlg.CreateDecryptor(key, iv)) + { + string result; + using (var msDecrypt = new MemoryStream(cipher)) + { + using (var csDecrypt = new CryptoStream(msDecrypt, decryptor, CryptoStreamMode.Read)) + { + using (var srDecrypt = new StreamReader(csDecrypt)) + { + result = srDecrypt.ReadToEnd(); + } + } + } + + return result; + } + } + } + catch (Exception) + { + return string.Empty; + } + } + } +} diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index c04acc0e5..b78e19ff3 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -4,6 +4,7 @@ using System.Linq; using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider.Events; @@ -24,7 +25,8 @@ namespace NzbDrone.Core.History public class HistoryService : IHistoryService, IHandle>, - IHandle + IHandle, + IHandle { private readonly IHistoryRepository _historyRepository; private readonly Logger _logger; @@ -99,6 +101,20 @@ namespace NzbDrone.Core.History _historyRepository.Insert(history); } + public void Handle(IndexerDownloadEvent message) + { + var history = new History + { + Date = DateTime.UtcNow, + IndexerId = message.IndexerId, + EventType = HistoryEventType.ReleaseGrabbed + }; + + history.Data.Add("Successful", message.Successful.ToString()); + + _historyRepository.Insert(history); + } + public void Handle(ProviderDeletedEvent message) { _historyRepository.DeleteForIndexers(new List { message.ProviderId }); diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index 3c8c57370..092ae247f 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -6,6 +6,7 @@ using NLog; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Common.TPL; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; diff --git a/src/NzbDrone.Core/Indexers/DownloadMappingService.cs b/src/NzbDrone.Core/Indexers/DownloadMappingService.cs new file mode 100644 index 000000000..8c5e7ea67 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/DownloadMappingService.cs @@ -0,0 +1,52 @@ +using System; +using System.Net; +using System.Text; +using Microsoft.AspNetCore.WebUtilities; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Authentication; +using NzbDrone.Core.Configuration; + +namespace NzbDrone.Core.Indexers +{ + public interface IDownloadMappingService + { + Uri ConvertToProxyLink(Uri link, string serverUrl, int indexerId, string file = "t"); + string ConvertToNormalLink(string link); + } + + public class DownloadMappingService : IDownloadMappingService + { + private readonly IProtectionService _protectionService; + private readonly IConfigFileProvider _configFileProvider; + + public DownloadMappingService(IProtectionService protectionService, IConfigFileProvider configFileProvider) + { + _protectionService = protectionService; + _configFileProvider = configFileProvider; + } + + public Uri ConvertToProxyLink(Uri link, string serverUrl, int indexerId, string file = "t") + { + var urlBase = _configFileProvider.UrlBase; + + if (urlBase.IsNotNullOrWhiteSpace() && !urlBase.StartsWith("/")) + { + urlBase = "/" + urlBase; + } + + var encryptedLink = _protectionService.Protect(link.ToString()); + var encodedLink = WebEncoders.Base64UrlEncode(Encoding.UTF8.GetBytes(encryptedLink)); + var urlEncodedFile = WebUtility.UrlEncode(file); + var proxyLink = $"{serverUrl}{urlBase}/api/v1/indexer/{indexerId}/download?apikey={_configFileProvider.ApiKey}&link={encodedLink}&file={urlEncodedFile}"; + return new Uri(proxyLink); + } + + public string ConvertToNormalLink(string link) + { + var encodedLink = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(link)); + var decryptedLink = _protectionService.UnProtect(encodedLink); + + return decryptedLink; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/DownloadService.cs b/src/NzbDrone.Core/Indexers/DownloadService.cs new file mode 100644 index 000000000..0f5ac3d75 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/DownloadService.cs @@ -0,0 +1,86 @@ +using System; +using NLog; +using NzbDrone.Common.EnsureThat; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.TPL; +using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Events; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.Indexers +{ + public interface IDownloadService + { + byte[] DownloadReport(string link, int indexerId); + } + + public class DownloadService : IDownloadService + { + private readonly IIndexerFactory _indexerFactory; + private readonly IIndexerStatusService _indexerStatusService; + private readonly IRateLimitService _rateLimitService; + private readonly IEventAggregator _eventAggregator; + private readonly Logger _logger; + + public DownloadService(IIndexerFactory indexerFactory, + IIndexerStatusService indexerStatusService, + IRateLimitService rateLimitService, + IEventAggregator eventAggregator, + Logger logger) + { + _indexerFactory = indexerFactory; + _indexerStatusService = indexerStatusService; + _rateLimitService = rateLimitService; + _eventAggregator = eventAggregator; + _logger = logger; + } + + public byte[] DownloadReport(string link, int indexerId) + { + var url = new HttpUri(link); + + // Limit grabs to 2 per second. + if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:")) + { + _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2)); + } + + var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(indexerId)); + bool success; + var downloadedBytes = Array.Empty(); + + try + { + downloadedBytes = indexer.Download(url); + _indexerStatusService.RecordSuccess(indexerId); + success = true; + } + catch (ReleaseUnavailableException) + { + _logger.Trace("Release {0} no longer available on indexer.", link); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, false)); + throw; + } + catch (ReleaseDownloadException ex) + { + var http429 = ex.InnerException as TooManyRequestsException; + if (http429 != null) + { + _indexerStatusService.RecordFailure(indexerId, http429.RetryAfter); + } + else + { + _indexerStatusService.RecordFailure(indexerId); + } + + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, false)); + throw; + } + + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success)); + return downloadedBytes; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs b/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs new file mode 100644 index 000000000..cb060b158 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NzbDrone.Common.Messaging; + +namespace NzbDrone.Core.Indexers.Events +{ + public class IndexerDownloadEvent : IEvent + { + public int IndexerId { get; set; } + public bool Successful { get; set; } + + public IndexerDownloadEvent(int indexerId, bool successful) + { + IndexerId = indexerId; + Successful = successful; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerQueryEvent.cs b/src/NzbDrone.Core/Indexers/Events/IndexerQueryEvent.cs similarity index 94% rename from src/NzbDrone.Core/Indexers/IndexerQueryEvent.cs rename to src/NzbDrone.Core/Indexers/Events/IndexerQueryEvent.cs index e83500c7c..e8c1477fd 100644 --- a/src/NzbDrone.Core/Indexers/IndexerQueryEvent.cs +++ b/src/NzbDrone.Core/Indexers/Events/IndexerQueryEvent.cs @@ -1,7 +1,7 @@ using NzbDrone.Common.Messaging; using NzbDrone.Core.IndexerSearch.Definitions; -namespace NzbDrone.Core.Indexers +namespace NzbDrone.Core.Indexers.Events { public class IndexerQueryEvent : IEvent { diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index e45694263..c1ff88afc 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -9,8 +9,10 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Http.CloudFlare; +using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; @@ -89,6 +91,32 @@ namespace NzbDrone.Core.Indexers return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } + public override byte[] Download(HttpUri link) + { + Cookies = GetCookies(); + + var requestBuilder = new HttpRequestBuilder(link.FullUri); + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + var downloadBytes = Array.Empty(); + + try + { + downloadBytes = _httpClient.Execute(requestBuilder.Build()).ResponseData; + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Download failed"); + } + + return downloadBytes; + } + protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator) { //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page. diff --git a/src/NzbDrone.Core/Indexers/IIndexer.cs b/src/NzbDrone.Core/Indexers/IIndexer.cs index f8a6e99e7..ce19c9dec 100644 --- a/src/NzbDrone.Core/Indexers/IIndexer.cs +++ b/src/NzbDrone.Core/Indexers/IIndexer.cs @@ -1,6 +1,5 @@ -using System.Collections.Generic; +using NzbDrone.Common.Http; using NzbDrone.Core.IndexerSearch.Definitions; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; namespace NzbDrone.Core.Indexers @@ -20,6 +19,8 @@ namespace NzbDrone.Core.Indexers IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria); IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria); + byte[] Download(HttpUri link); + IndexerCapabilities GetCapabilities(); } } diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 63fb54508..713765651 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -4,6 +4,7 @@ using System.Linq; using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; @@ -71,6 +72,7 @@ namespace NzbDrone.Core.Indexers public abstract IndexerPageableQueryResult Fetch(TvSearchCriteria searchCriteria); public abstract IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria); public abstract IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria); + public abstract byte[] Download(HttpUri searchCriteria); public abstract IndexerCapabilities GetCapabilities(); diff --git a/src/NzbDrone.Core/Prowlarr.Core.csproj b/src/NzbDrone.Core/Prowlarr.Core.csproj index 449b8e71f..1c9f1f4dd 100644 --- a/src/NzbDrone.Core/Prowlarr.Core.csproj +++ b/src/NzbDrone.Core/Prowlarr.Core.csproj @@ -5,6 +5,7 @@ + diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs index 5cb40608b..c2e2d8d05 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs @@ -1,10 +1,14 @@ +using System; using System.Collections.Generic; +using System.Text; using Nancy; using Nancy.ModelBinding; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch; using NzbDrone.Core.Parser; +using NzbDrone.Http.Extensions; +using Prowlarr.Http.Extensions; using Prowlarr.Http.REST; namespace Prowlarr.Api.V1.Indexers @@ -15,18 +19,26 @@ namespace Prowlarr.Api.V1.Indexers private IIndexerFactory _indexerFactory { get; set; } private ISearchForNzb _nzbSearchService { get; set; } + private IDownloadMappingService _downloadMappingService { get; set; } + private IDownloadService _downloadService { get; set; } - public IndexerModule(IndexerFactory indexerFactory, ISearchForNzb nzbSearchService) + public IndexerModule(IndexerFactory indexerFactory, ISearchForNzb nzbSearchService, IDownloadMappingService downloadMappingService, IDownloadService downloadService) : base(indexerFactory, "indexer", ResourceMapper) { _indexerFactory = indexerFactory; _nzbSearchService = nzbSearchService; + _downloadMappingService = downloadMappingService; + _downloadService = downloadService; Get("{id}/newznab", x => { var request = this.Bind(); return GetNewznabResponse(request); }); + Get("{id}/download", x => + { + return GetDownload(x.id); + }); } protected override void Validate(IndexerDefinition definition, bool includeWarnings) @@ -57,6 +69,7 @@ namespace Prowlarr.Api.V1.Indexers } var indexerInstance = _indexerFactory.GetInstance(indexer); + var serverUrl = Request.GetServerUrl(); switch (requestType) { @@ -69,12 +82,61 @@ namespace Prowlarr.Api.V1.Indexers case "music": case "book": case "movie": - Response searchResponse = _nzbSearchService.Search(request, new List { indexer.Id }, false).ToXml(indexerInstance.Protocol); + var results = _nzbSearchService.Search(request, new List { indexer.Id }, false); + + foreach (var result in results.Releases) + { + result.DownloadUrl = _downloadMappingService.ConvertToProxyLink(new Uri(result.DownloadUrl), serverUrl, indexer.Id, result.Title).ToString(); + } + + Response searchResponse = results.ToXml(indexerInstance.Protocol); searchResponse.ContentType = "application/rss+xml"; return searchResponse; default: throw new BadRequestException("Function Not Available"); } } + + private object GetDownload(int id) + { + var indexer = _indexerFactory.Get(id); + var link = Request.Query.Link; + var file = Request.Query.File; + + if (!link.HasValue || !file.HasValue) + { + throw new BadRequestException("Invalid Prowlarr link"); + } + + if (indexer == null) + { + throw new NotFoundException("Indexer Not Found"); + } + + var indexerInstance = _indexerFactory.GetInstance(indexer); + + var downloadBytes = Array.Empty(); + downloadBytes = _downloadService.DownloadReport(_downloadMappingService.ConvertToNormalLink(link), id); + + // handle magnet URLs + if (downloadBytes.Length >= 7 + && downloadBytes[0] == 0x6d + && downloadBytes[1] == 0x61 + && downloadBytes[2] == 0x67 + && downloadBytes[3] == 0x6e + && downloadBytes[4] == 0x65 + && downloadBytes[5] == 0x74 + && downloadBytes[6] == 0x3a) + { + var magnetUrl = Encoding.UTF8.GetString(downloadBytes); + return Response.AsRedirect(magnetUrl); + } + + var contentType = indexer.Protocol == DownloadProtocol.Torrent ? "application/x-bittorrent" : "application/x-nzb"; + var extension = indexer.Protocol == DownloadProtocol.Torrent ? "torrent" : "nzb"; + var filename = $"{file}.{extension}"; + + return Response.FromByteArray(downloadBytes, contentType).AsAttachment(filename, contentType); + } } } diff --git a/src/Prowlarr.Http/Extensions/RequestExtensions.cs b/src/Prowlarr.Http/Extensions/RequestExtensions.cs index 84dbe9bd0..baf76f19c 100644 --- a/src/Prowlarr.Http/Extensions/RequestExtensions.cs +++ b/src/Prowlarr.Http/Extensions/RequestExtensions.cs @@ -124,5 +124,34 @@ namespace Prowlarr.Http.Extensions return remoteAddress; } + + public static string GetServerUrl(this Request request) + { + var scheme = request.Url.Scheme; + var port = request.Url.Port; + + // Check for protocol headers added by reverse proxys + // X-Forwarded-Proto: A de facto standard for identifying the originating protocol of an HTTP request + var xForwardedProto = request.Headers.Where(x => x.Key == "X-Forwarded-Proto").Select(x => x.Value).FirstOrDefault(); + + if (xForwardedProto != null) + { + scheme = xForwardedProto.First(); + } + + // Front-End-Https: Non-standard header field used by Microsoft applications and load-balancers + else if (request.Headers.Where(x => x.Key == "Front-End-Https" && x.Value.FirstOrDefault() == "on").Any()) + { + scheme = "https"; + } + + //default to 443 if the Host header doesn't contain the port (needed for reverse proxy setups) + if (scheme == "https" && !request.Url.HostName.Contains(":")) + { + port = 443; + } + + return $"{scheme}://{request.Url.HostName}:{port}"; + } } } diff --git a/src/Prowlarr.Http/Extensions/ResponseExtensions.cs b/src/Prowlarr.Http/Extensions/ResponseExtensions.cs new file mode 100644 index 000000000..bc3eb3e98 --- /dev/null +++ b/src/Prowlarr.Http/Extensions/ResponseExtensions.cs @@ -0,0 +1,29 @@ +using System.IO; +using Nancy; + +namespace NzbDrone.Http.Extensions +{ + public static class ResponseExtensions + { + public static Response FromByteArray(this IResponseFormatter formatter, byte[] body, string contentType = null) + { + return new ByteArrayResponse(body, contentType); + } + } + + public class ByteArrayResponse : Response + { + public ByteArrayResponse(byte[] body, string contentType = null) + { + this.ContentType = contentType ?? "application/octet-stream"; + + this.Contents = stream => + { + using (var writer = new BinaryWriter(stream)) + { + writer.Write(body); + } + }; + } + } +}