diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index 99b66d2cc..d28b801c4 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -1,7 +1,6 @@ using System; using System.Threading.Tasks; using NLog; -using NzbDrone.Common.EnsureThat; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation.Extensions; @@ -61,14 +60,6 @@ namespace NzbDrone.Core.Download // Get the seed configuration for this release. // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie); - - // Limit grabs to 2 per second. - if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:")) - { - var url = new HttpUri(release.DownloadUrl); - _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2)); - } - var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId)); string downloadClientId; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 69540cf3e..5edfd8b5f 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -1,15 +1,12 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; -using System.Text; using System.Threading.Tasks; using FluentValidation.Results; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; @@ -53,7 +50,8 @@ namespace NzbDrone.Core.Indexers.Cardigann var generator = _generatorCache.Get(Settings.DefinitionFile, () => new CardigannRequestGenerator(_configService, _definitionService.GetCachedDefinition(Settings.DefinitionFile), - _logger) + _logger, + RateLimit) { HttpClient = _httpClient, Definition = Definition, @@ -180,63 +178,15 @@ namespace NzbDrone.Core.Indexers.Cardigann await generator.DoLogin(); } - public override async Task Download(Uri link) + protected override async Task GetDownloadRequest(Uri link) { var generator = (CardigannRequestGenerator)GetRequestGenerator(); var request = await generator.DownloadRequest(link); - if (request.Url.Scheme == "magnet") - { - ValidateMagnet(request.Url.FullUri); - - return Encoding.UTF8.GetBytes(request.Url.FullUri); - } - request.AllowAutoRedirect = true; - var downloadBytes = Array.Empty(); - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri); - throw new ReleaseUnavailableException("Downloading torrent failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri); - } - else - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); - } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading torrent failed"); - throw; - } - - ValidateTorrent(downloadBytes); - - return downloadBytes; + return request; } protected override async Task Test(List failures) diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 740b45ab1..0742c98aa 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -4,7 +4,6 @@ using System.Collections.Specialized; using System.Linq; using System.Net; using System.Net.Http; -using System.Text; using System.Threading.Tasks; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; @@ -29,11 +28,15 @@ namespace NzbDrone.Core.Indexers.Cardigann protected IHtmlDocument landingResultDocument; protected override string SiteLink => ResolveSiteLink(); + private readonly TimeSpan _rateLimit; + public CardigannRequestGenerator(IConfigService configService, CardigannDefinition definition, - Logger logger) + Logger logger, + TimeSpan rateLimit) : base(configService, definition, logger) { + _rateLimit = rateLimit; } public Func> GetCookies { get; set; } @@ -218,9 +221,12 @@ namespace NzbDrone.Core.Indexers.Cardigann requestBuilder.AddFormParameter(pair.Key, pair.Value); } - requestBuilder.Headers.Add("Referer", SiteLink); + var request = requestBuilder + .SetHeader("Referer", SiteLink) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); - var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); + var response = await HttpClient.ExecuteProxiedAsync(request, Definition); Cookies = response.GetCookies(); @@ -356,11 +362,13 @@ namespace NzbDrone.Core.Indexers.Cardigann Encoding = _encoding }; - requestBuilder.SetCookies(Cookies); + var request = requestBuilder + .SetCookies(Cookies) + .SetHeader("Referer", loginUrl) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); - requestBuilder.Headers.Add("Referer", loginUrl); - - var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); + var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(request, Definition); var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content); var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString(); @@ -424,10 +432,6 @@ namespace NzbDrone.Core.Indexers.Cardigann Encoding = _encoding }; - requestBuilder.Headers.Add("Referer", SiteLink); - - requestBuilder.SetCookies(Cookies); - foreach (var pair in pairs) { requestBuilder.AddFormParameter(pair.Key, pair.Value); @@ -438,7 +442,12 @@ namespace NzbDrone.Core.Indexers.Cardigann requestBuilder.SetHeader(header.Key, header.Value); } - var request = requestBuilder.Build(); + var request = requestBuilder + .SetCookies(Cookies) + .SetHeader("Referer", SiteLink) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); + request.SetContent(body); loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition); @@ -454,15 +463,18 @@ namespace NzbDrone.Core.Indexers.Cardigann Encoding = _encoding }; - requestBuilder.SetCookies(Cookies); - requestBuilder.Headers.Add("Referer", loginUrl); - foreach (var pair in pairs) { requestBuilder.AddFormParameter(pair.Key, pair.Value); } - loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); + var request = requestBuilder + .SetCookies(Cookies) + .SetHeader("Referer", loginUrl) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); + + loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition); } Cookies = loginResult.GetCookies(); @@ -496,9 +508,12 @@ namespace NzbDrone.Core.Indexers.Cardigann Encoding = _encoding }; - requestBuilder.Headers.Add("Referer", SiteLink); + var request = requestBuilder + .SetHeader("Referer", SiteLink) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); - var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); + var response = await HttpClient.ExecuteProxiedAsync(request, Definition); Cookies = response.GetCookies(); @@ -521,9 +536,12 @@ namespace NzbDrone.Core.Indexers.Cardigann Encoding = _encoding }; - requestBuilder.Headers.Add("Referer", SiteLink); + var request = requestBuilder + .SetHeader("Referer", SiteLink) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); - var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); + var response = await HttpClient.ExecuteProxiedAsync(request, Definition); Cookies = response.GetCookies(); @@ -594,14 +612,15 @@ namespace NzbDrone.Core.Indexers.Cardigann Encoding = _encoding }; - requestBuilder.Headers.Add("Referer", SiteLink); - if (Cookies != null) { requestBuilder.SetCookies(Cookies); } - var request = requestBuilder.Build(); + var request = requestBuilder + .SetHeader("Referer", SiteLink) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition); @@ -646,6 +665,7 @@ namespace NzbDrone.Core.Indexers.Cardigann .SetCookies(landingResult.GetCookies()) .SetHeader("Referer", loginUrl.AbsoluteUri) .SetEncoding(_encoding) + .WithRateLimit(_rateLimit.TotalSeconds) .Build(); var response = await HttpClient.ExecuteProxiedAsync(request, Definition); @@ -656,10 +676,8 @@ namespace NzbDrone.Core.Indexers.Cardigann ImageData = response.ResponseData }; } - else - { - _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id); - } + + _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id); } else { @@ -724,23 +742,28 @@ namespace NzbDrone.Core.Indexers.Cardigann requestLinkStr += queryCollection.GetQueryString(_encoding, separator: request.Queryseparator); } - var httpRequest = new HttpRequestBuilder(requestLinkStr) - .SetCookies(Cookies ?? new Dictionary()) - .SetEncoding(_encoding) - .SetHeader("Referer", referer); - - httpRequest.Method = method; + var httpRequestBuilder = new HttpRequestBuilder(requestLinkStr) + { + Method = method, + Encoding = _encoding + }; // Add form data for POST requests if (method == HttpMethod.Post) { foreach (var param in pairs) { - httpRequest.AddFormParameter(param.Key, param.Value); + httpRequestBuilder.AddFormParameter(param.Key, param.Value); } } - var response = await HttpClient.ExecuteProxiedAsync(httpRequest.Build(), Definition); + var httpRequest = httpRequestBuilder + .SetCookies(Cookies ?? new Dictionary()) + .SetHeader("Referer", referer) + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); + + var response = await HttpClient.ExecuteProxiedAsync(httpRequest, Definition); _logger.Debug("CardigannIndexer ({0}): handleRequest() remote server returned {1}", _definition.Id, response.StatusCode); return response; @@ -766,6 +789,7 @@ namespace NzbDrone.Core.Indexers.Cardigann .SetCookies(Cookies ?? new Dictionary()) .SetHeaders(headers ?? new Dictionary()) .SetEncoding(_encoding) + .WithRateLimit(_rateLimit.TotalSeconds) .Build(); request.AllowAutoRedirect = true; @@ -856,6 +880,7 @@ namespace NzbDrone.Core.Indexers.Cardigann .SetCookies(Cookies ?? new Dictionary()) .SetHeaders(headers ?? new Dictionary()) .SetEncoding(_encoding) + .WithRateLimit(_rateLimit.TotalSeconds) .Build(); response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition); @@ -875,6 +900,7 @@ namespace NzbDrone.Core.Indexers.Cardigann .SetCookies(Cookies ?? new Dictionary()) .SetHeaders(headers ?? new Dictionary()) .SetEncoding(_encoding) + .WithRateLimit(_rateLimit.TotalSeconds) .Build(); selectorDownloadRequest.Method = method; @@ -895,6 +921,7 @@ namespace NzbDrone.Core.Indexers.Cardigann .SetCookies(Cookies ?? new Dictionary()) .SetHeaders(headers ?? new Dictionary()) .SetEncoding(_encoding) + .WithRateLimit(_rateLimit.TotalSeconds) .Build(); downloadRequest.Method = method; @@ -1096,16 +1123,18 @@ namespace NzbDrone.Core.Indexers.Cardigann _logger.Info($"Adding request: {searchUrl}"); - var requestbuilder = new HttpRequestBuilder(searchUrl); - - requestbuilder.Method = method; + var requestBuilder = new HttpRequestBuilder(searchUrl) + { + Method = method, + Encoding = _encoding + }; // Add FormData for searchs that POST if (method == HttpMethod.Post) { foreach (var param in queryCollection) { - requestbuilder.AddFormParameter(param.Key, param.Value); + requestBuilder.AddFormParameter(param.Key, param.Value); } } @@ -1113,14 +1142,22 @@ namespace NzbDrone.Core.Indexers.Cardigann if (search.Headers != null) { var headers = ParseCustomHeaders(search.Headers, variables); - requestbuilder.SetHeaders(headers ?? new Dictionary()); + requestBuilder.SetHeaders(headers ?? new Dictionary()); } - var request = new CardigannRequest(requestbuilder.SetEncoding(_encoding).Build(), variables, searchPath); + var request = requestBuilder + .WithRateLimit(_rateLimit.TotalSeconds) + .Build(); - request.HttpRequest.AllowAutoRedirect = searchPath.Followredirect; + var cardigannRequest = new CardigannRequest(request, variables, searchPath) + { + HttpRequest = + { + AllowAutoRedirect = searchPath.Followredirect + } + }; - yield return request; + yield return cardigannRequest; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs b/src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs index efa3ff1a7..2750211b5 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs @@ -46,12 +46,17 @@ namespace NzbDrone.Core.Indexers.Definitions return new NebulanceParser(Settings); } - public override async Task Download(Uri link) + protected override Task GetDownloadRequest(Uri link) { - // Invalidate cookies before downloading to prevent redirect to login page. - UpdateCookies(null, null); + // Avoid using cookies to prevent redirects to login page + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri) + { + AllowAutoRedirect = FollowRedirect + }; - return await base.Download(link); + var request = requestBuilder.Build(); + + return Task.FromResult(request); } private IndexerCapabilities SetCapabilities() diff --git a/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs b/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs index fc69df97c..ecb9108f1 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs @@ -112,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions _logger.Error("Download failed"); } - ValidateTorrent(downloadBytes); + ValidateDownloadData(downloadBytes); return downloadBytes; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs index 2460c7f3d..e8fba9c96 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs @@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions return new RedactedParser(Settings, Capabilities.Categories); } + protected override Task GetDownloadRequest(Uri link) + { + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri) + { + AllowAutoRedirect = FollowRedirect + }; + + var request = requestBuilder + .SetHeader("Authorization", Settings.Apikey) + .Build(); + + return Task.FromResult(request); + } + private IndexerCapabilities SetCapabilities() { var caps = new IndexerCapabilities @@ -74,30 +88,6 @@ namespace NzbDrone.Core.Indexers.Definitions return caps; } - - public override async Task Download(Uri link) - { - var request = new HttpRequestBuilder(link.AbsoluteUri) - .SetHeader("Authorization", Settings.Apikey) - .Build(); - - var downloadBytes = Array.Empty(); - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Download failed"); - } - - ValidateTorrent(downloadBytes); - - return downloadBytes; - } } public class RedactedRequestGenerator : IIndexerRequestGenerator @@ -188,18 +178,12 @@ namespace NzbDrone.Core.Indexers.Definitions queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1")); } - var request = RequestBuilder() - .Resource($"/ajax.php?{parameters.GetQueryString()}") - .Build(); + var searchUrl = _settings.BaseUrl.TrimEnd('/') + $"/ajax.php?{parameters.GetQueryString()}"; - yield return new IndexerRequest(request); - } + var request = new IndexerRequest(searchUrl, HttpAccept.Json); + request.HttpRequest.Headers.Set("Authorization", _settings.Apikey); - private HttpRequestBuilder RequestBuilder() - { - return new HttpRequestBuilder($"{_settings.BaseUrl.TrimEnd('/')}") - .Accept(HttpAccept.Json) - .SetHeader("Authorization", _settings.Apikey); + yield return request; } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs b/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs index b7ffde0d6..91f4aa0c5 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs @@ -15,7 +15,6 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Annotations; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Indexers.Settings; using NzbDrone.Core.IndexerSearch.Definitions; @@ -104,70 +103,18 @@ namespace NzbDrone.Core.Indexers.Definitions request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}"); } - public override async Task Download(Uri link) + protected override Task GetDownloadRequest(Uri link) { - Cookies = GetCookies(); - - if (link.Scheme == "magnet") + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri) { - ValidateMagnet(link.OriginalString); + AllowAutoRedirect = FollowRedirect + }; - return Encoding.UTF8.GetBytes(link.OriginalString); - } + var request = requestBuilder + .SetHeader("Authorization", $"Bearer {Settings.ApiKey}") + .Build(); - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - - if (Cookies != null) - { - requestBuilder.SetCookies(Cookies); - } - - var request = requestBuilder.Build(); - request.AllowAutoRedirect = FollowRedirect; - request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}"); - - byte[] torrentData; - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - torrentData = response.ResponseData; - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); - throw new ReleaseUnavailableException("Downloading torrent failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); - } - else - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading torrent failed"); - throw; - } - - ValidateTorrent(torrentData); - - return torrentData; + return Task.FromResult(request); } protected virtual IndexerCapabilities SetCapabilities() @@ -276,7 +223,6 @@ namespace NzbDrone.Core.Indexers.Definitions var searchUrl = _settings.BaseUrl + "api/torrent?" + parameters.GetQueryString(duplicateKeysIfMulti: true); var request = new IndexerRequest(searchUrl, HttpAccept.Json); - request.HttpRequest.Headers.Set("Authorization", $"Bearer {_settings.ApiKey}"); yield return request; diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index dde3112b2..e1d61c2b6 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -6,10 +6,12 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; using FluentValidation.Results; +using MonoTorrent; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Http.CloudFlare; using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.Indexers.Exceptions; @@ -101,6 +103,95 @@ namespace NzbDrone.Core.Indexers return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } + public override async Task Download(Uri link) + { + Cookies = GetCookies(); + + var request = await GetDownloadRequest(link); + + if (request.Url.Scheme == "magnet") + { + ValidateMagnet(request.Url.FullUri); + return Encoding.UTF8.GetBytes(request.Url.FullUri); + } + + if (request.RateLimit < RateLimit) + { + request.RateLimit = RateLimit; + } + + byte[] fileData; + + try + { + var response = await _httpClient.ExecuteProxiedAsync(request, Definition); + fileData = response.ResponseData; + + _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri); + } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri); + throw new ReleaseUnavailableException("Download failed", ex); + } + + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) + { + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + } + else + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + } + + throw new ReleaseDownloadException("Download failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + + throw new ReleaseDownloadException("Download failed", ex); + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Download failed"); + throw; + } + + ValidateDownloadData(fileData); + + return fileData; + } + + protected virtual Task GetDownloadRequest(Uri link) + { + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri) + { + AllowAutoRedirect = FollowRedirect + }; + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + var request = requestBuilder.Build(); + + return Task.FromResult(request); + } + + protected virtual void ValidateDownloadData(byte[] fileData) + { + } + + protected void ValidateMagnet(string link) + { + MagnetLink.Parse(link); + } + protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator) { //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page. @@ -420,6 +511,11 @@ namespace NzbDrone.Core.Indexers request.RequestTimeout = TimeSpan.FromSeconds(15); } + if (request.RateLimit < RateLimit) + { + request.RateLimit = RateLimit; + } + var response = await _httpClient.ExecuteProxiedAsync(request, Definition); _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime)); diff --git a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs index dda66a713..aabf2c101 100644 --- a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs @@ -1,12 +1,7 @@ -using System; -using System.Net; using System.Text; -using System.Threading.Tasks; using MonoTorrent; using NLog; -using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers @@ -19,85 +14,15 @@ namespace NzbDrone.Core.Indexers { } - public override async Task Download(Uri link) - { - Cookies = GetCookies(); - - if (link.Scheme == "magnet") - { - ValidateMagnet(link.OriginalString); - - return Encoding.UTF8.GetBytes(link.OriginalString); - } - - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - - if (Cookies != null) - { - requestBuilder.SetCookies(Cookies); - } - - var request = requestBuilder.Build(); - request.AllowAutoRedirect = FollowRedirect; - - byte[] torrentData; - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - torrentData = response.ResponseData; - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); - throw new ReleaseUnavailableException("Downloading torrent failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); - } - else - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading torrent failed"); - throw; - } - - ValidateTorrent(torrentData); - - return torrentData; - } - - protected void ValidateMagnet(string link) - { - MagnetLink.Parse(link); - } - - protected void ValidateTorrent(byte[] torrentData) + protected override void ValidateDownloadData(byte[] fileData) { try { - Torrent.Load(torrentData); + Torrent.Load(fileData); } catch { - _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(torrentData)); + _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(fileData)); throw; } } diff --git a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs index f8104a844..b40d8fba8 100644 --- a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs @@ -1,11 +1,6 @@ -using System; -using System.Net; -using System.Threading.Tasks; using NLog; -using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers @@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers _nzbValidationService = nzbValidationService; } - public override async Task Download(Uri link) + protected override void ValidateDownloadData(byte[] fileData) { - Cookies = GetCookies(); - - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - - if (Cookies != null) - { - requestBuilder.SetCookies(Cookies); - } - - var request = requestBuilder.Build(); - request.AllowAutoRedirect = FollowRedirect; - - byte[] nzbData; - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - nzbData = response.ResponseData; - - _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri); - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri); - throw new ReleaseUnavailableException("Downloading nzb failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); - } - else - { - _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri); - } - - throw new ReleaseDownloadException("Downloading nzb failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri); - - throw new ReleaseDownloadException("Downloading nzb failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading nzb failed"); - throw; - } - - _nzbValidationService.Validate(nzbData); - - return nzbData; + _nzbValidationService.Validate(fileData); } } }