shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-09-17 17:14:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: (Indexers) Rate limit for download and auth

Browse Source
This commit is contained in:
Bogdan
2023-02-19 11:55:50 +02:00
parent 04276eb587
commit 99bc56efb6
10 changed files with 222 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/NzbDrone.Core/Download/DownloadService.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.EnsureThat;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Instrumentation.Extensions;
@@ -61,14 +60,6 @@ namespace NzbDrone.Core.Download
// Get the seed configuration for this release.
// remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
// Limit grabs to 2 per second.
if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
{
var url = new HttpUri(release.DownloadUrl);
_rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
}
var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
string downloadClientId;

58
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
View File

@@ -1,15 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
@@ -53,7 +50,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
new CardigannRequestGenerator(_configService,
_definitionService.GetCachedDefinition(Settings.DefinitionFile),
_logger)
_logger,
RateLimit)
{
HttpClient = _httpClient,
Definition = Definition,
@@ -180,63 +178,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
await generator.DoLogin();
}
public override async Task<byte[]> Download(Uri link)
protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
{
var generator = (CardigannRequestGenerator)GetRequestGenerator();
var request = await generator.DownloadRequest(link);
if (request.Url.Scheme == "magnet")
{
ValidateMagnet(request.Url.FullUri);
return Encoding.UTF8.GetBytes(request.Url.FullUri);
}
request.AllowAutoRedirect = true;
var downloadBytes = Array.Empty<byte>();
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
downloadBytes = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
ValidateTorrent(downloadBytes);
return downloadBytes;
return request;
}
protected override async Task Test(List<ValidationFailure> failures)

125
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
View File

@@ -4,7 +4,6 @@ using System.Collections.Specialized;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
@@ -29,11 +28,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
protected IHtmlDocument landingResultDocument;
protected override string SiteLink => ResolveSiteLink();
private readonly TimeSpan _rateLimit;
public CardigannRequestGenerator(IConfigService configService,
CardigannDefinition definition,
Logger logger)
Logger logger,
TimeSpan rateLimit)
: base(configService, definition, logger)
{
_rateLimit = rateLimit;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
@@ -218,9 +221,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.AddFormParameter(pair.Key, pair.Value);
}
requestBuilder.Headers.Add("Referer", SiteLink);
var request = requestBuilder
.SetHeader("Referer", SiteLink)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
Cookies = response.GetCookies();
@@ -356,11 +362,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
Encoding = _encoding
};
requestBuilder.SetCookies(Cookies);
var request = requestBuilder
.SetCookies(Cookies)
.SetHeader("Referer", loginUrl)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
requestBuilder.Headers.Add("Referer", loginUrl);
var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
@@ -424,10 +432,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
requestBuilder.SetCookies(Cookies);
foreach (var pair in pairs)
{
requestBuilder.AddFormParameter(pair.Key, pair.Value);
@@ -438,7 +442,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.SetHeader(header.Key, header.Value);
}
var request = requestBuilder.Build();
var request = requestBuilder
.SetCookies(Cookies)
.SetHeader("Referer", SiteLink)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
request.SetContent(body);
loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -454,15 +463,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
Encoding = _encoding
};
requestBuilder.SetCookies(Cookies);
requestBuilder.Headers.Add("Referer", loginUrl);
foreach (var pair in pairs)
{
requestBuilder.AddFormParameter(pair.Key, pair.Value);
}
loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
var request = requestBuilder
.SetCookies(Cookies)
.SetHeader("Referer", loginUrl)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
}
Cookies = loginResult.GetCookies();
@@ -496,9 +508,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
var request = requestBuilder
.SetHeader("Referer", SiteLink)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
Cookies = response.GetCookies();
@@ -521,9 +536,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
var request = requestBuilder
.SetHeader("Referer", SiteLink)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
Cookies = response.GetCookies();
@@ -594,14 +612,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
var request = requestBuilder
.SetHeader("Referer", SiteLink)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -646,6 +665,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
.SetCookies(landingResult.GetCookies())
.SetHeader("Referer", loginUrl.AbsoluteUri)
.SetEncoding(_encoding)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -656,10 +676,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
ImageData = response.ResponseData
};
}
else
{
_logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
}
_logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
}
else
{
@@ -724,23 +742,28 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestLinkStr += queryCollection.GetQueryString(_encoding, separator: request.Queryseparator);
}
var httpRequest = new HttpRequestBuilder(requestLinkStr)
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetEncoding(_encoding)
.SetHeader("Referer", referer);
httpRequest.Method = method;
var httpRequestBuilder = new HttpRequestBuilder(requestLinkStr)
{
Method = method,
Encoding = _encoding
};
// Add form data for POST requests
if (method == HttpMethod.Post)
{
foreach (var param in pairs)
{
httpRequest.AddFormParameter(param.Key, param.Value);
httpRequestBuilder.AddFormParameter(param.Key, param.Value);
}
}
var response = await HttpClient.ExecuteProxiedAsync(httpRequest.Build(), Definition);
var httpRequest = httpRequestBuilder
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeader("Referer", referer)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
var response = await HttpClient.ExecuteProxiedAsync(httpRequest, Definition);
_logger.Debug("CardigannIndexer ({0}): handleRequest() remote server returned {1}", _definition.Id, response.StatusCode);
return response;
@@ -766,6 +789,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.SetEncoding(_encoding)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
request.AllowAutoRedirect = true;
@@ -856,6 +880,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.SetEncoding(_encoding)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
@@ -875,6 +900,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.SetEncoding(_encoding)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
selectorDownloadRequest.Method = method;
@@ -895,6 +921,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.SetEncoding(_encoding)
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
downloadRequest.Method = method;
@@ -1096,16 +1123,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
_logger.Info($"Adding request: {searchUrl}");
var requestbuilder = new HttpRequestBuilder(searchUrl);
requestbuilder.Method = method;
var requestBuilder = new HttpRequestBuilder(searchUrl)
{
Method = method,
Encoding = _encoding
};
// Add FormData for searchs that POST
if (method == HttpMethod.Post)
{
foreach (var param in queryCollection)
{
requestbuilder.AddFormParameter(param.Key, param.Value);
requestBuilder.AddFormParameter(param.Key, param.Value);
}
}
@@ -1113,14 +1142,22 @@ namespace NzbDrone.Core.Indexers.Cardigann
if (search.Headers != null)
{
var headers = ParseCustomHeaders(search.Headers, variables);
requestbuilder.SetHeaders(headers ?? new Dictionary<string, string>());
requestBuilder.SetHeaders(headers ?? new Dictionary<string, string>());
}
var request = new CardigannRequest(requestbuilder.SetEncoding(_encoding).Build(), variables, searchPath);
var request = requestBuilder
.WithRateLimit(_rateLimit.TotalSeconds)
.Build();
request.HttpRequest.AllowAutoRedirect = searchPath.Followredirect;
var cardigannRequest = new CardigannRequest(request, variables, searchPath)
{
HttpRequest =
{
AllowAutoRedirect = searchPath.Followredirect
}
};
yield return request;
yield return cardigannRequest;
}
}
}

13
src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs
View File

@@ -46,12 +46,17 @@ namespace NzbDrone.Core.Indexers.Definitions
return new NebulanceParser(Settings);
}
public override async Task<byte[]> Download(Uri link)
protected override Task<HttpRequest> GetDownloadRequest(Uri link)
{
// Invalidate cookies before downloading to prevent redirect to login page.
UpdateCookies(null, null);
// Avoid using cookies to prevent redirects to login page
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
{
AllowAutoRedirect = FollowRedirect
};
return await base.Download(link);
var request = requestBuilder.Build();
return Task.FromResult(request);
}
private IndexerCapabilities SetCapabilities()

2
src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs
View File

@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions
_logger.Error("Download failed");
}
ValidateTorrent(downloadBytes);
ValidateDownloadData(downloadBytes);
return downloadBytes;
}

52
src/NzbDrone.Core/Indexers/Definitions/Redacted.cs
View File

@@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions
return new RedactedParser(Settings, Capabilities.Categories);
}
protected override Task<HttpRequest> GetDownloadRequest(Uri link)
{
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
{
AllowAutoRedirect = FollowRedirect
};
var request = requestBuilder
.SetHeader("Authorization", Settings.Apikey)
.Build();
return Task.FromResult(request);
}
private IndexerCapabilities SetCapabilities()
{
var caps = new IndexerCapabilities
@@ -74,30 +88,6 @@ namespace NzbDrone.Core.Indexers.Definitions
return caps;
}
public override async Task<byte[]> Download(Uri link)
{
var request = new HttpRequestBuilder(link.AbsoluteUri)
.SetHeader("Authorization", Settings.Apikey)
.Build();
var downloadBytes = Array.Empty<byte>();
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
downloadBytes = response.ResponseData;
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Download failed");
}
ValidateTorrent(downloadBytes);
return downloadBytes;
}
}
public class RedactedRequestGenerator : IIndexerRequestGenerator
@@ -188,18 +178,12 @@ namespace NzbDrone.Core.Indexers.Definitions
queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
}
var request = RequestBuilder()
.Resource($"/ajax.php?{parameters.GetQueryString()}")
.Build();
var searchUrl = _settings.BaseUrl.TrimEnd('/') + $"/ajax.php?{parameters.GetQueryString()}";
yield return new IndexerRequest(request);
}
var request = new IndexerRequest(searchUrl, HttpAccept.Json);
request.HttpRequest.Headers.Set("Authorization", _settings.Apikey);
private HttpRequestBuilder RequestBuilder()
{
return new HttpRequestBuilder($"{_settings.BaseUrl.TrimEnd('/')}")
.Accept(HttpAccept.Json)
.SetHeader("Authorization", _settings.Apikey);
yield return request;
}
}

70
src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs
View File

@@ -15,7 +15,6 @@ using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Indexers.Settings;
using NzbDrone.Core.IndexerSearch.Definitions;
@@ -104,70 +103,18 @@ namespace NzbDrone.Core.Indexers.Definitions
request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
}
public override async Task<byte[]> Download(Uri link)
protected override Task<HttpRequest> GetDownloadRequest(Uri link)
{
Cookies = GetCookies();
if (link.Scheme == "magnet")
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
{
ValidateMagnet(link.OriginalString);
AllowAutoRedirect = FollowRedirect
};
return Encoding.UTF8.GetBytes(link.OriginalString);
}
var request = requestBuilder
.SetHeader("Authorization", $"Bearer {Settings.ApiKey}")
.Build();
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
byte[] torrentData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
torrentData = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
ValidateTorrent(torrentData);
return torrentData;
return Task.FromResult(request);
}
protected virtual IndexerCapabilities SetCapabilities()
@@ -276,7 +223,6 @@ namespace NzbDrone.Core.Indexers.Definitions
var searchUrl = _settings.BaseUrl + "api/torrent?" + parameters.GetQueryString(duplicateKeysIfMulti: true);
var request = new IndexerRequest(searchUrl, HttpAccept.Json);
request.HttpRequest.Headers.Set("Authorization", $"Bearer {_settings.ApiKey}");
yield return request;

96
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@@ -6,10 +6,12 @@ using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using FluentValidation.Results;
using MonoTorrent;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.Indexers.Events;
using NzbDrone.Core.Indexers.Exceptions;
@@ -101,6 +103,95 @@ namespace NzbDrone.Core.Indexers
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
var request = await GetDownloadRequest(link);
if (request.Url.Scheme == "magnet")
{
ValidateMagnet(request.Url.FullUri);
return Encoding.UTF8.GetBytes(request.Url.FullUri);
}
if (request.RateLimit < RateLimit)
{
request.RateLimit = RateLimit;
}
byte[] fileData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
fileData = response.ResponseData;
_logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Download failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Download failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Download failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Download failed");
throw;
}
ValidateDownloadData(fileData);
return fileData;
}
protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
{
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
{
AllowAutoRedirect = FollowRedirect
};
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
return Task.FromResult(request);
}
protected virtual void ValidateDownloadData(byte[] fileData)
{
}
protected void ValidateMagnet(string link)
{
MagnetLink.Parse(link);
}
protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
{
//A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.
@@ -420,6 +511,11 @@ namespace NzbDrone.Core.Indexers
request.RequestTimeout = TimeSpan.FromSeconds(15);
}
if (request.RateLimit < RateLimit)
{
request.RateLimit = RateLimit;
}
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
_eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime));

81
src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs
View File

@@ -1,12 +1,7 @@
using System;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using MonoTorrent;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers
@@ -19,85 +14,15 @@ namespace NzbDrone.Core.Indexers
{
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
if (link.Scheme == "magnet")
{
ValidateMagnet(link.OriginalString);
return Encoding.UTF8.GetBytes(link.OriginalString);
}
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
byte[] torrentData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
torrentData = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
ValidateTorrent(torrentData);
return torrentData;
}
protected void ValidateMagnet(string link)
{
MagnetLink.Parse(link);
}
protected void ValidateTorrent(byte[] torrentData)
protected override void ValidateDownloadData(byte[] fileData)
{
try
{
Torrent.Load(torrentData);
Torrent.Load(fileData);
}
catch
{
_logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(torrentData));
_logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(fileData));
throw;
}
}

64
src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs
View File

@@ -1,11 +1,6 @@
using System;
using System.Net;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Download;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers
@@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
_nzbValidationService = nzbValidationService;
}
public override async Task<byte[]> Download(Uri link)
protected override void ValidateDownloadData(byte[] fileData)
{
Cookies = GetCookies();
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
byte[] nzbData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
nzbData = response.ResponseData;
_logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading nzb failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading nzb failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading nzb failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading nzb failed");
throw;
}
_nzbValidationService.Validate(nzbData);
return nzbData;
_nzbValidationService.Validate(fileData);
}
}
}