shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-09-17 17:14:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

New: Loads of Backend Updates to Clients and Indexers

Browse Source
This commit is contained in:
Qstick
2019-06-13 23:54:25 -04:00
parent c48838e5b6
commit 8a9e2dc90d
345 changed files with 5859 additions and 2669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@@ -17,7 +17,7 @@ using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.Indexers
{
public abstract class HttpIndexerBase<TSettings> : IndexerBase<TSettings>
where TSettings : IProviderConfig, new()
where TSettings : IIndexerSettings, new()
{
protected const int MaxNumResultsPerQuery = 1000;
@@ -45,8 +45,8 @@ namespace NzbDrone.Core.Indexers
{
return new List<ReleaseInfo>();
}
return FetchReleases(GetRequestChain(), true);
return FetchReleases(g => g.GetRecentRequests(), true);
}
public override IList<ReleaseInfo> Fetch(MovieSearchCriteria searchCriteria)
@@ -56,7 +56,7 @@ namespace NzbDrone.Core.Indexers
return new List<ReleaseInfo>();
}
return FetchReleases(GetRequestChain(searchCriteria));
return FetchReleases(g => g.GetSearchRequests(searchCriteria));
}
protected IndexerPageableRequestChain GetRequestChain(SearchCriteriaBase searchCriteria = null)
@@ -88,19 +88,22 @@ namespace NzbDrone.Core.Indexers
}
protected virtual IList<ReleaseInfo> FetchReleases(IndexerPageableRequestChain pageableRequestChain, bool isRecent = false)
protected virtual IList<ReleaseInfo> FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
{
var releases = new List<ReleaseInfo>();
var url = string.Empty;
var parser = GetParser();
parser.CookiesUpdater = (cookies, expiration) =>
{
_indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration);
};
try
{
var generator = GetRequestGenerator();
var parser = GetParser();
parser.CookiesUpdater = (cookies, expiration) =>
{
_indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration);
};
var pageableRequestChain = pageableRequestChainSelector(generator);
var fullyUpdated = false;
ReleaseInfo lastReleaseInfo = null;
if (isRecent)
@@ -156,7 +159,7 @@ namespace NzbDrone.Core.Indexers
}
}
releases.AddRange(pagedReleases);
releases.AddRange(pagedReleases.Where(IsValidRelease));
}
if (releases.Any())
@@ -203,18 +206,22 @@ namespace NzbDrone.Core.Indexers
_logger.Warn("{0} {1} {2}", this, url, webException.Message);
}
}
catch (HttpException httpException)
catch (TooManyRequestsException ex)
{
if ((int)httpException.Response.StatusCode == 429)
if (ex.RetryAfter != TimeSpan.Zero)
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
_logger.Warn("API Request Limit reached for {0}", this);
_indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, httpException.Message);
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (HttpException ex)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
}
catch (RequestLimitReachedException)
{
@@ -229,6 +236,7 @@ namespace NzbDrone.Core.Indexers
catch (CloudFlareCaptchaException ex)
{
_indexerStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
if (ex.IsExpired)
{
_logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this);
@@ -241,19 +249,28 @@ namespace NzbDrone.Core.Indexers
catch (IndexerException ex)
{
_indexerStatusService.RecordFailure(Definition.Id);
var message = string.Format("{0} - {1}", ex.Message, url);
_logger.Warn(ex, message);
_logger.Warn(ex, "{0}", url);
}
catch (Exception feedEx)
catch (Exception ex)
{
_indexerStatusService.RecordFailure(Definition.Id);
feedEx.Data.Add("FeedUrl", url);
_logger.Error(feedEx, "An error occurred while processing feed. " + url);
ex.WithData("FeedUrl", url);
_logger.Error(ex, "An error occurred while processing feed. {0}", url);
}
return CleanupReleases(releases);
}
protected virtual bool IsValidRelease(ReleaseInfo release)
{
if (release.DownloadUrl.IsNullOrWhiteSpace())
{
return false;
}
return true;
}
protected virtual bool IsFullPage(IList<ReleaseInfo> page)
{
return PageSize != 0 && page.Count >= PageSize;
@@ -263,7 +280,16 @@ namespace NzbDrone.Core.Indexers
{
var response = FetchIndexerResponse(request);
return parser.ParseResponse(response).ToList();
try
{
return parser.ParseResponse(response).ToList();
}
catch (Exception ex)
{
ex.WithData(response.HttpResponse, 128*1024);
_logger.Trace("Unexpected Response content ({0} bytes): {1}", response.HttpResponse.ResponseData.Length, response.HttpResponse.Content);
throw;
}
}
protected virtual IndexerResponse FetchIndexerResponse(IndexerRequest request)
@@ -294,12 +320,12 @@ namespace NzbDrone.Core.Indexers
{
_indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration);
};
var releases = FetchPage(GetRequestChain().GetAllTiers().First().First(), parser);
var generator = GetRequestGenerator();
var releases = FetchPage(generator.GetRecentRequests().GetAllTiers().First().First(), parser);
if (releases.Empty())
{
return new ValidationFailure(string.Empty,
"No results were returned from your indexer, please check your settings.");
return new ValidationFailure(string.Empty, "Query successful, but no results were returned from your indexer. This may be an issue with the indexer or your indexer category settings.");
}
}
catch (ApiKeyException)
@@ -320,8 +346,7 @@ namespace NzbDrone.Core.Indexers
}
else
{
return new ValidationFailure("CaptchaToken",
"Site protected by CloudFlare CAPTCHA. Valid CAPTCHA token required.");
return new ValidationFailure("CaptchaToken", "Site protected by CloudFlare CAPTCHA. Valid CAPTCHA token required.");
}
}
catch (UnsupportedFeedException ex)